From 9d9a0b33f3ff814f985973389763709ef9aab183 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:11:57 -0600 Subject: [PATCH 01/21] Add batched GraphQL history collection --- docs/commands.md | 2 +- docs/sync.md | 40 +++ internal/cli/help.go | 5 +- internal/cli/sync.go | 4 + internal/github/history.go | 345 ++++++++++++++++++++++++ internal/github/history_test.go | 124 +++++++++ internal/syncer/graphql_history_test.go | 89 ++++++ internal/syncer/syncer.go | 64 ++++- 8 files changed, 669 insertions(+), 4 deletions(-) create mode 100644 internal/github/history.go create mode 100644 internal/github/history_test.go create mode 100644 internal/syncer/graphql_history_test.go diff --git a/docs/commands.md b/docs/commands.md index a11c7ef0..aa4f11ef 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -42,7 +42,7 @@ These work on every command. | Command | Purpose | Docs | | --- | --- | --- | -| `gitcrawl sync owner/repo [--state --since --numbers --limit --include-comments --include-pr-details --with pr-details --force --progress-file --json]` | Sync issues and PRs from GitHub into local SQLite | [Sync](/sync/) | +| `gitcrawl sync owner/repo [--state --since --numbers --limit --include-comments --include-pr-details --with pr-details --graphql-history --force --progress-file --json]` | Sync issues and PRs from GitHub into local SQLite | [Sync](/sync/) | | `gitcrawl sync-failures owner/repo [--include-resolved --limit N --json]` | List failed issue, comment, and PR hydration attempts and optional resolved history | [Sync](/sync/#hydration-depth) | | `gitcrawl coverage [owner/repo \| --repos owner/a,owner/b] [--min-missing-pr-details N --json]` | Report archive, PR-detail, and enrichment coverage/freshness | — | | `gitcrawl fill-pr-details owner/repo [--limit --order --batch-size --reserve-rate-limit --include-comments --json-progress --json]` | Hydrate locally missing pull request detail rows in bounded batches | — | diff --git a/docs/sync.md b/docs/sync.md index 2bc0d836..d54ed0d6 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -84,6 +84,46 @@ The same rule applies to `threads --numbers` and to `embed` or `summarize ## Hydration depth +### Batched GraphQL history + +```bash +gitcrawl sync owner/repo --numbers 123,456 --state all --include-comments --with pr-metadata --graphql-history +``` + +This opt-in profile fetches exact selections in GraphQL batches of at most 25 +parents. It fully paginates labels, assignees, discussion comments, reviews and +each review's inline comments, then uses the existing per-thread transactions, +observation ordering, failure resolution and document persistence. It performs +no REST requests or fallback. The regular sync path is unchanged. + +The profile requires the flags above; it rejects `--since`, `--limit` and full +PR-detail hydration. It does not collect files, commit bodies, checks or Actions +logs. An incomplete GraphQL response, unavailable parent, missing identity, +duplicate child or nonadvancing cursor fails the batch before archive writes. +Supervisors should retry failed selections in isolation. Empty reviews remain +retained; the ordinary empty-comment filtering contract is unchanged. + +Stored raw maps are explicitly labelled projections: `_gitcrawl_source` is +`graphql`, and `_graphql` retains the hydrated provider object. Existing thread +database IDs and legacy `github_id` values are preserved. New threads use the +opaque GraphQL node ID because a PR's GraphQL database ID is different from its +REST issue ID. Comments/reviews use exact `fullDatabaseId` strings; PR metadata +uses its PR database ID. Bot logins receive REST-compatible `[bot]` suffixes in +normalized fields while raw actor names remain intact. GraphQL review creation, +update and submission timestamps are retained. + +Provider differences remain visible: for example GraphQL can return a null head +repository when REST previously named one. No missing value is invented from +that earlier observation. Keep a consistent archive backup when migrating +transports; this mode does not claim every REST-only field is available through +GraphQL or reconstruct uncaptured historical edits. + +Each command probes GraphQL's own quota and preserves a 500-point floor. +Sanitized `graphql budget` and `graphql cost` log records expose request sequence, +conservative unanswered-request charge and actual response cost. External +multi-process supervisors must additionally preserve a shared point budget +across token renewals and restarts. REST request counts are not GraphQL costs. + | Flag | What it adds | | --- | --- | | `--include-comments` | Issue comments, PR review comments, reviews | diff --git a/internal/cli/help.go b/internal/cli/help.go index b84a1ca4..a9d9c4d1 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -142,12 +142,15 @@ Usage: "sync": `gitcrawl sync mirrors GitHub issue and pull request metadata. Usage: - gitcrawl sync owner/repo [--state open|closed|all] [--numbers refs] [--with pr-metadata|pr-details] [--include-comments] [--include-pr-details] [--force] [--json] + gitcrawl sync owner/repo [--state open|closed|all] [--numbers refs] [--with pr-metadata|pr-details] [--include-comments] [--include-pr-details] [--graphql-history] [--force] [--json] pr-metadata fetches only the pull request object; pr-details also hydrates files, commits, checks, workflows, and review threads. Comments are selected separately. Unchanged issue comments are reused; --force downloads them again. PR reviews and PR details are always fetched when selected. +--graphql-history batches exact numbers through GraphQL with no REST fallback. +It requires --state all --include-comments --with pr-metadata, and does not +support --since, --limit, or full pr-details hydration. `, "sync-failures": `gitcrawl sync-failures lists failed sync hydration attempts. diff --git a/internal/cli/sync.go b/internal/cli/sync.go index 9fed2d72..16a8b1d7 100644 --- a/internal/cli/sync.go +++ b/internal/cli/sync.go @@ -25,6 +25,7 @@ func (a *App) runSync(ctx context.Context, args []string) error { numbersRaw := fs.String("numbers", "", "comma-separated issue or pull request numbers") limitRaw := fs.String("limit", "", "maximum issue/PR rows") jsonOut := fs.Bool("json", false, "write JSON output") + graphqlHistory := fs.Bool("graphql-history", false, "batch exact conversation and PR metadata collection through GraphQL") includeComments := fs.Bool("include-comments", false, "hydrate issue comments, PR reviews, and PR review comments") force := fs.Bool("force", false, "download selected data even when issue comments are unchanged") includePRDetails := fs.Bool("include-pr-details", false, "hydrate PR files, commits, checks, and workflow runs") @@ -71,6 +72,7 @@ func (a *App) runSync(ctx context.Context, args []string) error { } stats, target, err := a.syncRepository(ctx, owner, repo, syncOptions{ + GraphQLHistory: *graphqlHistory, Since: strings.TrimSpace(*since), State: strings.TrimSpace(*state), Limit: limit, @@ -97,6 +99,7 @@ func (a *App) runSync(ctx context.Context, args []string) error { } type syncOptions struct { + GraphQLHistory bool Since string State string Limit int @@ -428,6 +431,7 @@ func (a *App) syncRepository(ctx context.Context, owner, repo string, options sy }) service := syncer.New(client, rt.Store) stats, err := service.Sync(ctx, syncer.Options{ + GraphQLHistory: options.GraphQLHistory, Owner: owner, Repo: repo, State: strings.TrimSpace(options.State), diff --git a/internal/github/history.go b/internal/github/history.go new file mode 100644 index 00000000..7175b125 --- /dev/null +++ b/internal/github/history.go @@ -0,0 +1,345 @@ +package github + +// The history API batches complete conversations through GraphQL. The maps +// consumed by syncer are explicit projections; _graphql retains the unmodified +// provider object and _gitcrawl_source identifies its transport. No REST fallback. +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +type HistoryItem struct { + Thread, Pull map[string]any + Comments, Reviews, ReviewComments []map[string]any +} +type HistoryBatch struct { + Repository map[string]any + Items []HistoryItem +} + +const historyActor = `author { login __typename url }` +const historyComment = `id __typename fullDatabaseId body ` + historyActor + ` authorAssociation createdAt updatedAt publishedAt url isMinimized minimizedReason` +const historyInline = historyComment + ` path diffHunk line startLine originalLine originalStartLine position originalPosition state subjectType outdated commit { oid } originalCommit { oid } replyTo { id fullDatabaseId } pullRequestReview { id fullDatabaseId }` + +var historyReview = historyComment + ` state submittedAt commit { oid } ` + historyConnection("comments", historyInline, "") +var historyCommon = `id __typename fullDatabaseId number title body ` + historyActor + ` authorAssociation createdAt updatedAt closedAt url state locked activeLockReason repository { nameWithOwner } milestone { number title state dueOn createdAt updatedAt url } ` + historyConnection("labels", `id name color description`, "") + " " + historyConnection("assignees", `id login __typename url`, "") + " " + historyConnection("comments", historyComment, "") +var historyIssue = historyCommon + ` stateReason` +var historyPull = historyCommon + ` isDraft merged mergedAt mergedBy { login __typename url } mergeCommit { oid } mergeable mergeStateStatus maintainerCanModify additions deletions changedFiles headRefName headRefOid baseRefName baseRefOid headRepository { nameWithOwner } baseRepository { nameWithOwner } commits { totalCount } ` + historyConnection("reviews", historyReview, "") + +func historyConnection(name, fields, after string) string { + return name + `(first:20` + after + `) { totalCount pageInfo { hasNextPage endCursor } nodes { ` + fields + ` } }` +} + +type historySession struct { + client *Client + reporter Reporter + calls int + remaining int +} + +func (h *historySession) request(ctx context.Context, query string, variables map[string]any, estimate int) (map[string]any, error) { + if h.calls >= 1000 { + return nil, fmt.Errorf("GraphQL history pagination budget exceeded") + } + if h.remaining < 500+estimate { + return nil, fmt.Errorf("GraphQL history quota reserve reached") + } + h.calls++ + // An unanswered request is charged conservatively by external supervisors. + h.reporter.Printf("[github] graphql budget %d %d", h.calls, estimate) + var data map[string]any + if err := h.client.doGraphQL(ctx, query, variables, h.reporter, &data); err != nil { + return nil, err + } + rate := historyMap(data["rateLimit"]) + cost, ok := historyInt(rate["cost"]) + if !ok || cost < 0 { + return nil, fmt.Errorf("GraphQL history missing cost") + } + remaining, ok := historyInt(rate["remaining"]) + if !ok || remaining < 0 { + return nil, fmt.Errorf("GraphQL history missing remaining quota") + } + reset, err := time.Parse(time.RFC3339, historyString(rate["resetAt"])) + if err != nil { + return nil, fmt.Errorf("GraphQL history invalid reset") + } + h.remaining = min(h.remaining-cost, remaining) + h.reporter.Printf("[github] graphql cost %d %d remaining %d reset %d", h.calls, cost, remaining, reset.Unix()) + return data, nil +} + +// FetchGraphQLHistory supports exact selections and the conversation/metadata +// profile only. Full code/check hydration intentionally remains a separate mode. +func (c *Client) FetchGraphQLHistory(ctx context.Context, owner, repo string, numbers []int, reporter Reporter) (HistoryBatch, error) { + var result HistoryBatch + if len(numbers) == 0 || len(numbers) > 100 { + return result, fmt.Errorf("GraphQL history requires 1..100 numbers") + } + h := historySession{client: c, reporter: reporter, remaining: 20000} + if _, err := h.request(ctx, `query { rateLimit { cost remaining limit used resetAt } }`, nil, 1); err != nil { + return result, err + } + for offset := 0; offset < len(numbers); offset += 25 { + selected := numbers[offset:min(offset+25, len(numbers))] + var fields strings.Builder + for i, n := range selected { + if n <= 0 { + return result, fmt.Errorf("invalid history number") + } + fmt.Fprintf(&fields, "n%d: issueOrPullRequest(number:%d) { ... on Issue { %s } ... on PullRequest { %s } }\n", i, n, historyIssue, historyPull) + } + query := `query($owner:String!,$repo:String!){ repository(owner:$owner,name:$repo){ id databaseId nameWithOwner url description isPrivate createdAt updatedAt defaultBranchRef{name} ` + fields.String() + ` } rateLimit{cost remaining limit used resetAt}}` + data, err := h.request(ctx, query, map[string]any{"owner": owner, "repo": repo}, 16) + if err != nil { + return result, err + } + r := historyMap(data["repository"]) + if !strings.EqualFold(historyString(r["nameWithOwner"]), owner+"/"+repo) { + return result, fmt.Errorf("GraphQL history repository identity mismatch") + } + if result.Repository == nil { + raw := map[string]any{} + for k, v := range r { + if !strings.HasPrefix(k, "n") || k == "nameWithOwner" { + raw[k] = v + } + } + result.Repository = map[string]any{"id": r["databaseId"], "node_id": r["id"], "full_name": r["nameWithOwner"], "html_url": r["url"], "private": r["isPrivate"], "description": r["description"], "_gitcrawl_source": "graphql", "_graphql": raw} + } + for i, n := range selected { + node := historyMap(r[fmt.Sprintf("n%d", i)]) + got, _ := historyInt(node["number"]) + if got != n || !strings.EqualFold(historyString(historyMap(node["repository"])["nameWithOwner"]), owner+"/"+repo) || historyString(node["id"]) == "" { + return result, fmt.Errorf("GraphQL history item #%d unavailable or moved", n) + } + if err := h.hydrate(ctx, node); err != nil { + return result, fmt.Errorf("GraphQL history #%d: %w", n, err) + } + item, err := historyItem(node) + if err != nil { + return result, err + } + result.Items = append(result.Items, item) + } + } + return result, nil +} + +func historyFields(typ, key string) (string, error) { + switch { + case (typ == "Issue" || typ == "PullRequest") && key == "comments": + return historyComment, nil + case (typ == "Issue" || typ == "PullRequest") && key == "labels": + return `id name color description`, nil + case (typ == "Issue" || typ == "PullRequest") && key == "assignees": + return `id login __typename url`, nil + case typ == "PullRequest" && key == "reviews": + return historyReview, nil + case typ == "PullRequestReview" && key == "comments": + return historyInline, nil + } + return "", fmt.Errorf("unsupported history connection %s.%s", typ, key) +} + +func (h *historySession) hydrate(ctx context.Context, node map[string]any) error { + typ := historyString(node["__typename"]) + var required []string + switch typ { + case "Issue": + required = []string{"labels", "assignees", "comments"} + case "PullRequest": + required = []string{"labels", "assignees", "comments", "reviews"} + case "PullRequestReview": + required = []string{"comments"} + } + for _, key := range required { + if historyMap(node[key]) == nil { + return fmt.Errorf("missing history %s", key) + } + } + if typ == "Issue" || typ == "PullRequest" || typ == "PullRequestReview" || typ == "IssueComment" || typ == "PullRequestReviewComment" { + if node["fullDatabaseId"] == nil || historyString(node["id"]) == "" { + return fmt.Errorf("missing provider identity") + } + } + for _, key := range []string{"labels", "assignees", "comments", "reviews"} { + connection, exists := node[key] + if !exists { + continue + } + conn := historyMap(connection) + if conn == nil { + return fmt.Errorf("missing %s connection", key) + } + fields, err := historyFields(typ, key) + if err != nil { + return err + } + seen := map[string]bool{} + for { + page := historyMap(conn["pageInfo"]) + next, ok := page["hasNextPage"].(bool) + if !ok { + return fmt.Errorf("missing %s pageInfo", key) + } + if !next { + break + } + cursor := historyString(page["endCursor"]) + if cursor == "" || seen[cursor] { + return fmt.Errorf("nonadvancing %s cursor", key) + } + seen[cursor] = true + q := `query($id:ID!,$after:String!){node(id:$id){id ... on ` + typ + `{` + historyConnection(key, fields, `,after:$after`) + `}} rateLimit{cost remaining limit used resetAt}}` + data, err := h.request(ctx, q, map[string]any{"id": node["id"], "after": cursor}, 2) + if err != nil { + return err + } + parent := historyMap(data["node"]) + if parent["id"] != node["id"] { + return fmt.Errorf("history pagination identity mismatch") + } + nxt := historyMap(parent[key]) + a, ok := conn["nodes"].([]any) + if !ok { + return fmt.Errorf("missing history nodes") + } + b, ok := nxt["nodes"].([]any) + if !ok || len(b) == 0 { + return fmt.Errorf("empty history continuation") + } + conn["nodes"] = append(a, b...) + conn["pageInfo"] = nxt["pageInfo"] + } + children, ok := conn["nodes"].([]any) + if !ok { + return fmt.Errorf("missing history nodes") + } + ids := map[string]bool{} + for _, child := range children { + m := historyMap(child) + id := historyString(m["id"]) + if id == "" || ids[id] { + return fmt.Errorf("missing or duplicate history child identity") + } + ids[id] = true + if err := h.hydrate(ctx, m); err != nil { + return err + } + } + } + return nil +} + +func historyItem(node map[string]any) (HistoryItem, error) { + var item HistoryItem + typ := historyString(node["__typename"]) + if typ != "Issue" && typ != "PullRequest" { + return item, fmt.Errorf("invalid history item type") + } + row := historyProjection(node) + // GraphQL does not expose a PR's REST issue-database ID. New thread IDs + // therefore use the opaque node ID. Syncer preserves any existing legacy ID. + row["id"] = node["id"] + row["number"] = node["number"] + row["title"] = node["title"] + row["state"] = strings.ToLower(historyString(node["state"])) + if row["state"] == "merged" { + row["state"] = "closed" + } + row["closed_at"] = node["closedAt"] + row["locked"] = node["locked"] + row["active_lock_reason"] = node["activeLockReason"] + row["labels"] = historyNodes(node, "labels") + assignees := []map[string]any{} + for _, a := range historyNodes(node, "assignees") { + assignees = append(assignees, historyActorMap(a)) + } + row["assignees"] = assignees + row["comments"] = historyMap(node["comments"])["totalCount"] + for _, v := range historyNodes(node, "comments") { + item.Comments = append(item.Comments, historyProjection(v)) + } + if typ == "PullRequest" { + row["draft"] = node["isDraft"] + row["pull_request"] = map[string]any{"merged_at": node["mergedAt"], "html_url": node["url"]} + pull := historyProjection(node) + pull["number"] = node["number"] + pull["title"] = node["title"] + pull["state"] = row["state"] + pull["draft"] = node["isDraft"] + pull["merged"] = node["merged"] + pull["merged_at"] = node["mergedAt"] + pull["closed_at"] = node["closedAt"] + pull["merged_by"] = historyActorMap(historyMap(node["mergedBy"])) + pull["merge_commit_sha"] = historyMap(node["mergeCommit"])["oid"] + pull["mergeable_state"] = strings.ToLower(historyString(node["mergeStateStatus"])) + pull["additions"] = node["additions"] + pull["deletions"] = node["deletions"] + pull["changed_files"] = node["changedFiles"] + for _, side := range []string{"head", "base"} { + pull[side] = map[string]any{"sha": node[side+"RefOid"], "ref": node[side+"RefName"], "repo": map[string]any{"full_name": historyMap(node[side+"Repository"])["nameWithOwner"]}} + } + item.Pull = pull + for _, v := range historyNodes(node, "reviews") { + r := historyProjection(v) + r["state"] = v["state"] + r["submitted_at"] = v["submittedAt"] + r["commit_id"] = historyMap(v["commit"])["oid"] + item.Reviews = append(item.Reviews, r) + for _, comment := range historyNodes(v, "comments") { + p := historyProjection(comment) + for dest, src := range map[string]string{"path": "path", "diff_hunk": "diffHunk", "line": "line", "start_line": "startLine", "original_line": "originalLine", "original_start_line": "originalStartLine", "position": "position", "original_position": "originalPosition", "subject_type": "subjectType"} { + p[dest] = comment[src] + } + p["in_reply_to_id"] = historyMap(comment["replyTo"])["fullDatabaseId"] + p["pull_request_review_id"] = v["fullDatabaseId"] + item.ReviewComments = append(item.ReviewComments, p) + } + } + } + item.Thread = row + return item, nil +} + +func historyProjection(node map[string]any) map[string]any { + return map[string]any{"id": node["fullDatabaseId"], "node_id": node["id"], "body": node["body"], "user": historyActorMap(historyMap(node["author"])), "author_association": node["authorAssociation"], "created_at": node["createdAt"], "updated_at": node["updatedAt"], "html_url": node["url"], "_gitcrawl_source": "graphql", "_graphql": node} +} +func historyActorMap(a map[string]any) map[string]any { + if a == nil { + return nil + } + login := historyString(a["login"]) + typ := historyString(a["__typename"]) + if typ == "Bot" && !strings.HasSuffix(login, "[bot]") { + login += "[bot]" + } + return map[string]any{"login": login, "type": typ, "node_id": a["id"], "html_url": a["url"]} +} +func historyNodes(node map[string]any, key string) []map[string]any { + var out []map[string]any + values, _ := historyMap(node[key])["nodes"].([]any) + for _, v := range values { + out = append(out, historyMap(v)) + } + return out +} +func historyMap(v any) map[string]any { m, _ := v.(map[string]any); return m } +func historyString(v any) string { s, _ := v.(string); return s } +func historyInt(v any) (int, bool) { + switch n := v.(type) { + case json.Number: + i, e := strconv.Atoi(string(n)) + return i, e == nil + case int: + return n, true + case float64: + return int(n), n == float64(int(n)) + } + return 0, false +} diff --git a/internal/github/history_test.go b/internal/github/history_test.go new file mode 100644 index 00000000..563d794d --- /dev/null +++ b/internal/github/history_test.go @@ -0,0 +1,124 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func historyTestConnection(nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"totalCount": len(nodes), "nodes": nodes, "pageInfo": map[string]any{"hasNextPage": false, "endCursor": "end"}} +} +func historyTestNode() map[string]any { + return map[string]any{"id": "PR_fixture", "__typename": "PullRequest", "fullDatabaseId": "9007199254740993", "number": 1, "title": "history", "body": "body", "state": "MERGED", "author": map[string]any{"login": "helper", "__typename": "Bot"}, "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "labels": historyTestConnection(), "assignees": historyTestConnection(), "comments": historyTestConnection(), "reviews": historyTestConnection(), "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-02T00:00:00Z", "mergedAt": "2026-01-02T00:00:00Z"} +} +func TestGraphQLHistoryBatchNoRESTAndExactIDs(t *testing.T) { + node := historyTestNode() + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.URL.Path != "/graphql" || r.Method != "POST" { + t.Errorf("unexpected REST request %s", r.URL) + http.Error(w, "no REST", 400) + return + } + var request graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19999 - calls, "resetAt": "2099-01-01T01:00:00Z"}} + if strings.Contains(request.Query, "issueOrPullRequest") { + data["repository"] = map[string]any{"id": "R_fixture", "databaseId": 42, "nameWithOwner": "fixture/repo", "n0": node} + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + var log []string + batch, err := New(Options{BaseURL: server.URL}).FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, func(s string) { log = append(log, s) }) + if err != nil { + t.Fatal(err) + } + if calls != 2 || len(batch.Items) != 1 { + t.Fatalf("calls=%d batch=%+v", calls, batch) + } + item := batch.Items[0] + if item.Thread["id"] != "PR_fixture" || item.Pull["id"] != "9007199254740993" || item.Thread["state"] != "closed" { + t.Fatalf("identity/state %+v", item) + } + if historyMap(item.Thread["user"])["login"] != "helper[bot]" { + t.Fatal("bot normalization") + } + if historyMap(item.Thread["_graphql"])["fullDatabaseId"] != "9007199254740993" { + t.Fatal("raw payload lost") + } + if !strings.Contains(strings.Join(log, "\n"), "[github] graphql cost 2 1") { + t.Fatal("missing actual quota cost") + } +} +func TestGraphQLHistoryNestedPaginationAndRejection(t *testing.T) { + for _, mode := range []string{"complete", "repeated", "partial-error", "missing-child"} { + t.Run(mode, func(t *testing.T) { + node := historyTestNode() + comment := map[string]any{"id": "C1", "__typename": "PullRequestReviewComment", "fullDatabaseId": "9007199254740994", "body": "inline"} + review := map[string]any{"id": "V1", "__typename": "PullRequestReview", "fullDatabaseId": "9007199254740995", "body": "", "state": "APPROVED", "comments": historyTestConnection()} + review["comments"].(map[string]any)["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "first"} + node["reviews"] = historyTestConnection(review) + pages := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req graphqlEnvelope + json.NewDecoder(r.Body).Decode(&req) + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": "2099-01-01T01:00:00Z"}} + if strings.Contains(req.Query, "issueOrPullRequest") { + data["repository"] = map[string]any{"databaseId": 42, "nameWithOwner": "fixture/repo", "n0": node} + if mode == "missing-child" { + delete(node, "comments") + } + } + if strings.Contains(req.Query, "node(id:") { + pages++ + conn := historyTestConnection(comment) + if mode == "repeated" { + conn["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "first"} + } + data["node"] = map[string]any{"id": "V1", "comments": conn} + } + resp := map[string]any{"data": data} + if mode == "partial-error" && strings.Contains(req.Query, "issueOrPullRequest") { + resp["errors"] = []any{map[string]any{"message": "unavailable"}} + } + json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + batch, err := New(Options{BaseURL: server.URL}).FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if mode == "complete" { + if err != nil { + t.Fatal(err) + } + if pages != 1 || len(batch.Items[0].ReviewComments) != 1 || len(batch.Items[0].Reviews) != 1 { + t.Fatal("nested pagination incomplete") + } + } else if err == nil { + t.Fatalf("%s did not fail closed: %+v", mode, batch) + } + }) + } +} +func TestGraphQLHistoryReserveStopsBeforeBatch(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":1,"remaining":499,"resetAt":"2099-01-01T01:00:00Z"}}}`) + })) + defer server.Close() + _, err := New(Options{BaseURL: server.URL}).FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err == nil || calls != 1 { + t.Fatalf("reserve failed calls=%d error=%v", calls, err) + } +} diff --git a/internal/syncer/graphql_history_test.go b/internal/syncer/graphql_history_test.go new file mode 100644 index 00000000..d1ce8928 --- /dev/null +++ b/internal/syncer/graphql_history_test.go @@ -0,0 +1,89 @@ +package syncer + +import ( + "context" + "encoding/json" + "errors" + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" +) + +type historyFixtureClient struct { + *gh.Client + batch gh.HistoryBatch + err error +} + +func (f historyFixtureClient) FetchGraphQLHistory(context.Context, string, string, []int, gh.Reporter) (gh.HistoryBatch, error) { + return f.batch, f.err +} +func TestGraphQLHistoryUsesNativeTransactionsAndPreservesLegacyIdentity(t *testing.T) { + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + legacy := &metadataGitHub{} + options := Options{Owner: "openclaw", Repo: "gitcrawl", Numbers: []int{8}, State: "all", IncludePRMetadata: true, IncludeComments: true} + if _, err := New(legacy, st).Sync(ctx, options); err != nil { + t.Fatal(err) + } + repo, err := st.RepositoryByFullName(ctx, "openclaw/gitcrawl") + if err != nil { + t.Fatal(err) + } + before, err := st.ListThreads(ctx, repo.ID, true) + if err != nil || len(before) != 1 { + t.Fatalf("before=%+v err=%v", before, err) + } + var row map[string]any + if err := json.Unmarshal([]byte(before[0].RawJSON), &row); err != nil { + t.Fatal(err) + } + row["id"] = "PR_new_namespace" + row["_gitcrawl_source"] = "graphql" + pull, _ := legacy.GetPull(ctx, "openclaw", "gitcrawl", 8, nil) + comments, _ := legacy.ListIssueComments(ctx, "openclaw", "gitcrawl", 8, nil) + reviews, _ := legacy.ListPullReviews(ctx, "openclaw", "gitcrawl", 8, nil) + inline, _ := legacy.ListPullReviewComments(ctx, "openclaw", "gitcrawl", 8, nil) + rawRepo, _ := legacy.GetRepo(ctx, "openclaw", "gitcrawl", nil) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("REST fallback %s", r.URL) + http.Error(w, "REST forbidden", 500) + })) + defer server.Close() + client := historyFixtureClient{Client: gh.New(gh.Options{BaseURL: server.URL}), batch: gh.HistoryBatch{Repository: rawRepo, Items: []gh.HistoryItem{{Thread: row, Pull: pull, Comments: comments, Reviews: reviews, ReviewComments: inline}}}} + options.GraphQLHistory = true + stats, err := New(client, st).Sync(ctx, options) + if err != nil { + t.Fatal(err) + } + after, err := st.ListThreads(ctx, repo.ID, true) + if err != nil || len(after) != 1 { + t.Fatalf("after=%+v err=%v", after, err) + } + if after[0].ID != before[0].ID || after[0].GitHubID != before[0].GitHubID { + t.Fatalf("identity changed before=%+v after=%+v", before, after) + } + if stats.ThreadsSynced != 1 || stats.PRDetailsSynced != 1 || stats.CommentsSynced == 0 { + t.Fatalf("stats %+v", stats) + } + if _, err := New(client, st).Sync(ctx, options); err != nil { + t.Fatal(err) + } + assertTableRowCount(t, st, "threads", 1) + client.err = errors.New("partial GraphQL failure") + if _, err := New(client, st).Sync(ctx, options); err == nil { + t.Fatal("failed batch persisted") + } + assertTableRowCount(t, st, "threads", 1) + options.IncludePRDetails = true + if _, err := New(client, st).Sync(ctx, options); err == nil { + t.Fatal("unsupported hydration accepted") + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index ee978385..6ab6c239 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -43,6 +43,7 @@ type Syncer struct { } type Options struct { + GraphQLHistory bool Owner string Repo string State string @@ -136,7 +137,27 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { if err != nil { return Stats{}, err } - repoRaw, err := s.client.GetRepo(ctx, options.Owner, options.Repo, options.Reporter) + var history *gh.HistoryBatch + var repoRaw map[string]any + if options.GraphQLHistory { + if len(options.Numbers) == 0 || !options.IncludeComments || !options.IncludePRMetadata || options.IncludePRDetails || since != "" || options.Limit != 0 || state != "all" { + return Stats{}, fmt.Errorf("--graphql-history requires --numbers, --state all, --include-comments and --with pr-metadata; since/limit/pr-details are unsupported") + } + client, ok := s.client.(interface { + FetchGraphQLHistory(context.Context, string, string, []int, gh.Reporter) (gh.HistoryBatch, error) + }) + if !ok { + return Stats{}, fmt.Errorf("client does not support GraphQL history") + } + batch, fetchErr := client.FetchGraphQLHistory(ctx, options.Owner, options.Repo, uniquePositiveNumbers(options.Numbers), options.Reporter) + if fetchErr != nil { + return Stats{}, fetchErr + } + history = &batch + repoRaw = batch.Repository + } else { + repoRaw, err = s.client.GetRepo(ctx, options.Owner, options.Repo, options.Reporter) + } if err != nil { return Stats{}, err } @@ -167,7 +188,11 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { } numbers := uniquePositiveNumbers(options.Numbers) rows := make([]map[string]any, 0, len(numbers)) - if len(numbers) > 0 { + if history != nil { + for _, item := range history.Items { + rows = append(rows, item.Thread) + } + } else if len(numbers) > 0 { for _, number := range numbers { if reserveErr != nil { break @@ -239,6 +264,41 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { payload := threadSyncPayload{row: row} number := intValue(row["number"]) kind := issueKind(row) + if history != nil { + // Keep legacy REST identity stable when revisiting a previously saved + // thread. The exact GraphQL identity remains in the provider payload. + existing, err := s.store.ListThreadsFiltered(ctx, store.ThreadListOptions{RepoID: repoID, IncludeClosed: true, Numbers: []int{number}, Limit: 1}) + if err != nil { + return Stats{}, err + } + if len(existing) > 0 { + row["id"] = existing[0].GitHubID + } + for _, item := range history.Items { + if intValue(item.Thread["number"]) != number { + continue + } + for _, r := range item.Comments { + payload.commentRows = append(payload.commentRows, commentRow{kind: "issue_comment", raw: r}) + } + for _, r := range item.Reviews { + payload.commentRows = append(payload.commentRows, commentRow{kind: "pull_review", raw: r}) + } + for _, r := range item.ReviewComments { + payload.commentRows = append(payload.commentRows, commentRow{kind: "pull_review_comment", raw: r}) + } + if item.Pull != nil { + payload.hasPullDetails = true + payload.pullDetails = pullRequestDetailRows{pull: item.Pull, fetchedAt: s.now().Format(time.RFC3339Nano)} + } + } + received.CommentsReceived += len(payload.commentRows) + payloads = append(payloads, payload) + if err := reportSyncProgress(options.Progress, received); err != nil { + return Stats{}, err + } + continue + } if reserveErr != nil && (options.IncludeComments || kind == "pull_request" && (options.IncludePRMetadata || options.IncludePRDetails)) { continue From 6dc223eff0c97dca7968fa245218386e2b97db3f Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:01:32 -0600 Subject: [PATCH 02/21] Expose GraphQL request durations for supervised pacing --- docs/sync.md | 7 +++++++ internal/github/history.go | 5 ++++- internal/github/history_test.go | 3 +++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/sync.md b/docs/sync.md index d54ed0d6..d834995c 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -123,6 +123,13 @@ Sanitized `graphql budget` and `graphql cost` log records expose request sequenc conservative unanswered-request charge and actual response cost. External multi-process supervisors must additionally preserve a shared point budget across token renewals and restarts. REST request counts are not GraphQL costs. +`graphql timing ` records elapsed time for each history +request, including failed requests. Supervisors can use the sum across parallel +workers to pace work against GitHub's separate compute-time secondary limit. +Response time is an estimate, not a measurement of GitHub CPU usage; explicit +provider cooldowns always take precedence. Internal retry waits are included in +the timing, so consumers should avoid adding another pacing delay after a known +provider cooldown. | Flag | What it adds | | --- | --- | diff --git a/internal/github/history.go b/internal/github/history.go index 7175b125..7f34f220 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -52,7 +52,10 @@ func (h *historySession) request(ctx context.Context, query string, variables ma // An unanswered request is charged conservatively by external supervisors. h.reporter.Printf("[github] graphql budget %d %d", h.calls, estimate) var data map[string]any - if err := h.client.doGraphQL(ctx, query, variables, h.reporter, &data); err != nil { + started := time.Now() + err := h.client.doGraphQL(ctx, query, variables, h.reporter, &data) + h.reporter.Printf("[github] graphql timing %d %d", h.calls, time.Since(started).Milliseconds()) + if err != nil { return nil, err } rate := historyMap(data["rateLimit"]) diff --git a/internal/github/history_test.go b/internal/github/history_test.go index 563d794d..c55e37b6 100644 --- a/internal/github/history_test.go +++ b/internal/github/history_test.go @@ -58,6 +58,9 @@ func TestGraphQLHistoryBatchNoRESTAndExactIDs(t *testing.T) { if historyMap(item.Thread["_graphql"])["fullDatabaseId"] != "9007199254740993" { t.Fatal("raw payload lost") } + if !strings.Contains(strings.Join(log, "\n"), "[github] graphql timing 2 ") { + t.Fatal("missing request duration") + } if !strings.Contains(strings.Join(log, "\n"), "[github] graphql cost 2 1") { t.Fatal("missing actual quota cost") } From bb82876362e2393f2f7e29f0f091322747bb70fb Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:27:54 -0600 Subject: [PATCH 03/21] Retry transient GraphQL failures before discarding history batches --- docs/sync.md | 15 ++++++ internal/github/history.go | 67 +++++++++++++++++++++++++-- internal/github/history_test.go | 81 +++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 4 deletions(-) diff --git a/docs/sync.md b/docs/sync.md index d834995c..badad4f2 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -131,6 +131,21 @@ provider cooldowns always take precedence. Internal retry waits are included in the timing, so consumers should avoid adding another pacing delay after a known provider cooldown. +History reads retry transient HTTP 500/502/503/504 responses, transport timeouts +and truncated/empty response bodies up to two times in place. The same query and +variables are retained, so a gateway failure does not immediately discard a +whole selection and force individual-record retries. Delays are one and two +seconds; an explicit longer `Retry-After` takes precedence, and cancellation +stops the wait. Authentication, permission, missing-record and GraphQL semantic +errors do not get this additional retry policy. Existing rate-limit handling is +unchanged. Exhausted retries still fail acquisition and leave normal isolated +recovery available to the supervisor. + +Each transient attempt has a separate budget/timing identity. Unanswered attempts +retain their conservative charge even if a later quota receipt looks higher. +Transient backoff occurs outside the per-attempt timing; the underlying HTTP +client's rate-limit retry wait can still be included as described above. + | Flag | What it adds | | --- | --- | | `--include-comments` | Issue comments, PR review comments, reviews | diff --git a/internal/github/history.go b/internal/github/history.go index 7f34f220..b5d0a6ca 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -6,7 +6,10 @@ package github import ( "context" "encoding/json" + "errors" "fmt" + "io" + "net" "strconv" "strings" "time" @@ -35,13 +38,69 @@ func historyConnection(name, fields, after string) string { } type historySession struct { - client *Client - reporter Reporter - calls int - remaining int + client *Client + reporter Reporter + calls int + remaining int + retrySleep func(context.Context, time.Duration) error } func (h *historySession) request(ctx context.Context, query string, variables map[string]any, estimate int) (map[string]any, error) { + for attempt := 0; ; attempt++ { + data, err := h.requestOnce(ctx, query, variables, estimate) + if err == nil { + return data, nil + } + if attempt >= 2 || ctx.Err() != nil || !transientHistoryError(err) { + return nil, err + } + // An unanswered attempt may have consumed points. Do not let a retry + // refund that spending, even if the next provider receipt is higher. + h.remaining -= estimate + wait := time.Second << attempt + var response *RequestError + if errors.As(err, &response) { + if providerWait, ok := retryAfterWait(response.Headers.Get("Retry-After")); ok { + wait = max(wait, providerWait) + } + } + h.reporter.Printf("[github] transient retry wait=%s", wait) + sleep := h.retrySleep + if sleep == nil { + sleep = sleepHistoryRetry + } + if err := sleep(ctx, wait); err != nil { + return nil, err + } + } +} + +func transientHistoryError(err error) bool { + var response *RequestError + if errors.As(err, &response) { + switch response.Status { + case 500, 502, 503, 504: + return true + } + return false + } + var transport net.Error + return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + (errors.As(err, &transport) && (transport.Timeout() || transport.Temporary())) +} + +func sleepHistoryRetry(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (h *historySession) requestOnce(ctx context.Context, query string, variables map[string]any, estimate int) (map[string]any, error) { if h.calls >= 1000 { return nil, fmt.Errorf("GraphQL history pagination budget exceeded") } diff --git a/internal/github/history_test.go b/internal/github/history_test.go index c55e37b6..2e1fad75 100644 --- a/internal/github/history_test.go +++ b/internal/github/history_test.go @@ -3,11 +3,13 @@ package github import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" "strings" "testing" + "time" ) func historyTestConnection(nodes ...any) map[string]any { @@ -125,3 +127,82 @@ func TestGraphQLHistoryReserveStopsBeforeBatch(t *testing.T) { t.Fatalf("reserve failed calls=%d error=%v", calls, err) } } + +func TestGraphQLHistoryTransientRetriesKeepQueryAndAccountEveryAttempt(t *testing.T) { + for _, kind := range []string{"gateway", "truncated", "empty"} { + t.Run(kind, func(t *testing.T) { + calls := 0 + var queries []string + var waits []time.Duration + var logs []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + var req graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatal(err) + } + queries = append(queries, req.Query) + if calls < 3 { + switch kind { + case "gateway": + w.Header().Set("Retry-After", "7") + http.Error(w, "temporary gateway failure", 503) + case "truncated": + fmt.Fprint(w, `{"data":`) + case "empty": + w.WriteHeader(200) + } + return + } + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":6,"remaining":19999,"resetAt":"2099-01-01T01:00:00Z"},"marker":"complete"}}`) + })) + defer server.Close() + h := historySession{client: New(Options{BaseURL: server.URL}), remaining: 20000, reporter: func(s string) { logs = append(logs, s) }, retrySleep: func(_ context.Context, d time.Duration) error { waits = append(waits, d); return nil }} + data, err := h.request(context.Background(), "query { fixture }", nil, 16) + if err != nil || data["marker"] != "complete" || calls != 3 { + t.Fatalf("calls=%d data=%v err=%v", calls, data, err) + } + if len(waits) != 2 || queries[0] != queries[1] || queries[1] != queries[2] { + t.Fatal("retry changed query or bounds") + } + want := []time.Duration{time.Second, 2 * time.Second} + if kind == "gateway" { + want = []time.Duration{7 * time.Second, 7 * time.Second} + } + if waits[0] != want[0] || waits[1] != want[1] { + t.Fatalf("waits %v, want %v", waits, want) + } + if h.remaining != 19962 { + t.Fatalf("failed attempts refunded: %d", h.remaining) + } + text := strings.Join(logs, "\n") + if strings.Count(text, "[github] graphql budget ") != 3 || strings.Count(text, "[github] graphql timing ") != 3 || strings.Count(text, "[github] graphql cost ") != 1 { + t.Fatalf("invalid accounting: %s", text) + } + }) + } +} + +func TestGraphQLHistoryRetryBoundAndCancellation(t *testing.T) { + for _, status := range []int{502, 401, 404} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls++; http.Error(w, "failure", status) })) + defer server.Close() + h := historySession{client: New(Options{BaseURL: server.URL}), remaining: 20000, retrySleep: func(context.Context, time.Duration) error { return nil }} + _, err := h.request(context.Background(), "query { fixture }", nil, 16) + want := 1 + if status == 502 { + want = 3 + } + if err == nil || calls != want { + t.Fatalf("calls=%d want=%d err=%v", calls, want, err) + } + }) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := sleepHistoryRetry(ctx, time.Hour); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled retry: %v", err) + } +} From cdffe8769ac876a8cb2404fdc34076d939543c1a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 24 Sep 2026 08:47:08 -0700 Subject: [PATCH 04/21] fix: complete GraphQL review-thread history and prepare 0.12.0 --- CHANGELOG.md | 6 ++ docs/sync.md | 20 ++++- internal/github/history.go | 50 +++++++++--- internal/github/history_test.go | 138 +++++++++++++++++++++++++++++++- 4 files changed, 200 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc5bab92..6cb1dcbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +## 0.12.0 - 2026-09-24 + +**Highlights:** Batched GraphQL conversation history, including standalone inline review comments. + +- Add opt-in `sync --graphql-history` for batched issue and PR conversations with complete nested pagination, standalone review-thread comments, exact provider IDs, bounded retries, and GraphQL point receipts. Thanks @hannesrudolph. + ## 0.11.0 - 2026-09-22 **Highlights:** Lower-disk portable exports, preserved body-length metadata, and fewer redundant GitHub comment downloads. diff --git a/docs/sync.md b/docs/sync.md index badad4f2..f3cc291d 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -91,17 +91,24 @@ gitcrawl sync owner/repo --numbers 123,456 --state all --include-comments --with ``` This opt-in profile fetches exact selections in GraphQL batches of at most 25 -parents. It fully paginates labels, assignees, discussion comments, reviews and -each review's inline comments, then uses the existing per-thread transactions, +parents. It fully paginates labels, assignees, discussion comments, reviews, +review threads and both sources' nested inline comments. Review threads include +standalone comments with no review association; comments found through both +paths are deduplicated by provider identity while review bodies and metadata +remain separate. It then uses the existing per-thread transactions, observation ordering, failure resolution and document persistence. It performs no REST requests or fallback. The regular sync path is unchanged. The profile requires the flags above; it rejects `--since`, `--limit` and full PR-detail hydration. It does not collect files, commit bodies, checks or Actions logs. An incomplete GraphQL response, unavailable parent, missing identity, -duplicate child or nonadvancing cursor fails the batch before archive writes. +duplicate child within a connection, mismatched count or nonadvancing cursor +fails the batch before archive writes. Supervisors should retry failed selections in isolation. Empty reviews remain -retained; the ordinary empty-comment filtering contract is unchanged. +retained, including approvals without bodies. Minimized comments and null +(deleted) authors are retained as returned by GitHub; deleted comments that +GitHub no longer returns cannot be reconstructed. The ordinary empty-comment +filtering contract is unchanged. Stored raw maps are explicitly labelled projections: `_gitcrawl_source` is `graphql`, and `_graphql` retains the hydrated provider object. Existing thread @@ -112,6 +119,11 @@ uses its PR database ID. Bot logins receive REST-compatible `[bot]` suffixes in normalized fields while raw actor names remain intact. GraphQL review creation, update and submission timestamps are retained. +GitHub sometimes names the same bot differently between transports: Copilot +inline comments can use `Copilot` in REST and `copilot-pull-request-reviewer` +in GraphQL. This profile keeps the latter as `copilot-pull-request-reviewer[bot]`, +consistent with its review bodies, and preserves the original provider actor. + Provider differences remain visible: for example GraphQL can return a null head repository when REST previously named one. No missing value is invented from that earlier observation. Keep a consistent archive backup when migrating diff --git a/internal/github/history.go b/internal/github/history.go index b5d0a6ca..b0012a27 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -29,9 +29,10 @@ const historyComment = `id __typename fullDatabaseId body ` + historyActor + ` a const historyInline = historyComment + ` path diffHunk line startLine originalLine originalStartLine position originalPosition state subjectType outdated commit { oid } originalCommit { oid } replyTo { id fullDatabaseId } pullRequestReview { id fullDatabaseId }` var historyReview = historyComment + ` state submittedAt commit { oid } ` + historyConnection("comments", historyInline, "") +var historyReviewThread = `id __typename ` + historyConnection("comments", historyInline, "") var historyCommon = `id __typename fullDatabaseId number title body ` + historyActor + ` authorAssociation createdAt updatedAt closedAt url state locked activeLockReason repository { nameWithOwner } milestone { number title state dueOn createdAt updatedAt url } ` + historyConnection("labels", `id name color description`, "") + " " + historyConnection("assignees", `id login __typename url`, "") + " " + historyConnection("comments", historyComment, "") var historyIssue = historyCommon + ` stateReason` -var historyPull = historyCommon + ` isDraft merged mergedAt mergedBy { login __typename url } mergeCommit { oid } mergeable mergeStateStatus maintainerCanModify additions deletions changedFiles headRefName headRefOid baseRefName baseRefOid headRepository { nameWithOwner } baseRepository { nameWithOwner } commits { totalCount } ` + historyConnection("reviews", historyReview, "") +var historyPull = historyCommon + ` isDraft merged mergedAt mergedBy { login __typename url } mergeCommit { oid } mergeable mergeStateStatus maintainerCanModify additions deletions changedFiles headRefName headRefOid baseRefName baseRefOid headRepository { nameWithOwner } baseRepository { nameWithOwner } commits { totalCount } ` + historyConnection("reviews", historyReview, "") + " " + historyConnection("reviewThreads", historyReviewThread, "") func historyConnection(name, fields, after string) string { return name + `(first:20` + after + `) { totalCount pageInfo { hasNextPage endCursor } nodes { ` + fields + ` } }` @@ -202,7 +203,9 @@ func historyFields(typ, key string) (string, error) { return `id login __typename url`, nil case typ == "PullRequest" && key == "reviews": return historyReview, nil - case typ == "PullRequestReview" && key == "comments": + case typ == "PullRequest" && key == "reviewThreads": + return historyReviewThread, nil + case (typ == "PullRequestReview" || typ == "PullRequestReviewThread") && key == "comments": return historyInline, nil } return "", fmt.Errorf("unsupported history connection %s.%s", typ, key) @@ -215,8 +218,8 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error case "Issue": required = []string{"labels", "assignees", "comments"} case "PullRequest": - required = []string{"labels", "assignees", "comments", "reviews"} - case "PullRequestReview": + required = []string{"labels", "assignees", "comments", "reviews", "reviewThreads"} + case "PullRequestReview", "PullRequestReviewThread": required = []string{"comments"} } for _, key := range required { @@ -229,7 +232,7 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error return fmt.Errorf("missing provider identity") } } - for _, key := range []string{"labels", "assignees", "comments", "reviews"} { + for _, key := range []string{"labels", "assignees", "comments", "reviews", "reviewThreads"} { connection, exists := node[key] if !exists { continue @@ -282,6 +285,9 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error if !ok { return fmt.Errorf("missing history nodes") } + if total, ok := historyInt(conn["totalCount"]); !ok || total != len(children) { + return fmt.Errorf("incomplete history %s count", key) + } ids := map[string]bool{} for _, child := range children { m := historyMap(child) @@ -348,6 +354,7 @@ func historyItem(node map[string]any) (HistoryItem, error) { pull[side] = map[string]any{"sha": node[side+"RefOid"], "ref": node[side+"RefName"], "repo": map[string]any{"full_name": historyMap(node[side+"Repository"])["nameWithOwner"]}} } item.Pull = pull + inlineByID := map[string]map[string]any{} for _, v := range historyNodes(node, "reviews") { r := historyProjection(v) r["state"] = v["state"] @@ -355,12 +362,25 @@ func historyItem(node map[string]any) (HistoryItem, error) { r["commit_id"] = historyMap(v["commit"])["oid"] item.Reviews = append(item.Reviews, r) for _, comment := range historyNodes(v, "comments") { - p := historyProjection(comment) - for dest, src := range map[string]string{"path": "path", "diff_hunk": "diffHunk", "line": "line", "start_line": "startLine", "original_line": "originalLine", "original_start_line": "originalStartLine", "position": "position", "original_position": "originalPosition", "subject_type": "subjectType"} { - p[dest] = comment[src] + if _, exists := inlineByID[historyString(comment["id"])]; exists { + continue } - p["in_reply_to_id"] = historyMap(comment["replyTo"])["fullDatabaseId"] + p := historyInlineProjection(comment) p["pull_request_review_id"] = v["fullDatabaseId"] + inlineByID[historyString(comment["id"])] = p + item.ReviewComments = append(item.ReviewComments, p) + } + } + // A comment's review association is nullable. Threads independently + // supply standalone comments; review bodies and their metadata stay above. + for _, thread := range historyNodes(node, "reviewThreads") { + for _, comment := range historyNodes(thread, "comments") { + id := historyString(comment["id"]) + if _, exists := inlineByID[id]; exists { + continue + } + p := historyInlineProjection(comment) + inlineByID[id] = p item.ReviewComments = append(item.ReviewComments, p) } } @@ -369,6 +389,18 @@ func historyItem(node map[string]any) (HistoryItem, error) { return item, nil } +func historyInlineProjection(comment map[string]any) map[string]any { + p := historyProjection(comment) + for dest, src := range map[string]string{"path": "path", "diff_hunk": "diffHunk", "line": "line", "start_line": "startLine", "original_line": "originalLine", "original_start_line": "originalStartLine", "position": "position", "original_position": "originalPosition", "subject_type": "subjectType"} { + p[dest] = comment[src] + } + p["in_reply_to_id"] = historyMap(comment["replyTo"])["fullDatabaseId"] + p["pull_request_review_id"] = historyMap(comment["pullRequestReview"])["fullDatabaseId"] + p["commit_id"] = historyMap(comment["commit"])["oid"] + p["original_commit_id"] = historyMap(comment["originalCommit"])["oid"] + return p +} + func historyProjection(node map[string]any) map[string]any { return map[string]any{"id": node["fullDatabaseId"], "node_id": node["id"], "body": node["body"], "user": historyActorMap(historyMap(node["author"])), "author_association": node["authorAssociation"], "created_at": node["createdAt"], "updated_at": node["updatedAt"], "html_url": node["url"], "_gitcrawl_source": "graphql", "_graphql": node} } diff --git a/internal/github/history_test.go b/internal/github/history_test.go index 2e1fad75..3c3a88c8 100644 --- a/internal/github/history_test.go +++ b/internal/github/history_test.go @@ -19,7 +19,7 @@ func historyTestConnection(nodes ...any) map[string]any { return map[string]any{"totalCount": len(nodes), "nodes": nodes, "pageInfo": map[string]any{"hasNextPage": false, "endCursor": "end"}} } func historyTestNode() map[string]any { - return map[string]any{"id": "PR_fixture", "__typename": "PullRequest", "fullDatabaseId": "9007199254740993", "number": 1, "title": "history", "body": "body", "state": "MERGED", "author": map[string]any{"login": "helper", "__typename": "Bot"}, "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "labels": historyTestConnection(), "assignees": historyTestConnection(), "comments": historyTestConnection(), "reviews": historyTestConnection(), "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-02T00:00:00Z", "mergedAt": "2026-01-02T00:00:00Z"} + return map[string]any{"id": "PR_fixture", "__typename": "PullRequest", "fullDatabaseId": "9007199254740993", "number": 1, "title": "history", "body": "body", "state": "MERGED", "author": map[string]any{"login": "helper", "__typename": "Bot"}, "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "labels": historyTestConnection(), "assignees": historyTestConnection(), "comments": historyTestConnection(), "reviews": historyTestConnection(), "reviewThreads": historyTestConnection(), "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-02T00:00:00Z", "mergedAt": "2026-01-02T00:00:00Z"} } func TestGraphQLHistoryBatchNoRESTAndExactIDs(t *testing.T) { node := historyTestNode() @@ -73,6 +73,7 @@ func TestGraphQLHistoryNestedPaginationAndRejection(t *testing.T) { node := historyTestNode() comment := map[string]any{"id": "C1", "__typename": "PullRequestReviewComment", "fullDatabaseId": "9007199254740994", "body": "inline"} review := map[string]any{"id": "V1", "__typename": "PullRequestReview", "fullDatabaseId": "9007199254740995", "body": "", "state": "APPROVED", "comments": historyTestConnection()} + review["comments"].(map[string]any)["totalCount"] = 1 review["comments"].(map[string]any)["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "first"} node["reviews"] = historyTestConnection(review) pages := 0 @@ -128,6 +129,141 @@ func TestGraphQLHistoryReserveStopsBeforeBatch(t *testing.T) { } } +func TestGraphQLHistoryRejectsShortConnection(t *testing.T) { + node := historyTestNode() + node["comments"].(map[string]any)["totalCount"] = 1 + h := historySession{} + if err := h.hydrate(context.Background(), node); err == nil || !strings.Contains(err.Error(), "incomplete history comments count") { + t.Fatalf("short terminal connection accepted: %v", err) + } +} + +func TestGraphQLHistoryIssueDiscussion(t *testing.T) { + node := historyTestNode() + node["__typename"] = "Issue" + delete(node, "reviews") + delete(node, "reviewThreads") + node["comments"] = historyTestConnection(map[string]any{"id": "D1", "__typename": "IssueComment", "fullDatabaseId": json.Number("9007199254740993"), "body": "minimized discussion", "isMinimized": true, "author": nil}) + h := historySession{} + if err := h.hydrate(context.Background(), node); err != nil { + t.Fatal(err) + } + item, err := historyItem(node) + if err != nil || len(item.Comments) != 1 || item.Pull != nil || len(item.Reviews) != 0 { + t.Fatalf("issue discussion: %+v, %v", item, err) + } + if item.Comments[0]["id"] != json.Number("9007199254740993") || historyMap(item.Comments[0]["_graphql"])["isMinimized"] != true { + t.Fatal("issue comment identity or minimized state lost") + } +} + +func TestGraphQLHistoryBotIdentities(t *testing.T) { + for _, login := range []string{"helper", "helper[bot]", "copilot-pull-request-reviewer"} { + actor := map[string]any{"login": login, "__typename": "Bot"} + want := strings.TrimSuffix(login, "[bot]") + "[bot]" + if got := historyActorMap(actor); got["login"] != want || got["type"] != "Bot" || actor["login"] != login { + t.Fatalf("bot identity: raw=%+v normalized=%+v", actor, got) + } + } +} + +func TestGraphQLHistoryReviewThreadCompleteness(t *testing.T) { + for _, mode := range []string{"complete", "repeated-thread", "repeated-comment", "partial-error", "missing-threads", "missing-comments", "short-page", "missing-identity"} { + t.Run(mode, func(t *testing.T) { + node := historyTestNode() + standalone := map[string]any{"id": "C_standalone", "__typename": "PullRequestReviewComment", "fullDatabaseId": "9007199254740994", "body": "minimized standalone", "isMinimized": true, "minimizedReason": "OUTDATED", "author": nil, "pullRequestReview": nil} + associated := map[string]any{"id": "C_review", "__typename": "PullRequestReviewComment", "fullDatabaseId": "9007199254740995", "body": "inline", "pullRequestReview": map[string]any{"id": "V1", "fullDatabaseId": "9007199254740996"}, "author": map[string]any{"login": "helper[bot]", "__typename": "Bot"}} + review := map[string]any{"id": "V1", "__typename": "PullRequestReview", "fullDatabaseId": "9007199254740996", "body": "", "state": "APPROVED", "comments": historyTestConnection(associated)} + discussion := map[string]any{"id": "D1", "__typename": "IssueComment", "fullDatabaseId": "9007199254740997", "body": "discussion", "author": nil} + node["comments"] = historyTestConnection(discussion) + node["reviews"] = historyTestConnection(review) + thread1 := map[string]any{"id": "T1", "__typename": "PullRequestReviewThread", "comments": historyTestConnection(associated)} + thread2 := map[string]any{"id": "T2", "__typename": "PullRequestReviewThread", "comments": historyTestConnection()} + thread2["comments"].(map[string]any)["totalCount"] = 1 + thread2["comments"].(map[string]any)["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "comment-first"} + node["reviewThreads"] = historyTestConnection(thread1) + node["reviewThreads"].(map[string]any)["totalCount"] = 2 + node["reviewThreads"].(map[string]any)["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "thread-first"} + if mode == "missing-threads" { + delete(node, "reviewThreads") + } + if mode == "missing-comments" { + delete(thread2, "comments") + } + if mode == "missing-identity" { + delete(standalone, "fullDatabaseId") + } + pages := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": "2099-01-01T01:00:00Z"}} + if strings.Contains(req.Query, "issueOrPullRequest") { + if !strings.Contains(req.Query, "reviewThreads(first:20)") || !strings.Contains(req.Query, "pullRequestReview { id fullDatabaseId }") { + t.Error("query omits independent review-thread acquisition") + } + data["repository"] = map[string]any{"databaseId": 42, "nameWithOwner": "fixture/repo", "n0": node} + } + if strings.Contains(req.Query, "node(id:") { + pages++ + switch req.Variables["id"] { + case "PR_fixture": + conn := historyTestConnection(thread2) + if mode == "repeated-thread" { + conn["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "thread-first"} + } + data["node"] = map[string]any{"id": "PR_fixture", "reviewThreads": conn} + case "T2": + conn := historyTestConnection(standalone) + if mode == "repeated-comment" { + conn["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "comment-first"} + } + if mode == "short-page" { + conn["nodes"] = []any{} + } + data["node"] = map[string]any{"id": "T2", "comments": conn} + default: + t.Errorf("unexpected continuation: %v", req.Variables) + } + } + response := map[string]any{"data": data} + if mode == "partial-error" && pages > 0 { + response["errors"] = []any{map[string]any{"message": "partial continuation"}} + } + json.NewEncoder(w).Encode(response) + })) + defer server.Close() + batch, err := New(Options{BaseURL: server.URL}).FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if mode != "complete" { + if err == nil { + t.Fatalf("%s accepted incomplete history", mode) + } + return + } + if err != nil { + t.Fatal(err) + } + item := batch.Items[0] + if pages != 2 || len(item.ReviewComments) != 2 || len(item.Reviews) != 1 || len(item.Comments) != 1 { + t.Fatalf("incomplete or duplicate conversation: pages=%d item=%+v", pages, item) + } + if item.Reviews[0]["state"] != "APPROVED" || item.Reviews[0]["body"] != "" || item.Comments[0]["body"] != "discussion" { + t.Fatal("review metadata or discussion lost") + } + if item.ReviewComments[0]["pull_request_review_id"] != "9007199254740996" || historyMap(item.ReviewComments[0]["user"])["login"] != "helper[bot]" { + t.Fatal("review association or bot identity lost") + } + orphan := item.ReviewComments[1] + if orphan["id"] != "9007199254740994" || orphan["pull_request_review_id"] != nil || historyMap(orphan["user"]) != nil || historyMap(orphan["_graphql"])["isMinimized"] != true { + t.Fatalf("standalone/minimized/deleted-author comment lost: %+v", orphan) + } + }) + } +} + func TestGraphQLHistoryTransientRetriesKeepQueryAndAccountEveryAttempt(t *testing.T) { for _, kind := range []string{"gateway", "truncated", "empty"} { t.Run(kind, func(t *testing.T) { From a37c4f900b39f78c9a688a8c29ee87c522e2641f Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:18:13 -0600 Subject: [PATCH 05/21] Add analytics source evidence and ongoing GraphQL collection --- docs/analytics-source.md | 71 ++++ docs/commands.md | 1 + docs/sync.md | 7 + internal/cli/analytics.go | 466 ++++++++++++++++++++++++ internal/cli/analytics_test.go | 28 ++ internal/cli/app.go | 13 +- internal/cli/app_test.go | 6 +- internal/cli/gh_search_test.go | 4 +- internal/cli/help.go | 1 + internal/cli/sync.go | 3 + internal/github/analytics.go | 109 ++++++ internal/github/analytics_test.go | 46 +++ internal/github/history.go | 8 +- internal/store/analytics_source.go | 331 +++++++++++++++++ internal/store/analytics_source_test.go | 93 +++++ internal/store/comments.go | 27 +- internal/store/store.go | 5 +- internal/store/threads.go | 3 + 18 files changed, 1198 insertions(+), 24 deletions(-) create mode 100644 docs/analytics-source.md create mode 100644 internal/cli/analytics.go create mode 100644 internal/cli/analytics_test.go create mode 100644 internal/github/analytics.go create mode 100644 internal/github/analytics_test.go create mode 100644 internal/store/analytics_source.go create mode 100644 internal/store/analytics_source_test.go diff --git a/docs/analytics-source.md b/docs/analytics-source.md new file mode 100644 index 00000000..3777ac10 --- /dev/null +++ b/docs/analytics-source.md @@ -0,0 +1,71 @@ +--- +title: Analytics source preparation +nav_order: 8 +permalink: /analytics-source/ +--- + +# Analytics source preparation + +`gitcrawl analytics owner/repo` supports collector-owned publication repair, +identity enrichment and ongoing GraphQL collection for a full-history archive. +It uses the configured native source database and the existing token helper. +It does not run embeddings or classification models. + +```sh +gitcrawl --config SOURCE_CONFIG analytics owner/repo --json +gitcrawl --config SOURCE_CONFIG analytics owner/repo --apply --json +gitcrawl --config SOURCE_CONFIG --github-token-command TOKEN_HELPER \ + analytics owner/repo --enrich --watch --json +``` + +Without apply/enrich/watch/once, the command audits retained publication evidence +read-only. `--apply` performs an idempotent, bounded repair after taking an +operator-managed consistent backup. It stores `submitted_at_gh` and +`publication_at_gh` for current comments and source revisions. Review submission +is distinct from draft creation; pending reviews have no published event. Raw +payloads, existing IDs, genuine creation times and original observation times +remain unchanged, and normalization creates no fictional provider edit. + +Source schema 14 adds these fields and the actor/coverage evidence tables. +Historical normalization completion is recorded in +`analytics_collection_state.publication_repair_generation`; downstream consumers +must reconcile the affected datasets when this generation changes rather than +relying solely on append-only revision IDs. + +GraphQL history now retains actor node IDs. `--enrich` recovers missing identities +through original content nodes, not login guessing, and stores public actor +profiles. Unavailable nodes are explicit unknowns; permission or transport +failures are not converted to successful missing-data evidence. New source actors +enter the profile queue, and profiles become eligible for refresh after 24 hours. +Operational queue tables are separate from publishable evidence. + +`--watch` polls updated issues and PRs every two minutes, overlapping the prior +verified watermark by five minutes. It paginates without GitHub search's hit cap, +hydrates only relevant updated threads, and commits page checkpoints only after +native conversation collection succeeds. Up to sixteen two-thread requests progress independently per +page. Persistently failing transient requests split into smaller batches, +preserving the transport and complete-evidence requirement. Non-nested +continuation pages use 100 nodes to avoid repeated small round trips. Identity/profile enrichment uses eight disjoint 100-node requests with normal +quota guards and can run concurrently under +the same source owner. `--once` performs one resumable update cycle. + +Hydration workers reuse the watch owner's open store instead of repeating +full-archive migration checks for each batch. Independent guarded clients overlap +network work; native transactions still coordinate source writes. Enrichment +continues checking its queues every two minutes after the initial drain. New +observations enqueue unresolved content identities or known actor profiles +directly, avoiding repeated whole-archive scans during watch operation. + +A completed historical discovery receipt supplies the initial coverage baseline; +a later completed update cycle advances it. Fresh collector checks do not imply +complete historical/provider coverage. `analytics_coverage` exposes the verified +watermark independently of source row observation times. + +The permanent `runner.lock` coordinates ownership with the full-history backfill +supervisor. Do not unlink it or start a second supervisor against the same archive. +A finished backfill should be disabled as an automatic startup job when ongoing +collection takes ownership. The token-command result is held only in memory and +refreshed before its expected expiry, preserving one credential across quota +reservation and dispatch. Existing provider-rate protections remain active. + +See [sync](/sync/), [configuration](/configuration/) and the [command reference](/commands/). diff --git a/docs/commands.md b/docs/commands.md index aa4f11ef..56525b26 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -43,6 +43,7 @@ These work on every command. | Command | Purpose | Docs | | --- | --- | --- | | `gitcrawl sync owner/repo [--state --since --numbers --limit --include-comments --include-pr-details --with pr-details --graphql-history --force --progress-file --json]` | Sync issues and PRs from GitHub into local SQLite | [Sync](/sync/) | +| `gitcrawl analytics owner/repo [--apply --enrich --watch --once --json]` | Audit/repair source publication dates, enrich stable identities, and maintain GraphQL updates | [Analytics source preparation](/analytics-source/) | | `gitcrawl sync-failures owner/repo [--include-resolved --limit N --json]` | List failed issue, comment, and PR hydration attempts and optional resolved history | [Sync](/sync/#hydration-depth) | | `gitcrawl coverage [owner/repo \| --repos owner/a,owner/b] [--min-missing-pr-details N --json]` | Report archive, PR-detail, and enrichment coverage/freshness | — | | `gitcrawl fill-pr-details owner/repo [--limit --order --batch-size --reserve-rate-limit --include-comments --json-progress --json]` | Hydrate locally missing pull request detail rows in bounded batches | — | diff --git a/docs/sync.md b/docs/sync.md index f3cc291d..828655b1 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -369,3 +369,10 @@ gitcrawl sync owner/repo --numbers "$NUMS" --with pr-details - [Refresh and embed](/refresh-and-embed/) — the wrapper that runs sync, embed, and cluster end to end - [gh shim migration](/gh-shim/) — Octopool owns pooled `gh` reads now - [Portable stores](/portable-stores/) — sharing the synced cache across machines + +## Analytics source preparation + +The [analytics source command](/analytics-source/) repairs retained review publication +timestamps without rewriting raw evidence and maintains a full-history archive +through incremental GraphQL collection. Its actor IDs and coverage receipts are +source evidence for downstream analytics; account classifications remain derived. diff --git a/internal/cli/analytics.go b/internal/cli/analytics.go new file mode 100644 index 00000000..3af73561 --- /dev/null +++ b/internal/cli/analytics.go @@ -0,0 +1,466 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "github.com/openclaw/gitcrawl/internal/config" + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" + "github.com/openclaw/gitcrawl/internal/syncer" + "io" + "os" + "path/filepath" + "sync" + "time" +) + +func (a *App) analyticsClient(ctx context.Context, cfg config.Config) (*gh.Client, error) { + token := a.resolveGitHubToken(ctx, cfg) + var provider func(context.Context) (string, error) + var e error + if a.githubTokenCommand != nil { + provider, e = githubTokenProvider(*a.githubTokenCommand) + if e != nil { + return nil, e + } + } + if provider != nil { + fetch := provider + var mu sync.Mutex + var cached string + var expires time.Time + provider = func(ctx context.Context) (string, error) { + mu.Lock() + defer mu.Unlock() + if cached != "" && time.Now().Before(expires) { + return cached, nil + } + value, e := fetch(ctx) + if e != nil { + return "", e + } + cached = value + expires = time.Now().Add(45 * time.Minute) + return value, nil + } + a.analyticsTokenProvider = provider + } + if provider == nil && token.Value == "" { + return nil, fmt.Errorf("missing GitHub credential") + } + return gh.New(gh.Options{Token: token.Value, TokenProvider: provider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: 1500}), nil +} +func (a *App) runAnalytics(ctx context.Context, args []string) error { + for _, arg := range args { + if arg == "--help" || arg == "-h" { + _, err := fmt.Fprintln(a.Stdout, "Usage: gitcrawl [--config SOURCE_CONFIG] [--github-token-command TOKEN_HELPER] analytics owner/repo [--apply] [--enrich] [--watch|--once] [--json]\nWithout action flags: read-only publication audit. --apply repairs retained timestamps; --enrich collects actor evidence; --watch maintains GraphQL updates.") + return err + } + } + + fs := flag.NewFlagSet("analytics", flag.ContinueOnError) + fs.SetOutput(io.Discard) + apply := fs.Bool("apply", false, "apply source publication corrections") + enrich := fs.Bool("enrich", false, "collect missing provider identities and actor profiles") + watch := fs.Bool("watch", false, "maintain GraphQL updates every two minutes") + once := fs.Bool("once", false, "run one GraphQL update cycle") + fs.Bool("json", false, "JSON output") + if e := fs.Parse(normalizeCommandArgs(args, nil)); e != nil { + return e + } + if fs.NArg() != 1 { + return fmt.Errorf("analytics requires owner/repo") + } + owner, repo, e := parseOwnerRepo(fs.Arg(0)) + if e != nil { + return e + } + a.format = FormatJSON + cfg, e := config.LoadRuntime(a.configPath) + if e != nil { + return e + } + if !*apply && !*enrich && !*watch && !*once { + rt, e := a.openLocalRuntimeReadOnly(ctx) + if e != nil { + return e + } + defer rt.Store.Close() + r, e := rt.Store.RepairPublication(ctx, false) + if e != nil { + return e + } + return a.writeOutput("analytics_repair_dry_run", r, false) + } + lock, e := os.OpenFile(filepath.Join(filepath.Dir(cfg.DBPath), "runner.lock"), os.O_CREATE|os.O_RDWR, 0600) + if e != nil { + return e + } + defer lock.Close() + if e = lockPortableFile(lock); e != nil { + return fmt.Errorf("collector ownership lock busy: %w", e) + } + rt, e := a.openLocalRuntime(ctx) + if e != nil { + return e + } + defer rt.Store.Close() + if *apply { + r, e := rt.Store.RepairPublication(ctx, true) + if e != nil { + return e + } + if e = a.writeOutput("analytics_repair", r, false); e != nil { + return e + } + } + if !*enrich && !*watch && !*once { + return nil + } + client, e := a.analyticsClient(ctx, cfg) + if e != nil { + return e + } + if *watch || *once { + if e = rt.Store.SeedAnalyticsNodes(ctx, true); e != nil { + return e + } + var existing int + if e = rt.Store.DB().QueryRowContext(ctx, "SELECT count(*) FROM analytics_coverage WHERE repository=?", owner+"/"+repo).Scan(&existing); e != nil { + return e + } + if existing == 0 { + b, err := os.ReadFile(filepath.Join(filepath.Dir(a.configPath), "status.json")) + if err == nil { + var status struct { + Phase string `json:"phase"` + Discovery []struct { + Kind string `json:"kind"` + Done int `json:"done"` + Total int `json:"total"` + Updated string `json:"updated_at"` + } `json:"discovery"` + } + if json.Unmarshal(b, &status) == nil && status.Phase == "complete" && len(status.Discovery) == 2 { + through := "" + issues, prs := 0, 0 + valid := true + for _, d := range status.Discovery { + valid = valid && d.Done == 1 + if through == "" || d.Updated < through { + through = d.Updated + } + if d.Kind == "issues" { + issues = d.Total + } else if d.Kind == "pullRequests" { + prs = d.Total + } else { + valid = false + } + } + if valid && issues > 0 && prs > 0 { + if e = rt.Store.SaveAnalyticsCoverage(ctx, owner+"/"+repo, through, issues, prs); e != nil { + return e + } + } + } + } + } + } + // Independent guarded clients permit disjoint evidence batches to overlap; + // each preserves the normal quota reservation and uses the same cached token. + actorClients := make([]*gh.Client, 8) + for i := range actorClients { + token := a.resolveGitHubToken(ctx, cfg) + actorClients[i] = gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: 1500}) + } + parallelWatch := *watch && *enrich + if parallelWatch { + pollCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + defer func() { cancel(); <-done }() + go func() { + defer close(done) + for { + pollErr := a.analyticsCycle(pollCtx, rt.Store, client, owner, repo) + if pollErr != nil { + fmt.Fprintf(a.Stderr, "{\"event\":\"github_update_failed\",\"error\":%q}\n", pollErr.Error()) + } else { + fmt.Fprintln(a.Stderr, "{\"event\":\"github_update_complete\"}") + } + select { + case <-pollCtx.Done(): + return + case <-time.After(2 * time.Minute): + } + } + }() + } + if *enrich { + for _, profiles := range []bool{true, false} { + if e = rt.Store.SeedAnalyticsNodes(ctx, profiles); e != nil { + return e + } + } + e = maintainAnalyticsEnrichment(ctx, parallelWatch, 2*time.Minute, func() error { + for _, profiles := range []bool{true, false, true} { + for { + var ids []string + if profiles { + ids, e = rt.Store.AnalyticsProfileNodes(ctx, 800) + } else { + ids, e = rt.Store.AnalyticsIdentityNodes(ctx, 800) + } + if e != nil { + return e + } + if len(ids) == 0 { + break + } + var group sync.WaitGroup + var failures []error + var failureMu sync.Mutex + for offset := 0; offset < len(ids); offset += 100 { + part := append([]string(nil), ids[offset:min(offset+100, len(ids))]...) + worker := actorClients[offset/100] + group.Add(1) + go func() { + defer group.Done() + nodes, err := worker.AnalyticsNodes(ctx, part, profiles) + if err == nil { + at := time.Now().UTC().Format(time.RFC3339Nano) + if profiles { + err = rt.Store.SaveActorProfiles(ctx, nodes, at) + } else { + err = rt.Store.SaveActorEvidence(ctx, nodes, at) + } + } + if err != nil { + failureMu.Lock() + failures = append(failures, err) + failureMu.Unlock() + } + }() + } + group.Wait() + if len(failures) > 0 { + fmt.Fprintf(a.Stderr, "{\"event\":\"actor_enrichment_retry\",\"failed_batches\":%d}\n", len(failures)) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(30 * time.Second): + } + continue + } + + fmt.Fprintf(a.Stderr, "{\"event\":\"actor_enrichment\",\"profiles\":%t,\"nodes\":%d}\n", profiles, len(ids)) + + } + } + return nil + }) + if e != nil { + return e + } + } + + if parallelWatch { + <-ctx.Done() + return ctx.Err() + } + for *watch || *once { + e = a.analyticsCycle(ctx, rt.Store, client, owner, repo) + if e != nil { + if !*watch { + return e + } + fmt.Fprintf(a.Stderr, "{\"event\":\"github_update_failed\",\"error\":%q}\n", e.Error()) + } else { + fmt.Fprintln(a.Stderr, "{\"event\":\"github_update_complete\"}") + } + if !*watch { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(2 * time.Minute): + } + } + return nil +} + +type updateCheckpoint struct { + Started string `json:"started"` + Since string `json:"since"` + Kind int `json:"kind"` + Cursor string `json:"cursor"` + Issues int `json:"issues"` + PRs int `json:"prs"` +} + +func (a *App) analyticsCycle(ctx context.Context, s *store.Store, c *gh.Client, owner, repo string) error { + key := "updates:" + owner + "/" + repo + value, e := s.AnalyticsState(ctx, key) + if e != nil { + return e + } + var cp updateCheckpoint + if value != "" { + if e = json.Unmarshal([]byte(value), &cp); e != nil { + return e + } + } + if cp.Started == "" { + through, e := s.AnalyticsState(ctx, "through:"+owner+"/"+repo) + if e != nil { + return e + } + if through == "" { // Conservative baseline: earliest completed historical discovery, not the last row fetched. + b, e := os.ReadFile(filepath.Join(filepath.Dir(a.configPath), "status.json")) + if e == nil { + var st struct { + Discovery []struct { + Updated string `json:"updated_at"` + Done int `json:"done"` + } + } + if json.Unmarshal(b, &st) == nil { + for _, d := range st.Discovery { + if d.Done == 1 && (through == "" || d.Updated < through) { + through = d.Updated + } + } + } + } + } + if through == "" { + return fmt.Errorf("verified historical discovery watermark required") + } + at, e := time.Parse(time.RFC3339Nano, through) + if e != nil { + return e + } + cp = updateCheckpoint{Started: time.Now().UTC().Format(time.RFC3339Nano), Since: at.Add(-5 * time.Minute).Format(time.RFC3339Nano)} + } + since, _ := time.Parse(time.RFC3339Nano, cp.Since) + for cp.Kind < 2 { + kind := []string{"issues", "pullRequests"}[cp.Kind] + page, e := c.UpdatedNumbers(ctx, owner, repo, kind, cp.Cursor, since) + if e != nil { + return e + } + if cp.Kind == 0 { + cp.Issues = page.Total + } else { + cp.PRs = page.Total + } + var wg sync.WaitGroup + slots := make(chan struct{}, 16) + var firstErr error + var errMu sync.Mutex + for i := 0; i < len(page.Numbers); i += 2 { + numbers := append([]int(nil), page.Numbers[i:min(i+2, len(page.Numbers))]...) + slots <- struct{}{} + wg.Add(1) + go func() { + defer func() { <-slots }() + defer wg.Done() + e := a.syncAnalyticsBatch(ctx, s, owner, repo, numbers) + if e != nil { + errMu.Lock() + if firstErr == nil { + firstErr = e + } + errMu.Unlock() + } + }() + } + wg.Wait() + if firstErr != nil { + return firstErr + } + fmt.Fprintf(a.Stderr, "{\"event\":\"github_update_page\",\"kind\":%q,\"threads\":%d}\n", kind, len(page.Numbers)) + + if !page.More || (!page.Oldest.IsZero() && page.Oldest.Before(since)) { + cp.Kind++ + cp.Cursor = "" + } else { + cp.Cursor = page.Cursor + } + b, _ := json.Marshal(cp) + if e = s.SetAnalyticsState(ctx, key, string(b)); e != nil { + return e + } + for batch := 0; batch < 5; batch++ { + ids, e := s.AnalyticsProfileNodes(ctx, 100) + if e != nil { + return e + } + if len(ids) == 0 { + break + } + nodes, e := c.AnalyticsNodes(ctx, ids, true) + if e != nil { + return e + } + if e = s.SaveActorProfiles(ctx, nodes, time.Now().UTC().Format(time.RFC3339Nano)); e != nil { + return e + } + } + + } + if e = s.SaveAnalyticsCoverage(ctx, owner+"/"+repo, cp.Started, cp.Issues, cp.PRs); e != nil { + return e + } + if e = s.SetAnalyticsState(ctx, "through:"+owner+"/"+repo, cp.Started); e != nil { + return e + } + return s.SetAnalyticsState(ctx, key, "") +} + +// Smaller requests prevent high-fanout conversation queries exhausting GitHub's +// execution deadline. Split a persistently failing transient batch without +// accepting partial conversation evidence or changing transports. +func (a *App) syncAnalyticsBatch(ctx context.Context, s *store.Store, owner, repo string, numbers []int) error { + cfg, err := config.LoadRuntime(a.configPath) + if err != nil { + return err + } + token := a.resolveGitHubToken(ctx, cfg) + client := gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: 1500}) + // The watch owner has already validated and opened this store. Reopening it + // for every two threads repeats full-archive migration audits and serializes + // otherwise independent network work. Native transactions still own writes. + _, err = syncer.New(client, s).Sync(ctx, syncer.Options{Owner: owner, Repo: repo, GraphQLHistory: true, State: "all", Numbers: numbers, IncludeComments: true, IncludePRMetadata: true}) + if err == nil || ctx.Err() != nil || len(numbers) < 2 { + return err + } + var response *gh.RequestError + if errors.As(err, &response) && (response.Status == 502 || response.Status == 503 || response.Status == 504) { + middle := len(numbers) / 2 + return errors.Join(a.syncAnalyticsBatch(ctx, s, owner, repo, numbers[:middle]), a.syncAnalyticsBatch(ctx, s, owner, repo, numbers[middle:])) + } + return err +} + +// A drained queue is not the end of a watch: new actors and stale profiles +// become eligible later. Initial source scanning happens outside this loop. +func maintainAnalyticsEnrichment(ctx context.Context, watch bool, interval time.Duration, drain func() error) error { + for { + if err := drain(); err != nil { + return err + } + if !watch { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } +} diff --git a/internal/cli/analytics_test.go b/internal/cli/analytics_test.go new file mode 100644 index 00000000..5fd55cf4 --- /dev/null +++ b/internal/cli/analytics_test.go @@ -0,0 +1,28 @@ +package cli + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestAnalyticsEnrichmentContinuesAfterDraining(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + calls := 0 + err := maintainAnalyticsEnrichment(ctx, true, time.Millisecond, func() error { + calls++ + if calls == 2 { + cancel() + } + return nil + }) + if calls != 2 || !errors.Is(err, context.Canceled) { + t.Fatalf("calls=%d err=%v", calls, err) + } + calls = 0 + if err := maintainAnalyticsEnrichment(context.Background(), false, time.Millisecond, func() error { calls++; return nil }); err != nil || calls != 1 { + t.Fatalf("one-shot calls=%d err=%v", calls, err) + } +} diff --git a/internal/cli/app.go b/internal/cli/app.go index 0a390fc7..91462226 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -25,11 +25,12 @@ const ( ) type App struct { - githubTokenCommand *string - githubTokenMu sync.Mutex - observedGitHubToken string - Stdout io.Writer - Stderr io.Writer + analyticsTokenProvider func(context.Context) (string, error) + githubTokenCommand *string + githubTokenMu sync.Mutex + observedGitHubToken string + Stdout io.Writer + Stderr io.Writer configPath string format OutputFormat @@ -126,6 +127,8 @@ func (a *App) Run(ctx context.Context, args []string) error { return a.runDoctor(ctx, rest[1:]) case "status": return a.runStatus(ctx, rest[1:]) + case "analytics": + return a.runAnalytics(ctx, rest[1:]) case "sync": return a.runSync(ctx, rest[1:]) case "fill-pr-details": diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 1a3c3fdd..6f222382 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -4344,10 +4344,10 @@ func TestDoctorJSONReportsCurrentSchemaDiagnosticsWithoutMutation(t *testing.T) if got := schema["state"]; got != "current" { t.Fatalf("db_schema.state = %#v, payload=%#v", got, schema) } - if got := schema["current_version"]; got != float64(13) { + if got := schema["current_version"]; got != float64(14) { t.Fatalf("db_schema.current_version = %#v, payload=%#v", got, schema) } - if got := schema["supported_version"]; got != float64(13) { + if got := schema["supported_version"]; got != float64(14) { t.Fatalf("db_schema.supported_version = %#v, payload=%#v", got, schema) } if got := schema["child_observation_reservations"]; got != true { @@ -4609,7 +4609,7 @@ func TestDoctorJSONReportsLegacyPendingSchemaWithoutMutation(t *testing.T) { t.Fatalf("pr_details.duplicate_path_files_supported = %#v, payload=%#v", got, prDetails) } pending := doctorStringList(t, schema, "pending_migrations") - if !doctorListContains(pending, "schema_version_3_to_13") || + if !doctorListContains(pending, "schema_version_3_to_14") || !doctorListContains(pending, "pull_request_files_position_key") || !doctorListContains(pending, "thread_child_observation_reservations_table") { t.Fatalf("pending_migrations = %#v", pending) diff --git a/internal/cli/gh_search_test.go b/internal/cli/gh_search_test.go index aeb79dcc..5fe2fe7c 100644 --- a/internal/cli/gh_search_test.go +++ b/internal/cli/gh_search_test.go @@ -265,8 +265,8 @@ func TestGHSearchSyncIfStaleMigratesFreshPortableRuntime(t *testing.T) { if err := rt.Store.DB().QueryRowContext(ctx, `pragma user_version`).Scan(&schemaVersion); err != nil { t.Fatalf("read runtime schema version: %v", err) } - if schemaVersion != 13 { - t.Fatalf("runtime schema version = %d, want 13", schemaVersion) + if schemaVersion != 14 { + t.Fatalf("runtime schema version = %d, want 14", schemaVersion) } var tableName string if err := rt.Store.DB().QueryRowContext(ctx, `select name from sqlite_schema where type = 'table' and name = 'sync_runs'`).Scan(&tableName); err != nil { diff --git a/internal/cli/help.go b/internal/cli/help.go index a9d9c4d1..fe0c1b94 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -53,6 +53,7 @@ Core commands: init create config, optionally from a portable store doctor check config, token, and database readiness sync sync GitHub issue and pull request metadata + analytics repair publication dates, enrich identities, or watch GraphQL updates sync-failures list failed sync hydration attempts coverage report local archive PR-detail completeness fill-pr-details hydrate locally missing pull request detail rows diff --git a/internal/cli/sync.go b/internal/cli/sync.go index 16a8b1d7..ee53ccfe 100644 --- a/internal/cli/sync.go +++ b/internal/cli/sync.go @@ -400,6 +400,9 @@ func (a *App) syncRepository(ctx context.Context, owner, repo string, options sy return command(ctx) } } + if a.analyticsTokenProvider != nil { + provider = a.analyticsTokenProvider + } if provider == nil && token.Value == "" { return syncer.Stats{}, dbTargetInfo{}, fmt.Errorf("missing GitHub token: set %s or authenticate gh", cfg.GitHub.TokenEnv) } diff --git a/internal/github/analytics.go b/internal/github/analytics.go new file mode 100644 index 00000000..7d123b20 --- /dev/null +++ b/internal/github/analytics.go @@ -0,0 +1,109 @@ +package github + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" +) + +// AnalyticsNodes reads public provider evidence for the requested native node IDs. +func (c *Client) AnalyticsNodes(ctx context.Context, ids []string, profiles bool) ([]map[string]any, error) { + fields := `... on Issue {author{login __typename ... on Node{id}}} ... on PullRequest {author{login __typename ... on Node{id}}} ... on IssueComment {author{login __typename ... on Node{id}}} ... on PullRequestReview {author{login __typename ... on Node{id}}} ... on PullRequestReviewComment {author{login __typename ... on Node{id}}}` + if profiles { + fields = `... on User {login name bio url createdAt} ... on Bot {login url createdAt} ... on Mannequin {login url createdAt}` + } + payload, e := json.Marshal(graphqlEnvelope{Query: `query($ids:[ID!]!){rateLimit{cost remaining limit used resetAt} nodes(ids:$ids){id __typename ` + fields + `}}`, Variables: map[string]any{"ids": ids}}) + if e != nil { + return nil, e + } + var envelope struct { + Data json.RawMessage `json:"data"` + Errors []struct { + Message string `json:"message"` + Type string `json:"type"` + Path []any `json:"path"` + } `json:"errors"` + } + if e = c.doJSON(ctx, http.MethodPost, c.graphQLURL, bytes.NewReader(payload), nil, &envelope); e != nil { + return nil, e + } + for _, failure := range envelope.Errors { + if failure.Type != "NOT_FOUND" || len(failure.Path) < 2 || failure.Path[0] != "nodes" { + return nil, fmt.Errorf("actor evidence GraphQL error: %s", failure.Message) + } + } + var data map[string]any + if e = json.Unmarshal(envelope.Data, &data); e != nil { + return nil, e + } + + nodes, ok := data["nodes"].([]any) + if !ok || len(nodes) != len(ids) { + return nil, fmt.Errorf("incomplete actor identity response") + } + out := make([]map[string]any, 0, len(ids)) + for i, v := range nodes { + n, ok := v.(map[string]any) + if !ok { + n = map[string]any{"id": ids[i], "__typename": "Unavailable", "unavailable": true} + } + if n["id"] != ids[i] { + return nil, fmt.Errorf("actor node identity mismatch") + } + out = append(out, n) + } + return out, nil +} + +type UpdatedPage struct { + Numbers []int + Cursor string + More bool + Total int + Oldest time.Time +} + +func (c *Client) UpdatedNumbers(ctx context.Context, owner, repo, kind, after string, since time.Time) (UpdatedPage, error) { + if kind != "issues" && kind != "pullRequests" { + return UpdatedPage{}, fmt.Errorf("invalid discovery kind") + } + h := historySession{client: c, remaining: 20000} + var cursor any + if after != "" { + cursor = after + } + data, e := h.request(ctx, `query($owner:String!,$repo:String!,$after:String){rateLimit{cost remaining limit used resetAt} repository(owner:$owner,name:$repo){`+kind+`(first:100,after:$after,orderBy:{field:UPDATED_AT,direction:DESC}){totalCount pageInfo{hasNextPage endCursor} nodes{number updatedAt}}}}`, map[string]any{"owner": owner, "repo": repo, "after": cursor}, 1) + if e != nil { + return UpdatedPage{}, e + } + r := historyMap(historyMap(data["repository"])[kind]) + p := UpdatedPage{} + var valid bool + p.Total, valid = historyInt(r["totalCount"]) + if !valid || p.Total < 0 { + return p, fmt.Errorf("missing update-discovery count") + } + info := historyMap(r["pageInfo"]) + p.More, _ = info["hasNextPage"].(bool) + p.Cursor = historyString(info["endCursor"]) + for _, n := range historyNodes(historyMap(data["repository"]), kind) { + number, ok := historyInt(n["number"]) + at, e := time.Parse(time.RFC3339Nano, historyString(n["updatedAt"])) + if !ok || number < 1 || e != nil { + return p, fmt.Errorf("invalid update-discovery evidence") + } + if !at.Before(since) { + p.Numbers = append(p.Numbers, number) + } + if p.Oldest.IsZero() || at.Before(p.Oldest) { + p.Oldest = at + } + } + if p.More && (p.Cursor == "" || p.Cursor == after || p.Oldest.IsZero()) { + return p, fmt.Errorf("update cursor did not advance") + } + return p, nil +} diff --git a/internal/github/analytics_test.go b/internal/github/analytics_test.go new file mode 100644 index 00000000..0adc2dec --- /dev/null +++ b/internal/github/analytics_test.go @@ -0,0 +1,46 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestAnalyticsDiscoveryFiltersOldRowsAndRejectsBadCursors(t *testing.T) { + more := false + cursor := "end" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "limit": 20000, "used": 1000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}, "repository": map[string]any{"issues": map[string]any{"totalCount": 2, "pageInfo": map[string]any{"hasNextPage": more, "endCursor": cursor}, "nodes": []any{map[string]any{"number": 2, "updatedAt": "2026-09-24T00:00:00Z"}, map[string]any{"number": 1, "updatedAt": "2026-09-01T00:00:00Z"}}}}}}) + })) + defer server.Close() + c := New(Options{Token: "fixture", BaseURL: server.URL}) + since, _ := time.Parse(time.RFC3339, "2026-09-23T00:00:00Z") + p, e := c.UpdatedNumbers(context.Background(), "fixture", "repo", "issues", "", since) + if e != nil || len(p.Numbers) != 1 || p.Numbers[0] != 2 || p.Total != 2 { + t.Fatalf("%+v %v", p, e) + } + more = true + cursor = "same" + if _, e = c.UpdatedNumbers(context.Background(), "fixture", "repo", "issues", "same", since); e == nil { + t.Fatal("non-advancing cursor accepted") + } +} +func TestAnalyticsUnavailableNodesAreNotAuthorizationSuccess(t *testing.T) { + typ := "NOT_FOUND" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"nodes": []any{map[string]any{"id": "one", "__typename": "User", "login": "fixture"}, nil}}, "errors": []any{map[string]any{"type": typ, "message": "fixture unavailable", "path": []any{"nodes", 1}}}}) + })) + defer server.Close() + c := New(Options{Token: "fixture", BaseURL: server.URL}) + nodes, e := c.AnalyticsNodes(context.Background(), []string{"one", "two"}, true) + if e != nil || len(nodes) != 2 || nodes[1]["id"] != "two" || nodes[1]["__typename"] != "Unavailable" { + t.Fatalf("%+v %v", nodes, e) + } + typ = "FORBIDDEN" + if _, e = c.AnalyticsNodes(context.Background(), []string{"one", "two"}, true); e == nil { + t.Fatal("authorization failure became missing-data evidence") + } +} diff --git a/internal/github/history.go b/internal/github/history.go index b0012a27..3ed3d33c 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -24,7 +24,7 @@ type HistoryBatch struct { Items []HistoryItem } -const historyActor = `author { login __typename url }` +const historyActor = `author { login __typename url ... on Node { id } }` const historyComment = `id __typename fullDatabaseId body ` + historyActor + ` authorAssociation createdAt updatedAt publishedAt url isMinimized minimizedReason` const historyInline = historyComment + ` path diffHunk line startLine originalLine originalStartLine position originalPosition state subjectType outdated commit { oid } originalCommit { oid } replyTo { id fullDatabaseId } pullRequestReview { id fullDatabaseId }` @@ -35,7 +35,11 @@ var historyIssue = historyCommon + ` stateReason` var historyPull = historyCommon + ` isDraft merged mergedAt mergedBy { login __typename url } mergeCommit { oid } mergeable mergeStateStatus maintainerCanModify additions deletions changedFiles headRefName headRefOid baseRefName baseRefOid headRepository { nameWithOwner } baseRepository { nameWithOwner } commits { totalCount } ` + historyConnection("reviews", historyReview, "") + " " + historyConnection("reviewThreads", historyReviewThread, "") func historyConnection(name, fields, after string) string { - return name + `(first:20` + after + `) { totalCount pageInfo { hasNextPage endCursor } nodes { ` + fields + ` } }` + size := "20" + if after != "" && name != "reviews" && name != "reviewThreads" { + size = "100" + } + return name + `(first:` + size + after + `) { totalCount pageInfo { hasNextPage endCursor } nodes { ` + fields + ` } }` } type historySession struct { diff --git a/internal/store/analytics_source.go b/internal/store/analytics_source.go new file mode 100644 index 00000000..22f92f30 --- /dev/null +++ b/internal/store/analytics_source.go @@ -0,0 +1,331 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" +) + +// Publication describes provider-authored timestamps, not capture/repair time. +type Publication struct { + Submitted string `json:"submitted_at_gh,omitempty"` + Published string `json:"publication_at_gh,omitempty"` +} + +func ProviderTime(value any) string { + s, ok := value.(string) + if !ok || strings.TrimSpace(s) != s { + return "" + } + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return "" + } + return t.UTC().Format(time.RFC3339Nano) +} +func CommentPublication(kind, raw string) Publication { + var p map[string]any + if json.Unmarshal([]byte(raw), &p) != nil { + return Publication{} + } + g, _ := p["_graphql"].(map[string]any) + pick := func(values ...any) string { + for _, v := range values { + if s := ProviderTime(v); s != "" { + return s + } + } + return "" + } + if kind == "pull_review" { + if strings.EqualFold(fmt.Sprint(p["state"]), "pending") { + return Publication{} + } + at := pick(p["submitted_at"], g["submittedAt"]) + return Publication{Submitted: at, Published: at} + } + return Publication{Published: pick(g["publishedAt"], p["published_at"], p["created_at"], g["createdAt"])} +} +func (s *Store) ensureAnalyticsSourceSchema(ctx context.Context) error { + for _, table := range []string{"comments", "comment_revisions"} { + for _, col := range []string{"submitted_at_gh", "publication_at_gh"} { + if err := s.ensureColumn(ctx, table, col, "text"); err != nil { + return err + } + } + } + _, err := s.q().ExecContext(ctx, `CREATE TABLE IF NOT EXISTS actor_identity_evidence( + node_id TEXT PRIMARY KEY,actor_node_id TEXT,login TEXT,actor_type TEXT,observed_at TEXT NOT NULL,raw_json TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS actor_profiles(node_id TEXT PRIMARY KEY,login TEXT NOT NULL,actor_type TEXT NOT NULL,name TEXT,bio TEXT,url TEXT,created_at TEXT,observed_at TEXT NOT NULL,raw_json TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS analytics_coverage(repository TEXT PRIMARY KEY,through TEXT NOT NULL,issues INTEGER,pull_requests INTEGER,complete INTEGER NOT NULL,observed_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS analytics_repair_receipts(name TEXT PRIMARY KEY,cursor INTEGER NOT NULL DEFAULT 0,updated_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS analytics_pending_nodes(node_id TEXT NOT NULL,kind TEXT NOT NULL,PRIMARY KEY(kind,node_id)); + CREATE TABLE IF NOT EXISTS analytics_collection_state(name TEXT PRIMARY KEY,value TEXT NOT NULL,updated_at TEXT NOT NULL);`) + return err +} +func (s *Store) queueAnalyticsActor(ctx context.Context, raw string) error { + var payload struct { + NodeID string `json:"node_id"` + User struct { + NodeID string `json:"node_id"` + } `json:"user"` + } + if json.Unmarshal([]byte(raw), &payload) != nil { + return nil + } + id, kind := payload.User.NodeID, "profile" + if id == "" { + id, kind = payload.NodeID, "identity" + } + if id == "" { + return nil + } + _, err := s.q().ExecContext(ctx, "INSERT OR IGNORE INTO analytics_pending_nodes(node_id,kind) VALUES(?,?)", id, kind) + return err +} +func (s *Store) upsertCommentPublication(ctx context.Context, id int64, kind, raw string) error { + p := CommentPublication(kind, raw) + if err := s.queueAnalyticsActor(ctx, raw); err != nil { + return err + } + _, err := s.q().ExecContext(ctx, "UPDATE comments SET submitted_at_gh=?,publication_at_gh=? WHERE id=?", nullString(p.Submitted), nullString(p.Published), id) + return err +} + +type PublicationRepair struct { + Scanned int `json:"scanned"` + Changed int `json:"changed"` + WouldChange int `json:"would_change"` + Unknown int `json:"unknown"` +} + +// RepairPublication uses retained native payloads. Raw JSON, IDs and original +// recorded_at values are never rewritten, and no synthetic revision is appended. +func (s *Store) RepairPublication(ctx context.Context, apply bool) (PublicationRepair, error) { + result := PublicationRepair{} + pending := false + if apply { + value, e := s.AnalyticsState(ctx, "publication_repair_in_progress") + if e != nil { + return result, e + } + pending = value == "1" + if e = s.SetAnalyticsState(ctx, "publication_repair_in_progress", "1"); e != nil { + return result, e + } + } + for _, table := range []string{"comments", "comment_revisions"} { + var cursor int64 + var hasFields int + if err := s.q().QueryRowContext(ctx, "SELECT count(*) FROM pragma_table_info(?) WHERE name IN('submitted_at_gh','publication_at_gh')", table).Scan(&hasFields); err != nil { + return result, err + } + for { + fields := ",NULL,NULL" + if hasFields == 2 { + fields = ",c.submitted_at_gh,c.publication_at_gh" + } + query := "SELECT c.id,c.comment_type,c.raw_json" + fields + " FROM comments c WHERE c.id>? ORDER BY c.id LIMIT 1000" + if table == "comment_revisions" { + fields = ",NULL,NULL" + if hasFields == 2 { + fields = ",r.submitted_at_gh,r.publication_at_gh" + } + query = "SELECT r.id,c.comment_type,r.raw_json" + fields + " FROM comment_revisions r JOIN comments c ON c.id=r.comment_id WHERE r.id>? ORDER BY r.id LIMIT 1000" + } + rows, err := s.q().QueryContext(ctx, query, cursor) + if err != nil { + return result, err + } + type item struct { + id int64 + kind, raw string + submitted, published sql.NullString + } + var batch []item + for rows.Next() { + var r item + if err := rows.Scan(&r.id, &r.kind, &r.raw, &r.submitted, &r.published); err != nil { + rows.Close() + return result, err + } + batch = append(batch, r) + } + err = rows.Err() + rows.Close() + if err != nil { + return result, err + } + if len(batch) == 0 { + break + } + if apply { + err = s.WithTx(ctx, func(tx *Store) error { + for _, r := range batch { + p := CommentPublication(r.kind, r.raw) + if p.Submitted != r.submitted.String || p.Published != r.published.String { + result.WouldChange++ + } + result.Scanned++ + if r.kind == "pull_review" && p.Submitted == "" { + result.Unknown++ + } + res, e := tx.q().ExecContext(ctx, "UPDATE "+table+" SET submitted_at_gh=?,publication_at_gh=? WHERE id=? AND raw_json=? AND (submitted_at_gh IS NOT ? OR publication_at_gh IS NOT ?)", nullString(p.Submitted), nullString(p.Published), r.id, r.raw, nullString(p.Submitted), nullString(p.Published)) + if e != nil { + return e + } + n, e := res.RowsAffected() + if e != nil { + return e + } + result.Changed += int(n) + } + _, e := tx.q().ExecContext(ctx, "INSERT INTO analytics_repair_receipts(name,cursor,updated_at) VALUES(?,?,?) ON CONFLICT(name) DO UPDATE SET cursor=excluded.cursor,updated_at=excluded.updated_at", table, batch[len(batch)-1].id, time.Now().UTC().Format(time.RFC3339Nano)) + return e + }) + if err != nil { + return result, err + } + } else { + for _, r := range batch { + result.Scanned++ + p := CommentPublication(r.kind, r.raw) + if p.Submitted != r.submitted.String || p.Published != r.published.String { + result.WouldChange++ + } + if r.kind == "pull_review" && p.Submitted == "" { + result.Unknown++ + } + } + } + cursor = batch[len(batch)-1].id + } + } + if apply { + if result.Changed > 0 || pending { + if e := s.SetAnalyticsState(ctx, "publication_repair_generation", time.Now().UTC().Format(time.RFC3339Nano)); e != nil { + return result, e + } + } + if e := s.SetAnalyticsState(ctx, "publication_repair_in_progress", "0"); e != nil { + return result, e + } + } + return result, nil +} + +// Use the native node itself as the identity evidence key: login reuse is not an +// identity join. Deleted/unavailable actors are retained as explicit unknowns. +func (s *Store) SaveActorEvidence(ctx context.Context, nodes []map[string]any, at string) error { + return s.WithTx(ctx, func(tx *Store) error { + for _, n := range nodes { + id, _ := n["id"].(string) + if id == "" { + continue + } + a, _ := n["author"].(map[string]any) + actor, _ := a["id"].(string) + if actor != "" { + if _, e := tx.q().ExecContext(ctx, "INSERT OR IGNORE INTO analytics_pending_nodes(node_id,kind) VALUES(?,'profile')", actor); e != nil { + return e + } + } + login, _ := a["login"].(string) + typ, _ := a["__typename"].(string) + raw, e := json.Marshal(n) + if e != nil { + return e + } + if _, e = tx.q().ExecContext(ctx, `INSERT INTO actor_identity_evidence VALUES(?,?,?,?,?,?) ON CONFLICT(node_id) DO UPDATE SET actor_node_id=excluded.actor_node_id,login=excluded.login,actor_type=excluded.actor_type,observed_at=excluded.observed_at,raw_json=excluded.raw_json`, id, nullString(actor), nullString(login), nullString(typ), at, string(raw)); e != nil { + return e + } + } + return nil + }) +} +func (s *Store) SaveActorProfiles(ctx context.Context, nodes []map[string]any, at string) error { + return s.WithTx(ctx, func(tx *Store) error { + for _, n := range nodes { + id, _ := n["id"].(string) + login, _ := n["login"].(string) + typ, _ := n["__typename"].(string) + if id == "" { + continue + } + raw, e := json.Marshal(n) + if e != nil { + return e + } + name, _ := n["name"].(string) + bio, _ := n["bio"].(string) + url, _ := n["url"].(string) + _, e = tx.q().ExecContext(ctx, `INSERT INTO actor_profiles VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(node_id) DO UPDATE SET login=excluded.login,actor_type=excluded.actor_type,name=excluded.name,bio=excluded.bio,url=excluded.url,created_at=excluded.created_at,observed_at=excluded.observed_at,raw_json=excluded.raw_json`, id, login, typ, nullString(name), nullString(bio), nullString(url), nullString(ProviderTime(n["createdAt"])), at, string(raw)) + if e != nil { + return e + } + } + return nil + }) +} +func (s *Store) SeedAnalyticsNodes(ctx context.Context, profiles bool) error { + query := `INSERT OR IGNORE INTO analytics_pending_nodes(node_id,kind) SELECT json_extract(raw_json,'$.node_id'),'identity' FROM threads WHERE json_valid(raw_json) AND json_extract(raw_json,'$.user.node_id') IS NULL AND json_extract(raw_json,'$.node_id') IS NOT NULL + UNION SELECT json_extract(raw_json,'$.node_id'),'identity' FROM comments WHERE json_valid(raw_json) AND json_extract(raw_json,'$.user.node_id') IS NULL AND json_extract(raw_json,'$.node_id') IS NOT NULL` + if profiles { + query = `INSERT OR IGNORE INTO analytics_pending_nodes(node_id,kind) SELECT json_extract(raw_json,'$.user.node_id'),'profile' FROM threads WHERE json_valid(raw_json) AND json_extract(raw_json,'$.user.node_id') IS NOT NULL + UNION SELECT json_extract(raw_json,'$.user.node_id'),'profile' FROM comments WHERE json_valid(raw_json) AND json_extract(raw_json,'$.user.node_id') IS NOT NULL + UNION SELECT actor_node_id,'profile' FROM actor_identity_evidence WHERE actor_node_id IS NOT NULL` + } + _, err := s.q().ExecContext(ctx, query) + return err +} +func (s *Store) AnalyticsIdentityNodes(ctx context.Context, limit int) ([]string, error) { + return s.analyticsNodes(ctx, limit, false) +} +func (s *Store) AnalyticsProfileNodes(ctx context.Context, limit int) ([]string, error) { + return s.analyticsNodes(ctx, limit, true) +} +func (s *Store) analyticsNodes(ctx context.Context, limit int, profiles bool) ([]string, error) { + query := `SELECT q.node_id FROM analytics_pending_nodes q WHERE kind='identity' AND NOT EXISTS(SELECT 1 FROM actor_identity_evidence e WHERE e.node_id=q.node_id) LIMIT ?` + if profiles { + query = `SELECT q.node_id FROM analytics_pending_nodes q WHERE kind='profile' AND NOT EXISTS(SELECT 1 FROM actor_profiles p WHERE p.node_id=q.node_id AND p.observed_at>=?) LIMIT ?` + } + var rows *sql.Rows + var e error + if profiles { + rows, e = s.q().QueryContext(ctx, query, time.Now().UTC().Add(-24*time.Hour).Format(time.RFC3339Nano), limit) + } else { + rows, e = s.q().QueryContext(ctx, query, limit) + } + if e != nil { + return nil, e + } + defer rows.Close() + var out []string + for rows.Next() { + var id string + if e = rows.Scan(&id); e != nil { + return nil, e + } + out = append(out, id) + } + return out, rows.Err() +} +func (s *Store) AnalyticsState(ctx context.Context, key string) (string, error) { + var value string + err := s.q().QueryRowContext(ctx, "SELECT value FROM analytics_collection_state WHERE name=?", key).Scan(&value) + if err == sql.ErrNoRows { + return "", nil + } + return value, err +} +func (s *Store) SetAnalyticsState(ctx context.Context, key, value string) error { + _, err := s.q().ExecContext(ctx, "INSERT INTO analytics_collection_state VALUES(?,?,?) ON CONFLICT(name) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", key, value, time.Now().UTC().Format(time.RFC3339Nano)) + return err +} +func (s *Store) SaveAnalyticsCoverage(ctx context.Context, repo, through string, issues, prs int) error { + _, e := s.q().ExecContext(ctx, `INSERT INTO analytics_coverage VALUES(?,?,?,?,1,?) ON CONFLICT(repository) DO UPDATE SET through=excluded.through,issues=excluded.issues,pull_requests=excluded.pull_requests,complete=1,observed_at=excluded.observed_at`, repo, through, issues, prs, time.Now().UTC().Format(time.RFC3339Nano)) + return e +} diff --git a/internal/store/analytics_source_test.go b/internal/store/analytics_source_test.go new file mode 100644 index 00000000..bd2390a0 --- /dev/null +++ b/internal/store/analytics_source_test.go @@ -0,0 +1,93 @@ +package store + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" +) + +func TestPublicationRepairPreservesSourceEvidence(t *testing.T) { + ctx := context.Background() + s, e := Open(ctx, filepath.Join(t.TempDir(), "source.db")) + if e != nil { + t.Fatal(e) + } + defer s.Close() + _, e = s.db.Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo',1,'{}','2026-01-01'); + INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,body,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at) VALUES(1,1,'pr',1,'pull_request','open','','','','[]','[]','{}','h','2026-01-01')`) + if e != nil { + t.Fatal(e) + } + raw := `{"state":"APPROVED","created_at":"2026-01-01T00:00:00Z","submitted_at":"2026-01-03T12:34:56Z","user":{"login":"a","type":"User"}}` + id, e := s.UpsertComment(ctx, Comment{ThreadID: 1, GitHubID: "review", CommentType: "pull_review", RawJSON: raw, CreatedAtGitHub: "2026-01-01T00:00:00Z"}) + if e != nil { + t.Fatal(e) + } + _, e = s.db.Exec("UPDATE comments SET submitted_at_gh=NULL,publication_at_gh=NULL; UPDATE comment_revisions SET submitted_at_gh=NULL,publication_at_gh=NULL") + if e != nil { + t.Fatal(e) + } + var originalRecorded string + s.db.QueryRow("SELECT recorded_at FROM comment_revisions").Scan(&originalRecorded) + preview, e := s.RepairPublication(ctx, false) + if e != nil || preview.WouldChange != 2 || preview.Changed != 0 { + t.Fatalf("dry-run %+v %v", preview, e) + } + r, e := s.RepairPublication(ctx, true) + if e != nil || r.Changed != 2 { + t.Fatalf("%+v %v", r, e) + } + r, e = s.RepairPublication(ctx, true) + if e != nil || r.Changed != 0 { + t.Fatalf("replay %+v %v", r, e) + } + var published, submitted, created, stored string + s.db.QueryRow("SELECT publication_at_gh,submitted_at_gh,created_at_gh,raw_json FROM comments WHERE id=?", id).Scan(&published, &submitted, &created, &stored) + if published != "2026-01-03T12:34:56Z" || submitted != published || created != "2026-01-01T00:00:00Z" || stored != raw { + t.Fatalf("source timestamps/evidence changed incorrectly") + } + var n int + var recorded string + s.db.QueryRow("SELECT count(*),min(recorded_at) FROM comment_revisions").Scan(&n, &recorded) + if n != 1 || recorded != originalRecorded { + t.Fatal("repair manufactured a source revision") + } + for _, c := range []struct{ kind, raw, want string }{ + {"pull_review", `{"state":"PENDING","submitted_at":"2026-01-01T00:00:00Z"}`, ""}, + {"pull_review", `{"submitted_at":"not a date"}`, ""}, + {"pull_review", `{"_graphql":{"submittedAt":"2026-01-01T00:00:00+02:00"}}`, "2025-12-31T22:00:00Z"}, + {"pull_review_comment", `{"created_at":"2026-01-01T00:00:00Z","_graphql":{"publishedAt":"2026-01-02T00:00:00Z"}}`, "2026-01-02T00:00:00Z"}, + } { + if got := CommentPublication(c.kind, c.raw).Published; got != c.want { + t.Fatalf("publication=%q want %q", got, c.want) + } + } + nodes := []map[string]any{{"id": "comment-node", "author": map[string]any{"id": "user-one", "login": "same", "__typename": "User"}}, {"id": "other-node", "author": map[string]any{"id": "user-two", "login": "same", "__typename": "User"}}} + if e = s.SaveActorEvidence(ctx, nodes, "2026-01-01T00:00:00Z"); e != nil { + t.Fatal(e) + } + s.db.QueryRow("SELECT count(distinct actor_node_id) FROM actor_identity_evidence").Scan(&n) + if n != 2 { + t.Fatal("login reuse merged identities") + } + _, _ = json.Marshal(nodes) +} + +func TestAnalyticsQueuesNewUnresolvedSourceIdentities(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "source.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for _, raw := range []string{`{"node_id":"content","user":{"node_id":"actor"}}`, `{"node_id":"unresolved","user":null}`} { + if err := s.queueAnalyticsActor(ctx, raw); err != nil { + t.Fatal(err) + } + } + var count int + if err := s.DB().QueryRowContext(ctx, `SELECT count(*) FROM analytics_pending_nodes WHERE (node_id='actor' AND kind='profile') OR (node_id='unresolved' AND kind='identity')`).Scan(&count); err != nil || count != 2 { + t.Fatalf("count=%d err=%v", count, err) + } +} diff --git a/internal/store/comments.go b/internal/store/comments.go index f2c599ed..d523faf6 100644 --- a/internal/store/comments.go +++ b/internal/store/comments.go @@ -12,15 +12,16 @@ import ( ) type Comment struct { - ID int64 `json:"id"` - ThreadID int64 `json:"thread_id"` - GitHubID string `json:"github_id"` - CommentType string `json:"comment_type"` - AuthorLogin string `json:"author_login,omitempty"` - AuthorType string `json:"author_type,omitempty"` - Body string `json:"body"` - IsBot bool `json:"is_bot"` - ReviewState string `json:"review_state,omitempty"` + ID int64 `json:"id"` + ThreadID int64 `json:"thread_id"` + GitHubID string `json:"github_id"` + CommentType string `json:"comment_type"` + AuthorLogin string `json:"author_login,omitempty"` + AuthorType string `json:"author_type,omitempty"` + Body string `json:"body"` + IsBot bool `json:"is_bot"` + ReviewState string `json:"review_state,omitempty"` + Publication RawJSON string `json:"-"` CreatedAtGitHub string `json:"created_at_gh,omitempty"` UpdatedAtGitHub string `json:"updated_at_gh,omitempty"` @@ -71,6 +72,9 @@ func (s *Store) upsertComment(ctx context.Context, comment Comment) (int64, erro if err != nil { return 0, fmt.Errorf("upsert comment: %w", err) } + if err := s.upsertCommentPublication(ctx, id, comment.CommentType, comment.RawJSON); err != nil { + return 0, err + } if s.portableCommentBodyMetadata { if _, err := s.q().ExecContext(ctx, ` update comments @@ -131,10 +135,10 @@ func (s *Store) recordCommentRevision(ctx context.Context, commentID int64, reco if _, err := s.q().ExecContext(ctx, ` insert into comment_revisions( comment_id, author_login, author_type, body, is_bot, raw_json, - created_at_gh, updated_at_gh, deleted_at, deletion_reason, recorded_at + created_at_gh, updated_at_gh, deleted_at, deletion_reason, recorded_at,submitted_at_gh,publication_at_gh ) select c.id, c.author_login, c.author_type, c.body, c.is_bot, c.raw_json, - c.created_at_gh, c.updated_at_gh, c.deleted_at, c.deletion_reason, ? + c.created_at_gh, c.updated_at_gh, c.deleted_at, c.deletion_reason, ?,c.submitted_at_gh,c.publication_at_gh from comments c where c.id = ? and not exists ( @@ -180,6 +184,7 @@ func (s *Store) ListComments(ctx context.Context, threadID int64) ([]Comment, er Body: row.Body, IsBot: int64Bool(row.IsBot), ReviewState: reviewStateFromRawJSON(row.RawJson), + Publication: CommentPublication(row.CommentType, row.RawJson), RawJSON: row.RawJson, CreatedAtGitHub: stringValue(row.CreatedAtGh), UpdatedAtGitHub: stringValue(row.UpdatedAtGh), diff --git a/internal/store/store.go b/internal/store/store.go index 501a1e32..d0b42144 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,7 +17,7 @@ import ( ) const ( - schemaVersion = 13 + schemaVersion = 14 timeLayout = time.RFC3339Nano ) @@ -319,6 +319,9 @@ func (s *Store) migrate(ctx context.Context) error { if err := s.ensureFamilyTombstoneSchema(ctx); err != nil { return err } + if err := s.ensureAnalyticsSourceSchema(ctx); err != nil { + return err + } if err := s.ensureCanonicalObservationTables(ctx); err != nil { return err } diff --git a/internal/store/threads.go b/internal/store/threads.go index 0a3dd744..7b1b2a32 100644 --- a/internal/store/threads.go +++ b/internal/store/threads.go @@ -254,6 +254,9 @@ func (s *Store) upsertThreadObservation(ctx context.Context, thread Thread, opti if err != nil { return UpsertThreadResult{}, fmt.Errorf("upsert thread: %w", err) } + if err := s.queueAnalyticsActor(ctx, thread.RawJSON); err != nil { + return UpsertThreadResult{}, err + } if s.portableThreadBodyMetadata { if _, err := s.q().ExecContext(ctx, ` update threads From f58e55b03863145dccd7ef37860d2a5a81790738 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:37:23 -0600 Subject: [PATCH 06/21] Preserve review state and reconcile rejected GraphQL collection --- docs/analytics-source.md | 95 +++++- internal/cli/analytics.go | 157 ++------- internal/cli/analytics_integrity.go | 349 +++++++++++++++++++++ internal/cli/analytics_integrity_test.go | 284 +++++++++++++++++ internal/cli/app_test.go | 6 +- internal/cli/gh_search_test.go | 4 +- internal/github/analytics.go | 15 +- internal/github/analytics_test.go | 26 ++ internal/github/history.go | 18 +- internal/github/history_evidence.go | 145 +++++++++ internal/github/history_evidence_test.go | 60 ++++ internal/github/history_test.go | 12 +- internal/github/review_threads.go | 17 +- internal/store/analytics_integrity.go | 342 ++++++++++++++++++++ internal/store/analytics_integrity_test.go | 334 ++++++++++++++++++++ internal/store/review_threads.go | 38 +++ internal/store/store.go | 29 +- internal/syncer/graphql_history_test.go | 15 + internal/syncer/syncer.go | 42 ++- 19 files changed, 1819 insertions(+), 169 deletions(-) create mode 100644 internal/cli/analytics_integrity.go create mode 100644 internal/cli/analytics_integrity_test.go create mode 100644 internal/github/history_evidence.go create mode 100644 internal/github/history_evidence_test.go create mode 100644 internal/store/analytics_integrity.go create mode 100644 internal/store/analytics_integrity_test.go diff --git a/docs/analytics-source.md b/docs/analytics-source.md index 3777ac10..036ce63b 100644 --- a/docs/analytics-source.md +++ b/docs/analytics-source.md @@ -13,6 +13,7 @@ It does not run embeddings or classification models. ```sh gitcrawl --config SOURCE_CONFIG analytics owner/repo --json +gitcrawl --config SOURCE_CONFIG analytics owner/repo --status --json gitcrawl --config SOURCE_CONFIG analytics owner/repo --apply --json gitcrawl --config SOURCE_CONFIG --github-token-command TOKEN_HELPER \ analytics owner/repo --enrich --watch --json @@ -26,7 +27,7 @@ is distinct from draft creation; pending reviews have no published event. Raw payloads, existing IDs, genuine creation times and original observation times remain unchanged, and normalization creates no fictional provider edit. -Source schema 14 adds these fields and the actor/coverage evidence tables. +Source schema 14 added these fields and the actor/coverage evidence tables. Historical normalization completion is recorded in `analytics_collection_state.publication_repair_generation`; downstream consumers must reconcile the affected datasets when this generation changes rather than @@ -40,15 +41,27 @@ enter the profile queue, and profiles become eligible for refresh after 24 hours Operational queue tables are separate from publishable evidence. `--watch` polls updated issues and PRs every two minutes, overlapping the prior -verified watermark by five minutes. It paginates without GitHub search's hit cap, -hydrates only relevant updated threads, and commits page checkpoints only after -native conversation collection succeeds. Up to sixteen two-thread requests progress independently per -page. Persistently failing transient requests split into smaller batches, -preserving the transport and complete-evidence requirement. Non-nested +verified watermark by five minutes. It paginates without GitHub search's hit cap +and hydrates relevant updated threads. Issue and PR lanes have independent durable +checkpoints and process at most two discovery pages per cycle. Eight two-thread +requests progress independently per page. A rejected batch splits into individual +requests; an item may be passed only after its failure is durably queued. Unrelated +items and the other discovery lane continue. A cycle also retries at most sixteen +due items, with a two-minute request deadline and bounded exponential backoff. +No partial connection is accepted as complete evidence. Non-nested continuation pages use 100 nodes to avoid repeated small round trips. Identity/profile enrichment uses eight disjoint 100-node requests with normal quota guards and can run concurrently under the same source owner. `--once` performs one resumable update cycle. +Discovery skips unresolved core retry items. A review-only recovery queue does +not defer a newly discovered core edit: that edit is collected as core work. If +it fails, a core retry obligation prevents repeated discovery attempts until the +bounded scheduler selects it. This prevents enrichment backoff from hiding fresh +core data while its verified coverage watermark advances. +When both queues contain an item, its core retry owns the backoff; the review +retry pass cannot dispatch the same item. Recovery resolves only after an +accepted membership observation exists, including a proven empty set. + Hydration workers reuse the watch owner's open store instead of repeating full-archive migration checks for each batch. Independent guarded clients overlap network work; native transactions still coordinate source writes. Enrichment @@ -56,10 +69,72 @@ continues checking its queues every two minutes after the initial drain. New observations enqueue unresolved content identities or known actor profiles directly, avoiding repeated whole-archive scans during watch operation. -A completed historical discovery receipt supplies the initial coverage baseline; -a later completed update cycle advances it. Fresh collector checks do not imply -complete historical/provider coverage. `analytics_coverage` exposes the verified -watermark independently of source row observation times. +A completed historical discovery receipt supplies the initial discovery baseline. +The pre-upgrade checkpoint is retained; migration copies its in-flight cursor into +the corresponding independent lane. `through:owner/repo:issues` and +`through:owner/repo:pullRequests` record discovery progress; the aggregate watermark +is their minimum. These are not proof that queued failures have been repaired. +`analytics_coverage.complete` continues to describe verified core issue/PR/comment +traversal. Core failures clear it until reconciled; the previously verified +watermark remains available while another page is in progress. Historical +review-state enrichment does not invalidate core contributor/response coverage. + +## Review-state and failure contracts (schema 15) + +GraphQL collection requires explicit `isResolved` and `isOutdated` values and +retains review-thread IDs, source state, source comments and immediate reply IDs. +It writes the existing tables: + +- `pull_request_review_threads`: current observation, keyed by + `(thread_id, review_thread_id)`, including `is_resolved`, `is_outdated`, + `comments_json`, retained `raw_json`, and actual `fetched_at`. +- `pull_request_review_thread_revisions`: append-only changes, with integer `id` + and actual `recorded_at`; unchanged observations do not manufacture revisions. +- `pull_request_review_thread_syncs`: last complete observation per `thread_id`. + New nullable `review_thread_ids_json` contains its exact native ID membership. + NULL means not acquired under this contract; `[]` means a complete empty set. +- `analytics_review_state_coverage`: keyed by `repository`, with `cursor`, + `ceiling`, `scanned`, `queued`, `pending_items`, `scan_complete`, `complete`, + and `observed_at`. This separate completion contract requires a finished + targeted scan and no outstanding review-state recovery items. + +Absence from a complete membership set never invents a provider deletion or +removes retained history. Existing comment IDs/replies and +`thread_child_observation_memberships` keep their existing contract. Consumers +must distinguish historical rows from the latest provider membership. + +The watch gradually inspects 500 primary-key rows per cycle up to a captured +ceiling, queuing only PRs with retained review threads or unknown connection data. +Complete retained zero-count GraphQL connections materialize `[]` using their +actual retained observation time. Conditional writes preserve a newer live +observation; incomplete or undated evidence instead enters provider recovery. +`review_state_recovery:owner/repo` records cursor, ceiling, scanned/queued counts +and scan completion. Queue completion is separately required for review-state +coverage, never for the existing core coverage flag. This +targeted recovery does not restart historical discovery or enable remote syncing; +missing historical resolution states cannot be reconstructed from old payloads. + +Operational tables are **not public conversation datasets**: + +- `analytics_fetch_attempts(id, repository, number, operation, started_at, + finished_at, status, error_class, error_text, evidence_json)` retains successful + and rejected GraphQL work. `number=0` identifies a discovery-lane request. + Rejection evidence is bounded structural metadata, IDs, counts, pagination and + body lengths/hashes; it contains no credentials, response headers or prose bodies. +- `analytics_retries(repository, number, operation, first_seen_at, last_seen_at, + next_attempt_at, attempts, last_attempt_id, resolved_at)` is keyed by + `(repository, number, operation)`. Resolved entries and attempt history remain. + Operations include `graphql_history` (core traversal), `review_state` + (targeted enrichment), `discover_issues`, and + `discover_pullRequests`. Recovery queue rows start with zero attempts. + +`analytics --status --json` reads bounded recent receipts, outstanding retry count, +discovery coverage and review-state scan progress without credentials or collection. +Timestamped logs distinguish `github_update_complete`, `github_update_failed`, +`github_coverage_pending` (core), and `review_state_progress` (enrichment). +The older successful-only `sync_runs` table is not a +complete failure ledger. Existing embeddings are reused; this command does not +generate embeddings or reinterpret empty review bodies as missing replies. The permanent `runner.lock` coordinates ownership with the full-history backfill supervisor. Do not unlink it or start a second supervisor against the same archive. diff --git a/internal/cli/analytics.go b/internal/cli/analytics.go index 3af73561..a11b28be 100644 --- a/internal/cli/analytics.go +++ b/internal/cli/analytics.go @@ -3,7 +3,6 @@ package cli import ( "context" "encoding/json" - "errors" "flag" "fmt" "github.com/openclaw/gitcrawl/internal/config" @@ -56,7 +55,7 @@ func (a *App) analyticsClient(ctx context.Context, cfg config.Config) (*gh.Clien func (a *App) runAnalytics(ctx context.Context, args []string) error { for _, arg := range args { if arg == "--help" || arg == "-h" { - _, err := fmt.Fprintln(a.Stdout, "Usage: gitcrawl [--config SOURCE_CONFIG] [--github-token-command TOKEN_HELPER] analytics owner/repo [--apply] [--enrich] [--watch|--once] [--json]\nWithout action flags: read-only publication audit. --apply repairs retained timestamps; --enrich collects actor evidence; --watch maintains GraphQL updates.") + _, err := fmt.Fprintln(a.Stdout, "Usage: gitcrawl [--config SOURCE_CONFIG] [--github-token-command TOKEN_HELPER] analytics owner/repo [--status|--apply|--enrich] [--watch|--once] [--json]\nWithout action flags: read-only publication audit. --status reads bounded failure/recovery status; --apply repairs retained timestamps; --enrich collects actor evidence; --watch maintains GraphQL updates.") return err } } @@ -67,6 +66,7 @@ func (a *App) runAnalytics(ctx context.Context, args []string) error { enrich := fs.Bool("enrich", false, "collect missing provider identities and actor profiles") watch := fs.Bool("watch", false, "maintain GraphQL updates every two minutes") once := fs.Bool("once", false, "run one GraphQL update cycle") + status := fs.Bool("status", false, "read bounded collection, failure and recovery status") fs.Bool("json", false, "JSON output") if e := fs.Parse(normalizeCommandArgs(args, nil)); e != nil { return e @@ -83,6 +83,21 @@ func (a *App) runAnalytics(ctx context.Context, args []string) error { if e != nil { return e } + if *status { + if *apply || *enrich || *watch || *once { + return fmt.Errorf("--status cannot be combined with collection actions") + } + rt, err := a.openLocalRuntimeReadOnly(ctx) + if err != nil { + return err + } + defer rt.Store.Close() + result, err := rt.Store.AnalyticsIntegrityStatus(ctx, owner+"/"+repo) + if err != nil { + return err + } + return a.writeOutput("analytics_status", result, false) + } if !*apply && !*enrich && !*watch && !*once { rt, e := a.openLocalRuntimeReadOnly(ctx) if e != nil { @@ -186,11 +201,7 @@ func (a *App) runAnalytics(ctx context.Context, args []string) error { defer close(done) for { pollErr := a.analyticsCycle(pollCtx, rt.Store, client, owner, repo) - if pollErr != nil { - fmt.Fprintf(a.Stderr, "{\"event\":\"github_update_failed\",\"error\":%q}\n", pollErr.Error()) - } else { - fmt.Fprintln(a.Stderr, "{\"event\":\"github_update_complete\"}") - } + a.analyticsUpdateLog(pollErr) select { case <-pollCtx.Done(): return @@ -302,130 +313,10 @@ type updateCheckpoint struct { PRs int `json:"prs"` } -func (a *App) analyticsCycle(ctx context.Context, s *store.Store, c *gh.Client, owner, repo string) error { - key := "updates:" + owner + "/" + repo - value, e := s.AnalyticsState(ctx, key) - if e != nil { - return e - } - var cp updateCheckpoint - if value != "" { - if e = json.Unmarshal([]byte(value), &cp); e != nil { - return e - } - } - if cp.Started == "" { - through, e := s.AnalyticsState(ctx, "through:"+owner+"/"+repo) - if e != nil { - return e - } - if through == "" { // Conservative baseline: earliest completed historical discovery, not the last row fetched. - b, e := os.ReadFile(filepath.Join(filepath.Dir(a.configPath), "status.json")) - if e == nil { - var st struct { - Discovery []struct { - Updated string `json:"updated_at"` - Done int `json:"done"` - } - } - if json.Unmarshal(b, &st) == nil { - for _, d := range st.Discovery { - if d.Done == 1 && (through == "" || d.Updated < through) { - through = d.Updated - } - } - } - } - } - if through == "" { - return fmt.Errorf("verified historical discovery watermark required") - } - at, e := time.Parse(time.RFC3339Nano, through) - if e != nil { - return e - } - cp = updateCheckpoint{Started: time.Now().UTC().Format(time.RFC3339Nano), Since: at.Add(-5 * time.Minute).Format(time.RFC3339Nano)} - } - since, _ := time.Parse(time.RFC3339Nano, cp.Since) - for cp.Kind < 2 { - kind := []string{"issues", "pullRequests"}[cp.Kind] - page, e := c.UpdatedNumbers(ctx, owner, repo, kind, cp.Cursor, since) - if e != nil { - return e - } - if cp.Kind == 0 { - cp.Issues = page.Total - } else { - cp.PRs = page.Total - } - var wg sync.WaitGroup - slots := make(chan struct{}, 16) - var firstErr error - var errMu sync.Mutex - for i := 0; i < len(page.Numbers); i += 2 { - numbers := append([]int(nil), page.Numbers[i:min(i+2, len(page.Numbers))]...) - slots <- struct{}{} - wg.Add(1) - go func() { - defer func() { <-slots }() - defer wg.Done() - e := a.syncAnalyticsBatch(ctx, s, owner, repo, numbers) - if e != nil { - errMu.Lock() - if firstErr == nil { - firstErr = e - } - errMu.Unlock() - } - }() - } - wg.Wait() - if firstErr != nil { - return firstErr - } - fmt.Fprintf(a.Stderr, "{\"event\":\"github_update_page\",\"kind\":%q,\"threads\":%d}\n", kind, len(page.Numbers)) - - if !page.More || (!page.Oldest.IsZero() && page.Oldest.Before(since)) { - cp.Kind++ - cp.Cursor = "" - } else { - cp.Cursor = page.Cursor - } - b, _ := json.Marshal(cp) - if e = s.SetAnalyticsState(ctx, key, string(b)); e != nil { - return e - } - for batch := 0; batch < 5; batch++ { - ids, e := s.AnalyticsProfileNodes(ctx, 100) - if e != nil { - return e - } - if len(ids) == 0 { - break - } - nodes, e := c.AnalyticsNodes(ctx, ids, true) - if e != nil { - return e - } - if e = s.SaveActorProfiles(ctx, nodes, time.Now().UTC().Format(time.RFC3339Nano)); e != nil { - return e - } - } - - } - if e = s.SaveAnalyticsCoverage(ctx, owner+"/"+repo, cp.Started, cp.Issues, cp.PRs); e != nil { - return e - } - if e = s.SetAnalyticsState(ctx, "through:"+owner+"/"+repo, cp.Started); e != nil { - return e - } - return s.SetAnalyticsState(ctx, key, "") -} - // Smaller requests prevent high-fanout conversation queries exhausting GitHub's // execution deadline. Split a persistently failing transient batch without // accepting partial conversation evidence or changing transports. -func (a *App) syncAnalyticsBatch(ctx context.Context, s *store.Store, owner, repo string, numbers []int) error { +func (a *App) syncAnalyticsBatch(ctx context.Context, s *store.Store, owner, repo string, numbers []int, operation string) error { cfg, err := config.LoadRuntime(a.configPath) if err != nil { return err @@ -435,15 +326,7 @@ func (a *App) syncAnalyticsBatch(ctx context.Context, s *store.Store, owner, rep // The watch owner has already validated and opened this store. Reopening it // for every two threads repeats full-archive migration audits and serializes // otherwise independent network work. Native transactions still own writes. - _, err = syncer.New(client, s).Sync(ctx, syncer.Options{Owner: owner, Repo: repo, GraphQLHistory: true, State: "all", Numbers: numbers, IncludeComments: true, IncludePRMetadata: true}) - if err == nil || ctx.Err() != nil || len(numbers) < 2 { - return err - } - var response *gh.RequestError - if errors.As(err, &response) && (response.Status == 502 || response.Status == 503 || response.Status == 504) { - middle := len(numbers) / 2 - return errors.Join(a.syncAnalyticsBatch(ctx, s, owner, repo, numbers[:middle]), a.syncAnalyticsBatch(ctx, s, owner, repo, numbers[middle:])) - } + _, err = syncer.New(client, s).Sync(ctx, syncer.Options{Owner: owner, Repo: repo, GraphQLHistory: true, ReceiptOperation: operation, State: "all", Numbers: numbers, IncludeComments: true, IncludePRMetadata: true}) return err } diff --git a/internal/cli/analytics_integrity.go b/internal/cli/analytics_integrity.go new file mode 100644 index 00000000..88cd80d4 --- /dev/null +++ b/internal/cli/analytics_integrity.go @@ -0,0 +1,349 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" +) + +var errAnalyticsIncomplete = errors.New("analytics coverage remains incomplete") + +func (a *App) analyticsUpdateLog(err error) { + event := "github_update_complete" + fields := map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano)} + if err != nil { + event = "github_update_failed" + if errors.Is(err, errAnalyticsIncomplete) { + event = "github_coverage_pending" + fields["error"] = err.Error() + } else { + class, message, _ := gh.HistoryFailureDetails(err) + fields["error_class"] = class + fields["error"] = message + } + } + fields["event"] = event + encoded, _ := json.Marshal(fields) + fmt.Fprintln(a.Stderr, string(encoded)) +} + +func migrateAnalyticsLanes(ctx context.Context, s *store.Store, repository string) error { + // runAnalytics initializes coverage from a verified phase=complete discovery + // receipt before any cycle. A partial checkpoint alone cannot certify a + // historical baseline; missing baseline evidence must remain an error. + marker := "independent_lanes:" + repository + value, err := s.AnalyticsState(ctx, marker) + if err != nil || value != "" { + return err + } + value, err = s.AnalyticsState(ctx, "updates:"+repository) + if err != nil { + return err + } + var legacy updateCheckpoint + if value != "" { + if err = json.Unmarshal([]byte(value), &legacy); err != nil { + return err + } + } + through, err := s.AnalyticsState(ctx, "through:"+repository) + if err != nil { + return err + } + if through == "" { + if err = s.DB().QueryRowContext(ctx, "SELECT through FROM analytics_coverage WHERE repository=?", repository).Scan(&through); err != nil { + return fmt.Errorf("verified historical coverage watermark required: %w", err) + } + } + var verifiedThrough string + if err = s.DB().QueryRowContext(ctx, "SELECT through FROM analytics_coverage WHERE repository=?", repository).Scan(&verifiedThrough); err != nil { + return err + } + if _, err = time.Parse(time.RFC3339Nano, verifiedThrough); err != nil { + return fmt.Errorf("invalid verified historical watermark: %w", err) + } + return s.WithTx(ctx, func(tx *store.Store) error { + // through retains the previous verified baseline even when a transient + // post-upgrade failure temporarily clears complete before migration. + if err := tx.SetAnalyticsState(ctx, "core_baseline_verified:"+repository, "1"); err != nil { + return err + } + for i, kind := range []string{"issues", "pullRequests"} { + cp := updateCheckpoint{} + laneThrough := through + if legacy.Started != "" { + if legacy.Kind > i { + laneThrough = legacy.Started + total := legacy.Issues + if i == 1 { + total = legacy.PRs + } + encodedTotal, _ := json.Marshal(total) + if err := tx.SetAnalyticsState(ctx, "total:"+repository+":"+kind, string(encodedTotal)); err != nil { + return err + } + } else { + cp = legacy + cp.Kind = i + if legacy.Kind < i { + cp.Cursor = "" + } + } + } + encoded := "" + if cp.Started != "" { + b, _ := json.Marshal(cp) + encoded = string(b) + } + if err := tx.SetAnalyticsState(ctx, "updates:"+repository+":"+kind, encoded); err != nil { + return err + } + if err := tx.SetAnalyticsState(ctx, "through:"+repository+":"+kind, laneThrough); err != nil { + return err + } + } + // The original receipt/checkpoint is retained for historical evidence. + return tx.SetAnalyticsState(ctx, marker, time.Now().UTC().Format(time.RFC3339Nano)) + }) +} + +func (a *App) analyticsCycle(ctx context.Context, s *store.Store, c *gh.Client, owner, repo string) error { + repository := owner + "/" + repo + if err := migrateAnalyticsLanes(ctx, s, repository); err != nil { + return err + } + recovery, err := s.SeedReviewStateRecovery(ctx, repository, 500) + if err != nil { + return err + } + // Bounded retry work never replaces either independent discovery lane. + for _, operation := range []string{"graphql_history", "review_state"} { + due, e := s.DueAnalyticsRetries(ctx, repository, time.Now().UTC().Format(time.RFC3339Nano), 8, operation) + if e != nil { + return e + } + if e = a.analyticsNumbers(ctx, s, owner, repo, due, false, operation); e != nil { + return e + } + } + var failures []error + for i, kind := range []string{"issues", "pullRequests"} { + if err = a.analyticsLane(ctx, s, c, owner, repo, kind, i); err != nil { + failures = append(failures, err) + } + if ctx.Err() != nil { + return errors.Join(append(failures, ctx.Err())...) + } + } + var through string + baseline, err := s.AnalyticsState(ctx, "core_baseline_verified:"+repository) + if err != nil { + return err + } + complete := baseline == "1" && len(failures) == 0 + var totals [2]int + // Preserve last verified totals until this lane obtains a new provider count. + _ = s.DB().QueryRowContext(ctx, "SELECT issues,pull_requests FROM analytics_coverage WHERE repository=?", repository).Scan(&totals[0], &totals[1]) + for i, kind := range []string{"issues", "pullRequests"} { + at, e := s.AnalyticsState(ctx, "through:"+repository+":"+kind) + if e != nil { + return e + } + if through == "" || at < through { + through = at + } + value, e := s.AnalyticsState(ctx, "total:"+repository+":"+kind) + if e != nil { + return e + } + if value != "" { + if e = json.Unmarshal([]byte(value), &totals[i]); e != nil { + return e + } + } + } + outstanding, err := s.AnalyticsCoreOutstanding(ctx, repository) + if err != nil { + return err + } + complete = complete && outstanding == 0 + if err = s.WithTx(ctx, func(tx *store.Store) error { + if e := tx.SaveAnalyticsCoverage(ctx, repository, through, totals[0], totals[1]); e != nil { + return e + } + if e := tx.SetAnalyticsCoverageComplete(ctx, repository, complete); e != nil { + return e + } + return tx.SetAnalyticsState(ctx, "through:"+repository, through) + }); err != nil { + return err + } + if err = s.SaveReviewStateCoverage(ctx, repository, recovery); err != nil { + return err + } + totalPending, err := s.AnalyticsOutstanding(ctx, repository) + if err != nil { + return err + } + progress, _ := json.Marshal(map[string]any{"event": "review_state_progress", "at": time.Now().UTC().Format(time.RFC3339Nano), "scanned": recovery.Scanned, "ceiling": recovery.Ceiling, "queued": recovery.Queued, "pending_items": totalPending - outstanding, "scan_complete": recovery.Done, "complete": recovery.Done && totalPending == outstanding}) + fmt.Fprintln(a.Stderr, string(progress)) + if len(failures) > 0 { + return errors.Join(failures...) + } + if !complete { + return fmt.Errorf("%w: core_unresolved=%d", errAnalyticsIncomplete, outstanding) + } + return nil +} + +func (a *App) analyticsNumbers(ctx context.Context, s *store.Store, owner, repo string, numbers []int, discovery bool, operation string) error { + var wg sync.WaitGroup + defer wg.Wait() + slots := make(chan struct{}, 8) + var failures []error + var mu sync.Mutex + for i := 0; i < len(numbers); i += 2 { + var part []int + for _, n := range numbers[i:min(i+2, len(numbers))] { + if discovery { + // Review-only recovery must not defer a newly discovered core edit. + // If this core attempt fails, it creates graphql_history retry state + // and every later overlap skips it until the bounded scheduler retries. + deferred, e := s.AnalyticsItemQueued(ctx, owner+"/"+repo, n) + if e != nil { + return e + } + if deferred { + continue + } + } + part = append(part, n) + } + if len(part) == 0 { + continue + } + slots <- struct{}{} + wg.Add(1) + go func(part []int) { + defer wg.Done() + defer func() { <-slots }() + if e := a.analyticsIsolatedBatch(ctx, s, owner, repo, part, operation); e != nil { + mu.Lock() + failures = append(failures, e) + mu.Unlock() + } + }(part) + } + wg.Wait() + return errors.Join(failures...) +} + +func (a *App) analyticsIsolatedBatch(ctx context.Context, s *store.Store, owner, repo string, numbers []int, operation string) error { + bounded, cancel := context.WithTimeout(ctx, 2*time.Minute) + err := a.syncAnalyticsBatch(bounded, s, owner, repo, numbers, operation) + cancel() + if err == nil { + return nil + } + if ctx.Err() != nil { + return err + } + // Persisted failure receipts allow splitting all rejected batches, including + // partial/invalid connections. A poison item cannot block its healthy peer. + if len(numbers) > 1 { + var failures []error + for _, n := range numbers { + if e := a.analyticsIsolatedBatch(ctx, s, owner, repo, []int{n}, operation); e != nil { + failures = append(failures, e) + } + } + return errors.Join(failures...) + } + queued, e := s.AnalyticsItemQueued(ctx, owner+"/"+repo, numbers[0], operation) + if e != nil { + return errors.Join(err, e) + } + if !queued { + return err + } // Never advance past an unrecorded failure. + return nil +} + +func (a *App) analyticsLane(ctx context.Context, s *store.Store, c *gh.Client, owner, repo, kind string, index int) error { + repository := owner + "/" + repo + key := "updates:" + repository + ":" + kind + value, err := s.AnalyticsState(ctx, key) + if err != nil { + return err + } + var cp updateCheckpoint + if value != "" { + if err = json.Unmarshal([]byte(value), &cp); err != nil { + return err + } + } + if cp.Started == "" { + through, e := s.AnalyticsState(ctx, "through:"+repository+":"+kind) + if e != nil { + return e + } + at, e := time.Parse(time.RFC3339Nano, through) + if e != nil { + return e + } + cp = updateCheckpoint{Started: time.Now().UTC().Format(time.RFC3339Nano), Since: at.Add(-5 * time.Minute).Format(time.RFC3339Nano), Kind: index} + } + since, err := time.Parse(time.RFC3339Nano, cp.Since) + if err != nil { + return err + } + for pages := 0; pages < 2; pages++ { + started := time.Now().UTC().Format(time.RFC3339Nano) + page, e := c.UpdatedNumbers(ctx, owner, repo, kind, cp.Cursor, since) + attempt := store.AnalyticsAttempt{Repository: repository, Operation: "discover_" + kind, StartedAt: started, FinishedAt: time.Now().UTC().Format(time.RFC3339Nano), Status: "success", Evidence: json.RawMessage(`{}`)} + if e != nil { + attempt.Status = "failed" + attempt.ErrorClass, attempt.ErrorText, attempt.Evidence = gh.HistoryFailureDetails(e) + } + receiptCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + writeErr := s.RecordAnalyticsAttempt(receiptCtx, attempt) + cancel() + if e != nil || writeErr != nil { + return errors.Join(e, writeErr) + } + if e = a.analyticsNumbers(ctx, s, owner, repo, page.Numbers, true, "graphql_history"); e != nil { + return e + } + done := !page.More || (!page.Oldest.IsZero() && page.Oldest.Before(since)) + if !done { + cp.Cursor = page.Cursor + } + if e = s.WithTx(ctx, func(tx *store.Store) error { + count, _ := json.Marshal(page.Total) + if e := tx.SetAnalyticsState(ctx, "total:"+repository+":"+kind, string(count)); e != nil { + return e + } + if done { + if e := tx.SetAnalyticsState(ctx, "through:"+repository+":"+kind, cp.Started); e != nil { + return e + } + return tx.SetAnalyticsState(ctx, key, "") + } + encoded, _ := json.Marshal(cp) + return tx.SetAnalyticsState(ctx, key, string(encoded)) + }); e != nil { + return e + } + fmt.Fprintf(a.Stderr, "{\"event\":\"github_update_page\",\"at\":%q,\"kind\":%q,\"threads\":%d}\n", time.Now().UTC().Format(time.RFC3339Nano), kind, len(page.Numbers)) + if done { + return nil + } + } + return nil +} diff --git a/internal/cli/analytics_integrity_test.go b/internal/cli/analytics_integrity_test.go new file mode 100644 index 00000000..39122e9e --- /dev/null +++ b/internal/cli/analytics_integrity_test.go @@ -0,0 +1,284 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsPoisonItemAndDiscoveryLaneDoNotStarvePeers(t *testing.T) { + for _, brokenDiscovery := range []bool{false, true} { + t.Run(fmt.Sprint(brokenDiscovery), func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, "archive.db") + s, err := store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer func() { s.Close() }() + at := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano) + baseline := time.Now().UTC().Add(-10 * time.Minute).Format(time.RFC3339Nano) + if err = s.SetAnalyticsState(ctx, "through:fixture/repo", baseline); err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", baseline, 1, 1); err != nil { + t.Fatal(err) + } + var broken atomic.Bool + broken.Store(true) + var revised atomic.Bool + conn := func(nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"nodes": nodes, "totalCount": len(nodes), "pageInfo": map[string]any{"hasNextPage": false, "endCursor": "end"}} + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":50000,"remaining":49000,"reset":4102444800},"core":{"limit":50000,"remaining":49000,"reset":4102444800}}}`) + return + } + if r.URL.Path != "/graphql" { + t.Errorf("unexpected fallback: %s", r.URL.Path) + http.Error(w, "no fallback", 400) + return + } + var req struct{ Query string } + if decodeErr := json.NewDecoder(r.Body).Decode(&req); decodeErr != nil { + t.Error(decodeErr) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 48000, "resetAt": "2099-01-01T00:00:00Z"}} + if strings.Contains(req.Query, "orderBy") { + kind, n := "issues", 1 + if strings.Contains(req.Query, "pullRequests(first:") { + kind, n = "pullRequests", 2 + } + page := conn(map[string]any{"number": n, "updatedAt": at}) + if brokenDiscovery && broken.Load() && n == 1 { + delete(page, "pageInfo") + } + data["repository"] = map[string]any{kind: page} + } else if strings.Contains(req.Query, "issueOrPullRequest") { + n, typ := 1, "Issue" + if strings.Contains(req.Query, "number:2)") { + n, typ = 2, "PullRequest" + } + node := map[string]any{"id": fmt.Sprint("node-", n), "fullDatabaseId": fmt.Sprint(n), "__typename": typ, "number": n, "title": "fixture", "body": "retained body", "state": "OPEN", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": at, "url": fmt.Sprintf("https://github.com/fixture/repo/issues/%d", n), "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "labels": conn(), "assignees": conn(), "comments": conn()} + if revised.Load() && n == 2 { + node["body"] = "fresh core body" + } + if n == 2 { + node["reviews"] = conn() + node["reviewThreads"] = conn() + } + if !brokenDiscovery && broken.Load() && n == 1 { + node["comments"].(map[string]any)["totalCount"] = 1 + } + data["repository"] = map[string]any{"id": "repo", "databaseId": 1, "nameWithOwner": "fixture/repo", "n0": node} + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + a := New() + a.Stderr = io.Discard + a.configPath = writeDoctorTestConfig(t, dir, path) + client := gh.New(gh.Options{BaseURL: server.URL, Token: "test-token-placeholder"}) + reviewRetry := store.AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "review_state", StartedAt: "2099-01-01T00:00:00Z", FinishedAt: "2099-01-01T00:00:01Z", Status: "failed", Evidence: json.RawMessage(`{}`)} + if !brokenDiscovery { + if err = s.RecordAnalyticsAttempt(ctx, reviewRetry); err != nil { + t.Fatal(err) + } + } else { + // A transient core failure can precede first lane migration. Its + // incomplete flag must not erase the verified historical baseline. + failure := store.AnalyticsAttempt{Repository: "fixture/repo", Operation: "discover_issues", StartedAt: baseline, FinishedAt: baseline, Status: "failed", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, failure); err != nil { + t.Fatal(err) + } + var complete int + if err = s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&complete); err != nil || complete != 0 { + t.Fatalf("failure did not clear core coverage: complete=%d err=%v", complete, err) + } + } + if !brokenDiscovery { + // Exercise the real first-watch entry point with no coverage row or + // completed-update watermark: the completed discovery receipt owns it. + if _, err = s.DB().Exec("DELETE FROM analytics_coverage"); err != nil { + t.Fatal(err) + } + s.SetAnalyticsState(ctx, "through:fixture/repo", "") + receipt, _ := json.Marshal(map[string]any{"phase": "complete", "discovery": []map[string]any{{"kind": "issues", "done": 1, "total": 1, "updated_at": baseline}, {"kind": "pullRequests", "done": 1, "total": 1, "updated_at": baseline}}}) + if err = os.WriteFile(filepath.Join(dir, "status.json"), receipt, 0600); err != nil { + t.Fatal(err) + } + a.Stdout = io.Discard + err = a.runAnalytics(ctx, []string{"fixture/repo", "--once", "--json"}) + } else { + err = a.analyticsCycle(ctx, s, client, "fixture", "repo") + } + if err == nil { + t.Fatal("incomplete coverage reported complete") + } + if !brokenDiscovery && !errors.Is(err, errAnalyticsIncomplete) { + t.Fatal(err) + } + var count int + s.DB().QueryRow("SELECT count(*) FROM threads WHERE number=2").Scan(&count) + if count != 1 { + t.Fatal("independent PR never persisted") + } + s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&count) + if count != 0 { + t.Fatal("poison item hidden by complete=true") + } + if !brokenDiscovery { + var before, after int + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE number=1 AND operation='graphql_history'").Scan(&before) + if err = a.analyticsCycle(ctx, s, client, "fixture", "repo"); !errors.Is(err, errAnalyticsIncomplete) { + t.Fatalf("unexpected repeat outcome %v", err) + } + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE number=1 AND operation='graphql_history'").Scan(&after) + if before != after { + t.Fatal("review-only failure bypassed core retry backoff repeatedly") + } + } + broken.Store(false) + // Simulate a due retry after a process restart, without dropping its history. + if _, err = s.DB().Exec("UPDATE analytics_retries SET next_attempt_at='2000-01-01T00:00:00Z'"); err != nil { + t.Fatal(err) + } + s.Close() + s, err = store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if err = a.analyticsCycle(ctx, s, client, "fixture", "repo"); err != nil { + t.Fatal(err) + } + if outstanding, err := s.AnalyticsOutstanding(ctx, "fixture/repo"); err != nil || outstanding != 0 { + t.Fatalf("not recovered: %d %v", outstanding, err) + } + if err = s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&count); err != nil || count != 1 { + t.Fatalf("verified core baseline did not recover: complete=%d err=%v", count, err) + } + s.DB().QueryRow("SELECT count(*) FROM threads").Scan(&count) + if count != 2 { + t.Fatalf("wrong recovered content count %d", count) + } + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE status='failed'").Scan(&count) + if count < 1 { + t.Fatal("failure history erased") + } + reviewRetry.Number = 2 + if err = s.RecordAnalyticsAttempt(ctx, reviewRetry); err != nil { + t.Fatal(err) + } + revised.Store(true) + if err = a.analyticsCycle(ctx, s, client, "fixture", "repo"); err != nil { + t.Fatal(err) + } + var body string + s.DB().QueryRow("SELECT body FROM threads WHERE number=2").Scan(&body) + if body != "fresh core body" { + t.Fatal("review recovery backoff hid a newly discovered core update") + } + }) + } +} + +func TestAnalyticsLaneMigrationRetainsInflightCursorAndLegacyReceipt(t *testing.T) { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + legacy := `{"started":"2026-01-01T12:00:00Z","since":"2026-01-01T10:55:00Z","kind":1,"cursor":"opaque-provider-cursor","issues":10,"prs":20}` + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T11:00:00Z", 10, 20); err != nil { + t.Fatal(err) + } + s.SetAnalyticsState(ctx, "through:fixture/repo", "2026-01-01T11:00:00Z") + s.SetAnalyticsState(ctx, "updates:fixture/repo", legacy) + if err = migrateAnalyticsLanes(ctx, s, "fixture/repo"); err != nil { + t.Fatal(err) + } + value, err := s.AnalyticsState(ctx, "updates:fixture/repo:pullRequests") + if err != nil { + t.Fatal(err) + } + var cp updateCheckpoint + if err = json.Unmarshal([]byte(value), &cp); err != nil { + t.Fatal(err) + } + if cp.Cursor != "opaque-provider-cursor" || cp.Since != "2026-01-01T10:55:00Z" { + t.Fatal("in-flight provider cursor reset") + } + total, _ := s.AnalyticsState(ctx, "total:fixture/repo:issues") + if total != "10" { + t.Fatalf("completed lane total lost: %s", total) + } + old, _ := s.AnalyticsState(ctx, "updates:fixture/repo") + if old != legacy { + t.Fatal("legacy evidence overwritten") + } + s.SetAnalyticsState(ctx, "updates:fixture/repo:pullRequests", "new-progress") + if err = migrateAnalyticsLanes(ctx, s, "fixture/repo"); err != nil { + t.Fatal(err) + } + value, _ = s.AnalyticsState(ctx, "updates:fixture/repo:pullRequests") + if value != "new-progress" { + t.Fatal("restart reseeded an existing checkpoint") + } +} + +func TestAnalyticsDiscoveryLeavesDueItemsToBoundedRetryScheduler(t *testing.T) { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for i := 1; i <= 24; i++ { + a := store.AnalyticsAttempt{Repository: "fixture/repo", Number: i, Operation: "graphql_history", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + } + due, err := s.DueAnalyticsRetries(ctx, "fixture/repo", "2099-01-01T00:00:00Z", 8, "graphql_history") + if err != nil || len(due) != 8 { + t.Fatalf("retry bound: %v %v", due, err) + } + // This app has no usable configuration. Any attempted collection would fail; + // discovery must skip even due items not selected by the retry budget. + a := New() + numbers := []int{} + for i := 1; i <= 24; i++ { + numbers = append(numbers, i) + } + if err = a.analyticsNumbers(ctx, s, "fixture", "repo", numbers, true, "graphql_history"); err != nil { + t.Fatal(err) + } + var attempts int + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts").Scan(&attempts) + if attempts != 24 { + t.Fatal("discovery bypassed the retry scheduler") + } +} diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 6f222382..f2f9ba39 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -4344,10 +4344,10 @@ func TestDoctorJSONReportsCurrentSchemaDiagnosticsWithoutMutation(t *testing.T) if got := schema["state"]; got != "current" { t.Fatalf("db_schema.state = %#v, payload=%#v", got, schema) } - if got := schema["current_version"]; got != float64(14) { + if got := schema["current_version"]; got != float64(15) { t.Fatalf("db_schema.current_version = %#v, payload=%#v", got, schema) } - if got := schema["supported_version"]; got != float64(14) { + if got := schema["supported_version"]; got != float64(15) { t.Fatalf("db_schema.supported_version = %#v, payload=%#v", got, schema) } if got := schema["child_observation_reservations"]; got != true { @@ -4609,7 +4609,7 @@ func TestDoctorJSONReportsLegacyPendingSchemaWithoutMutation(t *testing.T) { t.Fatalf("pr_details.duplicate_path_files_supported = %#v, payload=%#v", got, prDetails) } pending := doctorStringList(t, schema, "pending_migrations") - if !doctorListContains(pending, "schema_version_3_to_14") || + if !doctorListContains(pending, "schema_version_3_to_15") || !doctorListContains(pending, "pull_request_files_position_key") || !doctorListContains(pending, "thread_child_observation_reservations_table") { t.Fatalf("pending_migrations = %#v", pending) diff --git a/internal/cli/gh_search_test.go b/internal/cli/gh_search_test.go index 5fe2fe7c..81e55eeb 100644 --- a/internal/cli/gh_search_test.go +++ b/internal/cli/gh_search_test.go @@ -265,8 +265,8 @@ func TestGHSearchSyncIfStaleMigratesFreshPortableRuntime(t *testing.T) { if err := rt.Store.DB().QueryRowContext(ctx, `pragma user_version`).Scan(&schemaVersion); err != nil { t.Fatalf("read runtime schema version: %v", err) } - if schemaVersion != 14 { - t.Fatalf("runtime schema version = %d, want 14", schemaVersion) + if schemaVersion != 15 { + t.Fatalf("runtime schema version = %d, want 15", schemaVersion) } var tableName string if err := rt.Store.DB().QueryRowContext(ctx, `select name from sqlite_schema where type = 'table' and name = 'sync_runs'`).Scan(&tableName); err != nil { diff --git a/internal/github/analytics.go b/internal/github/analytics.go index 7d123b20..c309183c 100644 --- a/internal/github/analytics.go +++ b/internal/github/analytics.go @@ -84,16 +84,23 @@ func (c *Client) UpdatedNumbers(ctx context.Context, owner, repo, kind, after st var valid bool p.Total, valid = historyInt(r["totalCount"]) if !valid || p.Total < 0 { - return p, fmt.Errorf("missing update-discovery count") + return p, historyFailure("discovery_validation", 0, r, fmt.Errorf("missing update-discovery count")) } info := historyMap(r["pageInfo"]) - p.More, _ = info["hasNextPage"].(bool) + var hasPageFlag bool + p.More, hasPageFlag = info["hasNextPage"].(bool) + nodes, hasNodes := r["nodes"].([]any) + // totalCount covers the whole connection. A resumed final page can become + // empty after deletion/reordering; it must not pin its cursor forever. + if !hasPageFlag || !hasNodes || (len(nodes) == 0 && p.Total > 0 && after == "") || len(nodes) > p.Total { + return p, historyFailure("discovery_validation", 0, r, fmt.Errorf("incomplete update-discovery page")) + } p.Cursor = historyString(info["endCursor"]) for _, n := range historyNodes(historyMap(data["repository"]), kind) { number, ok := historyInt(n["number"]) at, e := time.Parse(time.RFC3339Nano, historyString(n["updatedAt"])) if !ok || number < 1 || e != nil { - return p, fmt.Errorf("invalid update-discovery evidence") + return p, historyFailure("discovery_validation", 0, r, fmt.Errorf("invalid update-discovery evidence")) } if !at.Before(since) { p.Numbers = append(p.Numbers, number) @@ -103,7 +110,7 @@ func (c *Client) UpdatedNumbers(ctx context.Context, owner, repo, kind, after st } } if p.More && (p.Cursor == "" || p.Cursor == after || p.Oldest.IsZero()) { - return p, fmt.Errorf("update cursor did not advance") + return p, historyFailure("discovery_validation", 0, r, fmt.Errorf("update cursor did not advance")) } return p, nil } diff --git a/internal/github/analytics_test.go b/internal/github/analytics_test.go index 0adc2dec..889342e8 100644 --- a/internal/github/analytics_test.go +++ b/internal/github/analytics_test.go @@ -28,6 +28,32 @@ func TestAnalyticsDiscoveryFiltersOldRowsAndRejectsBadCursors(t *testing.T) { t.Fatal("non-advancing cursor accepted") } } +func TestAnalyticsDiscoveryEmptyContinuation(t *testing.T) { + for _, tc := range []struct { + name, after string + more, wantError bool + }{ + {"final continuation", "previous", false, false}, + {"incomplete initial page", "", false, true}, + {"empty advancing page", "previous", true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": "2099-01-01T00:00:00Z"}, "repository": map[string]any{"issues": map[string]any{"totalCount": 100, "pageInfo": map[string]any{"hasNextPage": tc.more, "endCursor": "next"}, "nodes": []any{}}}}}) + })) + defer server.Close() + c := New(Options{Token: "test-token-placeholder", BaseURL: server.URL}) + p, err := c.UpdatedNumbers(context.Background(), "fixture", "repo", "issues", tc.after, time.Time{}) + if (err != nil) != tc.wantError { + t.Fatalf("page=%+v error=%v, wantError=%v", p, err, tc.wantError) + } + if err == nil && (p.More || len(p.Numbers) != 0 || p.Total != 100) { + t.Fatalf("incorrect final page: %+v", p) + } + }) + } +} + func TestAnalyticsUnavailableNodesAreNotAuthorizationSuccess(t *testing.T) { typ := "NOT_FOUND" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/github/history.go b/internal/github/history.go index 3ed3d33c..8cfb1339 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -16,8 +16,8 @@ import ( ) type HistoryItem struct { - Thread, Pull map[string]any - Comments, Reviews, ReviewComments []map[string]any + Thread, Pull map[string]any + Comments, Reviews, ReviewComments, ReviewThreads []map[string]any } type HistoryBatch struct { Repository map[string]any @@ -29,7 +29,7 @@ const historyComment = `id __typename fullDatabaseId body ` + historyActor + ` a const historyInline = historyComment + ` path diffHunk line startLine originalLine originalStartLine position originalPosition state subjectType outdated commit { oid } originalCommit { oid } replyTo { id fullDatabaseId } pullRequestReview { id fullDatabaseId }` var historyReview = historyComment + ` state submittedAt commit { oid } ` + historyConnection("comments", historyInline, "") -var historyReviewThread = `id __typename ` + historyConnection("comments", historyInline, "") +var historyReviewThread = `id __typename path line startLine isResolved isOutdated viewerCanResolve viewerCanUnresolve viewerCanReply ` + historyConnection("comments", historyInline, "") var historyCommon = `id __typename fullDatabaseId number title body ` + historyActor + ` authorAssociation createdAt updatedAt closedAt url state locked activeLockReason repository { nameWithOwner } milestone { number title state dueOn createdAt updatedAt url } ` + historyConnection("labels", `id name color description`, "") + " " + historyConnection("assignees", `id login __typename url`, "") + " " + historyConnection("comments", historyComment, "") var historyIssue = historyCommon + ` stateReason` var historyPull = historyCommon + ` isDraft merged mergedAt mergedBy { login __typename url } mergeCommit { oid } mergeable mergeStateStatus maintainerCanModify additions deletions changedFiles headRefName headRefOid baseRefName baseRefOid headRepository { nameWithOwner } baseRepository { nameWithOwner } commits { totalCount } ` + historyConnection("reviews", historyReview, "") + " " + historyConnection("reviewThreads", historyReviewThread, "") @@ -182,10 +182,10 @@ func (c *Client) FetchGraphQLHistory(ctx context.Context, owner, repo string, nu node := historyMap(r[fmt.Sprintf("n%d", i)]) got, _ := historyInt(node["number"]) if got != n || !strings.EqualFold(historyString(historyMap(node["repository"])["nameWithOwner"]), owner+"/"+repo) || historyString(node["id"]) == "" { - return result, fmt.Errorf("GraphQL history item #%d unavailable or moved", n) + return result, historyFailure("identity", n, node, fmt.Errorf("GraphQL history item #%d unavailable or moved", n)) } if err := h.hydrate(ctx, node); err != nil { - return result, fmt.Errorf("GraphQL history #%d: %w", n, err) + return result, historyFailure("validation", n, node, fmt.Errorf("GraphQL history #%d: %w", n, err)) } item, err := historyItem(node) if err != nil { @@ -217,6 +217,13 @@ func historyFields(typ, key string) (string, error) { func (h *historySession) hydrate(ctx context.Context, node map[string]any) error { typ := historyString(node["__typename"]) + if typ == "PullRequestReviewThread" { + for _, field := range []string{"isResolved", "isOutdated", "viewerCanResolve", "viewerCanUnresolve", "viewerCanReply"} { + if _, ok := node[field].(bool); !ok { + return fmt.Errorf("missing or invalid review-thread %s", field) + } + } + } var required []string switch typ { case "Issue": @@ -378,6 +385,7 @@ func historyItem(node map[string]any) (HistoryItem, error) { // A comment's review association is nullable. Threads independently // supply standalone comments; review bodies and their metadata stay above. for _, thread := range historyNodes(node, "reviewThreads") { + item.ReviewThreads = append(item.ReviewThreads, thread) for _, comment := range historyNodes(thread, "comments") { id := historyString(comment["id"]) if _, exists := inlineByID[id]; exists { diff --git a/internal/github/history_evidence.go b/internal/github/history_evidence.go new file mode 100644 index 00000000..aaac59b4 --- /dev/null +++ b/internal/github/history_evidence.go @@ -0,0 +1,145 @@ +package github + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "strings" +) + +// HistoryFailure retains a bounded structural receipt, never credentials, HTTP +// headers, prose bodies or actor/profile text. Existing accepted raw evidence +// remains in the archive. A rejected body is represented by its length/hash. +type HistoryFailure struct { + Cause error + Stage string + Number int + Evidence json.RawMessage +} + +type historyResponseReader struct { + reader io.Reader + hash hash.Hash + read, hashed int64 +} + +func (r *historyResponseReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.read += int64(n) + keep := min(int64(n), 128*1024-r.hashed) + if keep > 0 { + _, _ = r.hash.Write(p[:keep]) + r.hashed += keep + } + return n, err +} + +func (r *historyResponseReader) failure(err error) error { + evidence, _ := json.Marshal(map[string]any{"version": 1, "response_bytes_read": r.read, "hashed_prefix_bytes": r.hashed, "prefix_sha256": hex.EncodeToString(r.hash.Sum(nil)), "complete_response": false}) + return &HistoryFailure{Cause: err, Stage: "response_decode", Evidence: evidence} +} + +func (e *HistoryFailure) Error() string { return e.Cause.Error() } +func (e *HistoryFailure) Unwrap() error { return e.Cause } + +func historyFailure(stage string, number int, data any, err error) error { + evidence := SafeHistoryEvidence(data) + var upstream *HistoryFailure + if errors.As(err, &upstream) { + stage = upstream.Stage + evidence, _ = json.Marshal(map[string]json.RawMessage{"context": evidence, "upstream_rejection": upstream.Evidence}) + } + return &HistoryFailure{Cause: err, Stage: stage, Number: number, Evidence: evidence} +} + +func SafeHistoryEvidence(data any) json.RawMessage { + raw, _ := json.Marshal(data) + sum := sha256.Sum256(raw) + var project func(any, int) any + project = func(value any, depth int) any { + if depth > 9 { + return map[string]any{"truncated": true} + } + switch v := value.(type) { + case map[string]any: + out := map[string]any{} + for k, child := range v { + if len(k) > 1 && k[0] == 'n' && strings.Trim(k[1:], "0123456789") == "" { + out[k] = project(child, depth+1) + continue + } + switch k { + case "id", "node_id", "fullDatabaseId", "number", "__typename", "totalCount", "hasNextPage", "endCursor", "createdAt", "updatedAt", "publishedAt", "submittedAt", "state", "isResolved", "isOutdated", "type": + switch scalar := child.(type) { + case string: + if len(scalar) <= 512 { + out[k] = scalar + } else { + out[k] = map[string]any{"truncated": true} + } + case bool, float64, int, int64, json.Number, nil: + out[k] = scalar + default: + out[k] = map[string]any{"invalid_type": true} + } + case "repository", "node", "data", "nodes", "pageInfo", "comments", "reviews", "reviewThreads", "labels", "assignees", "errors", "path": + out[k] = project(child, depth+1) + case "body": + if text, ok := child.(string); ok { + h := sha256.Sum256([]byte(text)) + out["body_evidence"] = map[string]any{"bytes": len(text), "sha256": hex.EncodeToString(h[:])} + } + } + } + return out + case []any: + items := make([]any, 0, min(len(v), 12)) + for _, item := range v[:min(len(v), 12)] { + items = append(items, project(item, depth+1)) + } + return map[string]any{"count": len(v), "sample": items, "truncated": len(v) > 12} + default: + // Unknown strings may be provider error messages or private prose. + return nil + } + } + out, _ := json.Marshal(map[string]any{"version": 1, "response_bytes": len(raw), "response_sha256": hex.EncodeToString(sum[:]), "structure": project(data, 0)}) + if len(out) > 128*1024 { + out, _ = json.Marshal(map[string]any{"version": 1, "response_bytes": len(raw), "response_sha256": hex.EncodeToString(sum[:]), "structure_truncated": true}) + } + return out +} + +// HistoryFailureDetails is safe to persist or log even if the original error +// included an HTTP body. It deliberately does not return that body/message. +func HistoryFailureDetails(err error) (string, string, json.RawMessage) { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "cancelled", "GraphQL attempt cancelled or timed out", json.RawMessage(`{}`) + } + var quota *RateLimitReserveError + if errors.As(err, "a) { + return "rate_limit", quota.Error(), json.RawMessage(`{}`) + } + var failure *HistoryFailure + if errors.As(err, &failure) { + message := fmt.Sprintf("GraphQL %s rejected for item %d", failure.Stage, failure.Number) + for _, connection := range []string{"comments", "reviews", "reviewThreads", "labels", "assignees"} { + for _, reason := range []string{"incomplete history " + connection + " count", "nonadvancing " + connection + " cursor", "missing history " + connection} { + if strings.HasSuffix(failure.Cause.Error(), reason) { + message += ": " + reason + } + } + } + return failure.Stage, message, failure.Evidence + } + var response *RequestError + if errors.As(err, &response) { + return "http", fmt.Sprintf("GitHub HTTP %d", response.Status), json.RawMessage(`{}`) + } + return "fetch", "GraphQL collection failed", json.RawMessage(`{}`) +} diff --git a/internal/github/history_evidence_test.go b/internal/github/history_evidence_test.go new file mode 100644 index 00000000..0c9a93c2 --- /dev/null +++ b/internal/github/history_evidence_test.go @@ -0,0 +1,60 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHistoryFailureEvidenceRetainsStructureWithoutSecretsOrBodies(t *testing.T) { + data := map[string]any{"id": "PR_fixture", "body": "private prose", "Authorization": "Bearer secret", "author": map[string]any{"login": "private-login"}, "comments": map[string]any{"totalCount": 2, "nodes": []any{map[string]any{"id": "C1", "body": "retained-by-hash"}}, "pageInfo": map[string]any{"hasNextPage": false}}} + evidence := SafeHistoryEvidence(data) + for _, secret := range []string{"private prose", "Bearer secret", "private-login", "retained-by-hash"} { + if strings.Contains(string(evidence), secret) { + t.Fatal("private data in safe receipt") + } + } + if !json.Valid(evidence) || !strings.Contains(string(evidence), `"totalCount":2`) || !strings.Contains(string(evidence), `"sha256"`) { + t.Fatal("missing structural evidence") + } + err := historyFailure("validation", 7, data, errors.New("GraphQL history #7: incomplete history comments count")) + class, message, stored := HistoryFailureDetails(err) + if class != "validation" || !strings.Contains(message, "incomplete history comments count") || string(stored) != string(evidence) { + t.Fatalf("receipt %s %s", class, message) + } +} + +func TestMalformedGraphQLResponseHasPrivateBoundedReceipt(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"data":{"body":"private malformed response"`)) + })) + defer server.Close() + var out map[string]any + err := New(Options{BaseURL: server.URL}).doGraphQL(context.Background(), "query { rateLimit { cost } }", nil, nil, &out) + if err == nil { + t.Fatal("malformed JSON accepted") + } + class, _, evidence := HistoryFailureDetails(err) + if class != "response_decode" || !strings.Contains(string(evidence), "prefix_sha256") || strings.Contains(string(evidence), "private malformed response") { + t.Fatalf("bad rejection receipt: %s %s", class, evidence) + } +} + +func TestReviewThreadMissingStateFailsClosed(t *testing.T) { + node := map[string]any{"id": "T1", "__typename": "PullRequestReviewThread", "comments": historyTestConnection()} + h := historySession{} + if err := h.hydrate(context.Background(), node); err == nil { + t.Fatal("unknown resolution became false") + } + for _, field := range []string{"isResolved", "isOutdated", "viewerCanResolve", "viewerCanUnresolve", "viewerCanReply"} { + node[field] = false + } + node["isResolved"] = true + if err := h.hydrate(context.Background(), node); err != nil { + t.Fatal(err) + } +} diff --git a/internal/github/history_test.go b/internal/github/history_test.go index 3c3a88c8..60036525 100644 --- a/internal/github/history_test.go +++ b/internal/github/history_test.go @@ -179,6 +179,13 @@ func TestGraphQLHistoryReviewThreadCompleteness(t *testing.T) { node["reviews"] = historyTestConnection(review) thread1 := map[string]any{"id": "T1", "__typename": "PullRequestReviewThread", "comments": historyTestConnection(associated)} thread2 := map[string]any{"id": "T2", "__typename": "PullRequestReviewThread", "comments": historyTestConnection()} + for _, thread := range []map[string]any{thread1, thread2} { + for _, field := range []string{"isResolved", "isOutdated", "viewerCanResolve", "viewerCanUnresolve", "viewerCanReply"} { + thread[field] = false + } + } + thread1["isResolved"] = true + thread2["isOutdated"] = true thread2["comments"].(map[string]any)["totalCount"] = 1 thread2["comments"].(map[string]any)["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "comment-first"} node["reviewThreads"] = historyTestConnection(thread1) @@ -247,9 +254,12 @@ func TestGraphQLHistoryReviewThreadCompleteness(t *testing.T) { t.Fatal(err) } item := batch.Items[0] - if pages != 2 || len(item.ReviewComments) != 2 || len(item.Reviews) != 1 || len(item.Comments) != 1 { + if pages != 2 || len(item.ReviewComments) != 2 || len(item.Reviews) != 1 || len(item.Comments) != 1 || len(item.ReviewThreads) != 2 { t.Fatalf("incomplete or duplicate conversation: pages=%d item=%+v", pages, item) } + if item.ReviewThreads[0]["isResolved"] != true || item.ReviewThreads[1]["isOutdated"] != true { + t.Fatal("review state lost") + } if item.Reviews[0]["state"] != "APPROVED" || item.Reviews[0]["body"] != "" || item.Comments[0]["body"] != "discussion" { t.Fatal("review metadata or discussion lost") } diff --git a/internal/github/review_threads.go b/internal/github/review_threads.go index 1d8cf1f2..0f439589 100644 --- a/internal/github/review_threads.go +++ b/internal/github/review_threads.go @@ -3,6 +3,7 @@ package github import ( "bytes" "context" + "crypto/sha256" "encoding/json" "fmt" "net/http" @@ -235,21 +236,29 @@ func (c *Client) doGraphQL(ctx context.Context, query string, variables map[stri return fmt.Errorf("encode graphql request: %w", err) } var envelope graphqlResponseEnvelope - if err := c.doJSON(ctx, http.MethodPost, c.graphQLURL, bytes.NewReader(payload), reporter, &envelope); err != nil { + response, err := c.do(ctx, http.MethodPost, c.graphQLURL, bytes.NewReader(payload), reporter) + if err != nil { return err } + defer response.Body.Close() + reader := &historyResponseReader{reader: response.Body, hash: sha256.New()} + if err := decodeJSON(reader, &envelope); err != nil { + return reader.failure(fmt.Errorf("decode github response: %w", err)) + } if len(envelope.Errors) > 0 { messages := make([]string, 0, len(envelope.Errors)) for _, graphqlErr := range envelope.Errors { messages = append(messages, graphqlErr.Message) } - return fmt.Errorf("github graphql: %s", strings.Join(messages, "; ")) + var rejected any + _ = decodeJSON(bytes.NewReader(envelope.Data), &rejected) + return historyFailure("partial_response", 0, rejected, fmt.Errorf("github graphql: %s", strings.Join(messages, "; "))) } if len(envelope.Data) == 0 || string(envelope.Data) == "null" { - return fmt.Errorf("github graphql response missing data") + return historyFailure("missing_data", 0, nil, fmt.Errorf("github graphql response missing data")) } if err := decodeJSON(bytes.NewReader(envelope.Data), out); err != nil { - return fmt.Errorf("decode github graphql data: %w", err) + return historyFailure("response_decode", 0, nil, fmt.Errorf("decode github graphql data: %w", err)) } return nil } diff --git a/internal/store/analytics_integrity.go b/internal/store/analytics_integrity.go new file mode 100644 index 00000000..e4482ee0 --- /dev/null +++ b/internal/store/analytics_integrity.go @@ -0,0 +1,342 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" +) + +// Diagnostic receipts and retry state are local operational evidence, not +// public conversation payloads. Resolved rows are retained, never reset/deleted. +func (s *Store) ensureAnalyticsIntegritySchema(ctx context.Context) error { + if err := s.ensureColumn(ctx, "pull_request_review_thread_syncs", "review_thread_ids_json", "text"); err != nil { + return err + } + _, err := s.q().ExecContext(ctx, ` +CREATE TABLE IF NOT EXISTS analytics_fetch_attempts( + id INTEGER PRIMARY KEY,repository TEXT NOT NULL,number INTEGER NOT NULL,operation TEXT NOT NULL, + started_at TEXT NOT NULL,finished_at TEXT NOT NULL,status TEXT NOT NULL, + error_class TEXT,error_text TEXT,evidence_json TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS analytics_attempt_item ON analytics_fetch_attempts(repository,number,id); +CREATE INDEX IF NOT EXISTS analytics_attempt_repository ON analytics_fetch_attempts(repository,id); +CREATE TABLE IF NOT EXISTS analytics_retries( + repository TEXT NOT NULL,number INTEGER NOT NULL,operation TEXT NOT NULL, + first_seen_at TEXT NOT NULL,last_seen_at TEXT NOT NULL,next_attempt_at TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0,last_attempt_id INTEGER,resolved_at TEXT, + PRIMARY KEY(repository,number,operation)); +CREATE INDEX IF NOT EXISTS analytics_retry_due ON analytics_retries(repository,resolved_at,next_attempt_at,number); +CREATE TABLE IF NOT EXISTS analytics_review_state_coverage( + repository TEXT PRIMARY KEY,cursor INTEGER NOT NULL,ceiling INTEGER NOT NULL, + scanned INTEGER NOT NULL,queued INTEGER NOT NULL,pending_items INTEGER NOT NULL, + scan_complete INTEGER NOT NULL,complete INTEGER NOT NULL,observed_at TEXT NOT NULL); +`) + return err +} + +type AnalyticsAttempt struct { + Repository string `json:"repository"` + Number int `json:"number"` + Operation string `json:"operation"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + Status string `json:"status"` + ErrorClass string `json:"error_class,omitempty"` + ErrorText string `json:"error_text,omitempty"` + Evidence json.RawMessage `json:"evidence"` +} + +func (s *Store) RecordAnalyticsAttempt(ctx context.Context, a AnalyticsAttempt) error { + if a.Repository == "" || a.Operation == "" || (a.Status != "success" && a.Status != "failed") || !json.Valid(a.Evidence) { + return fmt.Errorf("invalid analytics attempt") + } + finished, err := time.Parse(time.RFC3339Nano, a.FinishedAt) + if err != nil { + return err + } + return s.WithTx(ctx, func(tx *Store) error { + status, class, message := a.Status, a.ErrorClass, a.ErrorText + knownReview := true + if status == "success" && (a.Operation == "graphql_history" || a.Operation == "review_state") { + if err := tx.q().QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM threads t JOIN repositories r ON r.id=t.repo_id WHERE r.full_name=? COLLATE NOCASE AND t.number=? AND (t.kind<>'pull_request' OR EXISTS(SELECT 1 FROM pull_request_review_thread_syncs x WHERE x.thread_id=t.id AND x.review_thread_ids_json IS NOT NULL)))`, a.Repository, a.Number).Scan(&knownReview); err != nil { + return err + } + if a.Operation == "review_state" && !knownReview { + status = "failed" + class = "unapplied_review_state" + message = "Fetch did not establish a review-state membership observation" + } + } + r, err := tx.q().ExecContext(ctx, `INSERT INTO analytics_fetch_attempts(repository,number,operation,started_at,finished_at,status,error_class,error_text,evidence_json) VALUES(?,?,?,?,?,?,?,?,?)`, a.Repository, a.Number, a.Operation, a.StartedAt, a.FinishedAt, status, nullString(class), nullString(message), string(a.Evidence)) + if err != nil { + return err + } + id, err := r.LastInsertId() + if err != nil { + return err + } + if status == "success" { + _, err = tx.q().ExecContext(ctx, `UPDATE analytics_retries SET resolved_at=?,last_attempt_id=? WHERE repository=? AND number=? AND resolved_at IS NULL AND (operation=? OR (? IN ('graphql_history','review_state') AND operation IN ('graphql_history','review_state'))) AND (operation<>'review_state' OR ?)`, a.FinishedAt, id, a.Repository, a.Number, a.Operation, a.Operation, knownReview) + return err + } + if a.Operation != "review_state" { + if _, err = tx.q().ExecContext(ctx, "UPDATE analytics_coverage SET complete=0 WHERE repository=?", a.Repository); err != nil { + return err + } + } + var attempts int + if err = tx.q().QueryRowContext(ctx, `SELECT coalesce((SELECT attempts FROM analytics_retries WHERE repository=? AND number=? AND operation=?),0)`, a.Repository, a.Number, a.Operation).Scan(&attempts); err != nil { + return err + } + delay := time.Duration(1< 0 { + operation = operations[0] + } + rows, err := s.q().QueryContext(ctx, `SELECT r.number FROM analytics_retries r WHERE r.repository=? AND r.resolved_at IS NULL AND r.number>0 AND r.next_attempt_at<=? AND (?='' OR r.operation=?) + AND (r.operation<>'review_state' OR NOT EXISTS(SELECT 1 FROM analytics_retries core WHERE core.repository=r.repository AND core.number=r.number AND core.operation='graphql_history' AND core.resolved_at IS NULL)) + GROUP BY r.number ORDER BY min(r.next_attempt_at),r.number LIMIT ?`, repository, at, operation, operation, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []int + for rows.Next() { + var n int + if err = rows.Scan(&n); err != nil { + return nil, err + } + out = append(out, n) + } + return out, rows.Err() +} + +func (s *Store) AnalyticsOutstanding(ctx context.Context, repository string) (int, error) { + var n int + err := s.q().QueryRowContext(ctx, `SELECT count(*) FROM analytics_retries WHERE repository=? AND resolved_at IS NULL`, repository).Scan(&n) + return n, err +} + +func (s *Store) AnalyticsCoreOutstanding(ctx context.Context, repository string) (int, error) { + var n int + err := s.q().QueryRowContext(ctx, `SELECT count(*) FROM analytics_retries WHERE repository=? AND resolved_at IS NULL AND operation<>'review_state'`, repository).Scan(&n) + return n, err +} + +func (s *Store) AnalyticsItemQueued(ctx context.Context, repository string, number int, operations ...string) (bool, error) { + operation := "graphql_history" + if len(operations) > 0 { + operation = operations[0] + } + var n int + err := s.q().QueryRowContext(ctx, `SELECT count(*) FROM analytics_retries WHERE repository=? AND number=? AND operation=? AND resolved_at IS NULL`, repository, number, operation).Scan(&n) + return n > 0, err +} + +type ReviewStateRecovery struct { + Cursor int64 `json:"cursor"` + Ceiling int64 `json:"ceiling"` + Scanned int64 `json:"scanned"` + Queued int64 `json:"queued"` + Done bool `json:"done"` +} + +// SeedReviewStateRecovery walks at most limit primary-key rows per watch cycle. +// It queues only PRs with retained review threads (or unknown connection data), +// and never restarts historical discovery or invents missing resolution state. +func (s *Store) SeedReviewStateRecovery(ctx context.Context, repository string, limit int) (ReviewStateRecovery, error) { + key := "review_state_recovery:" + repository + value, err := s.AnalyticsState(ctx, key) + if err != nil { + return ReviewStateRecovery{}, err + } + var progress ReviewStateRecovery + if value != "" { + if err = json.Unmarshal([]byte(value), &progress); err != nil { + return progress, err + } + } else { + if err = s.q().QueryRowContext(ctx, "SELECT coalesce(max(id),0) FROM threads").Scan(&progress.Ceiling); err != nil { + return progress, err + } + } + if progress.Done { + return progress, nil + } + rows, err := s.q().QueryContext(ctx, `SELECT t.id,t.number,t.kind,r.full_name, + CASE WHEN json_valid(t.raw_json) THEN coalesce( + json_extract(t.raw_json,'$._gitcrawl_source')='graphql' AND + json_type(t.raw_json,'$._graphql.reviewThreads.totalCount')='integer' AND + json_extract(t.raw_json,'$._graphql.reviewThreads.totalCount')=0 AND + json_type(t.raw_json,'$._graphql.reviewThreads.nodes')='array' AND + json_array_length(t.raw_json,'$._graphql.reviewThreads.nodes')=0 AND + json_type(t.raw_json,'$._graphql.reviewThreads.pageInfo.hasNextPage')='false',0) ELSE 0 END, + t.observation_sequence,coalesce(t.last_pulled_at,''), + EXISTS(SELECT 1 FROM pull_request_review_thread_syncs x WHERE x.thread_id=t.id AND x.review_thread_ids_json IS NOT NULL) + FROM threads t JOIN repositories r ON r.id=t.repo_id WHERE t.id>? AND t.id<=? ORDER BY t.id LIMIT ?`, progress.Cursor, progress.Ceiling, limit) + if err != nil { + return progress, err + } + type item struct { + id int64 + number int + kind, repo string + empty bool + sequence int64 + pulled string + known bool + } + var items []item + for rows.Next() { + var v item + if err = rows.Scan(&v.id, &v.number, &v.kind, &v.repo, &v.empty, &v.sequence, &v.pulled, &v.known); err != nil { + rows.Close() + return progress, err + } + items = append(items, v) + } + err = rows.Err() + rows.Close() + if err != nil { + return progress, err + } + var committed ReviewStateRecovery + err = s.WithTx(ctx, func(tx *Store) error { + next := progress // A busy-transaction retry must not double-count progress. + at := time.Now().UTC().Format(time.RFC3339Nano) + for _, v := range items { + next.Cursor = v.id + next.Scanned++ + if !strings.EqualFold(v.repo, repository) || v.kind != "pull_request" || v.known { + continue + } + if _, e := time.Parse(time.RFC3339Nano, v.pulled); v.empty && e == nil { + // Materialize only an actual retained complete-empty observation. + // CAS checks prevent an older scan from overwriting a live refresh. + _, e = tx.q().ExecContext(ctx, `INSERT INTO pull_request_review_thread_syncs(thread_id,fetched_at,review_thread_ids_json) + SELECT id,last_pulled_at,'[]' FROM threads WHERE id=? AND observation_sequence=? AND last_pulled_at=? + ON CONFLICT(thread_id) DO UPDATE SET fetched_at=excluded.fetched_at,review_thread_ids_json=excluded.review_thread_ids_json + WHERE pull_request_review_thread_syncs.review_thread_ids_json IS NULL AND pull_request_review_thread_syncs.fetched_at<=excluded.fetched_at`, v.id, v.sequence, v.pulled) + if e != nil { + return e + } + var known bool + if e = tx.q().QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pull_request_review_thread_syncs WHERE thread_id=? AND review_thread_ids_json IS NOT NULL)`, v.id).Scan(&known); e != nil { + return e + } + if known { + continue + } + } + r, e := tx.q().ExecContext(ctx, `INSERT OR IGNORE INTO analytics_retries(repository,number,operation,first_seen_at,last_seen_at,next_attempt_at) VALUES(?,?,'review_state',?,?,?)`, repository, v.number, at, at, at) + if e != nil { + return e + } + n, e := r.RowsAffected() + if e != nil { + return e + } + next.Queued += n + } + next.Done = len(items) < limit || next.Cursor >= next.Ceiling + encoded, _ := json.Marshal(next) + if e := tx.SetAnalyticsState(ctx, key, string(encoded)); e != nil { + return e + } + committed = next + return nil + }) + if err == nil { + progress = committed + } + return progress, err +} + +// Core completeness concerns issue/PR/comment traversal. Review-state enrichment +// is deliberately independent so existing response/contributor KPIs remain usable. +func (s *Store) SetAnalyticsCoverageComplete(ctx context.Context, repository string, complete bool) error { + _, err := s.q().ExecContext(ctx, "UPDATE analytics_coverage SET complete=? WHERE repository=?", boolInt(complete), repository) + return err +} + +func (s *Store) SaveReviewStateCoverage(ctx context.Context, repository string, progress ReviewStateRecovery) error { + var pending int + if err := s.q().QueryRowContext(ctx, `SELECT count(*) FROM analytics_retries WHERE repository=? AND operation='review_state' AND resolved_at IS NULL`, repository).Scan(&pending); err != nil { + return err + } + _, err := s.q().ExecContext(ctx, `INSERT INTO analytics_review_state_coverage VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(repository) DO UPDATE SET cursor=excluded.cursor,ceiling=excluded.ceiling,scanned=excluded.scanned,queued=excluded.queued,pending_items=excluded.pending_items,scan_complete=excluded.scan_complete,complete=excluded.complete,observed_at=excluded.observed_at`, repository, progress.Cursor, progress.Ceiling, progress.Scanned, progress.Queued, pending, boolInt(progress.Done), boolInt(progress.Done && pending == 0), time.Now().UTC().Format(time.RFC3339Nano)) + return err +} + +func (s *Store) AnalyticsIntegrityStatus(ctx context.Context, repository string) (map[string]any, error) { + out := map[string]any{"repository": repository, "checked_at": time.Now().UTC().Format(time.RFC3339Nano)} + var through, observed string + var issues, prs, complete int + coverageErr := s.q().QueryRowContext(ctx, "SELECT through,issues,pull_requests,complete,observed_at FROM analytics_coverage WHERE repository=?", repository).Scan(&through, &issues, &prs, &complete, &observed) + if coverageErr == sql.ErrNoRows { + out["coverage"] = map[string]any{"state": "not_started", "through": nil, "issues": nil, "pull_requests": nil, "complete": false, "observed_at": nil} + } else if coverageErr != nil { + return nil, coverageErr + } else { + out["coverage"] = map[string]any{"state": "observed", "through": through, "issues": issues, "pull_requests": prs, "complete": complete == 1, "observed_at": observed} + } + n, err := s.AnalyticsOutstanding(ctx, repository) + if err != nil { + return nil, err + } + out["unresolved_retries"] = n + progress, err := s.AnalyticsState(ctx, "review_state_recovery:"+repository) + if err != nil { + return nil, err + } + if progress != "" { + out["review_state_recovery"] = json.RawMessage(progress) + var recovery ReviewStateRecovery + if err = json.Unmarshal([]byte(progress), &recovery); err != nil { + return nil, err + } + out["review_state_scan_complete"] = recovery.Done + } else { + out["review_state_recovery"] = nil + out["review_state_scan_complete"] = false + } + corePending, err := s.AnalyticsCoreOutstanding(ctx, repository) + if err != nil { + return nil, err + } + out["core_unresolved_retries"] = corePending + out["coverage"].(map[string]any)["complete"] = complete == 1 && corePending == 0 + var reviewPending, reviewComplete int + if err = s.q().QueryRowContext(ctx, `SELECT pending_items,complete FROM analytics_review_state_coverage WHERE repository=?`, repository).Scan(&reviewPending, &reviewComplete); err == nil { + out["review_state_coverage"] = map[string]any{"pending_items": reviewPending, "complete": reviewComplete == 1} + } else if err == sql.ErrNoRows { + out["review_state_coverage"] = nil + } else { + return nil, err + } + rows, err := s.q().QueryContext(ctx, `SELECT id,number,operation,started_at,finished_at,status,coalesce(error_class,'') FROM analytics_fetch_attempts WHERE repository=? ORDER BY id DESC LIMIT 10`, repository) + if err != nil { + return nil, err + } + defer rows.Close() + items := []map[string]any{} + for rows.Next() { + var id, number int64 + var operation, start, finish, status, class string + if err = rows.Scan(&id, &number, &operation, &start, &finish, &status, &class); err != nil { + return nil, err + } + items = append(items, map[string]any{"id": id, "number": number, "operation": operation, "started_at": start, "finished_at": finish, "status": status, "error_class": class}) + } + out["latest_attempts"] = items + return out, rows.Err() +} diff --git a/internal/store/analytics_integrity_test.go b/internal/store/analytics_integrity_test.go new file mode 100644 index 00000000..3f2502d3 --- /dev/null +++ b/internal/store/analytics_integrity_test.go @@ -0,0 +1,334 @@ +package store + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" +) + +func TestAnalyticsFailureRecoveryRetainsReceiptsAndFairRetryTimes(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "archive.db") + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + attempt := AnalyticsAttempt{Repository: "fixture/repo", Number: 7, Operation: "graphql_history", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", ErrorClass: "validation", ErrorText: "incomplete connection", Evidence: json.RawMessage(`{"totalCount":2,"received":1}`)} + if err = s.RecordAnalyticsAttempt(ctx, attempt); err != nil { + t.Fatal(err) + } + if due, err := s.DueAnalyticsRetries(ctx, "fixture/repo", "2026-01-01T00:00:10Z", 5); err != nil || len(due) != 0 { + t.Fatalf("early retry %v %v", due, err) + } + s.Close() + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if due, err := s.DueAnalyticsRetries(ctx, "fixture/repo", "2026-01-01T00:00:40Z", 5); err != nil || len(due) != 1 || due[0] != 7 { + t.Fatalf("restart lost failure %v %v", due, err) + } + attempt.Number = 8 + attempt.FinishedAt = "2026-01-01T00:00:02Z" + if err = s.RecordAnalyticsAttempt(ctx, attempt); err != nil { + t.Fatal(err) + } + attempt.Number = 7 + attempt.FinishedAt = "2026-01-01T00:00:40Z" + if err = s.RecordAnalyticsAttempt(ctx, attempt); err != nil { + t.Fatal(err) + } + if due, err := s.DueAnalyticsRetries(ctx, "fixture/repo", "2026-01-01T00:00:41Z", 1); err != nil || len(due) != 1 || due[0] != 8 { + t.Fatalf("poison item starved peer %v %v", due, err) + } + attempt.Status = "success" + attempt.FinishedAt = "2026-01-01T00:02:00Z" + attempt.ErrorClass = "" + attempt.ErrorText = "" + if err = s.RecordAnalyticsAttempt(ctx, attempt); err != nil { + t.Fatal(err) + } + var receipts, resolved int + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts").Scan(&receipts) + s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE number=7 AND resolved_at IS NOT NULL").Scan(&resolved) + if receipts != 4 || resolved != 1 { + t.Fatalf("history lost receipts=%d resolved=%d", receipts, resolved) + } +} + +func TestReviewStateRecoveryIsBoundedResumableAndPreservesHistory(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + _, err = s.DB().Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','Fixture/Repo',1,'{}','2026-01-01'); +INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at,last_pulled_at) VALUES +(1,1,'P1',1,'pull_request','open','','','[]','[]','{"_graphql":{"reviewThreads":{"totalCount":1}}}','h1','2026-01-01','2026-01-01T00:00:00Z'), +(2,1,'P2',2,'pull_request','open','','','[]','[]','{"_gitcrawl_source":"graphql","_graphql":{"reviewThreads":{"totalCount":0,"nodes":[],"pageInfo":{"hasNextPage":false}}}}','h2','2026-01-01','2026-01-01T00:00:00Z');`) + if err != nil { + t.Fatal(err) + } + p, err := s.SeedReviewStateRecovery(ctx, "fixture/repo", 1) + if err != nil || p.Done || p.Queued != 1 || p.Cursor != 1 { + t.Fatalf("first step %+v %v", p, err) + } + p, err = s.SeedReviewStateRecovery(ctx, "fixture/repo", 1) + if err != nil || !p.Done || p.Queued != 1 || p.Scanned != 2 { + t.Fatalf("resume %+v %v", p, err) + } + var emptyIDs, observed string + if err = s.DB().QueryRow("SELECT review_thread_ids_json,fetched_at FROM pull_request_review_thread_syncs WHERE thread_id=2").Scan(&emptyIDs, &observed); err != nil || emptyIDs != "[]" || observed != "2026-01-01T00:00:00Z" { + t.Fatalf("known empty evidence not materialized: %s %s %v", emptyIDs, observed, err) + } + threads := []PullRequestReviewThread{{ReviewThreadID: "T1", ThreadID: 1, IsResolved: true, IsOutdated: false, RawJSON: `{"id":"T1","isResolved":true}`, CommentsJSON: `[]`, FetchedAt: "2026-01-02T00:00:00Z"}} + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-02T00:00:00Z", threads); err != nil { + t.Fatal(err) + } + threads[0].IsResolved = false + threads[0].RawJSON = `{"id":"T1","isResolved":false}` + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-03T00:00:00Z", threads); err != nil { + t.Fatal(err) + } + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-04T00:00:00Z", nil); err != nil { + t.Fatal(err) + } + var revisions, retained int + var membership string + s.DB().QueryRow("SELECT count(*) FROM pull_request_review_thread_revisions WHERE thread_id=1").Scan(&revisions) + s.DB().QueryRow("SELECT count(*) FROM pull_request_review_threads WHERE thread_id=1 AND deleted_at IS NULL").Scan(&retained) + s.DB().QueryRow("SELECT review_thread_ids_json FROM pull_request_review_thread_syncs WHERE thread_id=1").Scan(&membership) + if revisions != 2 || retained != 1 || membership != "[]" { + t.Fatalf("absence became deletion or history lost: %d %d %s", revisions, retained, membership) + } + for _, invalid := range [][]PullRequestReviewThread{{{ReviewThreadID: ""}}, {{ReviewThreadID: "duplicate"}, {ReviewThreadID: "duplicate"}}} { + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-05T00:00:00Z", invalid); err == nil { + t.Fatal("invalid membership was certified") + } + } + s.DB().QueryRow("SELECT review_thread_ids_json FROM pull_request_review_thread_syncs WHERE thread_id=1").Scan(&membership) + if membership != "[]" { + t.Fatal("rejected input changed prior membership") + } + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-01T00:00:00Z", threads); err != nil { + t.Fatal(err) + } + s.DB().QueryRow("SELECT review_thread_ids_json,fetched_at FROM pull_request_review_thread_syncs WHERE thread_id=1").Scan(&membership, &observed) + if membership != "[]" || observed != "2026-01-04T00:00:00Z" { + t.Fatalf("older observation overwrote newer membership: %s %s", membership, observed) + } +} + +func TestAnalyticsV14MigrationPreservesReceiptsAndCoverageWatermark(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "archive.db") + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 2); err != nil { + t.Fatal(err) + } + if err = s.SetAnalyticsState(ctx, "updates:fixture/repo", `{"kind":1,"cursor":"provider-cursor"}`); err != nil { + t.Fatal(err) + } + if _, err = s.DB().Exec(`DROP TABLE analytics_fetch_attempts; DROP TABLE analytics_retries; ALTER TABLE pull_request_review_thread_syncs DROP COLUMN review_thread_ids_json; PRAGMA user_version=14`); err != nil { + t.Fatal(err) + } + if err = s.markObservationSchemaConverged(ctx); err != nil { + t.Fatal(err) + } + s.Close() + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var version, complete int + var through string + s.DB().QueryRow("PRAGMA user_version").Scan(&version) + s.DB().QueryRow("SELECT through,complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&through, &complete) + cp, err := s.AnalyticsState(ctx, "updates:fixture/repo") + if err != nil || version != 15 || through != "2026-01-01T00:00:00Z" || complete != 1 || cp != `{"kind":1,"cursor":"provider-cursor"}` { + t.Fatalf("migration changed evidence: %d %s %d %s %v", version, through, complete, cp, err) + } +} + +func TestReviewStateFailuresDoNotInvalidateVerifiedCoreCoverage(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 2); err != nil { + t.Fatal(err) + } + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 2, Operation: "review_state", StartedAt: "2026-01-02T00:00:00Z", FinishedAt: "2026-01-02T00:00:01Z", Status: "failed", ErrorClass: "validation", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + if err = s.SaveReviewStateCoverage(ctx, "fixture/repo", ReviewStateRecovery{Done: true, Scanned: 2, Ceiling: 2, Cursor: 2, Queued: 1}); err != nil { + t.Fatal(err) + } + var core, review int + s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&core) + s.DB().QueryRow("SELECT complete FROM analytics_review_state_coverage WHERE repository='fixture/repo'").Scan(&review) + if core != 1 || review != 0 { + t.Fatalf("enrichment invalidated core: core=%d review=%d", core, review) + } + if n, err := s.AnalyticsCoreOutstanding(ctx, "fixture/repo"); err != nil || n != 0 { + t.Fatalf("wrong core queue %d %v", n, err) + } + a.Operation = "graphql_history" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&core) + if core != 0 { + t.Fatal("core failure left core coverage complete") + } +} + +func TestAnalyticsStatusSelectsRepositoryBeforeLimit(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 2) + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "graphql_history", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "success", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + a.Repository = "fixture/other" + for i := 0; i < 60; i++ { + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + } + status, err := s.AnalyticsIntegrityStatus(ctx, "fixture/repo") + if err != nil { + t.Fatal(err) + } + if len(status["latest_attempts"].([]map[string]any)) != 1 { + t.Fatal("another repository hid scoped history") + } +} + +func TestAnalyticsStatusBeforeCollectionIsUnknown(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + status, err := s.AnalyticsIntegrityStatus(ctx, "fixture/repo") + if err != nil { + t.Fatal(err) + } + coverage := status["coverage"].(map[string]any) + if coverage["state"] != "not_started" || coverage["complete"] != false || coverage["issues"] != nil { + t.Fatalf("invented coverage: %+v", coverage) + } + var n int + s.DB().QueryRow("SELECT count(*) FROM analytics_coverage").Scan(&n) + if n != 0 { + t.Fatal("read-only status wrote a baseline") + } +} + +func TestAnalyticsV14WithoutExtensionTablesCanUpgrade(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "archive.db") + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if _, err = s.DB().Exec(`DROP TABLE actor_profiles; DROP TABLE analytics_collection_state; PRAGMA user_version=14`); err != nil { + t.Fatal(err) + } + if err = s.markObservationSchemaConverged(ctx); err != nil { + t.Fatal(err) + } + s.Close() + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if _, err = s.AnalyticsState(ctx, "missing"); err != nil { + t.Fatal(err) + } + if _, err = s.AnalyticsProfileNodes(ctx, 1); err != nil { + t.Fatal(err) + } +} + +func TestAnalyticsCoreRetryOwnsBackoffWhenReviewRetryAlsoExists(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "graphql_history", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + a.Operation = "review_state" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + if due, err := s.DueAnalyticsRetries(ctx, a.Repository, "2026-01-01T00:00:40Z", 8, "review_state"); err != nil || len(due) != 0 { + t.Fatalf("review bypassed due core scheduler: %v %v", due, err) + } + a.Operation = "graphql_history" + a.FinishedAt = "2026-01-01T00:00:50Z" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + if due, err := s.DueAnalyticsRetries(ctx, a.Repository, "2026-01-01T00:01:00Z", 8, "review_state"); err != nil || len(due) != 0 { + t.Fatalf("review bypassed core backoff: %v %v", due, err) + } +} + +func TestAnalyticsRecoveryRequiresAnAcceptedMembershipObservation(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "review_state", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "success", Evidence: json.RawMessage(`{"threads_skipped_stale":1}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + var status, class string + s.DB().QueryRow("SELECT status,error_class FROM analytics_fetch_attempts ORDER BY id DESC LIMIT 1").Scan(&status, &class) + if status != "failed" || class != "unapplied_review_state" { + t.Fatal("fetch success certified missing membership") + } + repo, err := s.UpsertRepository(ctx, Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", RawJSON: "{}", UpdatedAt: "2026-01-02T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + tid, err := s.UpsertThread(ctx, Thread{RepoID: repo, GitHubID: "P1", Number: 1, Kind: "pull_request", State: "open", Title: "fixture", HTMLURL: "https://github.com/fixture/repo/pull/1", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h", UpdatedAt: "2026-01-02T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + if err = s.UpsertPullRequestReviewThreads(ctx, tid, "2026-01-02T00:00:00Z", nil); err != nil { + t.Fatal(err) + } + a.FinishedAt = "2026-01-02T00:00:01Z" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + if n, err := s.AnalyticsOutstanding(ctx, a.Repository); err != nil || n != 0 { + t.Fatalf("proven empty membership did not reconcile: %d %v", n, err) + } +} diff --git a/internal/store/review_threads.go b/internal/store/review_threads.go index baf3d1c6..8bc50795 100644 --- a/internal/store/review_threads.go +++ b/internal/store/review_threads.go @@ -3,7 +3,9 @@ package store import ( "context" "database/sql" + "encoding/json" "fmt" + "strings" "time" "github.com/openclaw/gitcrawl/internal/store/storedb" @@ -34,7 +36,12 @@ type PullRequestReviewThread struct { } func (s *Store) UpsertPullRequestReviewThreads(ctx context.Context, threadID int64, fetchedAt string, threads []PullRequestReviewThread) error { + seen := map[string]bool{} for _, thread := range threads { + if thread.ReviewThreadID == "" || strings.TrimSpace(thread.ReviewThreadID) != thread.ReviewThreadID || seen[thread.ReviewThreadID] { + return fmt.Errorf("review-thread membership requires nonempty unique native IDs") + } + seen[thread.ReviewThreadID] = true if err := validateTombstone(thread.DeletedAt, thread.DeletionReason); err != nil { return fmt.Errorf("upsert pull request review thread %q: %w", thread.ReviewThreadID, err) } @@ -48,12 +55,43 @@ func (s *Store) UpsertPullRequestReviewThreads(ctx context.Context, threadID int } func (s *Store) upsertPullRequestReviewThreads(ctx context.Context, threadID int64, fetchedAt string, threads []PullRequestReviewThread) error { + observed, err := time.Parse(time.RFC3339Nano, fetchedAt) + if err != nil { + return fmt.Errorf("invalid review-thread observation time: %w", err) + } + var previous string + err = s.q().QueryRowContext(ctx, "SELECT fetched_at FROM pull_request_review_thread_syncs WHERE thread_id=?", threadID).Scan(&previous) + if err != nil && err != sql.ErrNoRows { + return err + } + if err == nil { + prior, parseErr := time.Parse(time.RFC3339Nano, previous) + if parseErr != nil { + return fmt.Errorf("invalid prior review-thread observation time: %w", parseErr) + } + // This runs inside the same native transaction as state/history writes. + // An older completion must not replace either membership or row state. + if observed.Before(prior) { + return nil + } + } if err := s.qsql().UpsertPullRequestReviewThreadSync(ctx, storedb.UpsertPullRequestReviewThreadSyncParams{ ThreadID: threadID, FetchedAt: fetchedAt, }); err != nil { return fmt.Errorf("mark pull request review threads fetched: %w", err) } + ids := make([]string, 0, len(threads)) + for _, thread := range threads { + ids = append(ids, thread.ReviewThreadID) + } + encoded, err := json.Marshal(ids) + if err != nil { + return err + } + if _, err = s.q().ExecContext(ctx, "UPDATE pull_request_review_thread_syncs SET review_thread_ids_json=? WHERE thread_id=? AND fetched_at=?", string(encoded), threadID, fetchedAt); err != nil { + return err + } for _, thread := range threads { if thread.ReviewThreadID == "" { continue diff --git a/internal/store/store.go b/internal/store/store.go index d0b42144..ca464e80 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,7 +17,7 @@ import ( ) const ( - schemaVersion = 14 + schemaVersion = 15 timeLayout = time.RFC3339Nano ) @@ -285,6 +285,30 @@ func (s *Store) migrate(ctx context.Context) error { if _, err := s.db.ExecContext(ctx, schemaSQL); err != nil { return fmt.Errorf("apply schema: %w", err) } + // Version 15 only adds operational receipts and an explicit review-thread + // membership column. A converged v14 archive needs no row/history rebuild. + if current == 14 { + structural, e := inspectStructuralCompatibilityMigrations(ctx, s, current, inspectPRDetailSchema(ctx, s)) + if e != nil { + return e + } + converged, e := s.observationSchemaConvergenceIsCurrent(ctx) + if e != nil { + return e + } + if len(structural) == 1 && structural[0] == "schema_version_14_to_15" && converged { + // Some v14 archives predate the optional analytics extension. These + // additive tables/columns are cheap to ensure and require no row scan. + if e = s.ensureAnalyticsSourceSchema(ctx); e != nil { + return e + } + if e = s.ensureAnalyticsIntegritySchema(ctx); e != nil { + return e + } + _, e = s.db.ExecContext(ctx, fmt.Sprintf("PRAGMA user_version=%d", schemaVersion)) + return e + } + } if current == schemaVersion { prDetails := inspectPRDetailSchema(ctx, s) structural, err := inspectStructuralCompatibilityMigrations( @@ -322,6 +346,9 @@ func (s *Store) migrate(ctx context.Context) error { if err := s.ensureAnalyticsSourceSchema(ctx); err != nil { return err } + if err := s.ensureAnalyticsIntegritySchema(ctx); err != nil { + return err + } if err := s.ensureCanonicalObservationTables(ctx); err != nil { return err } diff --git a/internal/syncer/graphql_history_test.go b/internal/syncer/graphql_history_test.go index d1ce8928..8938b0dc 100644 --- a/internal/syncer/graphql_history_test.go +++ b/internal/syncer/graphql_history_test.go @@ -59,6 +59,7 @@ func TestGraphQLHistoryUsesNativeTransactionsAndPreservesLegacyIdentity(t *testi defer server.Close() client := historyFixtureClient{Client: gh.New(gh.Options{BaseURL: server.URL}), batch: gh.HistoryBatch{Repository: rawRepo, Items: []gh.HistoryItem{{Thread: row, Pull: pull, Comments: comments, Reviews: reviews, ReviewComments: inline}}}} options.GraphQLHistory = true + client.batch.Items[0].ReviewThreads = []map[string]any{{"id": "RT_fixture", "isResolved": true, "isOutdated": false, "comments": map[string]any{"nodes": []any{map[string]any{"id": "inline-node", "body": "reply", "replyTo": map[string]any{"id": "parent-node"}}}}}} stats, err := New(client, st).Sync(ctx, options) if err != nil { t.Fatal(err) @@ -73,6 +74,16 @@ func TestGraphQLHistoryUsesNativeTransactionsAndPreservesLegacyIdentity(t *testi if stats.ThreadsSynced != 1 || stats.PRDetailsSynced != 1 || stats.CommentsSynced == 0 { t.Fatalf("stats %+v", stats) } + if stats.ReviewThreadsSynced != 1 { + t.Fatalf("review states missing: %+v", stats) + } + var resolved int + var members string + st.DB().QueryRow("SELECT is_resolved FROM pull_request_review_threads WHERE review_thread_id='RT_fixture'").Scan(&resolved) + st.DB().QueryRow("SELECT review_thread_ids_json FROM pull_request_review_thread_syncs WHERE thread_id=?", after[0].ID).Scan(&members) + if resolved != 1 || members != `["RT_fixture"]` { + t.Fatalf("review projection %d %s", resolved, members) + } if _, err := New(client, st).Sync(ctx, options); err != nil { t.Fatal(err) } @@ -82,6 +93,10 @@ func TestGraphQLHistoryUsesNativeTransactionsAndPreservesLegacyIdentity(t *testi t.Fatal("failed batch persisted") } assertTableRowCount(t, st, "threads", 1) + var failed int + if err := st.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE status='failed'").Scan(&failed); err != nil || failed != 1 { + t.Fatalf("missing durable fetch failure: %d %v", failed, err) + } options.IncludePRDetails = true if _, err := New(client, st).Sync(ctx, options); err == nil { t.Fatal("unsupported hydration accepted") diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 6ab6c239..e4694d1c 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -43,7 +43,10 @@ type Syncer struct { } type Options struct { - GraphQLHistory bool + GraphQLHistory bool + // ReceiptOperation separates targeted review-state recovery from verified + // core traversal; both still fetch and validate complete conversations. + ReceiptOperation string Owner string Repo string State string @@ -121,7 +124,7 @@ func New(client GitHubClient, st *store.Store) *Syncer { } } -func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { +func (s *Syncer) Sync(ctx context.Context, options Options) (result Stats, resultErr error) { startedAt := s.now() started := startedAt.Format(time.RFC3339Nano) if err := reportSyncProgress(options.Progress, SyncProgress{ @@ -143,6 +146,34 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { if len(options.Numbers) == 0 || !options.IncludeComments || !options.IncludePRMetadata || options.IncludePRDetails || since != "" || options.Limit != 0 || state != "all" { return Stats{}, fmt.Errorf("--graphql-history requires --numbers, --state all, --include-comments and --with pr-metadata; since/limit/pr-details are unsupported") } + operation := options.ReceiptOperation + if operation == "" { + operation = "graphql_history" + } + if operation != "graphql_history" && operation != "review_state" { + return Stats{}, fmt.Errorf("unsupported GraphQL receipt operation") + } + // Fetch/validation failures happen before conversation transactions and + // were previously invisible to durable run tables. Keep a receipt even + // when the request is cancelled; accepted content remains untouched. + defer func() { + receiptCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + finished := s.now().Format(time.RFC3339Nano) + status, class, message := "success", "", "" + evidence, _ := json.Marshal(result) + if resultErr != nil { + status = "failed" + class, message, evidence = gh.HistoryFailureDetails(resultErr) + } + for _, number := range uniquePositiveNumbers(options.Numbers) { + err := s.store.RecordAnalyticsAttempt(receiptCtx, store.AnalyticsAttempt{Repository: options.Owner + "/" + options.Repo, Number: number, Operation: operation, StartedAt: started, FinishedAt: finished, Status: status, ErrorClass: class, ErrorText: message, Evidence: evidence}) + if err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("persist GraphQL attempt: %w", err)) + return + } + } + }() client, ok := s.client.(interface { FetchGraphQLHistory(context.Context, string, string, []int, gh.Reporter) (gh.HistoryBatch, error) }) @@ -290,6 +321,8 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { if item.Pull != nil { payload.hasPullDetails = true payload.pullDetails = pullRequestDetailRows{pull: item.Pull, fetchedAt: s.now().Format(time.RFC3339Nano)} + payload.reviewThreads = item.ReviewThreads + payload.reviewThreadsFetchedAt = payload.pullDetails.fetchedAt } } received.CommentsReceived += len(payload.commentRows) @@ -516,6 +549,11 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { if err := reserveChild(store.ThreadChildPullRequestDetails); err != nil { return err } + if history != nil { + if err := reserveChild(store.ThreadChildReviewThreads); err != nil { + return err + } + } } if options.IncludePRDetails && thread.Kind == "pull_request" { for _, family := range []store.ThreadChildObservationFamily{ From 2644b642e1abee945c2e62fff64e50d70b687ea1 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:37:49 -0600 Subject: [PATCH 07/21] Use available GitHub quota for targeted review recovery --- docs/analytics-source.md | 19 +- internal/cli/analytics.go | 25 +- internal/cli/analytics_integrity.go | 192 ++++++++++++-- internal/cli/analytics_throughput_test.go | 292 ++++++++++++++++++++++ internal/syncer/graphql_history_test.go | 14 ++ internal/syncer/syncer.go | 4 +- 6 files changed, 509 insertions(+), 37 deletions(-) create mode 100644 internal/cli/analytics_throughput_test.go diff --git a/docs/analytics-source.md b/docs/analytics-source.md index 036ce63b..e2f56659 100644 --- a/docs/analytics-source.md +++ b/docs/analytics-source.md @@ -46,8 +46,18 @@ and hydrates relevant updated threads. Issue and PR lanes have independent durab checkpoints and process at most two discovery pages per cycle. Eight two-thread requests progress independently per page. A rejected batch splits into individual requests; an item may be passed only after its failure is durably queued. Unrelated -items and the other discovery lane continue. A cycle also retries at most sixteen -due items, with a two-minute request deadline and bounded exponential backoff. +items and the other discovery lane continue. Core retries process at most eight +due items before discovery, with a two-minute request deadline and bounded +exponential backoff. After persisting core coverage, targeted review recovery +uses the time until the next nominal two-minute core poll; it does not add a +two-minute idle wait after recovery. Each wave rechecks actual GraphQL quota and +admits up to sixteen items through the existing eight two-item workers. A +32-point-per-item admission margin and per-request native quota checks preserve +3,000 points for recovery, leaving 1,500 points above ordinary capture's floor. +Missing or expired quota stops provider recovery; local discovery can continue. +The request window yields before core polling and cancellation retains retry +receipts and the last committed scan. Quota and numeric per-query cost logs make +the actual spending observable without credentials or content bodies. No partial connection is accepted as complete evidence. Non-nested continuation pages use 100 nodes to avoid repeated small round trips. Identity/profile enrichment uses eight disjoint 100-node requests with normal quota guards and can run concurrently under @@ -103,7 +113,8 @@ removes retained history. Existing comment IDs/replies and `thread_child_observation_memberships` keep their existing contract. Consumers must distinguish historical rows from the latest provider membership. -The watch gradually inspects 500 primary-key rows per cycle up to a captured +The watch inspects bounded chunks of 5,000 primary-key rows within each recovery +window up to a captured ceiling, queuing only PRs with retained review threads or unknown connection data. Complete retained zero-count GraphQL connections materialize `[]` using their actual retained observation time. Conditional writes preserve a newer live @@ -132,6 +143,8 @@ Operational tables are **not public conversation datasets**: discovery coverage and review-state scan progress without credentials or collection. Timestamped logs distinguish `github_update_complete`, `github_update_failed`, `github_coverage_pending` (core), and `review_state_progress` (enrichment). +`review_state_quota` records fresh limit/remaining/reset/reserve and admitted wave +size; `review_state_cost` records actual provider points per recovery query. The older successful-only `sync_runs` table is not a complete failure ledger. Existing embeddings are reused; this command does not generate embeddings or reinterpret empty review bodies as missing replies. diff --git a/internal/cli/analytics.go b/internal/cli/analytics.go index a11b28be..c016fa76 100644 --- a/internal/cli/analytics.go +++ b/internal/cli/analytics.go @@ -50,7 +50,7 @@ func (a *App) analyticsClient(ctx context.Context, cfg config.Config) (*gh.Clien if provider == nil && token.Value == "" { return nil, fmt.Errorf("missing GitHub credential") } - return gh.New(gh.Options{Token: token.Value, TokenProvider: provider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: 1500}), nil + return gh.New(gh.Options{Token: token.Value, TokenProvider: provider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: analyticsCoreReserve}), nil } func (a *App) runAnalytics(ctx context.Context, args []string) error { for _, arg := range args { @@ -190,7 +190,7 @@ func (a *App) runAnalytics(ctx context.Context, args []string) error { actorClients := make([]*gh.Client, 8) for i := range actorClients { token := a.resolveGitHubToken(ctx, cfg) - actorClients[i] = gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: 1500}) + actorClients[i] = gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: analyticsCoreReserve}) } parallelWatch := *watch && *enrich if parallelWatch { @@ -200,12 +200,13 @@ func (a *App) runAnalytics(ctx context.Context, args []string) error { go func() { defer close(done) for { + nextPoll := time.Now().Add(analyticsPollInterval) pollErr := a.analyticsCycle(pollCtx, rt.Store, client, owner, repo) a.analyticsUpdateLog(pollErr) select { case <-pollCtx.Done(): return - case <-time.After(2 * time.Minute): + case <-time.After(time.Until(nextPoll)): } } }() @@ -283,6 +284,7 @@ func (a *App) runAnalytics(ctx context.Context, args []string) error { return ctx.Err() } for *watch || *once { + nextPoll := time.Now().Add(analyticsPollInterval) e = a.analyticsCycle(ctx, rt.Store, client, owner, repo) if e != nil { if !*watch { @@ -298,7 +300,7 @@ func (a *App) runAnalytics(ctx context.Context, args []string) error { select { case <-ctx.Done(): return ctx.Err() - case <-time.After(2 * time.Minute): + case <-time.After(time.Until(nextPoll)): } } return nil @@ -322,11 +324,22 @@ func (a *App) syncAnalyticsBatch(ctx context.Context, s *store.Store, owner, rep return err } token := a.resolveGitHubToken(ctx, cfg) - client := gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: 1500}) + reserve := analyticsCoreReserve + var reporter gh.Reporter + if operation == "review_state" { + reserve = analyticsReviewReserve + reporter = func(message string) { + var call, cost, remaining, reset int + if _, err := fmt.Sscanf(message, "[github] graphql cost %d %d remaining %d reset %d", &call, &cost, &remaining, &reset); err == nil { + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_cost\",\"at\":%q,\"points\":%d,\"remaining\":%d,\"reset_unix\":%d}\n", time.Now().UTC().Format(time.RFC3339Nano), cost, remaining, reset) + } + } + } + client := gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: reserve}) // The watch owner has already validated and opened this store. Reopening it // for every two threads repeats full-archive migration audits and serializes // otherwise independent network work. Native transactions still own writes. - _, err = syncer.New(client, s).Sync(ctx, syncer.Options{Owner: owner, Repo: repo, GraphQLHistory: true, ReceiptOperation: operation, State: "all", Numbers: numbers, IncludeComments: true, IncludePRMetadata: true}) + _, err = syncer.New(client, s).Sync(ctx, syncer.Options{Owner: owner, Repo: repo, GraphQLHistory: true, ReceiptOperation: operation, State: "all", Numbers: numbers, IncludeComments: true, IncludePRMetadata: true, Reporter: reporter}) return err } diff --git a/internal/cli/analytics_integrity.go b/internal/cli/analytics_integrity.go index 88cd80d4..f169808a 100644 --- a/internal/cli/analytics_integrity.go +++ b/internal/cli/analytics_integrity.go @@ -14,6 +14,13 @@ import ( var errAnalyticsIncomplete = errors.New("analytics coverage remains incomplete") +const ( + analyticsPollInterval = 2 * time.Minute + analyticsCoreReserve = 1500 + // Keep another 1500 points available to ordinary capture above its floor. + analyticsReviewReserve = 3000 +) + func (a *App) analyticsUpdateLog(err error) { event := "github_update_complete" fields := map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano)} @@ -114,23 +121,19 @@ func migrateAnalyticsLanes(ctx context.Context, s *store.Store, repository strin } func (a *App) analyticsCycle(ctx context.Context, s *store.Store, c *gh.Client, owner, repo string) error { + nextCore := time.Now().Add(analyticsPollInterval) repository := owner + "/" + repo if err := migrateAnalyticsLanes(ctx, s, repository); err != nil { return err } - recovery, err := s.SeedReviewStateRecovery(ctx, repository, 500) + // Ordinary capture and its retry obligations always go first. Historical + // review enrichment uses only the remaining time before the next core poll. + due, err := s.DueAnalyticsRetries(ctx, repository, time.Now().UTC().Format(time.RFC3339Nano), 8, "graphql_history") if err != nil { return err } - // Bounded retry work never replaces either independent discovery lane. - for _, operation := range []string{"graphql_history", "review_state"} { - due, e := s.DueAnalyticsRetries(ctx, repository, time.Now().UTC().Format(time.RFC3339Nano), 8, operation) - if e != nil { - return e - } - if e = a.analyticsNumbers(ctx, s, owner, repo, due, false, operation); e != nil { - return e - } + if err = a.analyticsNumbers(ctx, s, owner, repo, due, false, "graphql_history"); err != nil { + return err } var failures []error for i, kind := range []string{"issues", "pullRequests"} { @@ -184,31 +187,159 @@ func (a *App) analyticsCycle(ctx context.Context, s *store.Store, c *gh.Client, }); err != nil { return err } - if err = s.SaveReviewStateCoverage(ctx, repository, recovery); err != nil { - return err + reviewErr := a.analyticsReviewRecovery(ctx, s, c, owner, repo, nextCore.Add(-5*time.Second)) + if len(failures) > 0 { + return errors.Join(append(failures, reviewErr)...) } - totalPending, err := s.AnalyticsOutstanding(ctx, repository) - if err != nil { + if !complete { + return errors.Join(fmt.Errorf("%w: core_unresolved=%d", errAnalyticsIncomplete, outstanding), reviewErr) + } + return reviewErr +} + +// Fill the time formerly spent idle with bounded, quota-checked waves through +// the existing executor. Each scan chunk and item receipt remains durable. +func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh.Client, owner, repo string, deadline time.Time) (resultErr error) { + window, cancel := context.WithDeadline(ctx, deadline) + defer cancel() + yield := func(err error) error { + if ctx.Err() == nil && window.Err() == context.DeadlineExceeded && analyticsCancellationOnly(err) { + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_yield\",\"at\":%q}\n", time.Now().UTC().Format(time.RFC3339Nano)) + return nil + } return err } - progress, _ := json.Marshal(map[string]any{"event": "review_state_progress", "at": time.Now().UTC().Format(time.RFC3339Nano), "scanned": recovery.Scanned, "ceiling": recovery.Ceiling, "queued": recovery.Queued, "pending_items": totalPending - outstanding, "scan_complete": recovery.Done, "complete": recovery.Done && totalPending == outstanding}) - fmt.Fprintln(a.Stderr, string(progress)) - if len(failures) > 0 { - return errors.Join(failures...) + repository := owner + "/" + repo + var progress store.ReviewStateRecovery + haveProgress, quotaBlocked := false, false + defer func() { + if !haveProgress { + return + } + // Cancellation cannot erase the last committed scan or completed items. + receiptCtx, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + if err := s.SaveReviewStateCoverage(receiptCtx, repository, progress); err != nil { + resultErr = errors.Join(resultErr, err) + return + } + var pending int + if err := s.DB().QueryRowContext(receiptCtx, "SELECT pending_items FROM analytics_review_state_coverage WHERE repository=?", repository).Scan(&pending); err != nil { + resultErr = errors.Join(resultErr, err) + return + } + encoded, _ := json.Marshal(map[string]any{"event": "review_state_progress", "at": time.Now().UTC().Format(time.RFC3339Nano), "scanned": progress.Scanned, "ceiling": progress.Ceiling, "queued": progress.Queued, "pending_items": pending, "scan_complete": progress.Done, "complete": progress.Done && pending == 0}) + fmt.Fprintln(a.Stderr, string(encoded)) + }() + for time.Until(deadline) > 5*time.Second { + if err := window.Err(); err != nil { + return yield(err) + } + next, err := s.SeedReviewStateRecovery(window, repository, 5000) + if err != nil { + return yield(err) + } + progress, haveProgress = next, true + // Persist progress even if quota is exhausted or the process is stopped. + if err = s.SaveReviewStateCoverage(window, repository, progress); err != nil { + return yield(err) + } + if quotaBlocked { + if progress.Done { + return nil + } + continue + } + due, err := s.DueAnalyticsRetries(window, repository, time.Now().UTC().Format(time.RFC3339Nano), 16, "review_state") + if err != nil { + return yield(err) + } + if len(due) == 0 { + if progress.Done { + return nil + } + continue + } + limits, err := c.GetRateLimits(window, nil) + if err != nil { + if window.Err() != nil { + return yield(err) + } + class, message, _ := gh.HistoryFailureDetails(err) + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_quota_deferred\",\"at\":%q,\"error_class\":%q,\"error\":%q}\n", time.Now().UTC().Format(time.RFC3339Nano), class, message) + quotaBlocked = true + continue + } + budget, quota, err := analyticsReviewBudget(limits, time.Now()) + if err != nil { + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_quota_deferred\",\"at\":%q,\"error\":%q}\n", time.Now().UTC().Format(time.RFC3339Nano), err.Error()) + quotaBlocked = true + continue + } + encoded, _ := json.Marshal(map[string]any{"event": "review_state_quota", "at": time.Now().UTC().Format(time.RFC3339Nano), "limit": quota.Limit, "remaining": quota.Remaining, "reset_at": quota.ResetAt, "reserve": analyticsReviewReserve, "wave_items": min(len(due), budget)}) + fmt.Fprintln(a.Stderr, string(encoded)) + if budget == 0 { + quotaBlocked = true + continue + } + // Item-level reserve failures are durably queued and absorbed by + // analyticsIsolatedBatch; the next wave reprobes quota and keeps scanning. + // Remaining errors include unrecorded storage failures, not safe deferrals. + if err = a.analyticsNumbers(window, s, owner, repo, due[:min(len(due), budget)], false, "review_state"); err != nil { + return yield(err) + } } - if !complete { - return fmt.Errorf("%w: core_unresolved=%d", errAnalyticsIncomplete, outstanding) + return ctx.Err() +} + +// Joined storage/receipt errors must never disappear behind a window timeout. +func analyticsCancellationOnly(err error) bool { + if err == nil { + return false } - return nil + if joined, ok := err.(interface{ Unwrap() []error }); ok { + for _, child := range joined.Unwrap() { + if !analyticsCancellationOnly(child) { + return false + } + } + return true + } + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + return analyticsCancellationOnly(wrapped.Unwrap()) + } + return err == context.Canceled || err == context.DeadlineExceeded +} + +func analyticsReviewBudget(limits []gh.RateLimitSnapshot, now time.Time) (int, gh.RateLimitSnapshot, error) { + for _, quota := range limits { + if quota.Resource != "graphql" { + continue + } + if quota.Limit <= 0 || quota.Remaining < 0 || !quota.ResetAt.After(now) { + return 0, quota, fmt.Errorf("fresh GraphQL quota required for review recovery") + } + // A conservative admission margin limits in-flight overshoot. Native + // request guards recheck actual remaining quota before every request. + return min(16, max(0, quota.Remaining-analyticsReviewReserve)/32), quota, nil + } + return 0, gh.RateLimitSnapshot{}, fmt.Errorf("GraphQL quota unavailable for review recovery") } -func (a *App) analyticsNumbers(ctx context.Context, s *store.Store, owner, repo string, numbers []int, discovery bool, operation string) error { +func (a *App) analyticsNumbers(ctx context.Context, s *store.Store, owner, repo string, numbers []int, discovery bool, operation string) (resultErr error) { var wg sync.WaitGroup - defer wg.Wait() - slots := make(chan struct{}, 8) var failures []error + // Every return, including cancelled admission, waits for durable receipts. + defer func() { + wg.Wait() + resultErr = errors.Join(append(failures, resultErr)...) + }() + slots := make(chan struct{}, 8) var mu sync.Mutex for i := 0; i < len(numbers); i += 2 { + if err := ctx.Err(); err != nil { + return err + } var part []int for _, n := range numbers[i:min(i+2, len(numbers))] { if discovery { @@ -228,7 +359,15 @@ func (a *App) analyticsNumbers(ctx context.Context, s *store.Store, owner, repo if len(part) == 0 { continue } - slots <- struct{}{} + select { + case slots <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + if err := ctx.Err(); err != nil { + <-slots + return err + } wg.Add(1) go func(part []int) { defer wg.Done() @@ -240,8 +379,7 @@ func (a *App) analyticsNumbers(ctx context.Context, s *store.Store, owner, repo } }(part) } - wg.Wait() - return errors.Join(failures...) + return nil } func (a *App) analyticsIsolatedBatch(ctx context.Context, s *store.Store, owner, repo string, numbers []int, operation string) error { diff --git a/internal/cli/analytics_throughput_test.go b/internal/cli/analytics_throughput_test.go new file mode 100644 index 00000000..7cab71a0 --- /dev/null +++ b/internal/cli/analytics_throughput_test.go @@ -0,0 +1,292 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsRecoveryUsesQuotaAfterCoreAndResumesCancellation(t *testing.T) { + for _, mode := range []string{"available", "reserved", "quota_drops", "quota_races", "cancel"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + s, err := store.Open(ctx, filepath.Join(dir, "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + _, err = s.DB().Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo',1,'{}','2026-01-01')`) + if err != nil { + t.Fatal(err) + } + oldItems := 48 + if mode == "quota_races" { + oldItems = 6001 + } + tx, err := s.DB().BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + for n := 1; n <= oldItems; n++ { + _, err = tx.Exec(`INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at,last_pulled_at) VALUES(?,1,?,?,'pull_request','open','','','[]','[]','{}','fixture','2026-01-01','2026-01-01T00:00:00Z')`, n, fmt.Sprint(n), n) + if err != nil { + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + baseline := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano) + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", baseline, 0, oldItems); err != nil { + t.Fatal(err) + } + var coreSeen, cancelled atomic.Bool + var recovered, probes, recoveryProbes atomic.Int64 + quota := func() int { + if mode == "quota_races" && recoveryProbes.Load() > 1 { + return 2999 + } + if mode == "reserved" || (mode == "quota_drops" && recovered.Load() >= 16) { + return 3020 + } + return 19000 + } + conn := func(nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"totalCount": len(nodes), "nodes": nodes, "pageInfo": map[string]any{"hasNextPage": false, "endCursor": "end"}} + } + numbers := regexp.MustCompile(`n([0-9]+): issueOrPullRequest\(number:([0-9]+)\)`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/rate_limit" { + probes.Add(1) + if coreSeen.Load() { + recoveryProbes.Add(1) + } + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":%d,"reset":4102444800},"core":{"limit":20000,"remaining":19999,"reset":4102444800}}}`, quota()) + return + } + if r.URL.Path != "/graphql" { + t.Errorf("unexpected endpoint %s", r.URL.Path) + http.Error(w, "unexpected", 400) + return + } + var req struct{ Query string } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": quota(), "limit": 20000, "resetAt": "2099-01-01T00:00:00Z"}} + if strings.Contains(req.Query, "orderBy") { + kind, page := "issues", conn() + if strings.Contains(req.Query, "pullRequests(first:") { + kind, page = "pullRequests", conn(map[string]any{"number": 10000, "updatedAt": time.Now().UTC().Format(time.RFC3339Nano)}) + } + data["repository"] = map[string]any{kind: page} + } else if matches := numbers.FindAllStringSubmatch(req.Query, -1); len(matches) > 0 { + repo := map[string]any{"id": "repo", "databaseId": 1, "nameWithOwner": "fixture/repo"} + for _, m := range matches { + n, _ := strconv.Atoi(m[2]) + if n == 10000 { + coreSeen.Store(true) + } else { + if !coreSeen.Load() { + t.Error("recovery ran before ordinary capture") + } + var complete int + if e := s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&complete); e != nil || complete != 1 { + t.Errorf("core coverage unavailable during review: %d %v", complete, e) + } + if mode == "cancel" && cancelled.CompareAndSwap(false, true) { + cancel() + <-r.Context().Done() + return + } + recovered.Add(1) + } + node := map[string]any{"id": fmt.Sprint("PR-", n), "fullDatabaseId": fmt.Sprint(n), "__typename": "PullRequest", "number": n, "title": "fixture", "body": "retained", "state": "OPEN", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": baseline, "url": fmt.Sprintf("https://github.com/fixture/repo/pull/%d", n), "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "labels": conn(), "assignees": conn(), "comments": conn(), "reviews": conn(), "reviewThreads": conn()} + repo["n"+m[1]] = node + } + data["repository"] = repo + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + a := New() + a.Stderr = io.Discard + a.configPath = writeDoctorTestConfig(t, dir, filepath.Join(dir, "archive.db")) + client := gh.New(gh.Options{BaseURL: server.URL, Token: "test-token-placeholder"}) + err = a.analyticsCycle(ctx, s, client, "fixture", "repo") + if mode == "cancel" { + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + var failed int + if e := s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE operation='review_state' AND status='failed'").Scan(&failed); e != nil || failed == 0 { + t.Fatalf("missing cancellation receipts: %d %v", failed, e) + } + if _, err = s.DB().Exec("UPDATE analytics_retries SET next_attempt_at='2000-01-01T00:00:00Z' WHERE resolved_at IS NULL"); err != nil { + t.Fatal(err) + } + // Resume from the real committed cursor and retry rows; no reset/reseed. + err = a.analyticsCycle(context.Background(), s, client, "fixture", "repo") + } + if err != nil { + t.Fatal(err) + } + var resolved, pending, scanned, complete int + if err = s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE operation='review_state' AND resolved_at IS NOT NULL").Scan(&resolved); err != nil { + t.Fatal(err) + } + if err = s.DB().QueryRow("SELECT pending_items,scanned,complete FROM analytics_review_state_coverage").Scan(&pending, &scanned, &complete); err != nil { + t.Fatal(err) + } + want := oldItems + if mode == "reserved" || mode == "quota_races" { + want = 0 + } + if mode == "quota_drops" { + want = 16 + } + if resolved != want || pending != oldItems-want || scanned != oldItems+1 || (complete == 1) != (want == oldItems) { + t.Fatalf("resolved=%d pending=%d scanned=%d complete=%d, want recovered%d", resolved, pending, scanned, complete, want) + } + if mode == "quota_races" { + var deferred int + if e := s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE error_class='rate_limit'").Scan(&deferred); e != nil || deferred == 0 { + t.Fatalf("native reserve guard not exercised: %d %v", deferred, e) + } + } + if probes.Load() == 0 { + t.Fatal("never obtained actual quota") + } + if err = s.DB().QueryRow("SELECT complete FROM analytics_coverage").Scan(&complete); err != nil || complete != 1 { + t.Fatalf("core coverage affected by recovery: %d %v", complete, err) + } + }) + } +} + +func TestAnalyticsReviewBudgetRejectsMissingAndExpiredQuota(t *testing.T) { + now := time.Now() + for _, limits := range [][]gh.RateLimitSnapshot{nil, {{Resource: "core", Limit: 20000, Remaining: 19000, ResetAt: now.Add(time.Hour)}}, {{Resource: "graphql", Limit: 20000, Remaining: 19000, ResetAt: now.Add(-time.Second)}}} { + if _, _, err := analyticsReviewBudget(limits, now); err == nil { + t.Fatal("invalid quota admitted recovery") + } + } +} + +func TestAnalyticsRecoveryScansPastOneChunkWhenQuotaIsReserved(t *testing.T) { + for _, quotaJSON := range []string{`{"resources":{"graphql":{"limit":20000,"remaining":3020,"reset":4102444800}}}`, `{"resources":{}}`, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":1}}}`} { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + _, err = s.DB().Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo',1,'{}','2026-01-01'); + WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<6001) + INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at,last_pulled_at) + SELECT x,1,printf('%d',x),x,'pull_request','open','','','[]','[]','{}','fixture','2026-01-01','2026-01-01T00:00:00Z' FROM n`) + if err != nil { + t.Fatal(err) + } + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.Path != "/rate_limit" { + t.Errorf("provider work admitted inside reserve: %s", r.URL.Path) + } + fmt.Fprint(w, quotaJSON) + })) + defer server.Close() + a := New() + a.Stderr = io.Discard + c := gh.New(gh.Options{BaseURL: server.URL}) + if err = a.analyticsReviewRecovery(ctx, s, c, "fixture", "repo", time.Now().Add(time.Minute)); err != nil { + t.Fatal(err) + } + var scanned, pending, done, complete int + if err = s.DB().QueryRow("SELECT scanned,pending_items,scan_complete,complete FROM analytics_review_state_coverage").Scan(&scanned, &pending, &done, &complete); err != nil { + t.Fatal(err) + } + if scanned != 6001 || pending != 6001 || done != 1 || complete != 0 || requests.Load() != 1 { + t.Fatalf("scan stalled or reserve breached: %d %d %d %d requests=%d", scanned, pending, done, complete, requests.Load()) + } + } +} + +func TestAnalyticsRecoveryDeadlineYieldsWithDurableReceipt(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + s, err := store.Open(ctx, filepath.Join(dir, "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + _, err = s.DB().Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo',1,'{}','2026-01-01'); + INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at,last_pulled_at) VALUES(1,1,'1',1,'pull_request','open','','','[]','[]','{}','fixture','2026-01-01','2026-01-01T00:00:00Z')`) + if err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 0, 1); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":4102444800}}}`) + return + } + io.Copy(io.Discard, r.Body) + <-r.Context().Done() + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + a := New() + a.Stderr = io.Discard + a.configPath = writeDoctorTestConfig(t, dir, filepath.Join(dir, "archive.db")) + c := gh.New(gh.Options{BaseURL: server.URL, Token: "test-token-placeholder"}) + if err = a.analyticsReviewRecovery(ctx, s, c, "fixture", "repo", time.Now().Add(6*time.Second)); err != nil { + t.Fatal("window yield failed core cycle:", err) + } + var failed, pending, core int + if err = s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE status='failed' AND error_class='cancelled'").Scan(&failed); err != nil { + t.Fatal(err) + } + if err = s.DB().QueryRow("SELECT pending_items FROM analytics_review_state_coverage").Scan(&pending); err != nil { + t.Fatal(err) + } + if err = s.DB().QueryRow("SELECT complete FROM analytics_coverage").Scan(&core); err != nil { + t.Fatal(err) + } + if failed != 1 || pending != 1 || core != 1 { + t.Fatalf("lost cancellation state: failed=%d pending=%d core=%d", failed, pending, core) + } + if analyticsCancellationOnly(errors.Join(context.DeadlineExceeded, errors.New("receipt failed"))) { + t.Fatal("storage error hidden as yield") + } + if !analyticsCancellationOnly(errors.Join(fmt.Errorf("request: %w", context.DeadlineExceeded), context.Canceled)) { + t.Fatal("normal cancellation not recognized") + } +} diff --git a/internal/syncer/graphql_history_test.go b/internal/syncer/graphql_history_test.go index 8938b0dc..e8faa959 100644 --- a/internal/syncer/graphql_history_test.go +++ b/internal/syncer/graphql_history_test.go @@ -102,3 +102,17 @@ func TestGraphQLHistoryUsesNativeTransactionsAndPreservesLegacyIdentity(t *testi t.Fatal("unsupported hydration accepted") } } + +func TestGraphQLCancellationDoesNotHideReceiptPersistenceFailure(t *testing.T) { + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + st.Close() + client := historyFixtureClient{err: context.DeadlineExceeded} + _, err = New(client, st).Sync(ctx, Options{Owner: "fixture", Repo: "repo", Numbers: []int{1}, State: "all", GraphQLHistory: true, IncludeComments: true, IncludePRMetadata: true, ReceiptOperation: "review_state"}) + if !errors.Is(err, context.DeadlineExceeded) || !errors.Is(err, errAnalyticsReceipt) { + t.Fatalf("missing distinguishable receipt failure: %v", err) + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index e4694d1c..ce3d03c7 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -124,6 +124,8 @@ func New(client GitHubClient, st *store.Store) *Syncer { } } +var errAnalyticsReceipt = errors.New("persist GraphQL attempt") + func (s *Syncer) Sync(ctx context.Context, options Options) (result Stats, resultErr error) { startedAt := s.now() started := startedAt.Format(time.RFC3339Nano) @@ -169,7 +171,7 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (result Stats, resul for _, number := range uniquePositiveNumbers(options.Numbers) { err := s.store.RecordAnalyticsAttempt(receiptCtx, store.AnalyticsAttempt{Repository: options.Owner + "/" + options.Repo, Number: number, Operation: operation, StartedAt: started, FinishedAt: finished, Status: status, ErrorClass: class, ErrorText: message, Evidence: evidence}) if err != nil { - resultErr = errors.Join(resultErr, fmt.Errorf("persist GraphQL attempt: %w", err)) + resultErr = errors.Join(resultErr, fmt.Errorf("%w: %w", errAnalyticsReceipt, err)) return } } From 23f3ce67ae2849476741b94dbe6d280ff622b8d0 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:43:01 -0600 Subject: [PATCH 08/21] Use actual GraphQL quota for recovery admission and reserves --- docs/analytics-source.md | 5 +++- internal/cli/analytics_integrity.go | 4 +-- internal/cli/analytics_throughput_test.go | 17 +++++++++--- internal/github/analytics.go | 18 +++++++++++++ internal/github/analytics_test.go | 33 +++++++++++++++++++++++ internal/github/history.go | 10 +++++-- 6 files changed, 79 insertions(+), 8 deletions(-) diff --git a/docs/analytics-source.md b/docs/analytics-source.md index e2f56659..a7874f02 100644 --- a/docs/analytics-source.md +++ b/docs/analytics-source.md @@ -51,7 +51,10 @@ due items before discovery, with a two-minute request deadline and bounded exponential backoff. After persisting core coverage, targeted review recovery uses the time until the next nominal two-minute core poll; it does not add a two-minute idle wait after recovery. Each wave rechecks actual GraphQL quota and -admits up to sixteen items through the existing eight two-item workers. A +admits up to sixteen items through the existing eight two-item workers. Admission +uses the authoritative GraphQL `rateLimit` response rather than REST resource +counters, which can differ. Each history session also enforces its configured +floor against observed GraphQL balances before pagination. A 32-point-per-item admission margin and per-request native quota checks preserve 3,000 points for recovery, leaving 1,500 points above ordinary capture's floor. Missing or expired quota stops provider recovery; local discovery can continue. diff --git a/internal/cli/analytics_integrity.go b/internal/cli/analytics_integrity.go index f169808a..28c56a68 100644 --- a/internal/cli/analytics_integrity.go +++ b/internal/cli/analytics_integrity.go @@ -260,7 +260,7 @@ func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh } continue } - limits, err := c.GetRateLimits(window, nil) + observedQuota, err := c.AnalyticsRateLimit(window) if err != nil { if window.Err() != nil { return yield(err) @@ -270,7 +270,7 @@ func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh quotaBlocked = true continue } - budget, quota, err := analyticsReviewBudget(limits, time.Now()) + budget, quota, err := analyticsReviewBudget([]gh.RateLimitSnapshot{observedQuota}, time.Now()) if err != nil { fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_quota_deferred\",\"at\":%q,\"error\":%q}\n", time.Now().UTC().Format(time.RFC3339Nano), err.Error()) quotaBlocked = true diff --git a/internal/cli/analytics_throughput_test.go b/internal/cli/analytics_throughput_test.go index 7cab71a0..a505b2d3 100644 --- a/internal/cli/analytics_throughput_test.go +++ b/internal/cli/analytics_throughput_test.go @@ -94,6 +94,9 @@ func TestAnalyticsRecoveryUsesQuotaAfterCoreAndResumesCancellation(t *testing.T) t.Error(err) return } + if coreSeen.Load() && !strings.Contains(req.Query, "orderBy") && !strings.Contains(req.Query, "issueOrPullRequest") { + recoveryProbes.Add(1) + } data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": quota(), "limit": 20000, "resetAt": "2099-01-01T00:00:00Z"}} if strings.Contains(req.Query, "orderBy") { kind, page := "issues", conn() @@ -197,7 +200,7 @@ func TestAnalyticsReviewBudgetRejectsMissingAndExpiredQuota(t *testing.T) { } func TestAnalyticsRecoveryScansPastOneChunkWhenQuotaIsReserved(t *testing.T) { - for _, quotaJSON := range []string{`{"resources":{"graphql":{"limit":20000,"remaining":3020,"reset":4102444800}}}`, `{"resources":{}}`, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":1}}}`} { + for _, quotaJSON := range []string{`{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":3020,"resetAt":"2099-01-01T00:00:00Z"}}}`, `{"data":{}}`, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":19000,"resetAt":"2000-01-01T00:00:00Z"}}}`} { ctx := context.Background() s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) if err != nil { @@ -214,7 +217,7 @@ func TestAnalyticsRecoveryScansPastOneChunkWhenQuotaIsReserved(t *testing.T) { var requests atomic.Int64 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1) - if r.URL.Path != "/rate_limit" { + if r.URL.Path != "/graphql" { t.Errorf("provider work admitted inside reserve: %s", r.URL.Path) } fmt.Fprint(w, quotaJSON) @@ -257,7 +260,15 @@ func TestAnalyticsRecoveryDeadlineYieldsWithDurableReceipt(t *testing.T) { fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":4102444800}}}`) return } - io.Copy(io.Discard, r.Body) + var req struct{ Query string } + if e := json.NewDecoder(r.Body).Decode(&req); e != nil { + t.Error(e) + return + } + if !strings.Contains(req.Query, "issueOrPullRequest") { + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":19000,"resetAt":"2099-01-01T00:00:00Z"}}}`) + return + } <-r.Context().Done() })) defer server.Close() diff --git a/internal/github/analytics.go b/internal/github/analytics.go index c309183c..56edeb1a 100644 --- a/internal/github/analytics.go +++ b/internal/github/analytics.go @@ -66,6 +66,24 @@ type UpdatedPage struct { Oldest time.Time } +// AnalyticsRateLimit reads the same GraphQL balance charged by history queries. +// REST resource counters can differ and must not admit recovery on that basis. +func (c *Client) AnalyticsRateLimit(ctx context.Context) (RateLimitSnapshot, error) { + h := historySession{client: c, remaining: 20000} + data, err := h.request(ctx, `query { rateLimit { cost limit remaining used resetAt } }`, nil, 1) + if err != nil { + return RateLimitSnapshot{}, err + } + rate := historyMap(data["rateLimit"]) + limit, ok := historyInt(rate["limit"]) + if !ok || limit <= 0 { + return RateLimitSnapshot{}, fmt.Errorf("GraphQL quota limit unavailable") + } + remaining, _ := historyInt(rate["remaining"]) + reset, _ := time.Parse(time.RFC3339, historyString(rate["resetAt"])) + return RateLimitSnapshot{Resource: "graphql", Limit: limit, Remaining: remaining, ResetAt: reset}, nil +} + func (c *Client) UpdatedNumbers(ctx context.Context, owner, repo, kind, after string, since time.Time) (UpdatedPage, error) { if kind != "issues" && kind != "pullRequests" { return UpdatedPage{}, fmt.Errorf("invalid discovery kind") diff --git a/internal/github/analytics_test.go b/internal/github/analytics_test.go index 889342e8..257172aa 100644 --- a/internal/github/analytics_test.go +++ b/internal/github/analytics_test.go @@ -3,8 +3,10 @@ package github import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" + "strings" "testing" "time" ) @@ -70,3 +72,34 @@ func TestAnalyticsUnavailableNodesAreNotAuthorizationSuccess(t *testing.T) { t.Fatal("authorization failure became missing-data evidence") } } + +func TestAnalyticsQuotaAndHistoryReserveUseActualGraphQLBalance(t *testing.T) { + var contentRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19999,"reset":4102444800}}}`) + return + } + var req struct{ Query string } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + if strings.Contains(req.Query, "issueOrPullRequest") { + contentRequests++ + } + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":2990,"resetAt":"2099-01-01T00:00:00Z"}}}`) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, Token: "test-token-placeholder", RateLimitReserve: 3000}) + quota, err := c.AnalyticsRateLimit(context.Background()) + if err != nil || quota.Remaining != 2990 || quota.Limit != 20000 { + t.Fatalf("REST counter admitted work: %+v %v", quota, err) + } + if _, err = c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil); err == nil || !strings.Contains(err.Error(), "quota reserve reached") { + t.Fatalf("actual GraphQL reserve ignored: %v", err) + } + if contentRequests != 0 { + t.Fatal("content dispatched inside actual GraphQL reserve") + } +} diff --git a/internal/github/history.go b/internal/github/history.go index 8cfb1339..516579d5 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -109,8 +109,14 @@ func (h *historySession) requestOnce(ctx context.Context, query string, variable if h.calls >= 1000 { return nil, fmt.Errorf("GraphQL history pagination budget exceeded") } - if h.remaining < 500+estimate { - return nil, fmt.Errorf("GraphQL history quota reserve reached") + reserve := 500 + if h.client.reserve != nil { + reserve = max(reserve, h.client.reserve.reserve) + } + // Retain the configured floor against actual GraphQL responses as well as + // the existing REST quota guard, including every pagination request. + if h.remaining < reserve+estimate { + return nil, fmt.Errorf("GraphQL history quota reserve reached: %w", &RateLimitReserveError{RateLimit: RateLimitSnapshot{Resource: "graphql", Remaining: h.remaining}, Reserve: reserve}) } h.calls++ // An unanswered request is charged conservatively by external supervisors. From 195faf3fa61990e03b5048d43720298d8dce2569 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:59:16 -0600 Subject: [PATCH 09/21] Keep GraphQL quota conservative across anomalous samples --- docs/analytics-source.md | 4 ++ internal/cli/analytics.go | 9 ++- internal/cli/analytics_integrity.go | 4 +- internal/github/analytics.go | 9 +-- internal/github/analytics_test.go | 85 ++++++++++++++++++++++++++++- internal/github/client.go | 25 +++++++++ internal/github/history.go | 4 +- 7 files changed, 131 insertions(+), 9 deletions(-) diff --git a/docs/analytics-source.md b/docs/analytics-source.md index a7874f02..bc109322 100644 --- a/docs/analytics-source.md +++ b/docs/analytics-source.md @@ -55,6 +55,10 @@ admits up to sixteen items through the existing eight two-item workers. Admissio uses the authoritative GraphQL `rateLimit` response rather than REST resource counters, which can differ. Each history session also enforces its configured floor against observed GraphQL balances before pagination. A +client retains the lowest observed GraphQL balance until the reset boundary +passes. An upward sample or a shifted future reset cannot increase admission; +REST snapshot refreshes do not overwrite this evidence. Logs distinguish the +raw provider balance/reset from the conservative effective admission values. A 32-point-per-item admission margin and per-request native quota checks preserve 3,000 points for recovery, leaving 1,500 points above ordinary capture's floor. Missing or expired quota stops provider recovery; local discovery can continue. diff --git a/internal/cli/analytics.go b/internal/cli/analytics.go index c016fa76..fcf50aaf 100644 --- a/internal/cli/analytics.go +++ b/internal/cli/analytics.go @@ -328,10 +328,17 @@ func (a *App) syncAnalyticsBatch(ctx context.Context, s *store.Store, owner, rep var reporter gh.Reporter if operation == "review_state" { reserve = analyticsReviewReserve + // This call owns its client, history session and reporter. Synchronous + // callbacks are never shared with the other analyticsNumbers workers. + var effectiveRemaining, effectiveReset int reporter = func(message string) { + var providerRemaining, providerReset int + if _, err := fmt.Sscanf(message, "[github] graphql quota provider_remaining %d provider_reset %d effective_remaining %d effective_reset %d", &providerRemaining, &providerReset, &effectiveRemaining, &effectiveReset); err == nil { + return + } var call, cost, remaining, reset int if _, err := fmt.Sscanf(message, "[github] graphql cost %d %d remaining %d reset %d", &call, &cost, &remaining, &reset); err == nil { - fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_cost\",\"at\":%q,\"points\":%d,\"remaining\":%d,\"reset_unix\":%d}\n", time.Now().UTC().Format(time.RFC3339Nano), cost, remaining, reset) + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_cost\",\"at\":%q,\"points\":%d,\"remaining\":%d,\"reset_unix\":%d,\"provider_remaining\":%d,\"provider_reset_unix\":%d}\n", time.Now().UTC().Format(time.RFC3339Nano), cost, effectiveRemaining, effectiveReset, remaining, reset) } } } diff --git a/internal/cli/analytics_integrity.go b/internal/cli/analytics_integrity.go index 28c56a68..c657824f 100644 --- a/internal/cli/analytics_integrity.go +++ b/internal/cli/analytics_integrity.go @@ -260,7 +260,7 @@ func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh } continue } - observedQuota, err := c.AnalyticsRateLimit(window) + observedQuota, rawQuota, err := c.AnalyticsRateLimit(window) if err != nil { if window.Err() != nil { return yield(err) @@ -276,7 +276,7 @@ func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh quotaBlocked = true continue } - encoded, _ := json.Marshal(map[string]any{"event": "review_state_quota", "at": time.Now().UTC().Format(time.RFC3339Nano), "limit": quota.Limit, "remaining": quota.Remaining, "reset_at": quota.ResetAt, "reserve": analyticsReviewReserve, "wave_items": min(len(due), budget)}) + encoded, _ := json.Marshal(map[string]any{"event": "review_state_quota", "at": time.Now().UTC().Format(time.RFC3339Nano), "limit": quota.Limit, "remaining": quota.Remaining, "reset_at": quota.ResetAt, "provider_remaining": rawQuota.Remaining, "provider_reset_at": rawQuota.ResetAt, "reserve": analyticsReviewReserve, "wave_items": min(len(due), budget)}) fmt.Fprintln(a.Stderr, string(encoded)) if budget == 0 { quotaBlocked = true diff --git a/internal/github/analytics.go b/internal/github/analytics.go index 56edeb1a..ab17137c 100644 --- a/internal/github/analytics.go +++ b/internal/github/analytics.go @@ -68,20 +68,21 @@ type UpdatedPage struct { // AnalyticsRateLimit reads the same GraphQL balance charged by history queries. // REST resource counters can differ and must not admit recovery on that basis. -func (c *Client) AnalyticsRateLimit(ctx context.Context) (RateLimitSnapshot, error) { +func (c *Client) AnalyticsRateLimit(ctx context.Context) (effective, observed RateLimitSnapshot, err error) { h := historySession{client: c, remaining: 20000} data, err := h.request(ctx, `query { rateLimit { cost limit remaining used resetAt } }`, nil, 1) if err != nil { - return RateLimitSnapshot{}, err + return RateLimitSnapshot{}, RateLimitSnapshot{}, err } rate := historyMap(data["rateLimit"]) limit, ok := historyInt(rate["limit"]) if !ok || limit <= 0 { - return RateLimitSnapshot{}, fmt.Errorf("GraphQL quota limit unavailable") + return RateLimitSnapshot{}, RateLimitSnapshot{}, fmt.Errorf("GraphQL quota limit unavailable") } remaining, _ := historyInt(rate["remaining"]) reset, _ := time.Parse(time.RFC3339, historyString(rate["resetAt"])) - return RateLimitSnapshot{Resource: "graphql", Limit: limit, Remaining: remaining, ResetAt: reset}, nil + observed = RateLimitSnapshot{Resource: "graphql", Limit: limit, Remaining: remaining, ResetAt: reset} + return c.reserve.observeGraphQL(observed, time.Now()), observed, nil } func (c *Client) UpdatedNumbers(ctx context.Context, owner, repo, kind, after string, since time.Time) (UpdatedPage, error) { diff --git a/internal/github/analytics_test.go b/internal/github/analytics_test.go index 257172aa..60b36cac 100644 --- a/internal/github/analytics_test.go +++ b/internal/github/analytics_test.go @@ -92,7 +92,7 @@ func TestAnalyticsQuotaAndHistoryReserveUseActualGraphQLBalance(t *testing.T) { })) defer server.Close() c := New(Options{BaseURL: server.URL, Token: "test-token-placeholder", RateLimitReserve: 3000}) - quota, err := c.AnalyticsRateLimit(context.Background()) + quota, _, err := c.AnalyticsRateLimit(context.Background()) if err != nil || quota.Remaining != 2990 || quota.Limit != 20000 { t.Fatalf("REST counter admitted work: %+v %v", quota, err) } @@ -103,3 +103,86 @@ func TestAnalyticsQuotaAndHistoryReserveUseActualGraphQLBalance(t *testing.T) { t.Fatal("content dispatched inside actual GraphQL reserve") } } + +func TestAnalyticsQuotaDoesNotIncreaseOnAnomalousGraphQLSample(t *testing.T) { + reset := time.Now().UTC().Add(time.Hour).Truncate(time.Second) + balance := 3070 + shifted := reset + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":19999,"reset":%d}}}`, reset.Unix()) + return + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": balance, "resetAt": shifted.Format(time.RFC3339)}}}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, Token: "test-token-placeholder", RateLimitReserve: 1500}) + first, _, err := c.AnalyticsRateLimit(context.Background()) + if err != nil { + t.Fatal(err) + } + balance = 19985 + shifted = reset.Add(4 * time.Second) + high, raw, err := c.AnalyticsRateLimit(context.Background()) + if err != nil || first.Remaining != 3070 || high.Remaining != 3070 || raw.Remaining != 19985 || high.ResetAt != shifted { + t.Fatalf("anomaly increased admission: first=%+v effective=%+v raw=%+v err=%v", first, high, raw, err) + } + balance = 3050 + shifted = reset + low, _, err := c.AnalyticsRateLimit(context.Background()) + if err != nil || low.Remaining != 3050 || !low.ResetAt.Equal(reset.Add(4*time.Second)) { + t.Fatalf("lower balance/reset boundary lost: %+v %v", low, err) + } + // REST refreshes cannot replace this GraphQL evidence. Only a real rollover + // after the later observed reset boundary can replenish the balance. + c.reserve.replace([]RateLimitSnapshot{{Resource: "graphql", Remaining: 19999, ResetAt: reset.Add(time.Hour)}}) + same := c.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 19900, ResetAt: reset.Add(time.Hour)}, reset.Add(2*time.Second)) + if same.Remaining != 3050 || !same.ResetAt.Equal(reset.Add(4*time.Second)) { + t.Fatal("early rollover increased balance") + } + next := c.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 19900, ResetAt: reset.Add(time.Hour)}, reset.Add(5*time.Second)) + if next.Remaining != 19900 { + t.Fatal("actual rollover could not refill budget") + } +} + +func TestGraphQLQuotaJitterAfterPreviousBoundaryCannotRefill(t *testing.T) { + now := time.Now().UTC() + r := newRateLimitReserve(3000, nil) + r.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 3050, ResetAt: now.Add(time.Minute)}, now) + held := r.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 19985, ResetAt: now.Add(time.Minute + 4*time.Second)}, now.Add(time.Minute+time.Second)) + if held.Remaining != 3050 || !held.ResetAt.Equal(now.Add(time.Minute+4*time.Second)) { + t.Fatalf("post-boundary jitter refilled: %+v", held) + } + renewed := r.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 19980, ResetAt: now.Add(time.Hour)}, now.Add(time.Minute+5*time.Second)) + if renewed.Remaining != 19980 { + t.Fatal("genuine rollover remained clamped") + } +} + +func TestGraphQLHistoryQuotaLogLabelsProviderAndEffectiveBalance(t *testing.T) { + reset := time.Now().UTC().Add(time.Hour).Truncate(time.Second) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":19999,"reset":%d}}}`, reset.Unix()) + return + } + fmt.Fprintf(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":19985,"resetAt":%q}}}`, reset.Add(4*time.Second).Format(time.RFC3339)) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 3000}) + c.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 3070, ResetAt: reset}, time.Now()) + var messages []string + h := historySession{client: c, remaining: 20000, reporter: func(message string) { messages = append(messages, message) }} + data, err := h.request(context.Background(), "query { rateLimit { cost limit remaining resetAt } }", nil, 1) + if err != nil { + t.Fatal(err) + } + if raw, ok := historyInt(historyMap(data["rateLimit"])["remaining"]); !ok || raw != 19985 { + t.Fatal("raw provider evidence rewritten") + } + log := strings.Join(messages, "\n") + if !strings.Contains(log, "provider_remaining 19985") || !strings.Contains(log, "effective_remaining 3070") { + t.Fatal("raw and effective quota not labeled", log) + } +} diff --git a/internal/github/client.go b/internal/github/client.go index 612e0e92..ae5e5d27 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -79,6 +79,9 @@ type rateLimitReserve struct { mu sync.Mutex reserve int snapshots map[string]RateLimitSnapshot + // GraphQL response evidence must survive REST snapshot replacement and + // upward provider anomalies until the observed reset boundary has passed. + graphqlObserved RateLimitSnapshot } type rateLimitRequestLockKey struct{} @@ -187,6 +190,28 @@ func (r *rateLimitReserve) observe(snapshot RateLimitSnapshot) { r.snapshots[snapshot.Resource] = snapshot } +func (r *rateLimitReserve) observeGraphQL(snapshot RateLimitSnapshot, now time.Time) RateLimitSnapshot { + if r == nil || snapshot.Remaining < 0 || !snapshot.ResetAt.After(now) { + return snapshot + } + r.mu.Lock() + defer r.mu.Unlock() + previous := r.graphqlObserved + nearbyLaterReset := !previous.ResetAt.IsZero() && snapshot.ResetAt.After(previous.ResetAt) && snapshot.ResetAt.Sub(previous.ResetAt) <= time.Minute + if previous.ResetAt.After(now) || nearbyLaterReset { + snapshot.Remaining = min(snapshot.Remaining, previous.Remaining) + // A shifted reset timestamp before the prior boundary is not a new + // window. Wait through both boundaries before accepting a refill. + // An early sample of the next hourly window must not postpone the + // current boundary by another hour. Only nearby reset jitter extends it. + if previous.ResetAt.After(snapshot.ResetAt) || snapshot.ResetAt.Sub(previous.ResetAt) > time.Minute { + snapshot.ResetAt = previous.ResetAt + } + } + r.graphqlObserved = snapshot + return snapshot +} + func (r *rateLimitReserve) replace(snapshots []RateLimitSnapshot) { if r == nil { return diff --git a/internal/github/history.go b/internal/github/history.go index 516579d5..f5f847ba 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -141,7 +141,9 @@ func (h *historySession) requestOnce(ctx context.Context, query string, variable if err != nil { return nil, fmt.Errorf("GraphQL history invalid reset") } - h.remaining = min(h.remaining-cost, remaining) + effective := h.client.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: remaining, ResetAt: reset}, time.Now()) + h.remaining = min(h.remaining-cost, effective.Remaining) + h.reporter.Printf("[github] graphql quota provider_remaining %d provider_reset %d effective_remaining %d effective_reset %d", remaining, reset.Unix(), h.remaining, effective.ResetAt.Unix()) h.reporter.Printf("[github] graphql cost %d %d remaining %d reset %d", h.calls, cost, remaining, reset.Unix()) return data, nil } From 5ea6e005ccf982c2f463beecd2f3904190214c87 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:32:22 -0600 Subject: [PATCH 10/21] Retain safe GraphQL error codes and paths in rejection receipts --- docs/analytics-source.md | 5 + .../cli/analytics_partial_response_test.go | 167 ++++++++++++++++++ internal/github/history_evidence.go | 39 ++++ internal/github/history_evidence_test.go | 73 ++++++++ internal/github/review_threads.go | 14 +- 5 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 internal/cli/analytics_partial_response_test.go diff --git a/docs/analytics-source.md b/docs/analytics-source.md index bc109322..4cb5bf0a 100644 --- a/docs/analytics-source.md +++ b/docs/analytics-source.md @@ -139,6 +139,11 @@ Operational tables are **not public conversation datasets**: and rejected GraphQL work. `number=0` identifies a discovery-lane request. Rejection evidence is bounded structural metadata, IDs, counts, pagination and body lengths/hashes; it contains no credentials, response headers or prose bodies. + Partial-response receipts also retain `graphql_errors`: total count, up to eight + allowlisted provider codes and query paths of at most eight components, with + explicit truncation markers. Unknown codes/path components are null. Messages, + arbitrary field values and identities are excluded from this error metadata. + Older receipts are not rewritten to infer a cause from later provider reads. - `analytics_retries(repository, number, operation, first_seen_at, last_seen_at, next_attempt_at, attempts, last_attempt_id, resolved_at)` is keyed by `(repository, number, operation)`. Resolved entries and attempt history remain. diff --git a/internal/cli/analytics_partial_response_test.go b/internal/cli/analytics_partial_response_test.go new file mode 100644 index 00000000..e09baaee --- /dev/null +++ b/internal/cli/analytics_partial_response_test.go @@ -0,0 +1,167 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsPairedPartialRejectionIsolatesPeerAndPreservesUnavailableHistory(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dbpath := filepath.Join(dir, "archive.db") + s, err := store.Open(ctx, dbpath) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var reject atomic.Bool + beforeAt := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano) + afterAt := time.Now().UTC().Format(time.RFC3339Nano) + conn := func(nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"totalCount": len(nodes), "nodes": nodes, "pageInfo": map[string]any{"hasNextPage": false, "endCursor": "end"}} + } + aliases := regexp.MustCompile(`n([0-9]+): issueOrPullRequest\(number:([0-9]+)\)`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":4102444800},"core":{"limit":20000,"remaining":19000,"reset":4102444800}}}`) + return + } + var req struct{ Query string } + if e := json.NewDecoder(r.Body).Decode(&req); e != nil { + t.Error(e) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": "2099-01-01T00:00:00Z"}} + response := map[string]any{"data": data} + repo := map[string]any{"id": "repo", "databaseId": 1, "nameWithOwner": "fixture/repo"} + var failures []any + matches := aliases.FindAllStringSubmatch(req.Query, -1) + for _, m := range matches { + n, _ := strconv.Atoi(m[2]) + alias := "n" + m[1] + if reject.Load() && n == 16945 && len(matches) == 1 { + var retainedBody string + if e := s.DB().QueryRow("SELECT body FROM threads WHERE number=16945").Scan(&retainedBody); e != nil || retainedBody != "retained original" { + t.Errorf("paired partial response was applied before isolated success: %q %v", retainedBody, e) + } + } + if reject.Load() && n == 16944 { + repo[alias] = nil + failures = append(failures, map[string]any{"type": "NOT_FOUND", "path": []any{"repository", alias}, "message": "private provider prose"}) + continue + } + at, body := beforeAt, "retained original" + if reject.Load() { + at, body = afterAt, "isolated success" + } + comment := map[string]any{"id": fmt.Sprint("C", n), "__typename": "IssueComment", "fullDatabaseId": fmt.Sprint(n + 1000000), "body": "retained comment", "createdAt": beforeAt, "updatedAt": beforeAt, "publishedAt": beforeAt, "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "url": fmt.Sprintf("https://github.com/fixture/repo/pull/%d#comment", n)} + repo[alias] = map[string]any{"id": fmt.Sprint("PR", n), "fullDatabaseId": fmt.Sprint(n), "__typename": "PullRequest", "number": n, "title": "fixture", "body": body, "state": "CLOSED", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": at, "url": fmt.Sprintf("https://github.com/fixture/repo/pull/%d", n), "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "labels": conn(), "assignees": conn(), "comments": conn(comment), "reviews": conn(), "reviewThreads": conn()} + } + if len(repo) > 3 { + data["repository"] = repo + } + if len(failures) > 0 { + response["errors"] = failures + } + json.NewEncoder(w).Encode(response) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + a := New() + a.Stderr = io.Discard + a.configPath = writeDoctorTestConfig(t, dir, dbpath) + if err = a.syncAnalyticsBatch(ctx, s, "fixture", "repo", []int{16944, 16945}, "graphql_history"); err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", beforeAt, 0, 2); err != nil { + t.Fatal(err) + } + // Reproduce a retained legacy PR with unknown review membership. + if _, err = s.DB().Exec("DELETE FROM pull_request_review_thread_syncs WHERE thread_id=(SELECT id FROM threads WHERE number=16944)"); err != nil { + t.Fatal(err) + } + snapshot := func() string { + t.Helper() + tables := []string{"threads", "thread_revisions", "comments", "comment_revisions"} + out := map[string][][]any{} + for _, table := range tables { + where := "thread_id=(SELECT id FROM threads WHERE number=16944)" + if table == "threads" { + where = "number=16944" + } else if table == "comment_revisions" { + where = "comment_id IN (SELECT id FROM comments WHERE thread_id=(SELECT id FROM threads WHERE number=16944))" + } + rows, e := s.DB().Query("SELECT * FROM " + table + " WHERE " + where + " ORDER BY id") + if e != nil { + t.Fatal(e) + } + cols, _ := rows.Columns() + values := [][]any{} + for rows.Next() { + row := make([]any, len(cols)) + dest := make([]any, len(cols)) + for i := range row { + dest[i] = &row[i] + } + if e = rows.Scan(dest...); e != nil { + t.Fatal(e) + } + values = append(values, row) + } + if e = rows.Err(); e != nil { + t.Fatal(e) + } + rows.Close() + out[table] = values + } + b, _ := json.Marshal(out) + return string(b) + } + retained := snapshot() + reject.Store(true) + if err = a.analyticsIsolatedBatch(ctx, s, "fixture", "repo", []int{16944, 16945}, "review_state"); err != nil { + t.Fatal(err) + } + if snapshot() != retained { + t.Fatal("unavailable PR history changed") + } + var unknown, complete int + s.DB().QueryRow("SELECT count(*) FROM pull_request_review_thread_syncs WHERE thread_id=(SELECT id FROM threads WHERE number=16944)").Scan(&unknown) + s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&complete) + if unknown != 0 || complete != 1 { + t.Fatalf("fabricated membership or invalidated core: %d %d", unknown, complete) + } + var evidence string + if err = s.DB().QueryRow("SELECT evidence_json FROM analytics_fetch_attempts WHERE number=16944 AND operation='review_state' ORDER BY id DESC LIMIT 1").Scan(&evidence); err != nil { + t.Fatal(err) + } + if !strings.Contains(evidence, `"type":"NOT_FOUND"`) || !strings.Contains(evidence, `"path":["repository","n0"]`) || strings.Contains(evidence, "private provider prose") { + t.Fatal("missing or unsafe durable cause", evidence) + } + var pending, peerResolved, success int + s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE number=16944 AND operation='review_state' AND resolved_at IS NULL AND attempts=2").Scan(&pending) + s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE number=16945 AND operation='review_state' AND resolved_at IS NOT NULL").Scan(&peerResolved) + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE number=16945 AND operation='review_state' AND status='success'").Scan(&success) + var membership, body string + s.DB().QueryRow("SELECT x.review_thread_ids_json,t.body FROM threads t JOIN pull_request_review_thread_syncs x ON x.thread_id=t.id WHERE t.number=16945").Scan(&membership, &body) + if pending != 1 || peerResolved != 1 || success != 1 || membership != "[]" || body != "isolated success" { + t.Fatalf("isolation/retry proof failed: %d %d %d %s %s", pending, peerResolved, success, membership, body) + } +} diff --git a/internal/github/history_evidence.go b/internal/github/history_evidence.go index aaac59b4..016e1c47 100644 --- a/internal/github/history_evidence.go +++ b/internal/github/history_evidence.go @@ -47,6 +47,45 @@ func (r *historyResponseReader) failure(err error) error { func (e *HistoryFailure) Error() string { return e.Cause.Error() } func (e *HistoryFailure) Unwrap() error { return e.Cause } +// Error messages can contain prose or identities. Retain only bounded provider +// codes and known query path components, separately from the existing receipt. +func graphQLRejection(data any, failures []graphqlResponseError, cause error) error { + var evidence map[string]json.RawMessage + // SafeHistoryEvidence always wraps data in an object, including nil data. + _ = json.Unmarshal(SafeHistoryEvidence(data), &evidence) + items := make([]map[string]any, 0, min(len(failures), 8)) + for _, failure := range failures[:min(len(failures), 8)] { + var code any + switch failure.Type { + case "NOT_FOUND", "FORBIDDEN", "UNAUTHORIZED", "UNPROCESSABLE", "RATE_LIMITED", "INTERNAL", "INTERNAL_SERVER_ERROR", "SERVICE_UNAVAILABLE", "MAX_NODE_LIMIT_EXCEEDED", "EXCESSIVE_PAGINATION", "RESOURCE_LIMITS_EXCEEDED": + code = failure.Type + } + path := make([]any, 0, min(len(failure.Path), 8)) + for _, component := range failure.Path[:min(len(failure.Path), 8)] { + var safe any + switch value := component.(type) { + case string: + switch value { + case "query", "repository", "node", "nodes", "issue", "pullRequest", "issueOrPullRequest", "issues", "pullRequests", "comments", "reviews", "reviewThreads", "labels", "assignees", "edges", "pageInfo", "totalCount", "hasNextPage", "endCursor", "rateLimit", "id", "__typename", "body", "author", "state", "isResolved", "isOutdated": + safe = value + } + if len(value) >= 2 && len(value) <= 3 && value[0] == 'n' && strings.Trim(value[1:], "0123456789") == "" { + safe = value // Native generated aliases, never entity IDs. + } + case json.Number: + if index, err := value.Int64(); err == nil && index >= 0 && index <= 10000 { + safe = index + } + } + path = append(path, safe) + } + items = append(items, map[string]any{"type": code, "path": path, "path_truncated": len(failure.Path) > 8}) + } + evidence["graphql_errors"], _ = json.Marshal(map[string]any{"count": len(failures), "items": items, "truncated": len(failures) > 8}) + encoded, _ := json.Marshal(evidence) + return &HistoryFailure{Cause: cause, Stage: "partial_response", Evidence: encoded} +} + func historyFailure(stage string, number int, data any, err error) error { evidence := SafeHistoryEvidence(data) var upstream *HistoryFailure diff --git a/internal/github/history_evidence_test.go b/internal/github/history_evidence_test.go index 0c9a93c2..9161b5fa 100644 --- a/internal/github/history_evidence_test.go +++ b/internal/github/history_evidence_test.go @@ -58,3 +58,76 @@ func TestReviewThreadMissingStateFailsClosed(t *testing.T) { t.Fatal(err) } } + +func TestGraphQLRejectedResponseKeepsOnlyBoundedSafeErrorMetadata(t *testing.T) { + failures := []any{} + for i := 0; i < 12; i++ { + typ := "NOT_FOUND" + if i == 1 { + typ = "private-identity" + } + failures = append(failures, map[string]any{"type": typ, "message": "private provider prose", "path": []any{"repository", "n0", "comments", 0, "body", "private-identity", "IC_private_identity", 1000000000, "omitted"}}) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"repository": map[string]any{"n0": nil, "n1": map[string]any{"body": "private peer prose"}}}, "errors": failures}) + })) + defer server.Close() + out := map[string]any{"unchanged": true} + err := New(Options{BaseURL: server.URL}).doGraphQL(context.Background(), "query { repository { id } }", nil, nil, &out) + if err == nil || len(out) != 1 || out["unchanged"] != true { + t.Fatal("partial data accepted") + } + class, _, evidence := HistoryFailureDetails(err) + if class != "partial_response" { + t.Fatal(class) + } + for _, private := range []string{"private provider prose", "private peer prose", "private-identity", "IC_private_identity", "omitted"} { + if strings.Contains(string(evidence), private) { + t.Fatal("unsafe rejection metadata retained", private) + } + } + var receipt struct { + Errors struct { + Count int `json:"count"` + Items []struct { + Type *string `json:"type"` + Path []any `json:"path"` + Truncated bool `json:"path_truncated"` + } `json:"items"` + Truncated bool `json:"truncated"` + } `json:"graphql_errors"` + } + if err = json.Unmarshal(evidence, &receipt); err != nil { + t.Fatal(err) + } + if receipt.Errors.Count != 12 || len(receipt.Errors.Items) != 8 || !receipt.Errors.Truncated { + t.Fatal("error bound lost") + } + first := receipt.Errors.Items[0] + if first.Type == nil || *first.Type != "NOT_FOUND" || len(first.Path) != 8 || !first.Truncated || first.Path[0] != "repository" || first.Path[1] != "n0" || first.Path[3] != float64(0) || first.Path[5] != nil || first.Path[6] != nil || first.Path[7] != nil || receipt.Errors.Items[1].Type != nil { + t.Fatalf("incorrect safe metadata: %+v", receipt.Errors) + } +} + +func TestGraphQLRejectedNullOrAbsentDataRetainsSafeEnvelope(t *testing.T) { + for _, payload := range []string{ + `{"errors":[{"type":"NOT_FOUND","path":["repository","n0"],"message":"private prose"}]}`, + `{"data":null,"errors":[{"type":"NOT_FOUND","path":["repository","n0"],"message":"private prose"}]}`, + } { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(payload)) })) + var out map[string]any + err := New(Options{BaseURL: server.URL}).doGraphQL(context.Background(), "query { repository { id } }", nil, nil, &out) + server.Close() + if err == nil { + t.Fatal("error envelope accepted") + } + class, _, evidence := HistoryFailureDetails(err) + var record map[string]any + if e := json.Unmarshal(evidence, &record); e != nil { + t.Fatal(e) + } + if class != "partial_response" || record["version"] != float64(1) || record["graphql_errors"] == nil || record["structure"] != nil || strings.Contains(string(evidence), "private prose") { + t.Fatalf("missing/redaction-invalid envelope: %s", evidence) + } + } +} diff --git a/internal/github/review_threads.go b/internal/github/review_threads.go index 0f439589..4857d46b 100644 --- a/internal/github/review_threads.go +++ b/internal/github/review_threads.go @@ -113,10 +113,14 @@ type graphqlEnvelope struct { } type graphqlResponseEnvelope struct { - Data json.RawMessage `json:"data"` - Errors []struct { - Message string `json:"message"` - } `json:"errors"` + Data json.RawMessage `json:"data"` + Errors []graphqlResponseError `json:"errors"` +} + +type graphqlResponseError struct { + Message string `json:"message"` + Type string `json:"type"` + Path []any `json:"path"` } // ListPullReviewThreads fetches GitHub's review-thread graph for a pull request. @@ -252,7 +256,7 @@ func (c *Client) doGraphQL(ctx context.Context, query string, variables map[stri } var rejected any _ = decodeJSON(bytes.NewReader(envelope.Data), &rejected) - return historyFailure("partial_response", 0, rejected, fmt.Errorf("github graphql: %s", strings.Join(messages, "; "))) + return graphQLRejection(rejected, envelope.Errors, fmt.Errorf("github graphql: %s", strings.Join(messages, "; "))) } if len(envelope.Data) == 0 || string(envelope.Data) == "null" { return historyFailure("missing_data", 0, nil, fmt.Errorf("github graphql response missing data")) From 8f75d7fb6d046035a1deafe4e143908901be6bc8 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:32:15 -0600 Subject: [PATCH 11/21] Accelerate targeted review recovery with bounded parallel queries --- docs/analytics-source.md | 29 ++- internal/cli/analytics.go | 27 ++- internal/cli/analytics_integrity.go | 60 ++++++- .../cli/analytics_partial_response_test.go | 2 +- internal/cli/analytics_throughput_test.go | 39 +++- internal/github/analytics.go | 8 +- internal/github/client.go | 97 +++++++--- internal/github/history.go | 9 + internal/github/history_evidence.go | 2 +- internal/github/review_state.go | 68 +++++++ internal/github/review_state_test.go | 166 ++++++++++++++++++ internal/github/review_threads.go | 14 +- internal/store/analytics_integrity.go | 88 +++++++++- internal/store/analytics_integrity_test.go | 70 ++++++++ internal/syncer/review_state.go | 75 ++++++++ internal/syncer/review_state_test.go | 154 ++++++++++++++++ internal/syncer/syncer.go | 15 +- 17 files changed, 882 insertions(+), 41 deletions(-) create mode 100644 internal/github/review_state.go create mode 100644 internal/github/review_state_test.go create mode 100644 internal/syncer/review_state.go create mode 100644 internal/syncer/review_state_test.go diff --git a/docs/analytics-source.md b/docs/analytics-source.md index 4cb5bf0a..5345de4e 100644 --- a/docs/analytics-source.md +++ b/docs/analytics-source.md @@ -51,9 +51,17 @@ due items before discovery, with a two-minute request deadline and bounded exponential backoff. After persisting core coverage, targeted review recovery uses the time until the next nominal two-minute core poll; it does not add a two-minute idle wait after recovery. Each wave rechecks actual GraphQL quota and -admits up to sixteen items through the existing eight two-item workers. Admission +admits up to 256 items through 32 recovery workers, with up to eight PRs per +request. Waves are also limited to a 16 MiB estimated response budget, using a +conservative 64 KiB/item initial estimate that grows with measured bytes. Individual +recovery responses are capped at 32 MiB; rejected batches are isolated normally. +Ordinary capture retains its existing eight two-item workers. Admission uses the authoritative GraphQL `rateLimit` response rather than REST resource -counters, which can differ. Each history session also enforces its configured +counters, which can differ. Recovery and its quota probe omit redundant REST +`/rate_limit` preflights: an explicit GraphQL probe binds actual quota to the +selected credential, and every content/page request checks that credential and +unexpired balance against the reserve. Rotation requires a new probe. Ordinary +core transport retains its existing guards. Each history session also enforces its configured floor against observed GraphQL balances before pagination. A client retains the lowest observed GraphQL balance until the reset boundary passes. An upward sample or a shifted future reset cannot increase admission; @@ -79,6 +87,23 @@ When both queues contain an item, its core retry owns the backoff; the review retry pass cannot dispatch the same item. Recovery resolves only after an accepted membership observation exists, including a proven empty set. +Targeted recovery fetches only PR identity/update metadata and fully paginated +review threads with their complete inline comments and reply-to identities. It +does not request or project the PR body/title, issue comments, or review history. +An independent native child observation reserves only the review-thread family; +existing review state, revisions and exact membership publish in one transaction +after repository/node/number binding. Canonical threads, comments, revisions and +vectors remain untouched. Current content changes remain ordinary capture's job. +A review-only success cannot resolve a core traversal failure. + +Up to one quarter of a recovery wave is reserved for already-attempted due retries; +the remaining slots serve first-pass work, with unused capacity shared. This avoids +waiting behind the entire seeded census. Explicit retained `NOT_FOUND` evidence +has at least a 15-minute backoff; it never implies deletion or empty membership. +`review_state_wave` logs actual peak busy workers, worker/provider/database time, +items, response bytes and reported query points. Private success receipts identify +review-only work and its fetch/persistence timings. + Hydration workers reuse the watch owner's open store instead of repeating full-archive migration checks for each batch. Independent guarded clients overlap network work; native transactions still coordinate source writes. Enrichment diff --git a/internal/cli/analytics.go b/internal/cli/analytics.go index fcf50aaf..8d29a5bd 100644 --- a/internal/cli/analytics.go +++ b/internal/cli/analytics.go @@ -325,28 +325,51 @@ func (a *App) syncAnalyticsBatch(ctx context.Context, s *store.Store, owner, rep } token := a.resolveGitHubToken(ctx, cfg) reserve := analyticsCoreReserve + var responseLimit int64 + var responseBytes, points, providerMillis int64 var reporter gh.Reporter if operation == "review_state" { reserve = analyticsReviewReserve + responseLimit = 32 << 20 // This call owns its client, history session and reporter. Synchronous // callbacks are never shared with the other analyticsNumbers workers. var effectiveRemaining, effectiveReset int reporter = func(message string) { + var bytes, callNumber, millis int64 + if _, err := fmt.Sscanf(message, "[github] graphql bytes %d", &bytes); err == nil { + responseBytes += bytes + return + } + if _, err := fmt.Sscanf(message, "[github] graphql timing %d %d", &callNumber, &millis); err == nil { + providerMillis += millis + return + } var providerRemaining, providerReset int if _, err := fmt.Sscanf(message, "[github] graphql quota provider_remaining %d provider_reset %d effective_remaining %d effective_reset %d", &providerRemaining, &providerReset, &effectiveRemaining, &effectiveReset); err == nil { return } var call, cost, remaining, reset int if _, err := fmt.Sscanf(message, "[github] graphql cost %d %d remaining %d reset %d", &call, &cost, &remaining, &reset); err == nil { + points += int64(cost) fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_cost\",\"at\":%q,\"points\":%d,\"remaining\":%d,\"reset_unix\":%d,\"provider_remaining\":%d,\"provider_reset_unix\":%d}\n", time.Now().UTC().Format(time.RFC3339Nano), cost, effectiveRemaining, effectiveReset, remaining, reset) } } } - client := gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: reserve}) + client := gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: reserve, GraphQLResponseLimit: responseLimit, GraphQLQuotaGuard: operation == "review_state"}) // The watch owner has already validated and opened this store. Reopening it // for every two threads repeats full-archive migration audits and serializes // otherwise independent network work. Native transactions still own writes. - _, err = syncer.New(client, s).Sync(ctx, syncer.Options{Owner: owner, Repo: repo, GraphQLHistory: true, ReceiptOperation: operation, State: "all", Numbers: numbers, IncludeComments: true, IncludePRMetadata: true, Reporter: reporter}) + stats, err := syncer.New(client, s).Sync(ctx, syncer.Options{Owner: owner, Repo: repo, GraphQLHistory: true, ReviewStateOnly: operation == "review_state", ReceiptOperation: operation, State: "all", Numbers: numbers, IncludeComments: true, IncludePRMetadata: true, Reporter: reporter}) + if metrics, ok := ctx.Value(analyticsMetricsKey{}).(*analyticsWorkMetrics); ok { + metrics.bytes.Add(responseBytes) + metrics.points.Add(points) + metrics.providerMillis.Add(providerMillis) + metrics.dbMillis.Add(stats.PersistMillis) + metrics.attempted.Add(int64(len(numbers))) + if err == nil { + metrics.recovered.Add(int64(stats.ThreadsSynced)) + } + } return err } diff --git a/internal/cli/analytics_integrity.go b/internal/cli/analytics_integrity.go index c657824f..a3229ead 100644 --- a/internal/cli/analytics_integrity.go +++ b/internal/cli/analytics_integrity.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" gh "github.com/openclaw/gitcrawl/internal/github" @@ -18,9 +19,29 @@ const ( analyticsPollInterval = 2 * time.Minute analyticsCoreReserve = 1500 // Keep another 1500 points available to ordinary capture above its floor. - analyticsReviewReserve = 3000 + analyticsReviewReserve = 3000 + analyticsReviewWorkers = 32 + analyticsReviewBatch = 8 + analyticsReviewWave = analyticsReviewWorkers * analyticsReviewBatch + analyticsReviewWaveBytes = 16 << 20 ) +type analyticsMetricsKey struct{} +type analyticsWorkMetrics struct { + busy, maxBusy, bytes, points, attempted, recovered, providerMillis, dbMillis, workerMillis atomic.Int64 +} + +func (m *analyticsWorkMetrics) enter() func() { + start := time.Now() + busy := m.busy.Add(1) + for old := m.maxBusy.Load(); busy > old; old = m.maxBusy.Load() { + if m.maxBusy.CompareAndSwap(old, busy) { + break + } + } + return func() { m.workerMillis.Add(time.Since(start).Milliseconds()); m.busy.Add(-1) } +} + func (a *App) analyticsUpdateLog(err error) { event := "github_update_complete" fields := map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano)} @@ -211,6 +232,7 @@ func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh } repository := owner + "/" + repo var progress store.ReviewStateRecovery + bytesPerItem := int64(64 << 10) haveProgress, quotaBlocked := false, false defer func() { if !haveProgress { @@ -250,7 +272,7 @@ func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh } continue } - due, err := s.DueAnalyticsRetries(window, repository, time.Now().UTC().Format(time.RFC3339Nano), 16, "review_state") + due, err := s.DueReviewStateWork(window, repository, time.Now().UTC().Format(time.RFC3339Nano), analyticsReviewWave) if err != nil { return yield(err) } @@ -276,6 +298,15 @@ func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh quotaBlocked = true continue } + budget = min(budget, int(analyticsReviewWaveBytes/bytesPerItem)) + if budget > 0 && budget < len(due) { + // Retry fairness is relative to actual admitted work, including + // a smaller point/byte budget, not the maximum wave size. + due, err = s.DueReviewStateWork(window, repository, time.Now().UTC().Format(time.RFC3339Nano), budget) + if err != nil { + return yield(err) + } + } encoded, _ := json.Marshal(map[string]any{"event": "review_state_quota", "at": time.Now().UTC().Format(time.RFC3339Nano), "limit": quota.Limit, "remaining": quota.Remaining, "reset_at": quota.ResetAt, "provider_remaining": rawQuota.Remaining, "provider_reset_at": rawQuota.ResetAt, "reserve": analyticsReviewReserve, "wave_items": min(len(due), budget)}) fmt.Fprintln(a.Stderr, string(encoded)) if budget == 0 { @@ -285,7 +316,15 @@ func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh // Item-level reserve failures are durably queued and absorbed by // analyticsIsolatedBatch; the next wave reprobes quota and keeps scanning. // Remaining errors include unrecorded storage failures, not safe deferrals. - if err = a.analyticsNumbers(window, s, owner, repo, due[:min(len(due), budget)], false, "review_state"); err != nil { + metrics := &analyticsWorkMetrics{} + waveStarted := time.Now() + err = a.analyticsNumbers(context.WithValue(window, analyticsMetricsKey{}, metrics), s, owner, repo, due[:min(len(due), budget)], false, "review_state") + if metrics.attempted.Load() > 0 { + bytesPerItem = max(bytesPerItem, metrics.bytes.Load()/metrics.attempted.Load()) + } + measured, _ := json.Marshal(map[string]any{"event": "review_state_wave", "at": time.Now().UTC().Format(time.RFC3339Nano), "elapsed_ms": time.Since(waveStarted).Milliseconds(), "workers": analyticsReviewWorkers, "max_busy_workers": metrics.maxBusy.Load(), "busy_worker_ms": metrics.workerMillis.Load(), "attempted_items": metrics.attempted.Load(), "recovered_items": metrics.recovered.Load(), "response_bytes": metrics.bytes.Load(), "reported_points": metrics.points.Load(), "provider_ms": metrics.providerMillis.Load(), "db_ms": metrics.dbMillis.Load(), "byte_budget": analyticsReviewWaveBytes}) + fmt.Fprintln(a.Stderr, string(measured)) + if err != nil { return yield(err) } } @@ -321,7 +360,7 @@ func analyticsReviewBudget(limits []gh.RateLimitSnapshot, now time.Time) (int, g } // A conservative admission margin limits in-flight overshoot. Native // request guards recheck actual remaining quota before every request. - return min(16, max(0, quota.Remaining-analyticsReviewReserve)/32), quota, nil + return min(analyticsReviewWave, max(0, quota.Remaining-analyticsReviewReserve)/32), quota, nil } return 0, gh.RateLimitSnapshot{}, fmt.Errorf("GraphQL quota unavailable for review recovery") } @@ -334,14 +373,18 @@ func (a *App) analyticsNumbers(ctx context.Context, s *store.Store, owner, repo wg.Wait() resultErr = errors.Join(append(failures, resultErr)...) }() - slots := make(chan struct{}, 8) + workers, batchSize := 8, 2 + if operation == "review_state" { + workers, batchSize = analyticsReviewWorkers, analyticsReviewBatch + } + slots := make(chan struct{}, workers) var mu sync.Mutex - for i := 0; i < len(numbers); i += 2 { + for i := 0; i < len(numbers); i += batchSize { if err := ctx.Err(); err != nil { return err } var part []int - for _, n := range numbers[i:min(i+2, len(numbers))] { + for _, n := range numbers[i:min(i+batchSize, len(numbers))] { if discovery { // Review-only recovery must not defer a newly discovered core edit. // If this core attempt fails, it creates graphql_history retry state @@ -372,6 +415,9 @@ func (a *App) analyticsNumbers(ctx context.Context, s *store.Store, owner, repo go func(part []int) { defer wg.Done() defer func() { <-slots }() + if metrics, ok := ctx.Value(analyticsMetricsKey{}).(*analyticsWorkMetrics); ok { + defer metrics.enter()() + } if e := a.analyticsIsolatedBatch(ctx, s, owner, repo, part, operation); e != nil { mu.Lock() failures = append(failures, e) diff --git a/internal/cli/analytics_partial_response_test.go b/internal/cli/analytics_partial_response_test.go index e09baaee..853f52e2 100644 --- a/internal/cli/analytics_partial_response_test.go +++ b/internal/cli/analytics_partial_response_test.go @@ -161,7 +161,7 @@ func TestAnalyticsPairedPartialRejectionIsolatesPeerAndPreservesUnavailableHisto s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE number=16945 AND operation='review_state' AND status='success'").Scan(&success) var membership, body string s.DB().QueryRow("SELECT x.review_thread_ids_json,t.body FROM threads t JOIN pull_request_review_thread_syncs x ON x.thread_id=t.id WHERE t.number=16945").Scan(&membership, &body) - if pending != 1 || peerResolved != 1 || success != 1 || membership != "[]" || body != "isolated success" { + if pending != 1 || peerResolved != 1 || success != 1 || membership != "[]" || body != "retained original" { t.Fatalf("isolation/retry proof failed: %d %d %d %s %s", pending, peerResolved, success, membership, body) } } diff --git a/internal/cli/analytics_throughput_test.go b/internal/cli/analytics_throughput_test.go index a505b2d3..8b0f2976 100644 --- a/internal/cli/analytics_throughput_test.go +++ b/internal/cli/analytics_throughput_test.go @@ -21,7 +21,7 @@ import ( ) func TestAnalyticsRecoveryUsesQuotaAfterCoreAndResumesCancellation(t *testing.T) { - for _, mode := range []string{"available", "reserved", "quota_drops", "quota_races", "cancel"} { + for _, mode := range []string{"available", "reserved", "quota_drops", "quota_races", "concurrent", "cancel", "narrow_fair"} { t.Run(mode, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -36,6 +36,9 @@ func TestAnalyticsRecoveryUsesQuotaAfterCoreAndResumesCancellation(t *testing.T) t.Fatal(err) } oldItems := 48 + if mode == "quota_drops" || mode == "concurrent" { + oldItems = 300 + } if mode == "quota_races" { oldItems = 6001 } @@ -52,13 +55,29 @@ func TestAnalyticsRecoveryUsesQuotaAfterCoreAndResumesCancellation(t *testing.T) if err = tx.Commit(); err != nil { t.Fatal(err) } + if mode == "narrow_fair" { + for n := 1; n <= 16; n++ { + if _, err = s.DB().Exec("INSERT INTO analytics_retries(repository,number,operation,first_seen_at,last_seen_at,next_attempt_at,attempts) VALUES('fixture/repo',?,'review_state','2000-01-01T00:00:00Z','2000-01-01T00:00:00Z','2000-01-01T00:00:00Z',1)", n); err != nil { + t.Fatal(err) + } + } + } baseline := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano) if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", baseline, 0, oldItems); err != nil { t.Fatal(err) } + ready := make(chan struct{}) + var concurrentBatches atomic.Int64 var coreSeen, cancelled atomic.Bool var recovered, probes, recoveryProbes atomic.Int64 quota := func() int { + if mode == "narrow_fair" { + if recovered.Load() >= 16 { + return 3020 + } + return 3530 + } + if mode == "quota_races" && recoveryProbes.Load() > 1 { return 2999 } @@ -105,6 +124,17 @@ func TestAnalyticsRecoveryUsesQuotaAfterCoreAndResumesCancellation(t *testing.T) } data["repository"] = map[string]any{kind: page} } else if matches := numbers.FindAllStringSubmatch(req.Query, -1); len(matches) > 0 { + if mode == "concurrent" && !strings.Contains(req.Query, "number:10000)") { + if concurrentBatches.Add(1) == 32 { + close(ready) + } + select { + case <-ready: + case <-time.After(10 * time.Second): + t.Error("32 workers were not active") + return + } + } repo := map[string]any{"id": "repo", "databaseId": 1, "nameWithOwner": "fixture/repo"} for _, m := range matches { n, _ := strconv.Atoi(m[2]) @@ -169,7 +199,14 @@ func TestAnalyticsRecoveryUsesQuotaAfterCoreAndResumesCancellation(t *testing.T) want = 0 } if mode == "quota_drops" { + want = analyticsReviewWave + } + if mode == "narrow_fair" { want = 16 + var fresh int + if err = s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE operation='review_state' AND resolved_at IS NOT NULL AND attempts=0").Scan(&fresh); err != nil || fresh != 12 { + t.Fatalf("shrunken wave starved fresh work: %d %v", fresh, err) + } } if resolved != want || pending != oldItems-want || scanned != oldItems+1 || (complete == 1) != (want == oldItems) { t.Fatalf("resolved=%d pending=%d scanned=%d complete=%d, want recovered%d", resolved, pending, scanned, complete, want) diff --git a/internal/github/analytics.go b/internal/github/analytics.go index ab17137c..f6f13500 100644 --- a/internal/github/analytics.go +++ b/internal/github/analytics.go @@ -69,8 +69,12 @@ type UpdatedPage struct { // AnalyticsRateLimit reads the same GraphQL balance charged by history queries. // REST resource counters can differ and must not admit recovery on that basis. func (c *Client) AnalyticsRateLimit(ctx context.Context) (effective, observed RateLimitSnapshot, err error) { - h := historySession{client: c, remaining: 20000} - data, err := h.request(ctx, `query { rateLimit { cost limit remaining used resetAt } }`, nil, 1) + // Reuse the same credential-bound reserve state without changing ordinary + // content clients' existing transport policy. + quotaClient := *c + quotaClient.graphQLQuotaGuard = true + h := historySession{client: "aClient, remaining: 20000} + data, err := h.quota(ctx) if err != nil { return RateLimitSnapshot{}, RateLimitSnapshot{}, err } diff --git a/internal/github/client.go b/internal/github/client.go index ae5e5d27..6d0a0c5e 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -20,19 +20,23 @@ import ( type Reporter func(message string) type Client struct { - httpClient *http.Client - baseURL string - graphQLURL string - token string - tokenProvider func(context.Context) (string, error) - userAgent string - pageDelay time.Duration - rateLimit RateLimitObserver - reserve *rateLimitReserve + httpClient *http.Client + baseURL string + graphQLURL string + token string + tokenProvider func(context.Context) (string, error) + userAgent string + pageDelay time.Duration + rateLimit RateLimitObserver + reserve *rateLimitReserve + graphQLResponseLimit int64 + graphQLQuotaGuard bool } type Options struct { - Token string + GraphQLResponseLimit int64 + GraphQLQuotaGuard bool + Token string // TokenProvider exclusively selects credentials immediately before dispatch. TokenProvider func(context.Context) (string, error) BaseURL string @@ -41,7 +45,8 @@ type Options struct { PageDelay time.Duration RateLimit RateLimitObserver // RateLimitReserve preserves a best-effort observed floor for the shared - // token. Guarded requests refresh /rate_limit before dispatch so other token + // token. Unless GraphQLQuotaGuard uses explicit observed GraphQL quota, + // guarded requests refresh /rate_limit before dispatch so other token // consumers are observed, but unrelated consumers cannot be locked between // that probe and dispatch. RateLimitReserve int @@ -82,6 +87,41 @@ type rateLimitReserve struct { // GraphQL response evidence must survive REST snapshot replacement and // upward provider anomalies until the observed reset boundary has passed. graphqlObserved RateLimitSnapshot + graphqlToken string +} + +type graphQLQuotaProbeKey struct{} + +func (r *rateLimitReserve) bindGraphQLToken(token string) { + if r == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.graphqlToken != token { + r.graphqlObserved = RateLimitSnapshot{} + r.graphqlToken = token + } +} + +func (r *rateLimitReserve) beforeObservedGraphQL(token string, cost int) error { + if r == nil { + return fmt.Errorf("observed GraphQL quota guard required") + } + r.mu.Lock() + defer r.mu.Unlock() + q := r.graphqlObserved + if token != r.graphqlToken { + return fmt.Errorf("GraphQL credential changed; quota probe required") + } + if q.Resource != "graphql" || !q.ResetAt.After(time.Now()) { + return fmt.Errorf("fresh observed GraphQL quota required") + } + if q.Remaining-cost < r.reserve { + return &RateLimitReserveError{RateLimit: q, Reserve: r.reserve} + } + r.graphqlObserved.Remaining -= cost + return nil } type rateLimitRequestLockKey struct{} @@ -136,14 +176,16 @@ func New(options Options) *Client { userAgent = "gitcrawl" } client := &Client{ - httpClient: httpClient, - baseURL: baseURL, - graphQLURL: graphQLURLForBaseURL(baseURL), - token: options.Token, - tokenProvider: options.TokenProvider, - userAgent: userAgent, - pageDelay: options.PageDelay, - rateLimit: options.RateLimit, + httpClient: httpClient, + baseURL: baseURL, + graphQLURL: graphQLURLForBaseURL(baseURL), + token: options.Token, + tokenProvider: options.TokenProvider, + userAgent: userAgent, + pageDelay: options.PageDelay, + graphQLResponseLimit: options.GraphQLResponseLimit, + graphQLQuotaGuard: options.GraphQLQuotaGuard, + rateLimit: options.RateLimit, } if options.RateLimitReserve > 0 { client.reserve = newRateLimitReserve(options.RateLimitReserve, options.InitialRateLimits) @@ -563,7 +605,8 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader } } var probeToken string - if c.reserve != nil && cost > 0 { + observedGraphQL := c.graphQLQuotaGuard && resource == "graphql" + if c.reserve != nil && cost > 0 && !observedGraphQL { var err error _, probeToken, err = c.getRateLimits(ctx, reporter, nil) if err != nil { @@ -580,7 +623,7 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader return nil, err } } - if c.tokenProvider != nil && c.reserve != nil && cost > 0 && token != probeToken { + if c.tokenProvider != nil && c.reserve != nil && cost > 0 && !observedGraphQL && token != probeToken { // A new token cannot spend the previous token's quota. Allow one new // probe, then reject further rotation before the protected request. _, probeToken, err := c.getRateLimits(ctx, reporter, &token) @@ -595,7 +638,14 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader return nil, errors.New("GitHub token changed during rate limit reservation") } } - if err := c.reserve.beforeRequest(resource, cost); err != nil { + if observedGraphQL { + probe, _ := ctx.Value(graphQLQuotaProbeKey{}).(bool) + if probe { + c.reserve.bindGraphQLToken(token) + } else if err := c.reserve.beforeObservedGraphQL(token, cost); err != nil { + return nil, err + } + } else if err := c.reserve.beforeRequest(resource, cost); err != nil { var expired *rateLimitStatusExpiredError if c.tokenProvider != nil || !errors.As(err, &expired) { return nil, err @@ -608,6 +658,9 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader return nil, err } } + if resource == "graphql" && !observedGraphQL { + c.reserve.bindGraphQLToken(token) + } req, err := http.NewRequestWithContext(ctx, method, fullURL, body) if err != nil { return nil, err diff --git a/internal/github/history.go b/internal/github/history.go index f5f847ba..6b6a6f8e 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -50,6 +50,10 @@ type historySession struct { retrySleep func(context.Context, time.Duration) error } +func (h *historySession) quota(ctx context.Context) (map[string]any, error) { + return h.request(context.WithValue(ctx, graphQLQuotaProbeKey{}, true), `query { rateLimit {cost remaining limit used resetAt} }`, nil, 1) +} + func (h *historySession) request(ctx context.Context, query string, variables map[string]any, estimate int) (map[string]any, error) { for attempt := 0; ; attempt++ { data, err := h.requestOnce(ctx, query, variables, estimate) @@ -251,6 +255,11 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error return fmt.Errorf("missing provider identity") } } + return h.hydrateConnections(ctx, node) +} + +func (h *historySession) hydrateConnections(ctx context.Context, node map[string]any) error { + typ := historyString(node["__typename"]) for _, key := range []string{"labels", "assignees", "comments", "reviews", "reviewThreads"} { connection, exists := node[key] if !exists { diff --git a/internal/github/history_evidence.go b/internal/github/history_evidence.go index 016e1c47..e2f52f27 100644 --- a/internal/github/history_evidence.go +++ b/internal/github/history_evidence.go @@ -39,7 +39,7 @@ func (r *historyResponseReader) Read(p []byte) (int, error) { return n, err } -func (r *historyResponseReader) failure(err error) error { +func (r *historyResponseReader) failure(err error) *HistoryFailure { evidence, _ := json.Marshal(map[string]any{"version": 1, "response_bytes_read": r.read, "hashed_prefix_bytes": r.hashed, "prefix_sha256": hex.EncodeToString(r.hash.Sum(nil)), "complete_response": false}) return &HistoryFailure{Cause: err, Stage: "response_decode", Evidence: evidence} } diff --git a/internal/github/review_state.go b/internal/github/review_state.go new file mode 100644 index 00000000..0de13454 --- /dev/null +++ b/internal/github/review_state.go @@ -0,0 +1,68 @@ +package github + +import ( + "context" + "fmt" + "strings" + "time" +) + +// ReviewStateItem contains only review evidence. It must never enter the +// full-thread projection, since canonical bodies/comments were not requested. +type ReviewStateItem struct { + Number int + NodeID string + RepositoryID string + RepositoryNodeID string + UpdatedAt string + Threads []map[string]any +} + +func (c *Client) FetchGraphQLReviewState(ctx context.Context, owner, repo string, numbers []int, reporter Reporter) ([]ReviewStateItem, error) { + if len(numbers) == 0 || len(numbers) > 8 { + return nil, fmt.Errorf("review-state query requires 1..8 numbers") + } + h := historySession{client: c, reporter: reporter, remaining: 20000} + if _, err := h.quota(ctx); err != nil { + return nil, err + } + var fields strings.Builder + for i, n := range numbers { + if n < 1 { + return nil, fmt.Errorf("invalid review-state number") + } + fmt.Fprintf(&fields, `n%d: issueOrPullRequest(number:%d) {__typename ... on PullRequest{id number updatedAt repository{nameWithOwner} %s}} `, i, n, historyConnection("reviewThreads", historyReviewThread, "")) + } + data, err := h.request(ctx, `query($owner:String!,$repo:String!){repository(owner:$owner,name:$repo){id databaseId nameWithOwner `+fields.String()+`} rateLimit{cost remaining limit used resetAt}}`, map[string]any{"owner": owner, "repo": repo}, 16) + if err != nil { + return nil, err + } + r := historyMap(data["repository"]) + if !strings.EqualFold(historyString(r["nameWithOwner"]), owner+"/"+repo) || historyString(r["id"]) == "" { + return nil, historyFailure("identity", 0, r, fmt.Errorf("review-state repository identity mismatch")) + } + repoID, validRepoID := historyInt(r["databaseId"]) + if !validRepoID || repoID <= 0 { + return nil, historyFailure("identity", 0, r, fmt.Errorf("missing repository database identity")) + } + out := make([]ReviewStateItem, 0, len(numbers)) + for i, n := range numbers { + node := historyMap(r[fmt.Sprint("n", i)]) + got, valid := historyInt(node["number"]) + if !valid || got != n || historyString(node["__typename"]) != "PullRequest" || historyString(node["id"]) == "" || !strings.EqualFold(historyString(historyMap(node["repository"])["nameWithOwner"]), owner+"/"+repo) { + return nil, historyFailure("identity", n, node, fmt.Errorf("review-state PR identity mismatch")) + } + updated := historyString(node["updatedAt"]) + if _, err = time.Parse(time.RFC3339Nano, updated); err != nil { + return nil, historyFailure("validation", n, node, fmt.Errorf("invalid review-state source time")) + } + if historyMap(node["reviewThreads"]) == nil { + return nil, historyFailure("validation", n, node, fmt.Errorf("missing history reviewThreads")) + } + if err = h.hydrateConnections(ctx, map[string]any{"__typename": "PullRequest", "id": node["id"], "reviewThreads": node["reviewThreads"]}); err != nil { + return nil, historyFailure("validation", n, node, err) + } + out = append(out, ReviewStateItem{Number: n, RepositoryID: fmt.Sprint(repoID), RepositoryNodeID: historyString(r["id"]), NodeID: historyString(node["id"]), UpdatedAt: updated, Threads: historyNodes(node, "reviewThreads")}) + } + return out, nil +} diff --git a/internal/github/review_state_test.go b/internal/github/review_state_test.go new file mode 100644 index 00000000..a94101a2 --- /dev/null +++ b/internal/github/review_state_test.go @@ -0,0 +1,166 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestReviewStateOnlyStrictPaginationAndReplyFields(t *testing.T) { + for _, mode := range []string{"complete", "null", "short", "cursor", "wrong_parent", "missing_state"} { + t.Run(mode, func(t *testing.T) { + conn := func(total int, more bool, cursor string, nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"totalCount": total, "nodes": nodes, "pageInfo": map[string]any{"hasNextPage": more, "endCursor": cursor}} + } + comment := func(id string, reply any) map[string]any { + return map[string]any{"id": id, "__typename": "PullRequestReviewComment", "fullDatabaseId": id, "body": "inline retained", "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "url": "https://github.com/fixture/repo/pull/7#comment", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-02T00:00:00Z", "replyTo": reply} + } + thread := func(id string, comments any) map[string]any { + return map[string]any{"id": id, "__typename": "PullRequestReviewThread", "isResolved": true, "isOutdated": false, "viewerCanResolve": false, "viewerCanUnresolve": true, "viewerCanReply": true, "path": "file.go", "line": 9, "comments": comments} + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Query string + Variables map[string]any + } + json.NewDecoder(r.Body).Decode(&req) + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}} + if strings.Contains(req.Query, "issueOrPullRequest") { + if strings.Contains(req.Query, "title") || strings.Contains(req.Query, "labels") || strings.Contains(req.Query, "reviews(") { + t.Error("unrelated history requested") + } + first := thread("RT1", conn(2, true, "c1", comment("C1", nil))) + if mode == "missing_state" { + delete(first, "isResolved") + } + node := map[string]any{"__typename": "PullRequest", "id": "PR7", "number": 7, "updatedAt": "2026-01-02T00:00:00Z", "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "reviewThreads": conn(2, true, "rt1", first)} + if mode == "short" { + node["reviewThreads"] = conn(2, false, "end", first) + } + var selected any = node + if mode == "null" { + selected = nil + } + data["repository"] = map[string]any{"id": "R1", "databaseId": 1, "nameWithOwner": "fixture/repo", "n0": selected} + } else if req.Variables["id"] == "PR7" { + parent := "PR7" + if mode == "wrong_parent" { + parent = "OTHER" + } + data["node"] = map[string]any{"id": parent, "reviewThreads": conn(2, mode == "cursor", "rt1", thread("RT2", conn(0, false, "")))} + } else if req.Variables["id"] == "RT1" { + data["node"] = map[string]any{"id": "RT1", "comments": conn(2, false, "c2", comment("C2", map[string]any{"id": "C1", "fullDatabaseId": "C1"}))} + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + items, err := New(Options{BaseURL: server.URL}).FetchGraphQLReviewState(context.Background(), "fixture", "repo", []int{7}, nil) + if mode != "complete" { + if err == nil { + t.Fatal("incomplete evidence accepted", mode) + } + return + } + if err != nil || len(items) != 1 || len(items[0].Threads) != 2 { + t.Fatalf("%+v %v", items, err) + } + comments := historyNodes(items[0].Threads[0], "comments") + if len(comments) != 2 || historyMap(comments[1]["replyTo"])["id"] != "C1" || comments[0]["body"] != "inline retained" || historyMap(comments[0]["author"])["login"] != "fixture" { + t.Fatal("reply/body/author evidence lost") + } + }) + } +} +func TestReviewStateResponseByteLimitRejectsWithoutPartialResult(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"data":{"padding":"` + strings.Repeat("x", 8192) + `"}}`)) + })) + defer server.Close() + var out map[string]any + err := New(Options{BaseURL: server.URL, GraphQLResponseLimit: 1024}).doGraphQL(context.Background(), "query { rateLimit {cost} }", nil, nil, &out) + class, _, _ := HistoryFailureDetails(err) + if err == nil || class != "response_size" || out != nil { + t.Fatalf("oversize response accepted: %v %s", err, class) + } +} + +func TestReviewStateGraphQLGuardRequiresObservedCredentialQuota(t *testing.T) { + for _, mode := range []string{"normal", "low_probe", "low_page", "rotation", "expired", "unprobed"} { + t.Run(mode, func(t *testing.T) { + calls, rest, tokenCalls := 0, 0, 0 + reset := time.Now().UTC().Add(time.Hour) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/graphql" { + rest++ + t.Error("redundant REST quota request") + http.Error(w, "unexpected", 500) + return + } + calls++ + remaining := 19000 + if mode == "low_probe" || mode == "low_page" && calls == 2 { + remaining = 3000 + } + if mode == "expired" { + reset = time.Now().Add(-time.Second) + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": remaining, "resetAt": reset.Format(time.RFC3339)}}}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, GraphQLQuotaGuard: true, RateLimitReserve: 3000, TokenProvider: func(context.Context) (string, error) { + tokenCalls++ + if mode == "rotation" && tokenCalls > 1 { + return "changed-fixture", nil + } + return "fixture", nil + }}) + h := historySession{client: c, remaining: 20000} + if mode != "unprobed" { + if _, err := h.quota(context.Background()); err != nil { + t.Fatal(err) + } + } + _, err := h.request(context.Background(), `query { node(id:"fixture"){id} rateLimit{cost remaining limit resetAt}}`, nil, 16) + if mode == "normal" || mode == "low_page" { + if err != nil { + t.Fatal(err) + } + _, err = h.request(context.Background(), `query { node(id:"page"){id} rateLimit{cost remaining limit resetAt}}`, nil, 16) + } + if mode == "normal" { + if err != nil || calls != 3 { + t.Fatalf("calls=%d err=%v", calls, err) + } + } else if err == nil { + t.Fatal("unguarded content accepted") + } + expected := 1 + if mode == "normal" { + expected = 3 + } + if mode == "low_page" { + expected = 2 + } + if mode == "unprobed" { + expected = 0 + } + if calls != expected || rest != 0 { + t.Fatalf("calls=%d rest=%d", calls, rest) + } + if mode == "low_probe" || mode == "low_page" { + var reserve *RateLimitReserveError + if !errors.As(err, &reserve) { + t.Fatalf("not reserve error: %v", err) + } + } + }) + } +} diff --git a/internal/github/review_threads.go b/internal/github/review_threads.go index 4857d46b..27f116c9 100644 --- a/internal/github/review_threads.go +++ b/internal/github/review_threads.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "io" "net/http" "strings" ) @@ -246,7 +247,18 @@ func (c *Client) doGraphQL(ctx context.Context, query string, variables map[stri } defer response.Body.Close() reader := &historyResponseReader{reader: response.Body, hash: sha256.New()} - if err := decodeJSON(reader, &envelope); err != nil { + var input io.Reader = reader + if c.graphQLResponseLimit > 0 { + input = io.LimitReader(reader, c.graphQLResponseLimit+1) + } + decodeErr := decodeJSON(input, &envelope) + reporter.Printf("[github] graphql bytes %d", reader.read) + if c.graphQLResponseLimit > 0 && reader.read > c.graphQLResponseLimit { + failure := reader.failure(fmt.Errorf("review-state response exceeded byte limit")) + failure.Stage = "response_size" + return failure + } + if err := decodeErr; err != nil { return reader.failure(fmt.Errorf("decode github response: %w", err)) } if len(envelope.Errors) > 0 { diff --git a/internal/store/analytics_integrity.go b/internal/store/analytics_integrity.go index e4482ee0..f4046b9e 100644 --- a/internal/store/analytics_integrity.go +++ b/internal/store/analytics_integrity.go @@ -78,7 +78,7 @@ func (s *Store) RecordAnalyticsAttempt(ctx context.Context, a AnalyticsAttempt) return err } if status == "success" { - _, err = tx.q().ExecContext(ctx, `UPDATE analytics_retries SET resolved_at=?,last_attempt_id=? WHERE repository=? AND number=? AND resolved_at IS NULL AND (operation=? OR (? IN ('graphql_history','review_state') AND operation IN ('graphql_history','review_state'))) AND (operation<>'review_state' OR ?)`, a.FinishedAt, id, a.Repository, a.Number, a.Operation, a.Operation, knownReview) + _, err = tx.q().ExecContext(ctx, `UPDATE analytics_retries SET resolved_at=?,last_attempt_id=? WHERE repository=? AND number=? AND resolved_at IS NULL AND (operation=? OR (?='graphql_history' AND operation IN ('graphql_history','review_state'))) AND (operation<>'review_state' OR ?)`, a.FinishedAt, id, a.Repository, a.Number, a.Operation, a.Operation, knownReview) return err } if a.Operation != "review_state" { @@ -91,12 +91,98 @@ func (s *Store) RecordAnalyticsAttempt(ctx context.Context, a AnalyticsAttempt) return err } delay := time.Duration(1<0 AND next_attempt_at<=? AND NOT EXISTS(SELECT 1 FROM analytics_retries core WHERE core.repository=analytics_retries.repository AND core.number=analytics_retries.number AND core.operation='graphql_history' AND core.resolved_at IS NULL)` + read := func(attempted bool, n int) ([]int, error) { + predicate := "attempts=0" + if attempted { + predicate = "attempts>0" + } + rows, err := s.q().QueryContext(ctx, "SELECT number FROM analytics_retries WHERE "+eligible+" AND "+predicate+" ORDER BY next_attempt_at,number LIMIT ?", repository, at, n) + if err != nil { + return nil, err + } + defer rows.Close() + var out []int + for rows.Next() { + var number int + if err = rows.Scan(&number); err != nil { + return nil, err + } + out = append(out, number) + } + return out, rows.Err() + } + retries, err := read(true, max(1, limit/4)) + if err != nil { + return nil, err + } + fresh, err := read(false, limit-len(retries)) + if err != nil { + return nil, err + } + if len(fresh)+len(retries) < limit { + retries, err = read(true, limit-len(fresh)) + if err != nil { + return nil, err + } + } + return append(retries, fresh...), nil +} + +func (s *Store) ReviewStateParent(ctx context.Context, repository string, number int, providerRepositoryID, providerNodeID string) (Thread, error) { + var t Thread + var repoID, rawRepo string + err := s.q().QueryRowContext(ctx, `SELECT t.id,t.repo_id,t.github_id,t.kind,t.raw_json,r.github_repo_id,r.raw_json FROM threads t JOIN repositories r ON r.id=t.repo_id WHERE r.full_name=? COLLATE NOCASE AND t.number=?`, repository, number).Scan(&t.ID, &t.RepoID, &t.GitHubID, &t.Kind, &t.RawJSON, &repoID, &rawRepo) + if err != nil { + return t, err + } + if t.Kind != "pull_request" || repoID != providerRepositoryID { + return t, fmt.Errorf("review-state archived repository/PR identity mismatch") + } + var repo map[string]any + if err = json.Unmarshal([]byte(rawRepo), &repo); err != nil { + return t, err + } + if node, ok := repo["node_id"].(string); ok && node != "" && node != providerNodeID { + return t, fmt.Errorf("review-state repository node mismatch") + } + return t, nil +} + func (s *Store) DueAnalyticsRetries(ctx context.Context, repository, at string, limit int, operations ...string) ([]int, error) { operation := "" if len(operations) > 0 { diff --git a/internal/store/analytics_integrity_test.go b/internal/store/analytics_integrity_test.go index 3f2502d3..4b16c63d 100644 --- a/internal/store/analytics_integrity_test.go +++ b/internal/store/analytics_integrity_test.go @@ -332,3 +332,73 @@ func TestAnalyticsRecoveryRequiresAnAcceptedMembershipObservation(t *testing.T) t.Fatalf("proven empty membership did not reconcile: %d %v", n, err) } } + +func TestReviewRetryFairShareAndExplicitUnavailableBackoff(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for n := 1; n <= 100; n++ { + if _, err = s.DB().Exec("insert into analytics_retries(repository,number,operation,first_seen_at,last_seen_at,next_attempt_at) values('fixture/repo',?,'review_state','2000-01-01T00:00:00Z','2000-01-01T00:00:00Z','2000-01-01T00:00:00Z')", n); err != nil { + t.Fatal(err) + } + } + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 999, Operation: "review_state", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", ErrorClass: "partial_response", Evidence: json.RawMessage(`{"graphql_errors":{"items":[{"type":"NOT_FOUND"}]}}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + due, err := s.DueReviewStateWork(ctx, "fixture/repo", "2026-01-01T00:02:00Z", 16) + if err != nil { + t.Fatal(err) + } + for _, n := range due { + if n == 999 { + t.Fatal("unavailable backoff ignored") + } + } + due, err = s.DueReviewStateWork(ctx, "fixture/repo", "2026-01-01T00:16:00Z", 16) + if err != nil || len(due) != 16 || due[0] != 999 { + t.Fatalf("failed retry starved by bulk: %v %v", due, err) + } + a.Operation = "graphql_history" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + due, err = s.DueReviewStateWork(ctx, "fixture/repo", "2026-01-01T00:16:00Z", 16) + if err != nil { + t.Fatal(err) + } + for _, n := range due { + if n == 999 { + t.Fatal("review bypassed core obligation") + } + } + repoID, err := s.UpsertRepository(ctx, Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", RawJSON: "{}", UpdatedAt: a.FinishedAt}) + if err != nil { + t.Fatal(err) + } + tid, err := s.UpsertThread(ctx, Thread{RepoID: repoID, GitHubID: "P999", Number: 999, Kind: "pull_request", State: "open", Title: "fixture", HTMLURL: "https://github.com/fixture/repo/pull/999", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h", UpdatedAt: a.FinishedAt}) + if err != nil { + t.Fatal(err) + } + if err = s.UpsertPullRequestReviewThreads(ctx, tid, a.FinishedAt, nil); err != nil { + t.Fatal(err) + } + a.Operation = "review_state" + a.Status = "success" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + var status string + s.DB().QueryRow("select status from analytics_fetch_attempts order by id desc limit 1").Scan(&status) + if status != "success" { + t.Fatal("review success not proven", status) + } + var unresolved int + s.DB().QueryRow("select count(*) from analytics_retries where operation='graphql_history' and resolved_at is null").Scan(&unresolved) + if unresolved != 1 { + t.Fatal("review-only success cleared core failure") + } +} diff --git a/internal/syncer/review_state.go b/internal/syncer/review_state.go new file mode 100644 index 00000000..4a118c96 --- /dev/null +++ b/internal/syncer/review_state.go @@ -0,0 +1,75 @@ +package syncer + +import ( + "context" + "encoding/json" + "fmt" + "time" + + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" +) + +func (s *Syncer) syncReviewState(ctx context.Context, options Options, started string) (Stats, error) { + stats := Stats{Repository: options.Owner + "/" + options.Repo, Numbers: uniquePositiveNumbers(options.Numbers), StartedAt: started, ReviewStateOnly: true} + client, ok := s.client.(interface { + FetchGraphQLReviewState(context.Context, string, string, []int, gh.Reporter) ([]gh.ReviewStateItem, error) + }) + if !ok { + return stats, fmt.Errorf("client does not support targeted review state") + } + // Reserve an observation sequence before fetching, as in normal capture. + sequence, err := s.store.NextThreadObservationSequence(ctx, started) + if err != nil { + return stats, err + } + fetched := time.Now() + items, err := client.FetchGraphQLReviewState(ctx, options.Owner, options.Repo, stats.Numbers, options.Reporter) + stats.FetchMillis = time.Since(fetched).Milliseconds() + if err != nil { + return stats, err + } + if len(items) != len(stats.Numbers) { + return stats, fmt.Errorf("incomplete review-state batch") + } + persist := time.Now() + err = s.store.WithTx(ctx, func(tx *store.Store) error { + applied, threads := 0, 0 + for i, item := range items { + if item.Number != stats.Numbers[i] { + return fmt.Errorf("review-state selection mismatch") + } + t, err := tx.ReviewStateParent(ctx, stats.Repository, item.Number, item.RepositoryID, item.RepositoryNodeID) + if err != nil { + return err + } + var raw map[string]any + if err := json.Unmarshal([]byte(t.RawJSON), &raw); err != nil { + return err + } + if known := stringValue(raw["node_id"]); known != "" && known != item.NodeID { + return fmt.Errorf("review-state archived node identity mismatch") + } + reserved, err := tx.ReserveThreadChildObservation(ctx, t.ID, store.ThreadChildReviewThreads, item.UpdatedAt, sequence) + if err != nil { + return err + } + if !reserved { + continue + } + count, err := s.persistPullReviewThreads(ctx, tx, t, item.Threads, started) + if err != nil { + return err + } + applied++ + threads += count + } + stats.ThreadsSynced = applied + stats.PullRequestsSynced = applied + stats.ReviewThreadsSynced = threads + return nil + }) + stats.PersistMillis = time.Since(persist).Milliseconds() + stats.FinishedAt = s.now().Format(time.RFC3339Nano) + return stats, err +} diff --git a/internal/syncer/review_state_test.go b/internal/syncer/review_state_test.go new file mode 100644 index 00000000..8c207fce --- /dev/null +++ b/internal/syncer/review_state_test.go @@ -0,0 +1,154 @@ +package syncer + +import ( + "context" + "encoding/json" + "errors" + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" + "path/filepath" + "testing" + "time" +) + +type reviewOnlyFixture struct { + *gh.Client + items []gh.ReviewStateItem + err error +} + +func (f reviewOnlyFixture) FetchGraphQLReviewState(context.Context, string, string, []int, gh.Reporter) ([]gh.ReviewStateItem, error) { + return f.items, f.err +} +func TestReviewOnlyNeverOverwritesCanonicalContentHistoryOrVectors(t *testing.T) { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + legacy := &metadataGitHub{} + opts := Options{Owner: "openclaw", Repo: "gitcrawl", Numbers: []int{8}, State: "all", IncludeComments: true, IncludePRMetadata: true} + if _, err = New(legacy, s).Sync(ctx, opts); err != nil { + t.Fatal(err) + } + repo, err := s.RepositoryByFullName(ctx, "openclaw/gitcrawl") + if err != nil { + t.Fatal(err) + } + heads, err := s.ListThreads(ctx, repo.ID, true) + if err != nil { + t.Fatal(err) + } + head := heads[0] + if _, err = s.DB().Exec("update threads set raw_json=json_set(raw_json,'$.node_id','PR8') where id=?", head.ID); err != nil { + t.Fatal(err) + } + if _, err = s.DB().Exec("update repositories set raw_json=json_set(raw_json,'$.node_id','R1') where id=?", repo.ID); err != nil { + t.Fatal(err) + } + s.DB().QueryRow("select raw_json from threads where id=?", head.ID).Scan(&head.RawJSON) + s.DB().QueryRow("select raw_json from repositories where id=?", repo.ID).Scan(&repo.RawJSON) + if err = s.UpsertThreadVector(ctx, store.ThreadVector{ThreadID: head.ID, Basis: "title_original", Model: "fixture", Dimensions: 2, ContentHash: head.ContentHash, Vector: []float64{1, 2}, CreatedAt: "2026-01-01T00:00:00Z", UpdatedAt: "2026-01-01T00:00:00Z"}); err != nil { + t.Fatal(err) + } + snapshot := func() string { + tables := []string{"threads", "comments", "comment_revisions", "thread_revisions", "thread_fingerprints", "thread_vectors"} + out := map[string][][]any{} + for _, table := range tables { + rows, e := s.DB().Query("select * from " + table) + if e != nil { + t.Fatal(e) + } + cols, _ := rows.Columns() + for rows.Next() { + v := make([]any, len(cols)) + p := make([]any, len(v)) + for i := range v { + p[i] = &v[i] + } + if e = rows.Scan(p...); e != nil { + t.Fatal(e) + } + out[table] = append(out[table], v) + } + rows.Close() + } + b, _ := json.Marshal(out) + return string(b) + } + before := snapshot() + var raw map[string]any + json.Unmarshal([]byte(head.RawJSON), &raw) + nodeID := stringValue(raw["node_id"]) + if nodeID == "" { + nodeID = "PR8" + } + // Fixture repo supplies its retained database identity to the targeted path. + var repoRaw map[string]any + json.Unmarshal([]byte(repo.RawJSON), &repoRaw) + item := gh.ReviewStateItem{Number: 8, NodeID: nodeID, RepositoryID: repo.GitHubRepoID, RepositoryNodeID: stringValue(repoRaw["node_id"]), UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano), Threads: []map[string]any{{"id": "RT1", "isResolved": true, "isOutdated": true, "path": "file.go", "line": 3, "comments": map[string]any{"nodes": []any{map[string]any{"id": "RC1", "body": "inline body", "url": "https://github.com/openclaw/gitcrawl/pull/8#r1", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-02T00:00:00Z", "author": map[string]any{"login": "fixture", "__typename": "User"}, "replyTo": map[string]any{"id": "RC0"}}}}}}} + f := reviewOnlyFixture{items: []gh.ReviewStateItem{item}} + opts.GraphQLHistory = true + opts.ReviewStateOnly = true + opts.ReceiptOperation = "review_state" + stats, err := New(f, s).Sync(ctx, opts) + if err != nil { + t.Fatal(err) + } + if !stats.ReviewStateOnly || stats.CommentsSynced != 0 || stats.ReviewThreadsSynced != 1 { + t.Fatalf("unexpected stats %+v", stats) + } + if before != snapshot() { + t.Fatal("canonical content/history/vector changed") + } + var membership, body, login, comments, url, created, updated, authorType string + var resolved, outdated int + s.DB().QueryRow("select review_thread_ids_json from pull_request_review_thread_syncs where thread_id=?", head.ID).Scan(&membership) + s.DB().QueryRow("select first_comment_body,first_author_login,comments_json,is_resolved,is_outdated,first_comment_url,first_comment_created_at,first_comment_updated_at,first_author_type from pull_request_review_threads where thread_id=?", head.ID).Scan(&body, &login, &comments, &resolved, &outdated, &url, &created, &updated, &authorType) + if membership != `["RT1"]` || body != "inline body" || login != "fixture" || resolved != 1 || outdated != 1 { + t.Fatal("review contract lost", membership, body, login) + } + if url != "https://github.com/openclaw/gitcrawl/pull/8#r1" || created != "2026-01-01T00:00:00Z" || updated != "2026-01-02T00:00:00Z" || authorType != "User" { + t.Fatal("first-comment metadata lost") + } + saved := membership + comments + for _, failure := range []error{errors.New("partial response"), context.Canceled} { + f.err = failure + if _, err = New(f, s).Sync(ctx, opts); err == nil { + t.Fatal("failure accepted") + } + s.DB().QueryRow("select review_thread_ids_json from pull_request_review_thread_syncs where thread_id=?", head.ID).Scan(&membership) + s.DB().QueryRow("select comments_json from pull_request_review_threads where thread_id=?", head.ID).Scan(&comments) + if saved != membership+comments || before != snapshot() { + t.Fatal("failed observation overwrote retained data") + } + } + f.err = nil + f.items[0].RepositoryID = "wrong" + if _, err = New(f, s).Sync(ctx, opts); err == nil { + t.Fatal("wrong repository accepted") + } + f.items[0] = item + f.items[0].NodeID = "wrong" + if stringValue(raw["node_id"]) != "" { + if _, err = New(f, s).Sync(ctx, opts); err == nil { + t.Fatal("wrong node accepted") + } + } + // A later identity failure rolls back earlier state/history in the batch. + f.items = []gh.ReviewStateItem{item, item} + f.items[1].Number = 9 + f.items[0].Threads[0]["isResolved"] = false + opts.Numbers = []int{8, 9} + var revisionsBefore, revisionsAfter int + s.DB().QueryRow("select count(*) from pull_request_review_thread_revisions").Scan(&revisionsBefore) + if _, err = New(f, s).Sync(ctx, opts); err == nil { + t.Fatal("missing parent accepted") + } + s.DB().QueryRow("select count(*) from pull_request_review_thread_revisions").Scan(&revisionsAfter) + s.DB().QueryRow("select is_resolved from pull_request_review_threads where thread_id=?", head.ID).Scan(&resolved) + if resolved != 1 || revisionsAfter != revisionsBefore || before != snapshot() { + t.Fatal("partial batch publication or canonical mutation") + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index ce3d03c7..a538fd7c 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -43,7 +43,8 @@ type Syncer struct { } type Options struct { - GraphQLHistory bool + GraphQLHistory bool + ReviewStateOnly bool // ReceiptOperation separates targeted review-state recovery from verified // core traversal; both still fetch and validate complete conversations. ReceiptOperation string @@ -63,6 +64,9 @@ type Options struct { } type Stats struct { + ReviewStateOnly bool `json:"review_state_only,omitempty"` + FetchMillis int64 `json:"fetch_ms,omitempty"` + PersistMillis int64 `json:"persist_ms,omitempty"` Repository string `json:"repository"` ThreadsSynced int `json:"threads_synced"` IssuesSynced int `json:"issues_synced"` @@ -127,6 +131,9 @@ func New(client GitHubClient, st *store.Store) *Syncer { var errAnalyticsReceipt = errors.New("persist GraphQL attempt") func (s *Syncer) Sync(ctx context.Context, options Options) (result Stats, resultErr error) { + if options.ReviewStateOnly && !options.GraphQLHistory { + return Stats{}, fmt.Errorf("review-state-only requires GraphQL history transport") + } startedAt := s.now() started := startedAt.Format(time.RFC3339Nano) if err := reportSyncProgress(options.Progress, SyncProgress{ @@ -155,6 +162,9 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (result Stats, resul if operation != "graphql_history" && operation != "review_state" { return Stats{}, fmt.Errorf("unsupported GraphQL receipt operation") } + if options.ReviewStateOnly && operation != "review_state" { + return Stats{}, fmt.Errorf("review-state-only fetch requires review recovery operation") + } // Fetch/validation failures happen before conversation transactions and // were previously invisible to durable run tables. Keep a receipt even // when the request is cancelled; accepted content remains untouched. @@ -176,6 +186,9 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (result Stats, resul } } }() + if options.ReviewStateOnly { + return s.syncReviewState(ctx, options, started) + } client, ok := s.client.(interface { FetchGraphQLHistory(context.Context, string, string, []int, gh.Reporter) (gh.HistoryBatch, error) }) From 7e4d33e2c217d0724f4dac09d53e9053341f327f Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:54:28 -0600 Subject: [PATCH 12/21] Test analytics recovery ownership and atomicity contracts --- internal/cli/analytics_readonly_test.go | 148 ++++++++++++++++++ .../store/analytics_actor_recovery_test.go | 129 +++++++++++++++ .../store/analytics_recovery_contract_test.go | 145 +++++++++++++++++ 3 files changed, 422 insertions(+) create mode 100644 internal/cli/analytics_readonly_test.go create mode 100644 internal/store/analytics_actor_recovery_test.go create mode 100644 internal/store/analytics_recovery_contract_test.go diff --git a/internal/cli/analytics_readonly_test.go b/internal/cli/analytics_readonly_test.go new file mode 100644 index 00000000..d1ecffba --- /dev/null +++ b/internal/cli/analytics_readonly_test.go @@ -0,0 +1,148 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsCommandsRespectOwnershipAndExplicitRepair(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, "source.db") + s, err := store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + repoID, err := s.UpsertRepository(ctx, store.Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", RawJSON: `{}`, UpdatedAt: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + tid, err := s.UpsertThread(ctx, store.Thread{RepoID: repoID, GitHubID: "1", Number: 1, Kind: "pull_request", State: "open", Title: "retained", HTMLURL: "https://github.com/fixture/repo/pull/1", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{}`, ContentHash: "h", UpdatedAt: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + _, err = s.UpsertComment(ctx, store.Comment{ThreadID: tid, GitHubID: "review", CommentType: "pull_review", RawJSON: `{"submitted_at":"2026-01-02T00:00:00Z"}`, CreatedAtGitHub: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + if _, err = s.DB().ExecContext(ctx, "UPDATE comments SET submitted_at_gh=NULL,publication_at_gh=NULL; UPDATE comment_revisions SET submitted_at_gh=NULL,publication_at_gh=NULL"); err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 1); err != nil { + t.Fatal(err) + } + if err = s.RecordAnalyticsAttempt(ctx, store.AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "review_state", Status: "failed", ErrorClass: "validation", ErrorText: "private-rejection-sentinel", Evidence: json.RawMessage(`{"private":"private-rejection-sentinel"}`), StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z"}); err != nil { + t.Fatal(err) + } + cfg := writeDoctorTestConfig(t, dir, path) + lock, err := os.OpenFile(filepath.Join(dir, "runner.lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + if err = lockPortableFile(lock); err != nil { + t.Fatal(err) + } + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.Error(w, "unexpected provider request", 500) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + run := func(args ...string) (map[string]any, error) { + t.Helper() + a := New() + var out bytes.Buffer + a.Stdout = &out + a.Stderr = io.Discard + err := a.Run(ctx, append([]string{"--config", cfg, "analytics", "fixture/repo", "--json"}, args...)) + if err != nil { + return nil, err + } + if strings.Contains(out.String(), "private-rejection-sentinel") { + t.Fatal("status exposed private rejection evidence") + } + var payload map[string]any + if err = json.Unmarshal(out.Bytes(), &payload); err != nil { + t.Fatal(err) + } + return payload, nil + } + status, err := run("--status") + if err != nil { + t.Fatal(err) + } + coverage, ok := status["coverage"].(map[string]any) + if !ok || coverage["complete"] != true || status["unresolved_retries"] != float64(1) || status["core_unresolved_retries"] != float64(0) { + t.Fatalf("review failure hid core coverage: %+v", status) + } + audit, err := run() + if err != nil { + t.Fatal(err) + } + if audit["would_change"] != float64(2) || audit["changed"] != float64(0) { + t.Fatalf("audit mutated or missed repair: %+v", audit) + } + if _, err = run("--apply"); err == nil || !strings.Contains(err.Error(), "ownership lock busy") { + t.Fatalf("mutation bypassed collector owner: %v", err) + } + if _, err = run("--status", "--once"); err == nil || !strings.Contains(err.Error(), "cannot be combined") { + t.Fatalf("mixed read/write mode accepted: %v", err) + } + var modified, attempts int + if err = s.DB().QueryRowContext(ctx, "SELECT (SELECT count(*) FROM comments WHERE publication_at_gh IS NOT NULL)+(SELECT count(*) FROM comment_revisions WHERE publication_at_gh IS NOT NULL)").Scan(&modified); err != nil || modified != 0 { + t.Fatalf("audit applied publication repair: %d %v", modified, err) + } + if err = s.DB().QueryRowContext(ctx, "SELECT count(*) FROM analytics_fetch_attempts").Scan(&attempts); err != nil || attempts != 1 { + t.Fatalf("read created a collection receipt: %d %v", attempts, err) + } + // An explicit repair succeeds only after the fixture collector releases its + // lock. It normalizes retained evidence without inventing a new revision. + var rawBefore, recordedBefore string + if err = s.DB().QueryRowContext(ctx, "SELECT raw_json,recorded_at FROM comment_revisions").Scan(&rawBefore, &recordedBefore); err != nil { + t.Fatal(err) + } + if err = lock.Close(); err != nil { + t.Fatal(err) + } + repaired, err := run("--apply") + if err != nil || repaired["changed"] != float64(2) { + t.Fatalf("explicit repair=%+v err=%v", repaired, err) + } + generation, err := s.AnalyticsState(ctx, "publication_repair_generation") + if err != nil || generation == "" { + t.Fatalf("missing repair generation: %q %v", generation, err) + } + var rawAfter, recordedAfter, published string + var revisions int + if err = s.DB().QueryRowContext(ctx, "SELECT count(*),raw_json,recorded_at,publication_at_gh FROM comment_revisions").Scan(&revisions, &rawAfter, &recordedAfter, &published); err != nil { + t.Fatal(err) + } + if revisions != 1 || rawBefore != rawAfter || recordedBefore != recordedAfter || published != "2026-01-02T00:00:00Z" { + t.Fatal("repair changed retained source history or publication time") + } + replay, err := run("--apply") + if err != nil || replay["changed"] != float64(0) { + t.Fatalf("repair replay=%+v err=%v", replay, err) + } + again, err := s.AnalyticsState(ctx, "publication_repair_generation") + if err != nil || generation != again { + t.Fatalf("idempotent replay changed repair generation: %q %q %v", generation, again, err) + } + if requests.Load() != 0 { + t.Fatalf("read-only modes contacted provider %d times", requests.Load()) + } +} diff --git a/internal/store/analytics_actor_recovery_test.go b/internal/store/analytics_actor_recovery_test.go new file mode 100644 index 00000000..ffb024ef --- /dev/null +++ b/internal/store/analytics_actor_recovery_test.go @@ -0,0 +1,129 @@ +package store + +import ( + "context" + "encoding/json" + "path/filepath" + "reflect" + "slices" + "testing" + "time" +) + +func TestAnalyticsActorRecoverySeedsAndRefreshesNativeIdentitiesAtomically(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + // Historical rows predate the enqueue-on-capture path. + _, err = s.DB().ExecContext(ctx, `INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo','1','{}','2026-01-01'); + INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at) VALUES + (1,1,'1',1,'issue','open','fixture','','[]','[]','{"node_id":"content-unknown","user":null}','h','2026-01-01'), + (2,1,'2',2,'issue','open','fixture','','[]','[]','{"node_id":"content-known","user":{"node_id":"actor-known"}}','h','2026-01-01')`) + if err != nil { + t.Fatal(err) + } + for _, profiles := range []bool{false, true} { + if err = s.SeedAnalyticsNodes(ctx, profiles); err != nil { + t.Fatal(err) + } + } + assertNodes := func(profiles bool, want []string) { + t.Helper() + var got []string + var err error + if profiles { + got, err = s.AnalyticsProfileNodes(ctx, 100) + } else { + got, err = s.AnalyticsIdentityNodes(ctx, 100) + } + if err != nil { + t.Fatal(err) + } + slices.Sort(got) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Fatalf("profiles=%v nodes=%v want%v", profiles, got, want) + } + } + assertNodes(false, []string{"content-unknown"}) + assertNodes(true, []string{"actor-known"}) + now := time.Now().UTC() + at := now.Format(time.RFC3339Nano) + evidence := map[string]any{"id": "content-unknown", "author": map[string]any{"id": "actor-recovered", "login": "shared-login", "__typename": "User"}} + if err = s.SaveActorEvidence(ctx, []map[string]any{evidence}, at); err != nil { + t.Fatal(err) + } + assertNodes(false, nil) + assertNodes(true, []string{"actor-known", "actor-recovered"}) + profile := func(id, name string) map[string]any { + return map[string]any{"id": id, "login": "shared-login", "__typename": "User", "name": name, "bio": "fixture profile", "url": "https://github.com/shared-login", "createdAt": "2026-01-01T02:00:00+02:00"} + } + first, second := profile("actor-known", "Known"), profile("actor-recovered", "Recovered") + if err = s.SaveActorProfiles(ctx, []map[string]any{first, second}, at); err != nil { + t.Fatal(err) + } + assertNodes(true, nil) + var count int + if err = s.DB().QueryRowContext(ctx, "SELECT count(*) FROM actor_profiles WHERE login='shared-login'").Scan(&count); err != nil || count != 2 { + t.Fatalf("login reuse merged native actors: %d %v", count, err) + } + read := func(id string) (string, string, string) { + t.Helper() + var raw, created, observed string + if err := s.DB().QueryRowContext(ctx, "SELECT raw_json,created_at,observed_at FROM actor_profiles WHERE node_id=?", id).Scan(&raw, &created, &observed); err != nil { + t.Fatal(err) + } + return raw, created, observed + } + raw, created, observed := read("actor-known") + var retained map[string]any + if err = json.Unmarshal([]byte(raw), &retained); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(retained, first) || created != "2026-01-01T00:00:00Z" || observed != at { + t.Fatal("native profile evidence or provider date changed") + } + changed := profile("actor-known", "Changed") + invalid := profile("actor-recovered", "Must not apply") + invalid["unsupported"] = make(chan int) + if err = s.SaveActorProfiles(ctx, []map[string]any{changed, invalid}, at); err == nil { + t.Fatal("invalid batch accepted") + } + after, _, _ := read("actor-known") + if after != raw { + t.Fatal("failed later profile leaked an earlier update") + } + // Failed identity capture must roll back its newly enqueued profile as well. + invalidEvidence := map[string]any{"id": "other-content", "author": map[string]any{"id": "actor-leaked"}, "unsupported": make(chan int)} + if err = s.SaveActorEvidence(ctx, []map[string]any{invalidEvidence}, at); err == nil { + t.Fatal("invalid actor evidence accepted") + } + assertNodes(true, nil) + if err = s.DB().QueryRowContext(ctx, "SELECT count(*) FROM actor_identity_evidence").Scan(&count); err != nil || count != 1 { + t.Fatalf("failed identity batch changed evidence: %d %v", count, err) + } + // Fresh profiles are not polled repeatedly, but become eligible after 24 hours. + if err = s.SaveActorProfiles(ctx, []map[string]any{first}, now.Add(-25*time.Hour).Format(time.RFC3339Nano)); err != nil { + t.Fatal(err) + } + assertNodes(true, []string{"actor-known"}) + if err = s.SaveActorProfiles(ctx, []map[string]any{changed}, at); err != nil { + t.Fatal(err) + } + assertNodes(true, nil) + refreshed, _, _ := read("actor-known") + if err = json.Unmarshal([]byte(refreshed), &retained); err != nil || retained["name"] != "Changed" || retained["id"] != "actor-known" { + t.Fatalf("refresh lost native identity: %+v %v", retained, err) + } + if err = s.SeedAnalyticsNodes(ctx, true); err != nil { + t.Fatal(err) + } + if err = s.SeedAnalyticsNodes(ctx, false); err != nil { + t.Fatal(err) + } + assertNodes(true, nil) + assertNodes(false, nil) +} diff --git a/internal/store/analytics_recovery_contract_test.go b/internal/store/analytics_recovery_contract_test.go new file mode 100644 index 00000000..a564e9e6 --- /dev/null +++ b/internal/store/analytics_recovery_contract_test.go @@ -0,0 +1,145 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "path/filepath" + "testing" +) + +func recoveryContractStore(t *testing.T) (*Store, int64) { + t.Helper() + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + repoID, err := s.UpsertRepository(ctx, Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", GitHubRepoID: "101", RawJSON: `{"node_id":"R1"}`, UpdatedAt: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + id, err := s.UpsertThread(ctx, Thread{RepoID: repoID, GitHubID: "201", Number: 7, Kind: "pull_request", State: "open", Title: "retained", HTMLURL: "https://github.com/fixture/repo/pull/7", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{"node_id":"P7"}`, ContentHash: "retained-hash", UpdatedAt: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + return s, id +} + +func TestReviewStateParentBindsIdentityInOwningTransaction(t *testing.T) { + ctx := context.Background() + s, id := recoveryContractStore(t) + parent, err := s.ReviewStateParent(ctx, "FIXTURE/REPO", 7, "101", "R1") + if err != nil || parent.ID != id || parent.GitHubID != "201" || parent.RawJSON != `{"node_id":"P7"}` { + t.Fatalf("parent=%+v err=%v", parent, err) + } + for _, tc := range []struct { + repo string + number int + database, node string + }{ + {"fixture/other", 7, "101", "R1"}, {"fixture/repo", 8, "101", "R1"}, {"fixture/repo", 7, "102", "R1"}, {"fixture/repo", 7, "101", "R2"}, + } { + if _, err := s.ReviewStateParent(ctx, tc.repo, tc.number, tc.database, tc.node); err == nil { + t.Fatalf("mismatched identity accepted: %+v", tc) + } + } + rolledBack := errors.New("fixture rollback") + err = s.WithTx(ctx, func(tx *Store) error { + if _, err := tx.q().ExecContext(ctx, `UPDATE repositories SET raw_json='{"node_id":"R-new"}' WHERE id=?`, parent.RepoID); err != nil { + return err + } + if _, err := tx.ReviewStateParent(ctx, "fixture/repo", 7, "101", "R1"); err == nil { + t.Fatal("read stale identity outside owning transaction") + } + got, err := tx.ReviewStateParent(ctx, "fixture/repo", 7, "101", "R-new") + if err != nil || got.ID != id { + t.Fatalf("transaction-local binding failed: %+v %v", got, err) + } + return rolledBack + }) + if !errors.Is(err, rolledBack) { + t.Fatal(err) + } + if _, err = s.ReviewStateParent(ctx, "fixture/repo", 7, "101", "R1"); err != nil { + t.Fatal("identity mutation escaped rollback", err) + } + if _, err = s.DB().ExecContext(ctx, "UPDATE threads SET kind='issue' WHERE id=?", id); err != nil { + t.Fatal(err) + } + if _, err = s.ReviewStateParent(ctx, "fixture/repo", 7, "101", "R1"); err == nil { + t.Fatal("issue accepted as review-state parent") + } +} + +func TestReviewRecoveryCannotDischargeCoreRetryOrCertifyCoreCoverage(t *testing.T) { + ctx := context.Background() + s, id := recoveryContractStore(t) + through := "2026-01-01T00:00:00Z" + if err := s.SaveAnalyticsCoverage(ctx, "fixture/repo", through, 2, 1); err != nil { + t.Fatal(err) + } + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 7, Operation: "review_state", StartedAt: through, FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", ErrorClass: "validation", Evidence: json.RawMessage(`{}`)} + record := func() { + t.Helper() + if err := s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + } + queued := func(want bool, operations ...string) { + t.Helper() + got, err := s.AnalyticsItemQueued(ctx, a.Repository, a.Number, operations...) + if err != nil || got != want { + t.Fatalf("queued(%v)=%v want%v err%v", operations, got, want, err) + } + } + coverage := func(want int) { + t.Helper() + var complete int + var watermark string + if err := s.DB().QueryRowContext(ctx, "SELECT complete,through FROM analytics_coverage WHERE repository=?", a.Repository).Scan(&complete, &watermark); err != nil || complete != want || watermark != through { + t.Fatalf("coverage=%d through=%s err=%v", complete, watermark, err) + } + } + record() + queued(false) + queued(true, "review_state") + coverage(1) + a.Operation = "graphql_history" + record() + queued(true) + coverage(0) + if err := s.UpsertPullRequestReviewThreads(ctx, id, a.FinishedAt, nil); err != nil { + t.Fatal(err) + } + a.Operation = "review_state" + a.Status = "success" + record() + queued(false, "review_state") + queued(true) + coverage(0) + a.Operation = "graphql_history" + record() + queued(false) + coverage(0) + // A successful item is not itself a completed traversal watermark. + if err := s.SetAnalyticsCoverageComplete(ctx, a.Repository, true); err != nil { + t.Fatal(err) + } + coverage(1) + a.Status = "failed" + a.FinishedAt = "2026-01-01T00:01:00Z" + record() + queued(true) + coverage(0) + var receipts int + if err := s.DB().QueryRowContext(ctx, "SELECT count(*) FROM analytics_fetch_attempts WHERE repository=?", a.Repository).Scan(&receipts); err != nil || receipts != 5 { + t.Fatalf("lost recovery history: %d %v", receipts, err) + } + var resolved sql.NullString + if err := s.DB().QueryRowContext(ctx, "SELECT resolved_at FROM analytics_retries WHERE repository=? AND operation='review_state'", a.Repository).Scan(&resolved); err != nil || !resolved.Valid { + t.Fatalf("independent review resolution lost: %v %v", resolved, err) + } +} From abc4af886f1b73828e75c434e8d0e5d90d4df323 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:46:10 -0600 Subject: [PATCH 13/21] Retain bounded causes for rejected GraphQL requests --- docs/analytics-source.md | 8 + internal/github/client.go | 26 +-- internal/github/history.go | 28 +-- internal/github/history_cause.go | 145 ++++++++++++++ internal/github/history_cause_test.go | 234 +++++++++++++++++++++++ internal/github/history_evidence.go | 11 +- internal/github/history_evidence_test.go | 11 +- internal/github/review_threads.go | 2 +- 8 files changed, 431 insertions(+), 34 deletions(-) create mode 100644 internal/github/history_cause.go create mode 100644 internal/github/history_cause_test.go diff --git a/docs/analytics-source.md b/docs/analytics-source.md index 5345de4e..5cbf6f00 100644 --- a/docs/analytics-source.md +++ b/docs/analytics-source.md @@ -168,6 +168,14 @@ Operational tables are **not public conversation datasets**: allowlisted provider codes and query paths of at most eight components, with explicit truncation markers. Unknown codes/path components are null. Messages, arbitrary field values and identities are excluded from this error metadata. + Every rejection also has private `cause` metadata: allowlisted category/type, + guard or validation code, HTTP status when available, up to eight request or + pagination stages, and numeric quota/reset evidence for quota guards. The + primary error chain is bounded to sixteen links with explicit truncation. + Wrapping a transport or guard error as a validation failure retains this cause + without changing the existing error class, retry or acceptance behavior. No + error messages, URLs, tokens, headers, bodies, identities or certificate subjects + are copied into this diagnostic metadata. Unknown types remain `unclassified`. Older receipts are not rewritten to infer a cause from later provider reads. - `analytics_retries(repository, number, operation, first_seen_at, last_seen_at, next_attempt_at, attempts, last_attempt_id, resolved_at)` is keyed by diff --git a/internal/github/client.go b/internal/github/client.go index 6d0a0c5e..f0b1b7b4 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -106,16 +106,16 @@ func (r *rateLimitReserve) bindGraphQLToken(token string) { func (r *rateLimitReserve) beforeObservedGraphQL(token string, cost int) error { if r == nil { - return fmt.Errorf("observed GraphQL quota guard required") + return requestFailureAt("dispatch_guard", "quota_guard_missing", fmt.Errorf("observed GraphQL quota guard required")) } r.mu.Lock() defer r.mu.Unlock() q := r.graphqlObserved if token != r.graphqlToken { - return fmt.Errorf("GraphQL credential changed; quota probe required") + return requestFailureAt("dispatch_guard", "credential_changed", fmt.Errorf("GraphQL credential changed; quota probe required")) } if q.Resource != "graphql" || !q.ResetAt.After(time.Now()) { - return fmt.Errorf("fresh observed GraphQL quota required") + return requestFailureAt("dispatch_guard", "quota_observation_stale", fmt.Errorf("fresh observed GraphQL quota required")) } if q.Remaining-cost < r.reserve { return &RateLimitReserveError{RateLimit: q, Reserve: r.reserve} @@ -210,7 +210,7 @@ func (r *rateLimitReserve) beforeRequest(resource string, cost int) error { defer r.mu.Unlock() snapshot, ok := r.snapshots[resource] if !ok { - return fmt.Errorf("github %s rate limit status unavailable; cannot preserve reserve %d", resource, r.reserve) + return requestFailureAt("dispatch_guard", "quota_snapshot_missing", fmt.Errorf("github %s rate limit status unavailable; cannot preserve reserve %d", resource, r.reserve)) } if !snapshot.ResetAt.IsZero() && !time.Now().UTC().Before(snapshot.ResetAt) { return &rateLimitStatusExpiredError{RateLimit: snapshot} @@ -313,7 +313,7 @@ func (c *Client) getRateLimits(ctx context.Context, reporter Reporter, selectedT } defer resp.Body.Close() if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - return nil, "", fmt.Errorf("decode github response: %w", err) + return nil, "", requestFailureAt("rest_quota_decode", "", fmt.Errorf("decode github response: %w", err)) } token := strings.TrimPrefix(resp.Request.Header.Get("Authorization"), "Bearer ") host := rateLimitHostForBaseURL(c.baseURL) @@ -592,7 +592,7 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader if targetErr != nil || originErr != nil || target.User != nil || origin.Host == "" || (target.Scheme != "https" && target.Scheme != "http") || !strings.EqualFold(target.Scheme, origin.Scheme) || !strings.EqualFold(target.Host, origin.Host) { - return nil, errors.New("GitHub token provider requires the configured API origin") + return nil, requestFailureAt("dispatch_guard", "origin_mismatch", errors.New("GitHub token provider requires the configured API origin")) } } resource, cost := c.requestRateLimit(method, fullURL) @@ -610,7 +610,7 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader var err error _, probeToken, err = c.getRateLimits(ctx, reporter, nil) if err != nil { - return nil, fmt.Errorf("refresh GitHub rate limit status: %w", err) + return nil, requestFailureAt("rest_preflight", "", fmt.Errorf("refresh GitHub rate limit status: %w", err)) } } token := c.token @@ -628,14 +628,14 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader // probe, then reject further rotation before the protected request. _, probeToken, err := c.getRateLimits(ctx, reporter, &token) if err != nil { - return nil, fmt.Errorf("refresh GitHub rate limit status: %w", err) + return nil, requestFailureAt("rest_preflight", "", fmt.Errorf("refresh GitHub rate limit status: %w", err)) } token, err = c.requestToken(ctx) if err != nil { return nil, err } if token != probeToken { - return nil, errors.New("GitHub token changed during rate limit reservation") + return nil, requestFailureAt("dispatch_guard", "credential_changed", errors.New("GitHub token changed during rate limit reservation")) } } if observedGraphQL { @@ -648,11 +648,11 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader } else if err := c.reserve.beforeRequest(resource, cost); err != nil { var expired *rateLimitStatusExpiredError if c.tokenProvider != nil || !errors.As(err, &expired) { - return nil, err + return nil, requestFailureAt("dispatch_guard", "", err) } _, refreshErr := c.GetRateLimits(ctx, reporter) if refreshErr != nil { - return nil, fmt.Errorf("refresh GitHub rate limit status: %w", refreshErr) + return nil, requestFailureAt("rest_preflight", "", fmt.Errorf("refresh GitHub rate limit status: %w", refreshErr)) } if err := c.reserve.beforeRequest(resource, cost); err != nil { return nil, err @@ -677,7 +677,7 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader reporter.Printf("[github] request %s %s", method, path) resp, err := c.guardedHTTPClient().Do(req) if err != nil { - return nil, fmt.Errorf("github request: %w", err) + return nil, requestFailureAt("transport", "", fmt.Errorf("github request: %w", err)) } responseResource, responseCost := c.requestRateLimit(resp.Request.Method, resp.Request.URL.String()) responseToken := strings.TrimPrefix(resp.Request.Header.Get("Authorization"), "Bearer ") @@ -727,7 +727,7 @@ func (c *Client) requestToken(ctx context.Context) (string, error) { if ctx.Err() != nil { return "", ctx.Err() } - return "", errors.New("GitHub token provider failed") + return "", requestFailureAt("credential", "credential_provider_failed", errors.New("GitHub token provider failed")) } return token, nil } diff --git a/internal/github/history.go b/internal/github/history.go index 6b6a6f8e..cedfbaa5 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -111,7 +111,7 @@ func sleepHistoryRetry(ctx context.Context, duration time.Duration) error { func (h *historySession) requestOnce(ctx context.Context, query string, variables map[string]any, estimate int) (map[string]any, error) { if h.calls >= 1000 { - return nil, fmt.Errorf("GraphQL history pagination budget exceeded") + return nil, requestFailureAt("graphql_response", "pagination_budget", fmt.Errorf("GraphQL history pagination budget exceeded")) } reserve := 500 if h.client.reserve != nil { @@ -135,15 +135,15 @@ func (h *historySession) requestOnce(ctx context.Context, query string, variable rate := historyMap(data["rateLimit"]) cost, ok := historyInt(rate["cost"]) if !ok || cost < 0 { - return nil, fmt.Errorf("GraphQL history missing cost") + return nil, requestFailureAt("graphql_response", "quota_cost_missing", fmt.Errorf("GraphQL history missing cost")) } remaining, ok := historyInt(rate["remaining"]) if !ok || remaining < 0 { - return nil, fmt.Errorf("GraphQL history missing remaining quota") + return nil, requestFailureAt("graphql_response", "quota_remaining_missing", fmt.Errorf("GraphQL history missing remaining quota")) } reset, err := time.Parse(time.RFC3339, historyString(rate["resetAt"])) if err != nil { - return nil, fmt.Errorf("GraphQL history invalid reset") + return nil, requestFailureAt("graphql_response", "quota_reset_invalid", fmt.Errorf("GraphQL history invalid reset")) } effective := h.client.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: remaining, ResetAt: reset}, time.Now()) h.remaining = min(h.remaining-cost, effective.Remaining) @@ -267,7 +267,7 @@ func (h *historySession) hydrateConnections(ctx context.Context, node map[string } conn := historyMap(connection) if conn == nil { - return fmt.Errorf("missing %s connection", key) + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("missing %s connection", key)) } fields, err := historyFields(typ, key) if err != nil { @@ -278,50 +278,50 @@ func (h *historySession) hydrateConnections(ctx context.Context, node map[string page := historyMap(conn["pageInfo"]) next, ok := page["hasNextPage"].(bool) if !ok { - return fmt.Errorf("missing %s pageInfo", key) + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("missing %s pageInfo", key)) } if !next { break } cursor := historyString(page["endCursor"]) if cursor == "" || seen[cursor] { - return fmt.Errorf("nonadvancing %s cursor", key) + return requestFailureAt("pagination_"+key, "connection_cursor", fmt.Errorf("nonadvancing %s cursor", key)) } seen[cursor] = true q := `query($id:ID!,$after:String!){node(id:$id){id ... on ` + typ + `{` + historyConnection(key, fields, `,after:$after`) + `}} rateLimit{cost remaining limit used resetAt}}` data, err := h.request(ctx, q, map[string]any{"id": node["id"], "after": cursor}, 2) if err != nil { - return err + return requestFailureAt("pagination_"+key, "", err) } parent := historyMap(data["node"]) if parent["id"] != node["id"] { - return fmt.Errorf("history pagination identity mismatch") + return requestFailureAt("pagination_"+key, "connection_identity", fmt.Errorf("history pagination identity mismatch")) } nxt := historyMap(parent[key]) a, ok := conn["nodes"].([]any) if !ok { - return fmt.Errorf("missing history nodes") + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("missing history nodes")) } b, ok := nxt["nodes"].([]any) if !ok || len(b) == 0 { - return fmt.Errorf("empty history continuation") + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("empty history continuation")) } conn["nodes"] = append(a, b...) conn["pageInfo"] = nxt["pageInfo"] } children, ok := conn["nodes"].([]any) if !ok { - return fmt.Errorf("missing history nodes") + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("missing history nodes")) } if total, ok := historyInt(conn["totalCount"]); !ok || total != len(children) { - return fmt.Errorf("incomplete history %s count", key) + return requestFailureAt("pagination_"+key, "connection_count", fmt.Errorf("incomplete history %s count", key)) } ids := map[string]bool{} for _, child := range children { m := historyMap(child) id := historyString(m["id"]) if id == "" || ids[id] { - return fmt.Errorf("missing or duplicate history child identity") + return requestFailureAt("pagination_"+key, "connection_identity", fmt.Errorf("missing or duplicate history child identity")) } ids[id] = true if err := h.hydrate(ctx, m); err != nil { diff --git a/internal/github/history_cause.go b/internal/github/history_cause.go new file mode 100644 index 00000000..a3854a7d --- /dev/null +++ b/internal/github/history_cause.go @@ -0,0 +1,145 @@ +package github + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "io" + "net" + "net/url" +) + +// requestFailure adds source-owned diagnostic labels without changing the error +// text, unwrap chain, retry policy or acceptance decision. +type requestFailure struct { + cause error + stage, code string +} + +func (e *requestFailure) Error() string { return e.cause.Error() } +func (e *requestFailure) Unwrap() error { return e.cause } +func requestFailureAt(stage, code string, err error) error { + if err == nil { + return nil + } + return &requestFailure{cause: err, stage: stage, code: code} +} + +// All emitted strings are fixed labels. Never serialize Error(), URL, address, +// field/type names from JSON errors, certificate subjects, headers or tokens. +func safeHistoryCause(err error) map[string]any { + out := map[string]any{"category": "unknown", "type": "unclassified"} + stages := []string{} + stage := func(value string) { + switch value { + case "rest_preflight", "rest_quota_decode", "dispatch_guard", "credential", "transport", "graphql_request", "graphql_response", "validation", "identity", "partial_response", "response_decode", "response_size", "missing_data", "discovery_validation", "pagination_labels", "pagination_assignees", "pagination_comments", "pagination_reviews", "pagination_reviewThreads": + if len(stages) < 8 && (len(stages) == 0 || stages[len(stages)-1] != value) { + stages = append(stages, value) + } + } + } + set := func(category, typ, code string) { out["category"] = category; out["type"] = typ; out["code"] = code } + quota := func(q RateLimitSnapshot, reserve int) { + v := map[string]any{"remaining": q.Remaining, "limit": q.Limit} + switch q.Resource { + case "graphql", "core", "search": + v["resource"] = q.Resource + } + if !q.ResetAt.IsZero() { + v["reset_at"] = q.ResetAt.UTC().Format("2006-01-02T15:04:05Z07:00") + } + if reserve > 0 { + v["reserve"] = reserve + } + out["quota"] = v + } + for depth := 0; err != nil && depth < 16; depth++ { + switch e := err.(type) { + case *requestFailure: + stage(e.stage) + switch e.code { + case "quota_guard_missing", "quota_snapshot_missing", "quota_observation_stale", "credential_changed", "origin_mismatch": + set("guard", "native_guard", e.code) + case "credential_provider_failed": + set("credential", "native_provider", e.code) + case "quota_cost_missing", "quota_remaining_missing", "quota_reset_invalid", "pagination_budget": + set("validation", "native_validation", e.code) + case "connection_shape", "connection_cursor", "connection_identity", "connection_count": + set("validation", "native_validation", e.code) + } + case *HistoryFailure: + stage(e.Stage) + case *rateLimitStatusExpiredError: + set("guard", "quota_snapshot", "quota_snapshot_expired") + quota(e.RateLimit, 0) + case *RateLimitReserveError: + set("guard", "quota_reserve", "quota_reserve_reached") + quota(e.RateLimit, e.Reserve) + case *RequestError: + set("http", "github_http", "http_status") + if e.Status >= 100 && e.Status <= 599 { + out["http_status"] = e.Status + } + case *json.SyntaxError: + set("decode", "json_syntax", "invalid_json") + case *json.UnmarshalTypeError: + set("decode", "json_type", "invalid_json_type") + case *net.DNSError: + set("network", "dns", "dns_error") + out["timeout"] = e.Timeout() + case *net.OpError: + set("network", "net_operation", "network_error") + out["timeout"] = e.Timeout() + case *url.Error: + set("network", "url_request", "network_error") + out["timeout"] = e.Timeout() + case *tls.CertificateVerificationError: + set("tls", "certificate_verification", "certificate_invalid") + case x509.UnknownAuthorityError: + set("tls", "unknown_authority", "certificate_invalid") + case x509.HostnameError: + set("tls", "hostname", "certificate_invalid") + case x509.CertificateInvalidError: + set("tls", "certificate", "certificate_invalid") + } + switch err { + case context.Canceled: + set("cancelled", "context", "cancelled") + case context.DeadlineExceeded: + set("cancelled", "context", "deadline") + case io.EOF: + set("transport", "io", "eof") + case io.ErrUnexpectedEOF: + set("transport", "io", "unexpected_eof") + } + // A receipt describes its primary cause, not an unbounded joined error tree. + switch e := err.(type) { + case interface{ Unwrap() error }: + err = e.Unwrap() + case interface{ Unwrap() []error }: + children := e.Unwrap() + err = nil + if len(children) > 0 { + err = children[0] + } + default: + err = nil + } + } + if err != nil { + out["chain_truncated"] = true + } + out["stages"] = stages + return out +} + +func historyEvidenceWithCause(evidence json.RawMessage, err error) json.RawMessage { + var value map[string]json.RawMessage + if json.Unmarshal(evidence, &value) != nil || value == nil { + value = map[string]json.RawMessage{} + } + value["cause"], _ = json.Marshal(safeHistoryCause(err)) + out, _ := json.Marshal(value) + return out +} diff --git a/internal/github/history_cause_test.go b/internal/github/history_cause_test.go new file mode 100644 index 00000000..70e08d9b --- /dev/null +++ b/internal/github/history_cause_test.go @@ -0,0 +1,234 @@ +package github + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "testing" + "time" +) + +func TestHistoryCauseKeepsOnlyBoundedTypedMetadata(t *testing.T) { + canary := "private-token-host-body-identity" + cases := []struct { + name string + err error + category, typ, code string + }{ + {"unknown", errors.New(canary), "unknown", "unclassified", ""}, + {"cancel", context.Canceled, "cancelled", "context", "cancelled"}, + {"deadline", context.DeadlineExceeded, "cancelled", "context", "deadline"}, + {"eof", io.EOF, "transport", "io", "eof"}, + {"short", io.ErrUnexpectedEOF, "transport", "io", "unexpected_eof"}, + {"json syntax", &json.SyntaxError{}, "decode", "json_syntax", "invalid_json"}, + {"json type", &json.UnmarshalTypeError{Value: canary, Field: canary, Struct: canary, Type: reflect.TypeOf(0)}, "decode", "json_type", "invalid_json_type"}, + {"dns", &net.DNSError{Name: canary, Server: canary, Err: canary, IsTimeout: true}, "network", "dns", "dns_error"}, + {"operation", &net.OpError{Op: canary, Net: canary, Err: errors.New(canary)}, "network", "net_operation", "network_error"}, + {"url", &url.Error{Op: canary, URL: "https://" + canary, Err: errors.New(canary)}, "network", "url_request", "network_error"}, + {"tls", &tls.CertificateVerificationError{Err: errors.New(canary)}, "tls", "certificate_verification", "certificate_invalid"}, + {"authority", x509.UnknownAuthorityError{}, "tls", "unknown_authority", "certificate_invalid"}, + {"hostname", x509.HostnameError{Host: canary}, "tls", "hostname", "certificate_invalid"}, + {"certificate", x509.CertificateInvalidError{Detail: canary}, "tls", "certificate", "certificate_invalid"}, + {"http", &RequestError{Method: canary, URL: canary, Status: 401, Body: canary, Headers: http.Header{"Authorization": []string{canary}}}, "http", "github_http", "http_status"}, + {"expired", &rateLimitStatusExpiredError{RateLimit: RateLimitSnapshot{Resource: "graphql", Remaining: 19999, ResetAt: time.Unix(100, 0)}}, "guard", "quota_snapshot", "quota_snapshot_expired"}, + {"reserve", &RateLimitReserveError{RateLimit: RateLimitSnapshot{Resource: "graphql", Remaining: 1500}, Reserve: 1500}, "guard", "quota_reserve", "quota_reserve_reached"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := safeHistoryCause(fmt.Errorf("%s: %w", canary, tc.err)) + if got["category"] != tc.category || got["type"] != tc.typ { + t.Fatalf("%+v", got) + } + if tc.code != "" && got["code"] != tc.code { + t.Fatalf("code=%v", got["code"]) + } + b, _ := json.Marshal(got) + if strings.Contains(string(b), canary) || len(b) > 2048 { + t.Fatal("unsafe/unbounded diagnostic", string(b)) + } + }) + } + for code, category := range map[string]string{"quota_guard_missing": "guard", "quota_snapshot_missing": "guard", "quota_observation_stale": "guard", "credential_changed": "guard", "origin_mismatch": "guard", "credential_provider_failed": "credential", "quota_cost_missing": "validation", "quota_remaining_missing": "validation", "quota_reset_invalid": "validation", "pagination_budget": "validation", "connection_shape": "validation", "connection_cursor": "validation", "connection_identity": "validation", "connection_count": "validation"} { + base := errors.New(canary) + wrapped := requestFailureAt("dispatch_guard", code, base) + got := safeHistoryCause(wrapped) + if got["code"] != code || got["category"] != category || !errors.Is(wrapped, base) || wrapped.Error() != base.Error() { + t.Fatalf("diagnostic changed behavior: %s %+v", code, got) + } + } + var long error = errors.New(canary) + for i := 0; i < 30; i++ { + stage := "rest_preflight" + if i%2 == 0 { + stage = "graphql_request" + } + long = requestFailureAt(stage, "", long) + } + got := safeHistoryCause(long) + if got["chain_truncated"] != true || len(got["stages"].([]string)) != 8 { + t.Fatal("unbounded cause chain", got) + } + unknown := safeHistoryCause(requestFailureAt(canary, canary, errors.New(canary))) + b, _ := json.Marshal(unknown) + if strings.Contains(string(b), canary) { + t.Fatal("untrusted label leaked") + } + joined := safeHistoryCause(errors.Join(requestFailureAt("rest_preflight", "", io.ErrUnexpectedEOF), errors.New(canary))) + if joined["code"] != "unexpected_eof" { + t.Fatal("primary joined cause lost") + } + if requestFailureAt("transport", "", nil) != nil { + t.Fatal("nil error manufactured") + } + // Retry eligibility must stay identical after diagnostic wrapping. + for _, e := range []error{io.EOF, io.ErrUnexpectedEOF, &RequestError{Status: 503}, &RequestError{Status: 401}, context.Canceled} { + if transientHistoryError(e) != transientHistoryError(requestFailureAt("graphql_request", "", e)) { + t.Fatal("retry semantics changed") + } + } +} + +func TestHistoryCauseDiagnosesNativePreflightWithoutBypassingGuards(t *testing.T) { + for _, mode := range []string{"expired", "missing", "decode", "rotation", "http", "missing_cost", "missing_remaining", "invalid_reset"} { + t.Run(mode, func(t *testing.T) { + gql, credentials := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + if mode == "decode" { + fmt.Fprint(w, `{"resources":private-response`) + return + } + if mode == "missing" { + fmt.Fprint(w, `{"resources":{}}`) + return + } + if mode == "http" { + http.Error(w, "private-response", 401) + return + } + reset := time.Now().Add(time.Hour).Unix() + if mode == "expired" { + reset = time.Now().Add(-time.Minute).Unix() + } + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":%d}}}`, reset) + return + } + gql++ + rate := map[string]any{"cost": 1, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)} + if mode == "missing_cost" { + delete(rate, "cost") + } + if mode == "missing_remaining" { + delete(rate, "remaining") + } + if mode == "invalid_reset" { + rate["resetAt"] = "private-response" + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": rate}}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { + credentials++ + if mode == "rotation" { + return fmt.Sprint("test-token-", credentials), nil + } + return "test-token-placeholder", nil + }}) + batch, err := c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err == nil || len(batch.Items) != 0 { + t.Fatal("failed evidence accepted") + } + class, _, evidence := HistoryFailureDetails(err) + var record struct { + Cause struct { + Code string `json:"code"` + Status int `json:"http_status"` + Stages []string `json:"stages"` + } `json:"cause"` + } + if e := json.Unmarshal(evidence, &record); e != nil { + t.Fatal(e) + } + want := map[string]string{"expired": "quota_snapshot_expired", "missing": "quota_snapshot_missing", "decode": "invalid_json", "rotation": "credential_changed", "http": "http_status", "missing_cost": "quota_cost_missing", "missing_remaining": "quota_remaining_missing", "invalid_reset": "quota_reset_invalid"}[mode] + if record.Cause.Code != want { + t.Fatalf("mode=%s class=%s evidence=%s", mode, class, evidence) + } + if mode == "http" { + if class != "http" || record.Cause.Status != 401 { + t.Fatal("HTTP status lost") + } + } else if class != "fetch" { + t.Fatal("existing error class changed", class) + } + if strings.Contains(string(evidence), "private-response") || strings.Contains(string(evidence), "test-token") { + t.Fatal("private value leaked") + } + if mode == "expired" || mode == "missing" || mode == "decode" || mode == "rotation" || mode == "http" { + if gql != 0 { + t.Fatal("guard dispatched GraphQL content") + } + } + }) + } +} + +func TestHistoryCauseRetainsPaginationGuardAndAllowsNormalRetry(t *testing.T) { + expired := true + restCalls, gqlCalls := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + restCalls++ + reset := time.Now().Add(time.Hour).Unix() + if expired && restCalls == 3 { + reset = time.Now().Add(-time.Minute).Unix() + } + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":%d}}}`, reset) + return + } + gqlCalls++ + var req graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}} + if strings.Contains(req.Query, "issueOrPullRequest") { + node := historyTestNode() + node["labels"] = map[string]any{"totalCount": 2, "nodes": []any{map[string]any{"id": "L1", "name": "one"}}, "pageInfo": map[string]any{"hasNextPage": true, "endCursor": "first"}} + data["repository"] = map[string]any{"databaseId": 42, "nameWithOwner": "fixture/repo", "n0": node} + } + if req.Variables["after"] != nil { + data["node"] = map[string]any{"id": "PR_fixture", "labels": historyTestConnection(map[string]any{"id": "L2", "name": "two"})} + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "test-token-placeholder", nil }}) + batch, err := c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err == nil || len(batch.Items) != 0 || gqlCalls != 2 { + t.Fatal("partial pagination accepted") + } + class, _, evidence := HistoryFailureDetails(err) + var record map[string]json.RawMessage + if err = json.Unmarshal(evidence, &record); err != nil { + t.Fatal(err) + } + if class != "validation" || record["structure"] == nil || !strings.Contains(string(record["cause"]), "pagination_labels") || !strings.Contains(string(record["cause"]), "quota_snapshot_expired") { + t.Fatalf("lost wrapper or cause: %s", evidence) + } + expired = false + batch, err = c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err != nil || len(batch.Items) != 1 || len(historyNodes(historyMap(batch.Items[0].Thread["_graphql"]), "labels")) != 2 { + t.Fatalf("normal retry: items=%d err=%v", len(batch.Items), err) + } +} diff --git a/internal/github/history_evidence.go b/internal/github/history_evidence.go index e2f52f27..89e9cd6a 100644 --- a/internal/github/history_evidence.go +++ b/internal/github/history_evidence.go @@ -157,12 +157,13 @@ func SafeHistoryEvidence(data any) json.RawMessage { // HistoryFailureDetails is safe to persist or log even if the original error // included an HTTP body. It deliberately does not return that body/message. func HistoryFailureDetails(err error) (string, string, json.RawMessage) { + diagnostic := historyEvidenceWithCause(json.RawMessage(`{}`), err) if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return "cancelled", "GraphQL attempt cancelled or timed out", json.RawMessage(`{}`) + return "cancelled", "GraphQL attempt cancelled or timed out", diagnostic } var quota *RateLimitReserveError if errors.As(err, "a) { - return "rate_limit", quota.Error(), json.RawMessage(`{}`) + return "rate_limit", quota.Error(), diagnostic } var failure *HistoryFailure if errors.As(err, &failure) { @@ -174,11 +175,11 @@ func HistoryFailureDetails(err error) (string, string, json.RawMessage) { } } } - return failure.Stage, message, failure.Evidence + return failure.Stage, message, historyEvidenceWithCause(failure.Evidence, err) } var response *RequestError if errors.As(err, &response) { - return "http", fmt.Sprintf("GitHub HTTP %d", response.Status), json.RawMessage(`{}`) + return "http", fmt.Sprintf("GitHub HTTP %d", response.Status), diagnostic } - return "fetch", "GraphQL collection failed", json.RawMessage(`{}`) + return "fetch", "GraphQL collection failed", diagnostic } diff --git a/internal/github/history_evidence_test.go b/internal/github/history_evidence_test.go index 9161b5fa..c4a43226 100644 --- a/internal/github/history_evidence_test.go +++ b/internal/github/history_evidence_test.go @@ -23,7 +23,16 @@ func TestHistoryFailureEvidenceRetainsStructureWithoutSecretsOrBodies(t *testing } err := historyFailure("validation", 7, data, errors.New("GraphQL history #7: incomplete history comments count")) class, message, stored := HistoryFailureDetails(err) - if class != "validation" || !strings.Contains(message, "incomplete history comments count") || string(stored) != string(evidence) { + var preserved map[string]json.RawMessage + if err := json.Unmarshal(stored, &preserved); err != nil { + t.Fatal(err) + } + if preserved["cause"] == nil { + t.Fatal("missing safe cause metadata") + } + delete(preserved, "cause") + originalFields, _ := json.Marshal(preserved) + if class != "validation" || !strings.Contains(message, "incomplete history comments count") || string(originalFields) != string(evidence) { t.Fatalf("receipt %s %s", class, message) } } diff --git a/internal/github/review_threads.go b/internal/github/review_threads.go index 27f116c9..390f3ab5 100644 --- a/internal/github/review_threads.go +++ b/internal/github/review_threads.go @@ -243,7 +243,7 @@ func (c *Client) doGraphQL(ctx context.Context, query string, variables map[stri var envelope graphqlResponseEnvelope response, err := c.do(ctx, http.MethodPost, c.graphQLURL, bytes.NewReader(payload), reporter) if err != nil { - return err + return requestFailureAt("graphql_request", "", err) } defer response.Body.Close() reader := &historyResponseReader{reader: response.Body, hash: sha256.New()} From bd196fed26e05b74103c88e92387fdc0070a49e7 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Sat, 26 Sep 2026 06:02:56 -0600 Subject: [PATCH 14/21] fix: refresh expired GraphQL quota snapshots --- docs/analytics-source.md | 11 +- internal/github/client.go | 52 +++- internal/github/history.go | 2 +- internal/github/history_cause.go | 2 +- internal/github/history_cause_test.go | 17 +- internal/github/quota_refresh_test.go | 349 ++++++++++++++++++++++++++ 6 files changed, 416 insertions(+), 17 deletions(-) create mode 100644 internal/github/quota_refresh_test.go diff --git a/docs/analytics-source.md b/docs/analytics-source.md index 5cbf6f00..7049c56c 100644 --- a/docs/analytics-source.md +++ b/docs/analytics-source.md @@ -61,8 +61,15 @@ counters, which can differ. Recovery and its quota probe omit redundant REST `/rate_limit` preflights: an explicit GraphQL probe binds actual quota to the selected credential, and every content/page request checks that credential and unexpired balance against the reserve. Rotation requires a new probe. Ordinary -core transport retains its existing guards. Each history session also enforces its configured -floor against observed GraphQL balances before pagination. A +core transport retains its REST preflight. If that fresh response carries an +expired GraphQL snapshot (including crossing reset during preflight), it performs +a bounded quota-only GraphQL refresh under the existing request lock. The refresh +uses the selected credential and API origin; rotation before content dispatch +fails closed. A stale, invalid, failed, or below-reserve refresh cannot authorize +content. The refreshed balance is checked against the pending page's estimate, +without changing its identity, cursor, or completeness validation. Each history +session also enforces its configured floor against observed GraphQL balances +before pagination. A client retains the lowest observed GraphQL balance until the reset boundary passes. An upward sample or a shifted future reset cannot increase admission; REST snapshot refreshes do not overwrite this evidence. Logs distinguish the diff --git a/internal/github/client.go b/internal/github/client.go index f0b1b7b4..310f924e 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -91,6 +91,7 @@ type rateLimitReserve struct { } type graphQLQuotaProbeKey struct{} +type graphQLRequestEstimateKey struct{} func (r *rateLimitReserve) bindGraphQLToken(token string) { if r == nil { @@ -647,15 +648,21 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader } } else if err := c.reserve.beforeRequest(resource, cost); err != nil { var expired *rateLimitStatusExpiredError - if c.tokenProvider != nil || !errors.As(err, &expired) { - return nil, requestFailureAt("dispatch_guard", "", err) - } - _, refreshErr := c.GetRateLimits(ctx, reporter) - if refreshErr != nil { - return nil, requestFailureAt("rest_preflight", "", fmt.Errorf("refresh GitHub rate limit status: %w", refreshErr)) - } - if err := c.reserve.beforeRequest(resource, cost); err != nil { - return nil, err + if errors.As(err, &expired) && resource == "graphql" { + if err := c.refreshExpiredGraphQLQuota(ctx, token, cost); err != nil { + return nil, requestFailureAt("graphql_quota_refresh", "", err) + } + } else { + if c.tokenProvider != nil || !errors.As(err, &expired) { + return nil, requestFailureAt("dispatch_guard", "", err) + } + _, refreshErr := c.GetRateLimits(ctx, reporter) + if refreshErr != nil { + return nil, requestFailureAt("rest_preflight", "", fmt.Errorf("refresh GitHub rate limit status: %w", refreshErr)) + } + if err := c.reserve.beforeRequest(resource, cost); err != nil { + return nil, err + } } } if resource == "graphql" && !observedGraphQL { @@ -698,6 +705,33 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader } } +// refreshExpiredGraphQLQuota runs only after an expired REST GraphQL snapshot, +// with requestMu already held. A quota-only probe may cross that stale boundary; +// content may not. Reuse the authoritative probe and conservative same-window +// accounting, pinning its credential until the protected dispatch is rechecked. +func (c *Client) refreshExpiredGraphQLQuota(ctx context.Context, token string, cost int) error { + quotaClient := *c + quotaClient.tokenProvider = func(context.Context) (string, error) { return token, nil } + _, observed, err := quotaClient.AnalyticsRateLimit(ctx) + if err != nil { + return err + } + if !observed.ResetAt.After(time.Now()) { + return &rateLimitStatusExpiredError{RateLimit: observed} + } + current, err := c.requestToken(ctx) + if err != nil { + return err + } + if current != token { + return requestFailureAt("dispatch_guard", "credential_changed", errors.New("GitHub token changed during quota refresh")) + } + // A history session checked its estimate before entering transport, using + // the prior observation. Recheck against the newly refreshed balance too. + estimate, _ := ctx.Value(graphQLRequestEstimateKey{}).(int) + return c.reserve.beforeObservedGraphQL(token, max(cost, estimate)) +} + func (c *Client) guardedHTTPClient() *http.Client { if c.reserve == nil && c.tokenProvider == nil { return c.httpClient diff --git a/internal/github/history.go b/internal/github/history.go index cedfbaa5..99610e21 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -127,7 +127,7 @@ func (h *historySession) requestOnce(ctx context.Context, query string, variable h.reporter.Printf("[github] graphql budget %d %d", h.calls, estimate) var data map[string]any started := time.Now() - err := h.client.doGraphQL(ctx, query, variables, h.reporter, &data) + err := h.client.doGraphQL(context.WithValue(ctx, graphQLRequestEstimateKey{}, estimate), query, variables, h.reporter, &data) h.reporter.Printf("[github] graphql timing %d %d", h.calls, time.Since(started).Milliseconds()) if err != nil { return nil, err diff --git a/internal/github/history_cause.go b/internal/github/history_cause.go index a3854a7d..4257108c 100644 --- a/internal/github/history_cause.go +++ b/internal/github/history_cause.go @@ -33,7 +33,7 @@ func safeHistoryCause(err error) map[string]any { stages := []string{} stage := func(value string) { switch value { - case "rest_preflight", "rest_quota_decode", "dispatch_guard", "credential", "transport", "graphql_request", "graphql_response", "validation", "identity", "partial_response", "response_decode", "response_size", "missing_data", "discovery_validation", "pagination_labels", "pagination_assignees", "pagination_comments", "pagination_reviews", "pagination_reviewThreads": + case "graphql_quota_refresh", "rest_preflight", "rest_quota_decode", "dispatch_guard", "credential", "transport", "graphql_request", "graphql_response", "validation", "identity", "partial_response", "response_decode", "response_size", "missing_data", "discovery_validation", "pagination_labels", "pagination_assignees", "pagination_comments", "pagination_reviews", "pagination_reviewThreads": if len(stages) < 8 && (len(stages) == 0 || stages[len(stages)-1] != value) { stages = append(stages, value) } diff --git a/internal/github/history_cause_test.go b/internal/github/history_cause_test.go index 70e08d9b..032cd099 100644 --- a/internal/github/history_cause_test.go +++ b/internal/github/history_cause_test.go @@ -124,7 +124,10 @@ func TestHistoryCauseDiagnosesNativePreflightWithoutBypassingGuards(t *testing.T return } gql++ - rate := map[string]any{"cost": 1, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)} + rate := map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)} + if mode == "expired" { + rate["resetAt"] = time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + } if mode == "missing_cost" { delete(rate, "cost") } @@ -173,7 +176,10 @@ func TestHistoryCauseDiagnosesNativePreflightWithoutBypassingGuards(t *testing.T if strings.Contains(string(evidence), "private-response") || strings.Contains(string(evidence), "test-token") { t.Fatal("private value leaked") } - if mode == "expired" || mode == "missing" || mode == "decode" || mode == "rotation" || mode == "http" { + if mode == "expired" && gql != 1 { + t.Fatal("expected only the authoritative quota refresh") + } + if mode == "missing" || mode == "decode" || mode == "rotation" || mode == "http" { if gql != 0 { t.Fatal("guard dispatched GraphQL content") } @@ -201,7 +207,10 @@ func TestHistoryCauseRetainsPaginationGuardAndAllowsNormalRetry(t *testing.T) { t.Error(err) return } - data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}} + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}} + if expired && restCalls == 3 { + historyMap(data["rateLimit"])["resetAt"] = time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + } if strings.Contains(req.Query, "issueOrPullRequest") { node := historyTestNode() node["labels"] = map[string]any{"totalCount": 2, "nodes": []any{map[string]any{"id": "L1", "name": "one"}}, "pageInfo": map[string]any{"hasNextPage": true, "endCursor": "first"}} @@ -215,7 +224,7 @@ func TestHistoryCauseRetainsPaginationGuardAndAllowsNormalRetry(t *testing.T) { defer server.Close() c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "test-token-placeholder", nil }}) batch, err := c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) - if err == nil || len(batch.Items) != 0 || gqlCalls != 2 { + if err == nil || len(batch.Items) != 0 || gqlCalls != 3 { t.Fatal("partial pagination accepted") } class, _, evidence := HistoryFailureDetails(err) diff --git a/internal/github/quota_refresh_test.go b/internal/github/quota_refresh_test.go new file mode 100644 index 00000000..718c7eb8 --- /dev/null +++ b/internal/github/quota_refresh_test.go @@ -0,0 +1,349 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" +) + +// A fresh REST reply can still contain an expired GraphQL resource. The +// credential-bound GraphQL observation must be refreshed, never invented. +func TestExpiredGraphQLRESTSnapshotRefreshesFromProvider(t *testing.T) { + rest, probes, content := 0, 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + rest++ + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + var request graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + if request.Query == "query { viewer { id } }" { + content++ + } else { + probes++ + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"viewer": map[string]any{"id": "fixture"}, "rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}}}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "fixture", nil }}) + var result map[string]any + if err := c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &result); err != nil { + t.Fatal(err) + } + if rest != 1 || probes != 1 || content != 1 || historyString(historyMap(result["viewer"])["id"]) != "fixture" { + t.Fatalf("rest=%d probes=%d content=%d result=%v", rest, probes, content, result) + } +} + +func TestExpiredGraphQLQuotaRefreshFailsClosed(t *testing.T) { + for _, mode := range []string{"low", "stale", "missing_cost", "missing_remaining", "bad_reset", "missing_limit", "http", "partial", "decode", "rotation", "credential_failure", "cancel"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + probes, content, credentials := 0, 0, 0 + canary := "fixture-private-value" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + if strings.Contains(q.Query, "viewer") { + content++ + t.Error("content dispatched after failed refresh") + return + } + probes++ + if r.Header.Get("Authorization") != "Bearer fixture" { + t.Error("refresh credential changed") + } + switch mode { + case "http": + http.Error(w, canary, 401) + return + case "decode": + fmt.Fprint(w, "{") + return + case "cancel": + cancel() + return + } + rate := map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)} + switch mode { + case "low": + rate["remaining"] = 1500 + case "stale": + rate["resetAt"] = time.Now().UTC().Add(-time.Second).Format(time.RFC3339) + case "missing_cost": + delete(rate, "cost") + case "missing_remaining": + delete(rate, "remaining") + case "bad_reset": + rate["resetAt"] = canary + case "missing_limit": + delete(rate, "limit") + } + envelope := map[string]any{"data": map[string]any{"rateLimit": rate}} + if mode == "partial" { + envelope["errors"] = []any{map[string]any{"type": "FORBIDDEN", "message": canary, "path": []any{"rateLimit"}}} + } + json.NewEncoder(w).Encode(envelope) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { + credentials++ + if credentials == 3 { + if mode == "rotation" { + return "replacement", nil + } + if mode == "credential_failure" { + return "", errors.New(canary) + } + } + return "fixture", nil + }}) + var out map[string]any + err := c.doGraphQL(ctx, "query { viewer { id } }", nil, nil, &out) + wantProbes := 1 + if mode == "decode" { + wantProbes = 3 + } + if err == nil || probes != wantProbes || content != 0 || out != nil { + t.Fatalf("err=%v probes=%d content=%d out=%v", err, probes, content, out) + } + cause := safeHistoryCause(err) + code, _ := cause["code"].(string) + wants := map[string]string{"low": "quota_reserve_reached", "stale": "quota_snapshot_expired", "missing_cost": "quota_cost_missing", "missing_remaining": "quota_remaining_missing", "bad_reset": "quota_reset_invalid", "http": "http_status", "decode": "unexpected_eof", "rotation": "credential_changed", "credential_failure": "credential_provider_failed", "cancel": "cancelled"} + if want := wants[mode]; want != "" && code != want { + t.Fatalf("cause=%v want=%s", cause, want) + } + _, _, evidence := HistoryFailureDetails(err) + if strings.Contains(string(evidence), canary) || !strings.Contains(string(evidence), "graphql_quota_refresh") { + t.Fatalf("unsafe or missing refresh evidence: %s", evidence) + } + }) + } +} + +func TestExpiredGraphQLQuotaRefreshBindsRotatedCredential(t *testing.T) { + var probes, content []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + credential := r.Header.Get("Authorization") + if r.URL.Path == "/rate_limit" { + probes = append(probes, credential) + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + if strings.Contains(q.Query, "viewer") { + content = append(content, credential) + } else { + probes = append(probes, credential) + } + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":19000,"resetAt":"2099-01-01T00:00:00Z"}}}`) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: sequenceTokenProvider(t, "first", "second", "second", "second")}) + // Observations from the old credential must not constrain or admit the new one. + c.reserve.bindGraphQLToken("first") + c.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 0, ResetAt: time.Now().Add(time.Hour)}, time.Now()) + var out map[string]any + if err := c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &out); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(probes, []string{"Bearer first", "Bearer second", "Bearer second"}) || !reflect.DeepEqual(content, []string{"Bearer second"}) { + t.Fatalf("wrong binding: probes=%v content=%v", probes, content) + } +} + +func TestExpiredGraphQLQuotaRefreshConcurrentReserve(t *testing.T) { + var active, peak, probes, content atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := active.Add(1) + defer active.Add(-1) + for p := peak.Load(); n > p; p = peak.Load() { + if peak.CompareAndSwap(p, n) { + break + } + } + if r.URL.Path == "/rate_limit" { + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + balance := 19999 + if strings.Contains(q.Query, "viewer") { + content.Add(1) + } else if probes.Add(1) == 1 { + balance = 1501 + } + fmt.Fprintf(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":%d,"resetAt":"2099-01-01T00:00:00Z"}}}`, balance) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "fixture", nil }}) + results := make(chan error, 8) + for range 8 { + go func() { + var out map[string]any + results <- c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &out) + }() + } + success, blocked := 0, 0 + for range 8 { + err := <-results + var floor *RateLimitReserveError + if err == nil { + success++ + } else if errors.As(err, &floor) { + blocked++ + } else { + t.Fatal(err) + } + } + if success != 1 || blocked != 7 || content.Load() != 1 || probes.Load() != 8 || peak.Load() != 1 { + t.Fatalf("success=%d blocked=%d content=%d probes=%d simultaneous=%d", success, blocked, content.Load(), probes.Load(), peak.Load()) + } +} + +func TestExpiredGraphQLQuotaRefreshRefusesRedirect(t *testing.T) { + var leaked atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { leaked.Add(1) })) + defer target.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "fixture", nil }}) + var out map[string]any + err := c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &out) + var req *RequestError + if !errors.As(err, &req) || req.Status != 302 || leaked.Load() != 0 { + t.Fatalf("redirect: err=%v target=%d", err, leaked.Load()) + } +} + +func TestExpiredGraphQLQuotaRefreshPreservesPaginationValidation(t *testing.T) { + for _, mode := range []string{"complete", "short_page", "low_page", "stale_refresh"} { + t.Run(mode, func(t *testing.T) { + rest, probes, pages := 0, 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + rest++ + reset := time.Now().Add(time.Hour) + if rest == 3 { + reset = time.Now().Add(-time.Second) + } + writeRateLimits(t, w, reset, 19983, 19983) + return + } + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + rate := map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": "2099-01-01T00:00:00Z"} + data := map[string]any{"rateLimit": rate} + if strings.Contains(q.Query, "issueOrPullRequest") { + n := historyTestNode() + n["labels"] = map[string]any{"totalCount": 2, "nodes": []any{map[string]any{"id": "L1", "name": "one"}}, "pageInfo": map[string]any{"hasNextPage": true, "endCursor": "first"}} + data["repository"] = map[string]any{"databaseId": 42, "nameWithOwner": "fixture/repo", "n0": n} + } else if q.Variables["after"] != nil { + pages++ + if q.Variables["after"] != "first" || q.Variables["id"] != "PR_fixture" { + t.Error("continuation identity/cursor changed") + } + page := historyTestConnection(map[string]any{"id": "L2", "name": "two"}) + if mode == "short_page" { + page = historyTestConnection() + } + data["node"] = map[string]any{"id": "PR_fixture", "labels": page} + } else if rest == 3 { + probes++ + if mode == "low_page" { + rate["remaining"] = 1501 + } + if mode == "stale_refresh" { + rate["resetAt"] = time.Now().UTC().Add(-time.Second).Format(time.RFC3339) + } + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "fixture", nil }}) + batch, err := c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if probes != 1 || rest != 3 { + t.Fatalf("unbounded refresh: probes=%d rest=%d", probes, rest) + } + if mode == "complete" { + if err != nil || pages != 1 || len(batch.Items) != 1 { + t.Fatalf("items=%d pages=%d err=%v", len(batch.Items), pages, err) + } + item := batch.Items[0] + if item.Thread["id"] != "PR_fixture" || item.Thread["body"] != "body" || item.Pull["id"] != "9007199254740993" || len(historyNodes(historyMap(item.Thread["_graphql"]), "labels")) != 2 { + t.Fatal("content/identity/membership changed") + } + } else { + if err == nil || len(batch.Items) != 0 { + t.Fatal("incomplete membership accepted") + } + if mode == "low_page" { + var floor *RateLimitReserveError + if !errors.As(err, &floor) || pages != 0 { + t.Fatalf("refreshed quota ignored page estimate: pages=%d err=%v", pages, err) + } + } + if mode == "stale_refresh" && pages != 0 { + t.Fatal("old valid observation masked invalid refresh") + } + } + }) + } +} + +func TestFreshGraphQLQuotaDoesNotTriggerRefresh(t *testing.T) { + for _, provider := range []bool{false, true} { + t.Run(fmt.Sprint(provider), func(t *testing.T) { + rest, content := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + rest++ + writeRateLimits(t, w, time.Now().Add(time.Hour), 19000, 19000) + return + } + content++ + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + if q.Query != "query { viewer { id } }" { + t.Error("unexpected refresh") + } + fmt.Fprint(w, `{"data":{"viewer":{"id":"fixture"}}}`) + })) + defer server.Close() + opts := Options{BaseURL: server.URL, RateLimitReserve: 1500, Token: "fixture"} + if provider { + opts.TokenProvider = func(context.Context) (string, error) { return "fixture", nil } + } + c := New(opts) + var out map[string]any + if err := c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &out); err != nil || rest != 1 || content != 1 { + t.Fatalf("rest=%d content=%d err=%v", rest, content, err) + } + }) + } +} From a6f57998ebfb167b778cb9da85498509881533b2 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Sat, 26 Sep 2026 06:07:36 -0600 Subject: [PATCH 15/21] test: synchronize quota refresh cancellation fixture --- internal/github/quota_refresh_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/github/quota_refresh_test.go b/internal/github/quota_refresh_test.go index 718c7eb8..7de4ff06 100644 --- a/internal/github/quota_refresh_test.go +++ b/internal/github/quota_refresh_test.go @@ -79,6 +79,9 @@ func TestExpiredGraphQLQuotaRefreshFailsClosed(t *testing.T) { return case "cancel": cancel() + // Wait until the client closes this request before the handler + // can return an empty 200 and race cancellation with EOF. + <-r.Context().Done() return } rate := map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)} From 290d943b1c0c3b9d16dd50342c8650e92a0a7a43 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 27 Sep 2026 17:38:39 -0700 Subject: [PATCH 16/21] perf(vector): prepare embeddings once and add portable SIMD scoring (#218) --- .github/workflows/ci.yml | 7 + .goreleaser.yaml | 2 + CHANGELOG.md | 1 + Dockerfile | 2 +- docs/installation.md | 6 + internal/cli/cluster_graph.go | 46 +++--- internal/cli/cluster_graph_bench_test.go | 133 ++++++++++++++++ internal/cli/cluster_graph_scoring_test.go | 58 +++++++ internal/vector/exact.go | 7 +- internal/vector/prepared.go | 52 +++++++ internal/vector/prepared_nosimd.go | 11 ++ internal/vector/prepared_simd.go | 115 ++++++++++++++ internal/vector/prepared_simd_test.go | 22 +++ internal/vector/prepared_test.go | 168 +++++++++++++++++++++ 14 files changed, 610 insertions(+), 20 deletions(-) create mode 100644 internal/cli/cluster_graph_bench_test.go create mode 100644 internal/cli/cluster_graph_scoring_test.go create mode 100644 internal/vector/prepared.go create mode 100644 internal/vector/prepared_nosimd.go create mode 100644 internal/vector/prepared_simd.go create mode 100644 internal/vector/prepared_simd_test.go create mode 100644 internal/vector/prepared_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb5bd21..17d142a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,6 +76,13 @@ jobs: echo "total coverage: ${total}%" awk -v total="$total" 'BEGIN { if (total + 0 < 85.0) { printf("coverage %.1f%% is below 85.0%%\n", total); exit 1 } }' + - name: Test portable SIMD and scalar fallback + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + # Release binaries are built with this experiment, so test everything they ship. + GOEXPERIMENT=simd go test ./... + GOEXPERIMENT=simd GODEBUG=simd=0 go test ./internal/vector ./internal/cli + - name: Build run: go build -ldflags "-X github.com/openclaw/gitcrawl/internal/cli.version=${GITHUB_SHA:0:7}" -o bin/gitcrawl ./cmd/gitcrawl diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 676f374e..c4ed3986 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -11,6 +11,7 @@ builds: binary: gitcrawl env: - CGO_ENABLED=0 + - GOEXPERIMENT=simd ldflags: - -s -w -X github.com/openclaw/gitcrawl/internal/cli.version={{ .Version }} targets: @@ -23,6 +24,7 @@ builds: binary: gitcrawl env: - CGO_ENABLED=0 + - GOEXPERIMENT=simd ldflags: - -s -w -X github.com/openclaw/gitcrawl/internal/cli.version={{ .Version }} targets: diff --git a/CHANGELOG.md b/CHANGELOG.md index d4714a60..7a72f533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Prepare vectors once for 2.6× faster cluster scoring and 2.0× faster exact neighbors; enable `GOEXPERIMENT=simd` in release builds for a further 2.6×/1.7× on Apple M3 Ultra (1,024 dimensions). - Disable automatic Git maintenance during portable-refresh fixture setup so temporary repositories do not launch detached cleanup work. ## 0.12.0 - 2026-09-24 diff --git a/Dockerfile b/Dockerfile index 2c30ff53..53f1fce5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . ARG VERSION=dev -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath \ +RUN CGO_ENABLED=0 GOEXPERIMENT=simd GOOS=linux go build -trimpath \ -ldflags="-s -w -X github.com/openclaw/gitcrawl/internal/cli.version=${VERSION}" \ -o /out/gitcrawl ./cmd/gitcrawl diff --git a/docs/installation.md b/docs/installation.md index 6ef6fa5e..c5bf7f04 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -76,6 +76,12 @@ go build \ Symlink or copy `bin/gitcrawl` somewhere on your `PATH` (`~/bin`, `/usr/local/bin`, `~/.local/bin`). +Source builds use scalar vector scoring by default. Set `GOEXPERIMENT=simd` on +`go build` to enable Go 1.27's experimental portable SIMD kernels for clustering +and exact neighbors. The API may change in Go 1.28; unsupported hardware and +`GODEBUG=simd=0` use the scalar kernels. Release archives and Docker builds enable +this SIMD path. + ## GitHub CLI shim migration `gitcrawl gh` moved to Octopool: diff --git a/internal/cli/cluster_graph.go b/internal/cli/cluster_graph.go index 688ad413..72d59eb6 100644 --- a/internal/cli/cluster_graph.go +++ b/internal/cli/cluster_graph.go @@ -81,24 +81,8 @@ func buildDurableClusterInputs(ctx context.Context, st *store.Store, repoID int6 } nodes = append(nodes, clusterer.Node{ThreadID: stored.ThreadID, Number: thread.Number, Title: thread.Title}) } - candidateByPair := map[string]clusterer.Edge{} - for left := 0; left < len(nodes); left++ { - for right := left + 1; right < len(nodes); right++ { - leftID := nodes[left].ThreadID - rightID := nodes[right].ThreadID - score := vector.Cosine(vectorByThreadID[leftID], vectorByThreadID[rightID]) - if score < options.Threshold { - continue - } - if score < highConfidenceEdgeScore && titleTokenOverlap(threads[leftID].Title, threads[rightID].Title) < weakEdgeMinTitleOverlap { - continue - } - if threads[leftID].Kind != threads[rightID].Kind && score < options.CrossKindThreshold { - continue - } - upsertClusterEdge(candidateByPair, leftID, rightID, score) - } - } + candidateByPair := scoreClusterEdges(nodes, threads, vectorByThreadID, options) + repoFullName, err := repositoryFullNameByID(ctx, st, repoID) if err != nil { return nil, 0, err @@ -155,6 +139,32 @@ func buildDurableClusterInputs(ctx context.Context, st *store.Store, repoID int6 return inputs, len(edges), nil } +func scoreClusterEdges(nodes []clusterer.Node, threads map[int64]store.Thread, vectorByThreadID map[int64][]float64, options clusterBuildOptions) map[string]clusterer.Edge { + prepared := make([]vector.Prepared, len(nodes)) + for i, node := range nodes { + prepared[i] = vector.Prepare(vectorByThreadID[node.ThreadID]) + } + candidateByPair := map[string]clusterer.Edge{} + for left := 0; left < len(nodes); left++ { + for right := left + 1; right < len(nodes); right++ { + leftID := nodes[left].ThreadID + rightID := nodes[right].ThreadID + score := prepared[left].Cosine(prepared[right]) + if score < options.Threshold { + continue + } + if score < highConfidenceEdgeScore && titleTokenOverlap(threads[leftID].Title, threads[rightID].Title) < weakEdgeMinTitleOverlap { + continue + } + if threads[leftID].Kind != threads[rightID].Kind && score < options.CrossKindThreshold { + continue + } + upsertClusterEdge(candidateByPair, leftID, rightID, score) + } + } + return candidateByPair +} + func upsertClusterEdge(edges map[string]clusterer.Edge, leftID, rightID int64, score float64) { if leftID == rightID { return diff --git a/internal/cli/cluster_graph_bench_test.go b/internal/cli/cluster_graph_bench_test.go new file mode 100644 index 00000000..b7a6924c --- /dev/null +++ b/internal/cli/cluster_graph_bench_test.go @@ -0,0 +1,133 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "math/rand/v2" + "path/filepath" + "testing" + "time" + + clusterer "github.com/openclaw/gitcrawl/internal/cluster" + "github.com/openclaw/gitcrawl/internal/config" + "github.com/openclaw/gitcrawl/internal/store" +) + +func clusterBenchmarkData(count int) ([]clusterer.Node, map[int64]store.Thread, map[int64][]float64) { + rng := rand.New(rand.NewPCG(7, 11)) + dims := config.Default().OpenAI.EmbedDimensions + nodes := make([]clusterer.Node, count) + threads := make(map[int64]store.Thread, count) + vectors := make(map[int64][]float64, count) + var center []float64 + for i := range nodes { + if i%8 == 0 { + center = make([]float64, dims) + for j := range center { + center[j] = rng.NormFloat64() + } + } + values := make([]float64, dims) + for j := range values { + values[j] = center[j] + 0.15*rng.NormFloat64() + } + id := int64(i + 1) + title := fmt.Sprintf("Synthetic embedding group %d", i/8) + nodes[i] = clusterer.Node{ThreadID: id, Number: i + 1, Title: title} + kind := "issue" + if i%3 == 0 { + kind = "pull_request" + } + threads[id] = store.Thread{Number: i + 1, Kind: kind, State: "open", Title: title} + vectors[id] = values + } + return nodes, threads, vectors +} + +var benchmarkEdges map[string]clusterer.Edge + +func BenchmarkClusterEdges2000(b *testing.B) { + nodes, threads, vectors := clusterBenchmarkData(2000) + opts := clusterBuildOptions{Threshold: 0.90, CrossKindThreshold: defaultCrossKindMinScore} + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + benchmarkEdges = scoreClusterEdges(nodes, threads, vectors, opts) + } +} + +func BenchmarkClusterCommand2000(b *testing.B) { + ctx := context.Background() + b.Setenv("GITCRAWL_NO_UPDATE_CHECK", "1") + dir := b.TempDir() + configPath, dbPath := filepath.Join(dir, "config.toml"), filepath.Join(dir, "gitcrawl.db") + app := New() + app.Stdout, app.Stderr = io.Discard, io.Discard + if err := app.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + b.Fatal(err) + } + st, err := store.Open(ctx, dbPath) + if err != nil { + b.Fatal(err) + } + defer st.Close() + now := time.Now().UTC().Format(time.RFC3339Nano) + repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "synthetic", Name: "bench", FullName: "synthetic/bench", UpdatedAt: now}) + if err != nil { + b.Fatal(err) + } + nodes, threads, vectors := clusterBenchmarkData(2000) + if err := st.WithTx(ctx, func(st *store.Store) error { + for _, node := range nodes { + thread := threads[node.ThreadID] + thread.RepoID, thread.GitHubID = repoID, fmt.Sprint(node.ThreadID) + thread.LabelsJSON, thread.AssigneesJSON, thread.RawJSON = "[]", "[]", "{}" + thread.ContentHash, thread.UpdatedAt = fmt.Sprint(node.ThreadID), now + if _, err := st.UpsertThread(ctx, thread); err != nil { + return err + } + } + return nil + }); err != nil { + b.Fatal(err) + } + tasks, err := st.ListEmbeddingTasks(ctx, store.EmbeddingTaskOptions{RepoID: repoID, Basis: "title_original", Model: "text-embedding-3-small", Force: true}) + if err != nil { + b.Fatal(err) + } + if len(tasks) != len(nodes) { + b.Fatalf("tasks: %d want %d", len(tasks), len(nodes)) + } + // Bulk fixture insertion stays outside the timed command and avoids per-row fsync. + tx, err := st.DB().BeginTx(ctx, nil) + if err != nil { + b.Fatal(err) + } + defer tx.Rollback() + for _, task := range tasks { + values := vectors[task.ThreadID] + data, err := json.Marshal(values) + if err != nil { + b.Fatal(err) + } + if _, err := tx.ExecContext(ctx, `insert into thread_vectors(thread_id,basis,model,dimensions,content_hash,vector_json,vector_backend,created_at,updated_at) values(?,?,?,?,?,?,?,?,?)`, task.ThreadID, "title_original", "text-embedding-3-small", len(values), task.ContentHash, string(data), "exact", now, now); err != nil { + b.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + + if err := st.Close(); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if err := app.Run(ctx, []string{"--config", configPath, "cluster", "synthetic/bench", "--threshold", "0.90", "--json"}); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/cli/cluster_graph_scoring_test.go b/internal/cli/cluster_graph_scoring_test.go new file mode 100644 index 00000000..a37030d8 --- /dev/null +++ b/internal/cli/cluster_graph_scoring_test.go @@ -0,0 +1,58 @@ +package cli + +import ( + clusterer "github.com/openclaw/gitcrawl/internal/cluster" + "github.com/openclaw/gitcrawl/internal/store" + "github.com/openclaw/gitcrawl/internal/vector" + "math" + "testing" +) + +func TestPreparedClusterEdgesMatchOriginal(t *testing.T) { + nodes, threads, vectors := clusterBenchmarkData(96) + vectors[1] = []float64{math.NaN()} + vectors[2] = nil + vectors[3] = []float64{0} + vectors[4] = []float64{1, 2} + vectors[5] = []float64{math.MaxFloat64, math.MaxFloat64} + vectors[6] = []float64{math.SmallestNonzeroFloat64, math.SmallestNonzeroFloat64} + var maxDeviation float64 + for _, threshold := range []float64{-1, 0, 0.3, 0.9, 1} { + opts := clusterBuildOptions{Threshold: threshold, CrossKindThreshold: defaultCrossKindMinScore} + got, want := scoreClusterEdges(nodes, threads, vectors, opts), originalClusterEdges(nodes, threads, vectors, opts) + if len(got) != len(want) { + t.Fatalf("threshold %g: %d edges want %d", threshold, len(got), len(want)) + } + for pair, expected := range want { + actual, ok := got[pair] + deviation := math.Abs(actual.Score - expected.Score) + maxDeviation = max(maxDeviation, deviation) + if !ok || actual.LeftThreadID != expected.LeftThreadID || actual.RightThreadID != expected.RightThreadID || deviation > 1e-12 { + t.Fatalf("threshold %g pair %s: %+v want %+v", threshold, pair, actual, expected) + } + } + } + t.Logf("max cluster-edge score deviation: %.17g", maxDeviation) +} + +func originalClusterEdges(nodes []clusterer.Node, threads map[int64]store.Thread, vectorByThreadID map[int64][]float64, options clusterBuildOptions) map[string]clusterer.Edge { + candidateByPair := map[string]clusterer.Edge{} + for left := 0; left < len(nodes); left++ { + for right := left + 1; right < len(nodes); right++ { + leftID := nodes[left].ThreadID + rightID := nodes[right].ThreadID + score := vector.Cosine(vectorByThreadID[leftID], vectorByThreadID[rightID]) + if score < options.Threshold { + continue + } + if score < highConfidenceEdgeScore && titleTokenOverlap(threads[leftID].Title, threads[rightID].Title) < weakEdgeMinTitleOverlap { + continue + } + if threads[leftID].Kind != threads[rightID].Kind && score < options.CrossKindThreshold { + continue + } + upsertClusterEdge(candidateByPair, leftID, rightID, score) + } + } + return candidateByPair +} diff --git a/internal/vector/exact.go b/internal/vector/exact.go index 60f58639..3b236605 100644 --- a/internal/vector/exact.go +++ b/internal/vector/exact.go @@ -56,6 +56,8 @@ func queryExact(ctx context.Context, items []Item, query []float64, limit int, e if err := validateExactQuery(query); err != nil { return nil, err } + preparedQuery := Prepare(query) + scratch := make([]float64, len(query)) scored := make([]crawlvector.Scored[Neighbor], 0, len(items)) for _, item := range items { if err := ctx.Err(); err != nil { @@ -64,7 +66,10 @@ func queryExact(ctx context.Context, items []Item, query []float64, limit int, e if item.ThreadID == excludeThreadID { continue } - score := Cosine(query, item.Vector) + if len(item.Vector) != len(query) { + continue + } + score := preparedQuery.Cosine(prepareInto(scratch, item.Vector)) if math.IsNaN(score) || math.IsInf(score, 0) || score <= 0 { continue } diff --git a/internal/vector/prepared.go b/internal/vector/prepared.go new file mode 100644 index 00000000..654eb075 --- /dev/null +++ b/internal/vector/prepared.go @@ -0,0 +1,52 @@ +package vector + +import "math" + +// Prepared holds an immutable, validated copy of a vector for repeated scoring. +// Its zero value scores zero against every vector. +type Prepared struct { + values []float64 + magnitude float64 +} + +// Prepare scales by max-abs before computing the magnitude, so even very large +// or subnormal finite inputs are safe. Empty, zero and non-finite inputs are invalid. +func Prepare(values []float64) Prepared { + return prepareInto(make([]float64, len(values)), values) +} + +func prepareScalar(dst, values []float64) Prepared { + var maxAbs float64 + for _, value := range values { + if math.IsNaN(value) || math.IsInf(value, 0) { + return Prepared{} + } + maxAbs = max(maxAbs, math.Abs(value)) + } + if maxAbs == 0 { + return Prepared{} + } + var magnitude float64 + for i, value := range values { + scaled := value / maxAbs + dst[i] = scaled + magnitude += scaled * scaled + } + return Prepared{values: dst[:len(values)], magnitude: math.Sqrt(magnitude)} +} + +// Cosine returns zero for invalid or mismatched vectors and clamps to [-1, 1]. +func (left Prepared) Cosine(right Prepared) float64 { + if len(left.values) == 0 || len(left.values) != len(right.values) { + return 0 + } + return max(-1, min(1, preparedDot(left.values, right.values)/(left.magnitude*right.magnitude))) +} + +func dotScalar(left, right []float64) float64 { + var dot float64 + for i, value := range left { + dot += value * right[i] + } + return dot +} diff --git a/internal/vector/prepared_nosimd.go b/internal/vector/prepared_nosimd.go new file mode 100644 index 00000000..6836af00 --- /dev/null +++ b/internal/vector/prepared_nosimd.go @@ -0,0 +1,11 @@ +//go:build !goexperiment.simd + +package vector + +func preparedDot(left, right []float64) float64 { + return dotScalar(left, right) +} + +func prepareInto(dst, values []float64) Prepared { + return prepareScalar(dst, values) +} diff --git a/internal/vector/prepared_simd.go b/internal/vector/prepared_simd.go new file mode 100644 index 00000000..8816dab5 --- /dev/null +++ b/internal/vector/prepared_simd.go @@ -0,0 +1,115 @@ +//go:build goexperiment.simd + +package vector + +import ( + "math" + "simd" +) + +// Emulation is slower than the scalar kernels. +var emulatedSIMD = simd.Emulated() + +// Lane buffers cover 2048-bit vectors (arm64 SVE is planned for Go 1.28), since +// release builds enable this experiment and Store panics on a short slice. +const maxFloat64Lanes = 32 + +func preparedDot(left, right []float64) float64 { + if emulatedSIMD { + return dotScalar(left, right) + } + var a, b, c, d simd.Float64s + lanes := a.Len() + i := 0 + for ; i+4*lanes <= len(left); i += 4 * lanes { + a = simd.LoadFloat64s(left[i:]).MulAdd(simd.LoadFloat64s(right[i:]), a) + b = simd.LoadFloat64s(left[i+lanes:]).MulAdd(simd.LoadFloat64s(right[i+lanes:]), b) + c = simd.LoadFloat64s(left[i+2*lanes:]).MulAdd(simd.LoadFloat64s(right[i+2*lanes:]), c) + d = simd.LoadFloat64s(left[i+3*lanes:]).MulAdd(simd.LoadFloat64s(right[i+3*lanes:]), d) + } + for ; i+lanes <= len(left); i += lanes { + a = simd.LoadFloat64s(left[i:]).MulAdd(simd.LoadFloat64s(right[i:]), a) + } + // Reduce once, outside the loop. + var sums [maxFloat64Lanes]float64 + a.Add(b).Add(c.Add(d)).Store(sums[:]) + var dot float64 + for _, value := range sums[:lanes] { + dot += value + } + for ; i < len(left); i++ { + dot += left[i] * right[i] + } + return dot +} + +func prepareInto(dst, values []float64) Prepared { + if emulatedSIMD { + return prepareScalar(dst, values) + } + var maxima simd.Float64s + var invalid simd.Mask64s + finiteLimit := simd.BroadcastFloat64s(math.MaxFloat64) + lanes := maxima.Len() + i := 0 + for ; i+lanes <= len(values); i += lanes { + v := simd.LoadFloat64s(values[i:]) + abs := v.Abs() + invalid = invalid.Or(v.NotEqual(v)).Or(abs.Greater(finiteLimit)) + maxima = maxima.Max(abs) + } + var maxValues [maxFloat64Lanes]float64 + var bad [maxFloat64Lanes]int64 + maxima.Store(maxValues[:]) + invalid.ToInt64s().Store(bad[:]) + var maxAbs float64 + for lane := 0; lane < lanes; lane++ { + if bad[lane] != 0 { + return Prepared{} + } + maxAbs = max(maxAbs, maxValues[lane]) + } + for _, value := range values[i:] { + if math.IsNaN(value) || math.IsInf(value, 0) { + return Prepared{} + } + maxAbs = max(maxAbs, math.Abs(value)) + } + if maxAbs == 0 { + return Prepared{} + } + divisor := simd.BroadcastFloat64s(maxAbs) + var a, b, c, d simd.Float64s + i = 0 + for ; i+4*lanes <= len(values); i += 4 * lanes { + v0 := simd.LoadFloat64s(values[i:]).Div(divisor) + v1 := simd.LoadFloat64s(values[i+lanes:]).Div(divisor) + v2 := simd.LoadFloat64s(values[i+2*lanes:]).Div(divisor) + v3 := simd.LoadFloat64s(values[i+3*lanes:]).Div(divisor) + v0.Store(dst[i:]) + v1.Store(dst[i+lanes:]) + v2.Store(dst[i+2*lanes:]) + v3.Store(dst[i+3*lanes:]) + a = v0.MulAdd(v0, a) + b = v1.MulAdd(v1, b) + c = v2.MulAdd(v2, c) + d = v3.MulAdd(v3, d) + } + for ; i+lanes <= len(values); i += lanes { + v := simd.LoadFloat64s(values[i:]).Div(divisor) + v.Store(dst[i:]) + a = v.MulAdd(v, a) + } + var sums [maxFloat64Lanes]float64 + a.Add(b).Add(c.Add(d)).Store(sums[:]) + var magnitude float64 + for _, value := range sums[:lanes] { + magnitude += value + } + for ; i < len(values); i++ { + scaled := values[i] / maxAbs + dst[i] = scaled + magnitude += scaled * scaled + } + return Prepared{values: dst[:len(values)], magnitude: math.Sqrt(magnitude)} +} diff --git a/internal/vector/prepared_simd_test.go b/internal/vector/prepared_simd_test.go new file mode 100644 index 00000000..1cce7355 --- /dev/null +++ b/internal/vector/prepared_simd_test.go @@ -0,0 +1,22 @@ +//go:build goexperiment.simd + +package vector + +import ( + "simd" + "testing" +) + +func TestSIMDMode(t *testing.T) { + var lanes simd.Float64s + t.Logf("float64 lanes=%d emulated=%t", lanes.Len(), emulatedSIMD) + if !emulatedSIMD { + return + } + items := benchmarkItems(2, 1024) + left := Prepare(items[0].Vector) + right := Prepare(items[1].Vector) + if got, want := left.Cosine(right), Cosine(items[0].Vector, items[1].Vector); got != want { + t.Fatalf("emulated fallback: got %.17g want scalar %.17g", got, want) + } +} diff --git a/internal/vector/prepared_test.go b/internal/vector/prepared_test.go new file mode 100644 index 00000000..3d90bcdf --- /dev/null +++ b/internal/vector/prepared_test.go @@ -0,0 +1,168 @@ +package vector + +import ( + "context" + "math" + "math/rand/v2" + "sort" + "testing" + + "github.com/openclaw/gitcrawl/internal/config" +) + +// Lane reduction and FMA change rounding; normalized scores must agree within 1e-12. +func TestPreparedCosine(t *testing.T) { + rng := rand.New(rand.NewPCG(17, 29)) + lengths := []int{1024, 1536, 3072} + // Cover three times the maximum float32 lane count, including odd tails. + for n := 0; n <= 3*16+7; n++ { + lengths = append(lengths, n) + } + var maxDeviation, maxDotDeviation float64 + for _, n := range lengths { + left, right := make([]float64, n), make([]float64, n) + for trial := 0; trial < 24; trial++ { + for i := range left { + left[i], right[i] = rng.NormFloat64(), rng.NormFloat64() + if trial%3 == 0 { + right[i] = left[i] + right[i]*0.01 + } + if trial%3 == 1 { + right[i] = -left[i] + } + if trial%4 == 0 { + left[i] *= 1e300 + right[i] *= 1e-300 + } + } + l, r := Prepare(left), Prepare(right) + want, got := Cosine(left, right), l.Cosine(r) + deviation := math.Abs(want - got) + maxDeviation = max(maxDeviation, deviation) + if math.IsNaN(got) || deviation > 1e-12 { + t.Fatalf("n=%d trial=%d: got %.17g want %.17g", n, trial, got, want) + } + if n > 0 { + scalar := dotScalar(l.values, r.values) / (l.magnitude * r.magnitude) + actual := preparedDot(l.values, r.values) / (l.magnitude * r.magnitude) + maxDotDeviation = max(maxDotDeviation, math.Abs(scalar-actual)) + if math.Abs(scalar-actual) > 1e-12 { + t.Fatalf("dot n=%d: got %.17g want %.17g", n, actual, scalar) + } + } + } + } + t.Logf("max cosine deviation from original: %.17g; dispatch-vs-scalar normalized dot: %.17g", maxDeviation, maxDotDeviation) +} + +func TestPreparedSpecialValues(t *testing.T) { + cases := [][]float64{nil, {}, {0}, {0, 0}, {1}, {-1}, {1, -1}, {math.MaxFloat64, -math.MaxFloat64}, {math.SmallestNonzeroFloat64, -math.SmallestNonzeroFloat64}, {math.NaN(), 1}, {1, math.Inf(1)}, {math.Inf(-1), 1}} + for _, left := range cases { + for _, right := range cases { + want, got := Cosine(left, right), Prepare(left).Cosine(Prepare(right)) + if math.IsNaN(got) || math.Abs(want-got) > 1e-12 { + t.Fatalf("%v / %v: got %.17g want %.17g", left, right, got, want) + } + } + } + for n := 1; n <= 3*16+7; n++ { + for i := 0; i < n; i++ { + for _, bad := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + values := make([]float64, n) + values[0], values[i] = 1, bad + if got := Prepare(values).Cosine(Prepare(values)); got != 0 { + t.Fatalf("n=%d index=%d: %g", n, i, got) + } + } + } + } + values := []float64{1, 2, 3} + prepared := Prepare(values) + values[0] = math.NaN() + if got := prepared.Cosine(Prepare([]float64{1, 2, 3})); math.Abs(got-1) > 1e-12 { + t.Fatalf("Prepare aliases input: %g", got) + } + if got := preparedDot(nil, nil); got != 0 { + t.Fatalf("empty dot = %g", got) + } + if got := preparedDot(make([]float64, 55), make([]float64, 55)); got != 0 { + t.Fatalf("zero dot = %g", got) + } +} + +func TestPreparedQueryMatchesOriginal(t *testing.T) { + items := benchmarkItems(256, 1024) + query := items[0].Vector + var want []Neighbor + for _, item := range items[1:] { + score := Cosine(query, item.Vector) + if score > 0 { + want = append(want, Neighbor{ThreadID: item.ThreadID, Score: score}) + } + } + sort.Slice(want, func(i, j int) bool { + if want[i].Score == want[j].Score { + return want[i].ThreadID < want[j].ThreadID + } + return want[i].Score > want[j].Score + }) + want = want[:20] + got := Query(items, query, 20, items[0].ThreadID) + if len(got) != len(want) { + t.Fatalf("neighbors: %d want %d", len(got), len(want)) + } + for i := range want { + if got[i].ThreadID != want[i].ThreadID || math.Abs(got[i].Score-want[i].Score) > 1e-12 { + t.Fatalf("neighbor %d: %+v want %+v", i, got[i], want[i]) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := queryExact(ctx, items, query, 20, 0); err != context.Canceled { + t.Fatalf("cancellation: %v", err) + } +} + +func benchmarkItems(count, dims int) []Item { + rng := rand.New(rand.NewPCG(7, 11)) + items := make([]Item, count) + for i := range items { + values := make([]float64, dims) + for j := range values { + values[j] = rng.NormFloat64() + } + items[i] = Item{ThreadID: int64(i + 1), Vector: values} + } + return items +} + +var benchmarkScore float64 +var benchmarkNeighbors []Neighbor + +func BenchmarkCosine(b *testing.B) { + items := benchmarkItems(2, config.Default().OpenAI.EmbedDimensions) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + benchmarkScore = Cosine(items[0].Vector, items[1].Vector) + } +} + +func BenchmarkPreparedCosine(b *testing.B) { + items := benchmarkItems(2, config.Default().OpenAI.EmbedDimensions) + left, right := Prepare(items[0].Vector), Prepare(items[1].Vector) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + benchmarkScore = left.Cosine(right) + } +} + +func BenchmarkQueryExact20000(b *testing.B) { + items := benchmarkItems(20000, config.Default().OpenAI.EmbedDimensions) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + benchmarkNeighbors = Query(items, items[0].Vector, 20, items[0].ThreadID) + } +} From b20b4f3cc327dc5a9bf4624c38038edc7e73edad Mon Sep 17 00:00:00 2001 From: Hannes Rudolph Date: Mon, 28 Sep 2026 05:16:20 -0600 Subject: [PATCH 17/21] feat: collect repository metrics in an independent store (#206) * feat: collect repository metrics in an independent store * docs: describe isolated source-built metrics installation * test: use synthetic multi-project metrics examples * fix: serialize metrics writers before scheduled collection * docs: describe local signing for metrics LaunchAgents * docs: distinguish macOS volume prompts from access probes * style: fix cloud ingest formatting gate Restore gofmt indentation in the oversized-row error path. The previous Ubuntu CI run stopped here before executing tests; no runtime behavior changes. * fix(metrics): protect store identity and historical observations Recover returned first-initialization failures without removing pre-existing files, reject database hardlink aliases, compare imported timestamps and UTC days chronologically, and enforce the config size limit. Preserve original import IDs, timestamp spelling, NULLs, zeroes, and corrections. * fix(metrics): preserve partial results and stop exhausted collection Stop on exhausted quota while keeping completed reads, including cancellation after a response. Reject missing provider lists and emit structured partial results and explicit zero status totals. Document archive ownership inspection, permissions, recovery limits, and collection request costs. * fix(metrics): validate opened database before schema writes Keep read-only prechecks, then pin the writable connection and validate ownership under BEGIN IMMEDIATE before applying schema and metadata atomically. Require new databases to remain empty, reject replaced file identities, and retain guarded cleanup. Cover file and parent swaps, in-place changes, and rollback when metadata insertion fails. --------- Co-authored-by: Peter Steinberger --- README.md | 1 + docs/commands.md | 11 + docs/installation.md | 5 + docs/metrics.md | 303 +++++++++++ internal/cli/app.go | 4 + internal/cli/control.go | 5 +- internal/cli/help.go | 14 + internal/cli/metrics.go | 84 +++ internal/cli/metrics_test.go | 224 ++++++++ internal/cli/releasecheck.go | 5 + internal/github/headline_metrics.go | 70 +++ internal/headlinemetrics/github.go | 171 ++++++ internal/headlinemetrics/github_test.go | 310 +++++++++++ internal/headlinemetrics/lock.go | 55 ++ internal/headlinemetrics/lock_other.go | 16 + internal/headlinemetrics/lock_test.go | 152 ++++++ internal/headlinemetrics/lock_unix.go | 33 ++ internal/headlinemetrics/lock_windows.go | 33 ++ internal/headlinemetrics/metrics.go | 562 ++++++++++++++++++++ internal/headlinemetrics/metrics_test.go | 646 +++++++++++++++++++++++ scripts/build-docs-site.mjs | 2 +- 21 files changed, 2704 insertions(+), 2 deletions(-) create mode 100644 docs/metrics.md create mode 100644 internal/cli/metrics.go create mode 100644 internal/cli/metrics_test.go create mode 100644 internal/github/headline_metrics.go create mode 100644 internal/headlinemetrics/github.go create mode 100644 internal/headlinemetrics/github_test.go create mode 100644 internal/headlinemetrics/lock.go create mode 100644 internal/headlinemetrics/lock_other.go create mode 100644 internal/headlinemetrics/lock_test.go create mode 100644 internal/headlinemetrics/lock_unix.go create mode 100644 internal/headlinemetrics/lock_windows.go create mode 100644 internal/headlinemetrics/metrics.go create mode 100644 internal/headlinemetrics/metrics_test.go diff --git a/README.md b/README.md index 9fd7ae0c..7488818e 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Octopool owns pooled live `gh` reads. Gitcrawl keeps local mirror, search, clust | Check archive health | `gitcrawl status` / `gitcrawl doctor` | [Configuration](docs/configuration.md) | | Refresh a portable subscriber | `gitcrawl portable refresh --expected-remote URL` | [Portable stores](docs/portable-stores.md#routine-subscriber-refresh) | | Mirror GitHub threads | `gitcrawl sync owner/repo` | [Sync](docs/sync.md) | +| Collect repository counters and releases | `gitcrawl metrics collect\|import\|status --config metrics.json` | [Repository metrics](docs/metrics.md) | | Search threads or indexed code | `gitcrawl search ...` | [Search](docs/search.md) | | Build and inspect clusters | `gitcrawl refresh`, `clusters`, `tui` | [Clustering](docs/clustering.md) | | Export a code-free conversation snapshot | `gitcrawl capture owner/repo` | [Capture](docs/capture.md) | diff --git a/docs/commands.md b/docs/commands.md index aa4f11ef..3d3b740e 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -38,6 +38,17 @@ These work on every command. | `gitcrawl configure [--summary-model --embed-model --embedding-basis --json]` | Update model fields in `config.toml` | [Configuration](/configuration/#gitcrawl-configure) | | `gitcrawl version` | Print version | — | +## Repository metrics + +| Command | Purpose | Docs | +| --- | --- | --- | +| `gitcrawl metrics collect --config metrics.json [--json]` | Observe stars, forks, actual watchers, open PRs/issues, optional daily clones, and stable releases in a separate database | [Repository metrics](/metrics/) | +| `gitcrawl metrics import --config metrics.json [--json]` | Atomically import scoped NDJSON history from stdin, preserving NULLs and IDs | [Repository metrics](/metrics/#storage-imports-and-failures) | +| `gitcrawl metrics status --config metrics.json [--json]` | Inspect the metrics database without writes or network calls | [Repository metrics](/metrics/) | + +For `metrics`, `--config` selects an independent JSON config; it never selects or +initializes the normal thread archive. No embedding or model calls are made. + ## Sync | Command | Purpose | Docs | diff --git a/docs/installation.md b/docs/installation.md index c5bf7f04..23bb9b75 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -76,6 +76,11 @@ go build \ Symlink or copy `bin/gitcrawl` somewhere on your `PATH` (`~/bin`, `/usr/local/bin`, `~/.local/bin`). +For a metrics-only source deployment alongside an existing portable subscriber, +use an [isolated versioned metrics runtime](/metrics/#isolated-source-built-installation) +and verify it with `metrics status` and its separate config. Keep the subscriber's +binary selection and refresh job unchanged. + Source builds use scalar vector scoring by default. Set `GOEXPERIMENT=simd` on `go build` to enable Go 1.27's experimental portable SIMD kernels for clustering and exact neighbors. The API may change in Go 1.28; unsupported hardware and diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 00000000..9b803c12 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,303 @@ +--- +title: Repository metrics +nav_order: 16 +permalink: /metrics/ +--- + +# Repository metrics + +`gitcrawl metrics` collects repository headline counters and stable release events +into a separate, private SQLite database. It does not sync the thread archive, +refresh a portable store, generate embeddings, call a model, or start a scheduler. + +## Configuration and commands + +Create a metrics JSON config. `database` must be an absolute filesystem path to a +new file or an existing Gitcrawl metrics database, outside your archive and portable +store. The example path is illustrative; choose a private directory on your host. + +```json +{ + "database": "/private/metrics/gitcrawl/metrics.sqlite", + "targets": [ + {"entity": "OpenClaw", "target": "openclaw/openclaw"}, + {"entity": "Example", "target": "example/project"} + ] +} +``` + +```sh +gitcrawl metrics collect --config /private/metrics/github.json --json +gitcrawl metrics import --config /private/metrics/github.json --json < history.ndjson +gitcrawl metrics status --config /private/metrics/github.json --json +gitcrawl help metrics +``` + +The config is independent of `config.toml`, `GITCRAWL_CONFIG`, and `GITCRAWL_DB_PATH`. For these commands, +`--config` selects the metrics JSON file; it can appear before `metrics` or after +the subcommand. Global output flags and command-local `--json` work normally. + +Public counters and releases can be read without authentication. Authenticated +collection uses `GITHUB_TOKEN`, then the normal native `gh auth token` resolver. +An optional `tokenEnv` selects another environment variable. A global +`--github-token-command /absolute/executable` selects the normal managed credential +provider exclusively on supported platforms. Tokens never appear in results. +The metrics config contains no token values; cookie authentication is not used. + +## Isolated source-built installation + +A source-built metrics runtime can coexist with an official Gitcrawl installation +that refreshes a portable mirror. Give the metrics binary a private, versioned +directory and invoke that exact executable. The installed metrics CLI path is: + +```text +$HOME/.local/share/gitcrawl/metrics-runtimes//gitcrawl +``` + +Copy an already validated artifact into a new directory; do not overwrite an +existing version. Record its full source commit and expected SHA-256 from the +validation handoff, verify the hash before and after copying, and use directory +mode `0700` and executable mode `0500`. This installation does not replace a +`current` symlink, the command on `PATH`, an archive config, or a refresh job. + +Local source builds do not require official release signing credentials under the +[installation policy](/installation/#install-from-source). On macOS, verify the +source artifact's signature with `codesign --verify --strict` before and after +copying. This is source-build verification, not official release notarization; +official release signing and notarization remain the [release workflow's](/releasing/) +responsibility. Do not change signing policies or remove quarantine to force an +untrusted artifact to run. + +The installation check is read-only and uses the separately provided metrics config: + +```sh +metrics_revision=SOURCE_COMMIT +metrics_binary="$HOME/.local/share/gitcrawl/metrics-runtimes/$metrics_revision/gitcrawl" +"$metrics_binary" --version +"$metrics_binary" metrics status \ + --config "$HOME/.local/share/gitcrawl/metrics.json" --json +``` + +Keep a machine-local installation receipt at the path returned by +`git rev-parse --git-path metrics-runtime-installation.json`. Record the exact +resolved CLI/config/database paths, source commit, artifact hash, signature result, +read-only status, and preservation checks there. Git metadata keeps these private +host details out of the public documentation and PR. The coordinator's handoff +should contain the same exact CLI path. Installation alone does not authorize +collection, imports, scheduling, or a final cutover. + +### Local signing for background collection + +A directly launched macOS job has its own file-access identity; permission granted +to a terminal does not prove the background executable has access. When the owner +authorizes an existing local signing certificate, sign a copy of the verified +runtime with a stable identifier and an explicit requirement bound to that +certificate. Use the already selected public SHA-1 fingerprint, without searching +for another identity or changing keychain trust or access controls. + +```sh +metrics_revision=SOURCE_COMMIT +metrics_identity=SELECTED_PUBLIC_CERTIFICATE_SHA1 +metrics_identifier=com.example.gitcrawl.metrics +metrics_source="$HOME/.local/share/gitcrawl/metrics-runtimes/$metrics_revision/gitcrawl" +metrics_install="$HOME/.local/libexec/gitcrawl-metrics/$metrics_revision" +metrics_requirement="identifier \"$metrics_identifier\" and certificate leaf = H\"$metrics_identity\"" +codesign --verify --strict "$metrics_source" +# Check the source SHA-256 against the validation receipt before copying. +umask 077 +mkdir -p "$(dirname "$metrics_install")" +mkdir "$metrics_install" +cp "$metrics_source" "$metrics_install/gitcrawl" +chmod 0700 "$metrics_install/gitcrawl" +codesign --force --sign "$metrics_identity" --identifier "$metrics_identifier" \ + --requirements "=designated => $metrics_requirement" --timestamp=none \ + "$metrics_install/gitcrawl" +codesign --verify --strict --test-requirement "=$metrics_requirement" \ + "$metrics_install/gitcrawl" +codesign --display --requirements - "$metrics_install/gitcrawl" +chmod 0500 "$metrics_install/gitcrawl" +shasum -a 256 "$metrics_source" "$metrics_install/gitcrawl" +``` + +Record both hashes, the source commit, certificate fingerprint, identifier and +requirement in the private receipt. Preserve the original runtime. Point only the +metrics LaunchAgent's first argument at the signed copy, preserving its other +settings. If this job was disabled, enable its exact label before bootstrapping. +Request one collection and let the user approve normal macOS file-access prompts. +Follow the specific permission request through its prompt and decision: a default +`SystemPolicyAllFiles` probe denial can precede a normal +`SystemPolicyRemovableVolumes` prompt. That probe alone does not establish a Full +Disk Access requirement. Keep the same authorized attempt running while the user +answers its normal prompt, then verify the terminal result and metrics database. +If permission remains blocked, stop and disable that job and report the evidence; +do not retry repeatedly, grant Full Disk Access, export keys, add privileged +wrappers, or relocate data. Local signing is not official release notarization. + +## Hourly collection on macOS + +After authorizing local collection, create a separate user LaunchAgent that calls +the pinned native binary directly. Use absolute paths; launchd does not expand +`~`, `$HOME`, or shell variables in a plist. Keep the config and logs outside Git +and shared publication. Give their directories mode `0700` and files mode `0600`. +Create both log files before bootstrapping the job. + +The following is a template for `~/Library/LaunchAgents/org.openclaw.gitcrawl.metrics.plist`. +Replace `/Users/you` and `SOURCE_COMMIT` with your actual installation paths: + +```xml + + + + + Labelorg.openclaw.gitcrawl.metrics + ProgramArguments + + /Users/you/.local/share/gitcrawl/metrics-runtimes/SOURCE_COMMIT/gitcrawl + metricscollect + --config/Users/you/.local/share/gitcrawl/metrics.json + --json + + WorkingDirectory/Users/you/.local/share/gitcrawl + EnvironmentVariables + + HOME/Users/you + PATH/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + GITCRAWL_NO_UPDATE_CHECK1 + + StartCalendarIntervalMinute6 + KeepAlive + Umask63 + StandardOutPath/Users/you/.local/share/gitcrawl/metrics-logs/stdout.log + StandardErrorPath/Users/you/.local/share/gitcrawl/metrics-logs/stderr.log + + +``` + +The job runs hourly at minute 6 in the host's local time, while observations use +UTC. `63` is the decimal representation of umask `0077`. The explicit minimal +`PATH` supports the existing native `gh` credential resolver without interactive +shell initialization. Do not put credentials in the plist. The versioned runtime +is not auto-updated; `GITCRAWL_NO_UPDATE_CHECK` also disables release notices. + +Validate and bootstrap this new job once, then request one immediate collection: + +```sh +metrics_plist="$HOME/Library/LaunchAgents/org.openclaw.gitcrawl.metrics.plist" +plutil -lint "$metrics_plist" +launchctl bootstrap "gui/$(id -u)" "$metrics_plist" +launchctl kickstart "gui/$(id -u)/org.openclaw.gitcrawl.metrics" +launchctl print "gui/$(id -u)/org.openclaw.gitcrawl.metrics" +``` + +Wait for the job to exit and check its last exit code, private logs, and the latest +`metric_runs` row using read-only SQLite. Confirm all five required counters for +every configured target have non-NULL values at that run's timestamp. A successful +bootstrap or a PID alone is not collection evidence. An unavailable optional clone +report is not a required-counter failure. On a provider failure, inspect the +recorded result before retrying; `KeepAlive` is disabled to avoid rapid restarts. +Existing archive refresh jobs are independent and need no changes. + +`collect` and `import` hold a nonblocking native OS lock on +`.writer.lock` from before database initialization through database +close. A second writer exits nonzero before collection or import; `status` remains +read-only and available. Ownership is released even if the process is killed. +The private lock file remains in place: never delete or replace it while writers +can run. Use the locking runtime for every writer; older binaries do not honor +this lock. + +## What is collected + +Each invocation observes all configured targets at a single UTC timestamp. Run +`collect` hourly with an external scheduler when hourly history is required. + +| Metric | Meaning | +| --- | --- | +| `stars` | Repository `stargazers_count` | +| `forks` | Repository `forks_count`; changes between snapshots are net changes | +| `watchers` | Actual subscribers, `subscribers_count`; **not** `watchers_count`, which aliases stars | +| `open_prs` | Open pull-request search count, only when results are complete | +| `open_issues` | Combined repository issue/PR count minus the valid open-PR count | +| `clones` | Optional daily clone counts from the authenticated traffic endpoint | + +Missing, invalid, or incomplete required counts remain SQL `NULL`; they never +become zero. If the PR count is unavailable or exceeds the combined count, +`open_issues` remains unknown. Real zeroes and decreases are retained. Individual +fork creation events are not crawled. + +Clone traffic requires repository push access (or a fine-grained token with repository +administration read permission). Permission-denied HTTP 403/404 means optional +unavailability and does not fail otherwise healthy collection. A rate-limit +response is a collection failure, including on the traffic endpoint. Only completed UTC +days are recorded: `ts` is that day's final millisecond and `observed_at` is the +actual read time. Unchanged daily values are not re-appended; corrected values +receive a new sequence. Imported daily timestamps retain their original spelling; +group them by UTC day. Sum only the latest observation for each day, never all +revisions. GitHub's traffic window limits how far a missed day can be backfilled. + +Stable releases exclude drafts and prereleases. All release pages are read; +events use stable GitHub release IDs so repeated collection is idempotent. +`published_at` is preferred, with `created_at` as the fallback for older records. + +Each target costs one repository request, one search request for the open-PR +count, optionally one traffic request, and one request per 100-release page +(including a final empty page when the total is a multiple of 100). Every +collection rereads the release history. Search has its own GitHub quota; there +is no metrics-specific quota reserve or incremental release checkpoint. The +shared client retries a rate-limited request once with a wait capped at five +minutes. An exhausted rate limit stops collection, retaining completed reads and +leaving later targets unattempted. Choose the schedule and target count accordingly. + +## Storage, imports, and failures + +New database files are private (`0600`). Existing files must identify themselves +with `metric_meta.owner = gitcrawl` and `metric_meta.version = 1`. Databases with +foreign tables, another owner/version, database symlinks or hard-link aliases, and +pre-existing empty files are rejected before a writable open. Existing databases +are inspected read-only for this check; no archive runtime or config is loaded. +Before applying the metrics schema or ownership metadata, the writable connection +rechecks ownership under a write transaction. Newly created databases must still +be empty, and changed file identities are rejected. Schema and ownership metadata +commit together on that same connection. +A failed first initialization removes only the newly created file so it can be +retried; pre-existing files are never removed. A process killed during that first +initialization can still leave an unowned file requiring operator inspection. `status` checks identity read-only and +does not create a missing database. Never point this config at the thread archive. + +The delivery tables are: + +- `metric_observations(sequence, id, entity, target, metric, kind, ts, value, + observed_at, provenance)` — counters and daily observations; `value` is nullable. +- `metric_events(sequence, id, entity, target, kind, ts, label, url, observed_at, + provenance)` — release history. +- `metric_runs(sequence, ts, status, rows_written)` — completed collection attempts. + +`status` reports total observations and events. Its `last_observed` is the latest +observation instant, comparing parsed timestamps even when imported offsets or +fractional precision differ; it is absent until an observation exists. + +Observation and event sequences advance independently. Read each table using its +own delivery cursor. Daily revisions supersede by latest sequence. Counter values +are snapshots, not increments; derive net change from consecutive observations. + +Import accepts one JSON object per line on stdin: + +```json +{"type":"metric","id":"history:github:watchers:1","entity":"OpenClaw","target":"openclaw/openclaw","metric":"watchers","kind":"counter","ts":"2026-09-14T00:00:00Z","value":null,"observed_at":"2026-09-15T00:00:00Z","provenance":"historical-import"} +{"type":"event","id":"history:github:release:1","entity":"Example","target":"example/project","kind":"release","ts":"2026-09-14T00:00:00Z","label":"v1","url":"https://github.com/example/project/releases/tag/v1","observed_at":"2026-09-15T00:00:00Z","provenance":"historical-import"} +``` + +IDs are required and idempotent within each destination table. Imported explicit +IDs preserve distinct observations even when values match. `entity` and `target` +must match a configured pair. Config files are limited to 1 MiB. Import validates every row and commits the whole +input atomically; a malformed or out-of-scope late row rolls everything back. +Daily imports must identify a day completed before `observed_at`'s UTC day. + +On a partial source failure, successful reads and explicit unknown counter values +are committed together with a `partial` run; stdout contains the result and the +command exits nonzero. A canceled collection gets a bounded opportunity to retain +already completed reads, without starting another network request. Diagnostics +never include GitHub response bodies or credential output. + +This command does not install schedules, migrate another application's history, +change existing refresh jobs, or publish the private database. Those are explicit +operator/integration responsibilities. diff --git a/internal/cli/app.go b/internal/cli/app.go index 0a390fc7..df486939 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -30,6 +30,7 @@ type App struct { observedGitHubToken string Stdout io.Writer Stderr io.Writer + Stdin io.Reader configPath string format OutputFormat @@ -53,6 +54,7 @@ func New() *App { return &App{ Stdout: os.Stdout, Stderr: os.Stderr, + Stdin: os.Stdin, format: FormatText, getWorkingDirectory: os.Getwd, } @@ -126,6 +128,8 @@ func (a *App) Run(ctx context.Context, args []string) error { return a.runDoctor(ctx, rest[1:]) case "status": return a.runStatus(ctx, rest[1:]) + case "metrics": + return a.runMetrics(ctx, rest[1:]) case "sync": return a.runSync(ctx, rest[1:]) case "fill-pr-details": diff --git a/internal/cli/control.go b/internal/cli/control.go index 90863dad..af94af7a 100644 --- a/internal/cli/control.go +++ b/internal/cli/control.go @@ -37,9 +37,12 @@ func (a *App) runMetadata(args []string) error { DefaultCache: cfg.CacheDir, DefaultLogs: cfg.LogDir, } - manifest.Capabilities = []string{"metadata", "status", "doctor", "sync", "capture", "coverage", "search", "code-index", "tui", "portable", "remote", "cloud-publish", "clusters", "summaries", "embeddings"} + manifest.Capabilities = []string{"metadata", "status", "metrics", "doctor", "sync", "capture", "coverage", "search", "code-index", "tui", "portable", "remote", "cloud-publish", "clusters", "summaries", "embeddings"} manifest.Privacy = control.Privacy{ContainsPrivateMessages: true, ExportsSecrets: false, LocalOnlyScopes: []string{"github", "git", "sqlite", "portable"}} manifest.Commands = map[string]control.Command{ + "metrics-collect": {Title: "Collect repository metrics", Argv: []string{"gitcrawl", "metrics", "collect", "--config", "METRICS_CONFIG", "--json"}, JSON: true, Mutates: true}, + "metrics-import": {Title: "Import metric history from stdin", Argv: []string{"gitcrawl", "metrics", "import", "--config", "METRICS_CONFIG", "--json"}, JSON: true, Mutates: true}, + "metrics-status": {Title: "Metrics database status", Argv: []string{"gitcrawl", "metrics", "status", "--config", "METRICS_CONFIG", "--json"}, JSON: true}, "status": {Title: "Status", Argv: []string{"gitcrawl", "status", "--json"}, JSON: true}, "remote-status": {Title: "Remote archive status", Argv: []string{"gitcrawl", "remote", "status", "--json"}, JSON: true}, "remote-archives": {Title: "Remote archive list", Argv: []string{"gitcrawl", "remote", "archives", "--json"}, JSON: true}, diff --git a/internal/cli/help.go b/internal/cli/help.go index a9d9c4d1..95464122 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -45,6 +45,7 @@ Core commands: metadata print crawlkit control metadata check-update check for a newer gitcrawl release status print fast read-only archive status + metrics collect, import, or inspect an independent metrics database remote status print remote archive status remote archives list remote archives visible to the current identity remote login authenticate with GitHub org access for a remote archive @@ -90,6 +91,19 @@ No API server is provided. There is intentionally no serve command. ` var commandUsageTexts = map[string]string{ + "metrics": `gitcrawl metrics collects repository headline metrics in a separate SQLite database. + +Usage: + gitcrawl metrics collect --config /absolute/metrics.json [--json] + gitcrawl metrics import --config /absolute/metrics.json [--json] < history.ndjson + gitcrawl metrics status --config /absolute/metrics.json [--json] + +The JSON config requires database (absolute path) and targets [{entity,target}]. +Optional tokenEnv overrides GITHUB_TOKEN; native gh auth and --github-token-command +are supported. No archive, portable-store, embedding, or model operations occur. +Import validates target scope and IDs atomically. Unknown values remain SQL NULL. +Status is read-only. Clone traffic 403/404 is optional unavailability. +`, "metadata": `gitcrawl metadata prints crawlkit control metadata. Usage: diff --git a/internal/cli/metrics.go b/internal/cli/metrics.go new file mode 100644 index 00000000..bc940913 --- /dev/null +++ b/internal/cli/metrics.go @@ -0,0 +1,84 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "net/http" + "time" + + "github.com/openclaw/gitcrawl/internal/config" + "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/headlinemetrics" +) + +func (a *App) runMetrics(ctx context.Context, args []string) error { + if len(args) == 0 || args[0] == "help" || args[0] == "--help" || args[0] == "-h" { + return a.printCommandUsage("metrics") + } + command := args[0] + if command != "collect" && command != "import" && command != "status" { + return usageErr(fmt.Errorf("unknown metrics command %q", command)) + } + fs := flag.NewFlagSet("metrics "+command, flag.ContinueOnError) + fs.SetOutput(io.Discard) + path := fs.String("config", a.configPath, "metrics JSON config path") + jsonOut := fs.Bool("json", false, "write JSON output") + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return a.printCommandUsage("metrics") + } + return usageErr(err) + } + if *path == "" || fs.NArg() != 0 { + return usageErr(errors.New("metrics requires --config and no positional arguments")) + } + c, err := headlinemetrics.ReadConfig(*path) + if err != nil { + return usageErr(err) + } + a.applyCommandJSON(*jsonOut) + var collect headlinemetrics.Collector + if command == "collect" { + cfg := config.Default() + if c.TokenEnv != "" { + cfg.GitHub.TokenEnv = c.TokenEnv + } + token := a.resolveGitHubToken(ctx, cfg) + var provider func(context.Context) (string, error) + if a.githubTokenCommand != nil { + provider, err = githubTokenProvider(*a.githubTokenCommand) + if err != nil { + return usageErr(err) + } + } + client := github.New(github.Options{Token: token.Value, TokenProvider: provider, BaseURL: githubBaseURL(), HTTPClient: &http.Client{ + Timeout: 30 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }}) + collect = headlinemetrics.GitHubCollector(client, token.Value != "" || provider != nil) + } + in := a.Stdin + if in == nil { + in = http.NoBody + } + result, err := headlinemetrics.Execute(ctx, command, c, collect, in) + // Partial collection is still a useful structured result; diagnostics stay + // on stderr and a nonzero exit communicates unavailable required metrics. + if err == nil || errors.Is(err, headlinemetrics.ErrPartialCollection) { + var output any = result + if command == "status" { + output = struct { + headlinemetrics.Result + Observations int `json:"observations"` + Events int `json:"events"` + }{result, result.Observations, result.Events} + } + if writeErr := a.writeOutput("metrics "+command, output, false); writeErr != nil { + return writeErr + } + } + return err +} diff --git a/internal/cli/metrics_test.go b/internal/cli/metrics_test.go new file mode 100644 index 00000000..4892996a --- /dev/null +++ b/internal/cli/metrics_test.go @@ -0,0 +1,224 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/openclaw/gitcrawl/internal/headlinemetrics" +) + +func metricsConfigFixture(t *testing.T) (string, headlinemetrics.Config) { + t.Helper() + dir := t.TempDir() + c := headlinemetrics.Config{Database: filepath.Join(dir, "metrics.sqlite"), Targets: []headlinemetrics.Target{{Entity: "OpenClaw", Target: "openclaw/openclaw"}}} + raw, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "metrics.json") + if err := os.WriteFile(path, raw, 0600); err != nil { + t.Fatal(err) + } + return path, c +} +func metricsApp(t *testing.T) (*App, *bytes.Buffer) { + t.Helper() + app := New() + out := new(bytes.Buffer) + app.Stdout = out + app.Stderr = new(bytes.Buffer) + app.githubAuthTokenLookup = func(context.Context) (string, error) { + t.Error("unexpected credential lookup") + return "", errors.New("no credential") + } + return app, out +} + +func TestMetricsNativeHelpMetadataAndUsage(t *testing.T) { + for _, args := range [][]string{{"--help"}, {"help", "metrics"}, {"metrics"}, {"metrics", "help"}, {"metrics", "-h"}, {"metrics", "collect", "--help"}, {"--json", "metrics", "status", "-h"}, {"metrics", "import", "--help"}} { + app, out := metricsApp(t) + if err := app.Run(context.Background(), args); err != nil || !strings.Contains(out.String(), "metrics") { + t.Fatalf("%v: %v %s", args, err, out) + } + } + app, out := metricsApp(t) + if err := app.Run(context.Background(), []string{"metadata", "--json"}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"metrics-collect", "metrics-import", "metrics-status"} { + if !strings.Contains(out.String(), name) { + t.Fatalf("metadata missing %s", name) + } + } + for _, args := range [][]string{{"metrics", "bad"}, {"metrics", "collect"}, {"metrics", "status", "--unknown"}, {"metrics", "import", "--config", "/missing", "extra"}, {"metrics", "collect", "--config", "/missing"}} { + app, _ := metricsApp(t) + if err := app.Run(context.Background(), args); err == nil || ExitCode(err) != 2 { + t.Fatalf("%v: usage error %v", args, err) + } + } + if releaseNotificationAllowed([]string{"metrics", "status"}) { + t.Fatal("metrics triggers release side effects") + } +} + +func TestMetricsImportStatusNativeJSONAndArchiveIsolation(t *testing.T) { + path, c := metricsConfigFixture(t) + archive := filepath.Join(t.TempDir(), "archive.db") + if err := os.WriteFile(archive, []byte("untouched archive"), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("GITCRAWL_DB_PATH", archive) + t.Setenv("GITCRAWL_CONFIG", filepath.Join(t.TempDir(), "does-not-exist.toml")) + row := `{"type":"metric","id":"history:1","entity":"OpenClaw","target":"openclaw/openclaw","metric":"watchers","kind":"counter","ts":"2026-09-14T00:00:00Z","value":null,"observed_at":"2026-09-15T00:00:00Z","provenance":"claw-track"}` + for i, args := range [][]string{{"--json", "metrics", "import", "--config", path}, {"metrics", "import", "--config", path, "--json"}} { + app, out := metricsApp(t) + app.Stdin = strings.NewReader(row + "\n") + if err := app.Run(context.Background(), args); err != nil { + t.Fatal(err) + } + var r headlinemetrics.Result + if err := json.Unmarshal(out.Bytes(), &r); err != nil { + t.Fatal(err) + } + if r.RowsWritten != 1-i || !r.OK { + t.Fatalf("import=%+v", r) + } + } + for _, args := range [][]string{{"metrics", "status", "--config", path, "--json"}, {"--config", path, "--json", "metrics", "status"}, {"--format", "json", "metrics", "status", "--config", path}} { + app, out := metricsApp(t) + if err := app.Run(context.Background(), args); err != nil { + t.Fatal(err) + } + var r headlinemetrics.Result + if err := json.Unmarshal(out.Bytes(), &r); err != nil || r.Observations != 1 { + t.Fatalf("status=%s %v", out, err) + } + } + if b, _ := os.ReadFile(archive); string(b) != "untouched archive" { + t.Fatal("archive changed") + } + info, err := os.Stat(c.Database) + if err != nil || info.Size() == 0 { + t.Fatalf("metrics DB absent: %v", err) + } + app, _ := metricsApp(t) + app.Stdin = strings.NewReader("bad") + if err := app.Run(context.Background(), []string{"metrics", "import", "--config", path}); err == nil || ExitCode(err) != 1 { + t.Fatal("invalid history accepted") + } +} + +func TestMetricsCollectUsesNativeCredentialsAndKeepsPartialOutput(t *testing.T) { + for _, mode := range []string{"environment", "gh-fallback", "managed", "partial", "quota"} { + t.Run(mode, func(t *testing.T) { + if mode == "managed" && runtime.GOOS == "windows" { + t.Skip("managed helper is Unix-only") + } + path, c := metricsConfigFixture(t) + c.TokenEnv = "GITCRAWL_METRICS_TEST_TOKEN" + b, _ := json.Marshal(c) + if err := os.WriteFile(path, b, 0600); err != nil { + t.Fatal(err) + } + t.Setenv("GITHUB_TOKEN", "ambient-must-not-win") + t.Setenv(c.TokenEnv, "") + token := "fixture-token" + if mode == "environment" || mode == "partial" || mode == "quota" { + t.Setenv(c.TokenEnv, token) + } + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.Header.Get("Authorization") != "Bearer "+token { + t.Error("wrong credential selected") + } + switch r.URL.Path { + case "/repos/openclaw/openclaw": + if mode == "quota" { + w.WriteHeader(http.StatusTooManyRequests) + return + } + fmt.Fprint(w, `{"stargazers_count":4,"forks_count":2,"subscribers_count":1,"open_issues_count":8}`) + case "/search/issues": + if mode == "partial" { + fmt.Fprint(w, `{"incomplete_results":true,"total_count":3}`) + } else { + fmt.Fprint(w, `{"total_count":3}`) + } + case "/repos/openclaw/openclaw/traffic/clones": + w.WriteHeader(403) + case "/repos/openclaw/openclaw/releases": + fmt.Fprint(w, `[]`) + default: + t.Errorf("unexpected API/model/archive request %s", r.URL.Path) + w.WriteHeader(404) + } + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITCRAWL_OPENAI_BASE_URL", server.URL) + app, out := metricsApp(t) + args := []string{"metrics", "collect", "--config", path, "--json"} + if mode == "gh-fallback" { + app.githubAuthTokenLookup = func(context.Context) (string, error) { return token, nil } + } + if mode == "managed" { + helper := filepath.Join(t.TempDir(), "credential-helper") + if err := os.WriteFile(helper, []byte("#!/bin/sh\nprintf '%s\\n' 'fixture-token'\n"), 0700); err != nil { + t.Fatal(err) + } + args = append([]string{"--github-token-command", helper}, args...) + } + err := app.Run(context.Background(), args) + if (err != nil) != (mode == "partial" || mode == "quota") { + t.Fatalf("error=%v", err) + } + var result headlinemetrics.Result + if e := json.Unmarshal(out.Bytes(), &result); e != nil { + t.Fatal(e) + } + wantRows, wantRequests := 5, 4 + if mode == "quota" { + wantRows, wantRequests = 0, 1 + } + if result.RowsWritten != wantRows || result.OK == (mode == "partial" || mode == "quota") || requests != wantRequests { + t.Fatalf("result=%+v requests=%d", result, requests) + } + if strings.Contains(out.String(), token) { + t.Fatal("token printed") + } + }) + } +} + +func TestMetricsEmptyStatusReportsZeroTotals(t *testing.T) { + path, _ := metricsConfigFixture(t) + app, _ := metricsApp(t) + app.Stdin = strings.NewReader("") + if err := app.Run(context.Background(), []string{"metrics", "import", "--config", path, "--json"}); err != nil { + t.Fatal(err) + } + app, out := metricsApp(t) + if err := app.Run(context.Background(), []string{"metrics", "status", "--config", path, "--json"}); err != nil { + t.Fatal(err) + } + var result map[string]any + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatal(err) + } + for _, key := range []string{"observations", "events"} { + if value, present := result[key]; !present || value != float64(0) { + t.Fatalf("status must report %s=0: %s", key, out) + } + } +} diff --git a/internal/cli/releasecheck.go b/internal/cli/releasecheck.go index 67d285a1..73b93d15 100644 --- a/internal/cli/releasecheck.go +++ b/internal/cli/releasecheck.go @@ -37,6 +37,11 @@ func (a *App) maybeNotifyRelease(ctx context.Context, args []string) { } func releaseNotificationAllowed(args []string) bool { + // Metrics commands must not touch archive/runtime state or make unrelated + // network requests, including during read-only imports and status checks. + if len(args) > 0 && args[0] == "metrics" { + return false + } if len(args) == 0 || args[0] != "fill-pr-details" { return true } diff --git a/internal/github/headline_metrics.go b/internal/github/headline_metrics.go new file mode 100644 index 00000000..626dd569 --- /dev/null +++ b/internal/github/headline_metrics.go @@ -0,0 +1,70 @@ +package github + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" +) + +// Pointer counts distinguish an unavailable field from an actual zero. +type RepositoryHeadline struct { + Stars *float64 `json:"stargazers_count"` + Forks *float64 `json:"forks_count"` + Watchers *float64 `json:"subscribers_count"` + OpenIssuesAndPulls *float64 `json:"open_issues_count"` +} + +type PullCount struct { + Count *float64 `json:"total_count"` + Incomplete bool `json:"incomplete_results"` +} + +type CloneTraffic struct { + Clones []struct { + Timestamp string `json:"timestamp"` + Count *float64 `json:"count"` + } `json:"clones"` +} + +type Release struct { + ID int64 `json:"id"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + Published string `json:"published_at"` + Created string `json:"created_at"` + Name string `json:"name"` + Tag string `json:"tag_name"` + URL string `json:"html_url"` +} + +func (c *Client) RepositoryHeadline(ctx context.Context, repository string) (RepositoryHeadline, error) { + var out RepositoryHeadline + err := c.doJSON(ctx, http.MethodGet, "/repos/"+repository, nil, nil, &out) + return out, err +} + +func (c *Client) OpenPullCount(ctx context.Context, repository string) (PullCount, error) { + var out PullCount + err := c.doJSON(ctx, http.MethodGet, "/search/issues?q="+url.QueryEscape("repo:"+repository+" is:pr is:open")+"&per_page=1", nil, nil, &out) + return out, err +} + +func (c *Client) CloneTraffic(ctx context.Context, repository string) (CloneTraffic, error) { + var out CloneTraffic + err := c.doJSON(ctx, http.MethodGet, "/repos/"+repository+"/traffic/clones?per=day", nil, nil, &out) + if err == nil && out.Clones == nil { + err = errors.New("missing GitHub clone traffic list") + } + return out, err +} + +func (c *Client) ReleasePage(ctx context.Context, repository string, page int) ([]Release, error) { + var out []Release + err := c.doJSON(ctx, http.MethodGet, fmt.Sprintf("/repos/%s/releases?per_page=100&page=%d", repository, page), nil, nil, &out) + if err == nil && out == nil { + err = errors.New("missing GitHub release list") + } + return out, err +} diff --git a/internal/headlinemetrics/github.go b/internal/headlinemetrics/github.go new file mode 100644 index 00000000..6c2c67a8 --- /dev/null +++ b/internal/headlinemetrics/github.go @@ -0,0 +1,171 @@ +package headlinemetrics + +import ( + "context" + "errors" + "fmt" + "math" + "net/http" + "strings" + "time" + + "github.com/openclaw/gitcrawl/internal/github" +) + +// GitHubCollector uses the same HTTP client and credential provider as native +// Gitcrawl commands. Clone traffic is optional and requires authentication. +func GitHubCollector(client *github.Client, trafficEnabled bool) Collector { + return func(ctx context.Context, c Config, ts string) ([]Row, error) { + if err := c.Validate(); err != nil { + return nil, err + } + observed, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + return nil, errors.New("invalid collection timestamp") + } + rows := []Row{} + failed := false + for _, t := range c.Targets { + if err := ctx.Err(); err != nil { + return rows, err + } + repo, err := client.RepositoryHeadline(ctx, t.Target) + if stopCollection(ctx, err) { + return rows, errors.Join(err, ctx.Err()) + } + if err != nil { + failed = true + repo = github.RepositoryHeadline{} + } + prs, err := client.OpenPullCount(ctx, t.Target) + if err != nil || prs.Incomplete { + failed = true + prs = github.PullCount{} + } + var issues *float64 + if validCount(repo.OpenIssuesAndPulls) != nil && validCount(prs.Count) != nil { + issues = Value(*repo.OpenIssuesAndPulls - *prs.Count) + } + for _, m := range []struct { + Name string + Value *float64 + }{ + {"stars", repo.Stars}, {"forks", repo.Forks}, {"watchers", repo.Watchers}, {"open_prs", prs.Count}, {"open_issues", issues}, + } { + value := validCount(m.Value) + if value == nil { + failed = true + } + rows = append(rows, Counter(t, m.Name, value, ts, "github_rest")) + } + if stopCollection(ctx, err) { + return rows, errors.Join(err, ctx.Err()) + } + if trafficEnabled { + traffic, err := client.CloneTraffic(ctx, t.Target) + if stopCollection(ctx, err) { + return rows, errors.Join(err, ctx.Err()) + } + var requestErr *github.RequestError + optional := errors.As(err, &requestErr) && (requestErr.Status == 403 || requestErr.Status == 404) + if err != nil && !optional { + failed = true + } + if err == nil { + for _, v := range traffic.Clones { + day, err := time.Parse(time.RFC3339Nano, v.Timestamp) + if err != nil { + failed = true + continue + } + day = day.UTC().Truncate(24 * time.Hour) + if !day.Before(observed.UTC().Truncate(24 * time.Hour)) { + continue + } + value := validCount(v.Count) + if value == nil { + failed = true + } + r := Counter(t, "clones", value, day.Add(24*time.Hour-time.Millisecond).Format(time.RFC3339Nano), "github_traffic") + r.Kind = "daily" + r.ObservedAt = ts + rows = append(rows, r) + } + } + } + // Follow all release pages. Do not silently truncate stable release history. + for page := 1; ; page++ { + releases, err := client.ReleasePage(ctx, t.Target, page) + if stopCollection(ctx, err) { + return rows, errors.Join(err, ctx.Err()) + } + if err != nil { + failed = true + break + } + for _, v := range releases { + if v.Draft || v.Prerelease { + continue + } + at := v.Published + if at == "" { + at = v.Created + } + when, err := time.Parse(time.RFC3339Nano, at) + label := v.Name + if label == "" { + label = v.Tag + } + if err != nil || v.ID <= 0 || label == "" { + failed = true + continue + } + rows = append(rows, Row{Type: "event", ID: fmt.Sprintf("github-release:%d", v.ID), Entity: t.Entity, Target: t.Target, Kind: "release", TS: when.UTC().Format(time.RFC3339Nano), ObservedAt: ts, Provenance: "github_releases", Label: label, URL: v.URL}) + } + if len(releases) < 100 { + break + } + } + } + if err := ctx.Err(); err != nil { + return rows, err + } + if failed { + return rows, errors.New("one or more GitHub metrics unavailable") + } + return rows, nil + } +} + +func validCount(value *float64) *float64 { + if value == nil || math.Trunc(*value) != *value { + return nil + } + return Value(*value) +} + +// Permission-denied traffic is optional; exhausted quota and cancellation stop +// all acquisition after the shared HTTP client's bounded retry. +func stopCollection(ctx context.Context, err error) bool { + // Retain a successfully decoded response even if cancellation arrived as + // it finished; the next request uses the canceled context. + if err == nil { + return false + } + if ctx.Err() != nil { + return true + } + var reserve *github.RateLimitReserveError + if errors.As(err, &reserve) { + return true + } + var response *github.RequestError + if !errors.As(err, &response) { + return false + } + return response.Status == http.StatusTooManyRequests || + (response.Status == http.StatusForbidden && + (response.Headers.Get("X-RateLimit-Remaining") == "0" || + response.Headers.Get("Retry-After") != "" || + strings.Contains(strings.ToLower(response.Body), "rate limit"))) +} diff --git a/internal/headlinemetrics/github_test.go b/internal/headlinemetrics/github_test.go new file mode 100644 index 00000000..60920f13 --- /dev/null +++ b/internal/headlinemetrics/github_test.go @@ -0,0 +1,310 @@ +package headlinemetrics + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/openclaw/gitcrawl/internal/github" +) + +func fixtureCollector(t *testing.T, override func(http.ResponseWriter, *http.Request) bool, traffic bool) Collector { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + t.Errorf("unexpected mutation %s", r.Method) + } + if r.Header.Get("Authorization") != "Bearer fixture-token" { + t.Error("missing normal client authorization") + } + if override != nil && override(w, r) { + return + } + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/search/issues": + if !strings.Contains(r.URL.Query().Get("q"), "is:pr is:open") { + t.Error("incorrect pull query") + } + fmt.Fprint(w, `{"total_count":3,"incomplete_results":false}`) + case strings.HasSuffix(r.URL.Path, "/traffic/clones"): + fmt.Fprint(w, `{"clones":[{"timestamp":"2026-09-14T00:00:00Z","count":0},{"timestamp":"2026-09-15T00:00:00Z","count":7}]}`) + case strings.HasSuffix(r.URL.Path, "/releases"): + fmt.Fprint(w, `[{"id":42,"published_at":"2026-09-14T15:00:00Z","name":"Stable","tag_name":"v1","html_url":"https://example.test/v1"},{"id":43,"draft":true},{"id":44,"prerelease":true}]`) + case strings.HasPrefix(r.URL.Path, "/repos/"): + fmt.Fprint(w, `{"stargazers_count":100,"forks_count":5,"watchers_count":100,"subscribers_count":7,"open_issues_count":11}`) + default: + t.Errorf("unexpected API %s", r.URL.String()) + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + return GitHubCollector(github.New(github.Options{BaseURL: server.URL, Token: "fixture-token", HTTPClient: server.Client()}), traffic) +} + +func TestGitHubMetricsUseActualWatchersSeparateIssuesAndCompletedDays(t *testing.T) { + c := testConfig(t) + rows, err := fixtureCollector(t, nil, true)(context.Background(), c, "2026-09-15T01:00:00Z") + if err != nil { + t.Fatal(err) + } + if len(rows) != 14 { + t.Fatalf("rows=%d", len(rows)) + } + for _, target := range c.Targets { + values := map[string]float64{} + events := 0 + for _, r := range rows { + if r.Target != target.Target { + continue + } + if r.Type == "event" { + events++ + if r.Label != "Stable" { + t.Fatal(r) + } + continue + } + if r.Value == nil { + t.Fatal("unexpected unknown", r) + } + values[r.Metric] = *r.Value + if r.Kind == "daily" && (r.TS != "2026-09-14T23:59:59.999Z" || r.ObservedAt != "2026-09-15T01:00:00Z") { + t.Fatal("incorrect UTC day", r) + } + } + if values["stars"] != 100 || values["forks"] != 5 || values["watchers"] != 7 || values["open_prs"] != 3 || values["open_issues"] != 8 || values["clones"] != 0 || events != 1 { + t.Fatalf("%s values=%v events=%d", target.Target, values, events) + } + } +} + +func TestMissingOrIncompletePRCountNeverFallsBackToCombinedIssues(t *testing.T) { + for _, body := range []string{`{}`, `{"total_count":3,"incomplete_results":true}`, `{"total_count":-1}`, `{"total_count":30}`, `{"total_count":3,"incomplete_results":"invalid"}`} { + t.Run(body, func(t *testing.T) { + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path == "/search/issues" { + fmt.Fprint(w, body) + return true + } + return false + }, false) + rows, err := collect(context.Background(), testConfig(t), "2026-09-15T01:00:00Z") + if err == nil { + t.Fatal("missing count reported healthy") + } + for _, r := range rows { + if r.Metric == "open_issues" && r.Value != nil { + t.Fatal("combined count used", r) + } + } + }) + } +} + +func TestOptionalCloneTrafficAndOtherPartialFailures(t *testing.T) { + for _, status := range []int{403, 404} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if strings.HasSuffix(r.URL.Path, "/traffic/clones") { + w.WriteHeader(status) + fmt.Fprint(w, "private response") + return true + } + return false + }, true) + rows, err := collect(context.Background(), testConfig(t), "2026-09-15T01:00:00Z") + if err != nil { + t.Fatal(err) + } + if len(rows) != 12 { + t.Fatalf("unexpected optional traffic rows %d", len(rows)) + } + }) + } + cases := []struct { + name string + match func(*http.Request) bool + body string + missing string + }{ + {"missing watchers", func(r *http.Request) bool { return r.URL.Path == "/repos/openclaw/openclaw" }, `{"stargazers_count":1,"forks_count":1,"watchers_count":900,"open_issues_count":10}`, "watchers"}, + {"malformed repo", func(r *http.Request) bool { return r.URL.Path == "/repos/openclaw/openclaw" }, `{"stargazers_count":12,`, "stars"}, + {"malformed traffic", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/traffic/clones") }, `{"clones":[{"timestamp":"bad","count":3}]}`, ""}, + {"missing clone count", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/traffic/clones") }, `{"clones":[{"timestamp":"2026-09-14T00:00:00Z"}]}`, "clones"}, + {"invalid stable release", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/releases") }, `[{"id":9,"published_at":"bad","name":"v1"}]`, ""}, + {"malformed releases", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/releases") }, `{`, ""}, + {"null releases", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/releases") }, `null`, ""}, + {"missing traffic", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/traffic/clones") }, `{}`, ""}, + {"null traffic", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/traffic/clones") }, `{"clones":null}`, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if tc.match(r) { + fmt.Fprint(w, tc.body) + return true + } + return false + }, true) + rows, err := collect(context.Background(), testConfig(t), "2026-09-15T01:00:00Z") + if err == nil || len(rows) < 10 { + t.Fatalf("partial=%d,%v", len(rows), err) + } + if tc.missing != "" { + found := false + for _, r := range rows { + if r.Target == "openclaw/openclaw" && r.Metric == tc.missing { + found = true + if r.Value != nil { + t.Fatal("invalid value retained", r) + } + } + } + if !found { + t.Fatal("missing unknown observation") + } + } + }) + } +} + +func TestReleasePaginationBeyondFivePagesAndNoUnauthenticatedTraffic(t *testing.T) { + pages := 0 + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if strings.HasSuffix(r.URL.Path, "/traffic/clones") { + t.Error("unauthenticated traffic request") + return true + } + if !strings.HasSuffix(r.URL.Path, "/releases") { + return false + } + pages++ + size := 100 + if pages == 6 { + size = 1 + } + releases := make([]github.Release, size) + for i := range releases { + releases[i] = github.Release{ID: int64(pages*100 + i), Created: "2026-09-14T00:00:00Z", Tag: "v1"} + } + if err := json.NewEncoder(w).Encode(releases); err != nil { + t.Error(err) + } + return true + }, false) + c := testConfig(t) + c.Targets = c.Targets[:1] + rows, err := collect(context.Background(), c, "2026-09-15T01:00:00Z") + if err != nil { + t.Fatal(err) + } + if pages != 6 || len(rows) != 506 { + t.Fatalf("pages=%d rows=%d", pages, len(rows)) + } + if _, err := collect(context.Background(), c, "bad"); err == nil { + t.Fatal("invalid clock accepted") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := collect(ctx, c, "2026-09-15T01:00:00Z"); err == nil { + t.Fatal("cancellation accepted") + } +} + +func TestQuotaFailureStopsCollectionAndRetainsCompletedReads(t *testing.T) { + for _, endpoint := range []string{"/repos/openclaw/openclaw", "/search/issues", "/repos/openclaw/openclaw/traffic/clones", "/repos/openclaw/openclaw/releases"} { + t.Run(endpoint, func(t *testing.T) { + limited := false + requestsAfter := 0 + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if limited { + requestsAfter++ + } + if r.URL.Path == endpoint { + limited = true + w.Header().Set("X-RateLimit-Remaining", "0") + w.WriteHeader(http.StatusForbidden) + return true + } + return false + }, true) + c := testConfig(t) + result, err := Execute(context.Background(), "collect", c, func(ctx context.Context, cfg Config, _ string) ([]Row, error) { + return collect(ctx, cfg, "2026-09-15T01:00:00Z") + }, nil) + if err == nil || result.OK || requestsAfter != 0 { + t.Fatalf("quota stop = %+v, %v; extra requests = %d", result, err, requestsAfter) + } + want := 5 + if endpoint == "/repos/openclaw/openclaw" { + want = 0 + } else if strings.HasSuffix(endpoint, "/releases") { + want = 6 + } + if result.RowsWritten != want { + t.Fatalf("retained %d rows; want %d", result.RowsWritten, want) + } + }) + } +} + +type metricsRoundTripper func(*http.Request) (*http.Response, error) + +func (f metricsRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type cancelMetricsBody struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (b cancelMetricsBody) Close() error { + err := b.ReadCloser.Close() + b.cancel() + return err +} + +func TestCancellationRetainsTheResponseJustRead(t *testing.T) { + for _, endpoint := range []string{"/repos/openclaw/openclaw", "/repos/openclaw/openclaw/traffic/clones", "/repos/openclaw/openclaw/releases"} { + t.Run(endpoint, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := github.New(github.Options{BaseURL: "https://example.test", HTTPClient: &http.Client{Transport: metricsRoundTripper(func(r *http.Request) (*http.Response, error) { + if err := r.Context().Err(); err != nil { + return nil, err + } + payload := `{"stargazers_count":4,"forks_count":0,"subscribers_count":2,"open_issues_count":1}` + switch r.URL.Path { + case "/search/issues": + payload = `{"total_count":0}` + case "/repos/openclaw/openclaw/traffic/clones": + payload = `{"clones":[{"timestamp":"2026-09-14T00:00:00Z","count":3}]}` + case "/repos/openclaw/openclaw/releases": + payload = `[{"id":1,"published_at":"2026-09-14T00:00:00Z","tag_name":"v1"}]` + } + var body io.ReadCloser = io.NopCloser(strings.NewReader(payload)) + if r.URL.Path == endpoint { + body = cancelMetricsBody{body, cancel} + } + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body, Request: r}, nil + })}}) + c := testConfig(t) + c.Targets = c.Targets[:1] + rows, err := GitHubCollector(client, true)(ctx, c, "2026-09-15T01:00:00Z") + want := 5 + if strings.HasSuffix(endpoint, "/clones") { + want = 6 + } else if strings.HasSuffix(endpoint, "/releases") { + want = 7 + } + if err == nil || len(rows) != want || rows[0].Value == nil || *rows[0].Value != 4 { + t.Fatalf("completed reads = %+v, %v; want %d rows", rows, err, want) + } + }) + } +} diff --git a/internal/headlinemetrics/lock.go b/internal/headlinemetrics/lock.go new file mode 100644 index 00000000..4a7ea3af --- /dev/null +++ b/internal/headlinemetrics/lock.go @@ -0,0 +1,55 @@ +package headlinemetrics + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +var errWriterBusy = errors.New("another metrics writer is running") + +// The persistent sidecar must never be unlinked: all writers lock the same inode. +// Closing the file (including process exit) releases ownership without stale PIDs. +func acquireWriter(database string) (*os.File, error) { + if err := os.MkdirAll(filepath.Dir(database), 0700); err != nil { + return nil, err + } + path := filepath.Clean(database) + ".writer.lock" + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if errors.Is(err, os.ErrExist) { + info, statErr := os.Lstat(path) + if statErr != nil { + return nil, statErr + } + if !info.Mode().IsRegular() { + return nil, errors.New("metrics writer lock must be a regular file") + } + f, err = os.OpenFile(path, os.O_RDWR, 0600) + } + if err != nil { + return nil, err + } + fail := func(err error) (*os.File, error) { + _ = f.Close() + return nil, err + } + info, err := f.Stat() + if err != nil { + return fail(err) + } + entry, err := os.Lstat(path) + if err != nil { + return fail(err) + } + if !info.Mode().IsRegular() || !entry.Mode().IsRegular() || !os.SameFile(info, entry) { + return fail(errors.New("metrics writer lock path changed or is not regular")) + } + if err := lockWriterFile(f); err != nil { + return fail(fmt.Errorf("lock metrics database: %w", err)) + } + if err := f.Chmod(0600); err != nil { + return fail(err) + } + return f, nil +} diff --git a/internal/headlinemetrics/lock_other.go b/internal/headlinemetrics/lock_other.go new file mode 100644 index 00000000..3f6ca899 --- /dev/null +++ b/internal/headlinemetrics/lock_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows + +package headlinemetrics + +import ( + "errors" + "os" +) + +func lockWriterFile(*os.File) error { + return errors.New("metrics writer locking is unsupported on this platform") +} + +func checkSingleLink(*os.File) error { + return errors.New("metrics file identity checks are unsupported on this platform") +} diff --git a/internal/headlinemetrics/lock_test.go b/internal/headlinemetrics/lock_test.go new file mode 100644 index 00000000..caf87cd0 --- /dev/null +++ b/internal/headlinemetrics/lock_test.go @@ -0,0 +1,152 @@ +package headlinemetrics + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +func TestExecuteSerializesWriterProcesses(t *testing.T) { + for _, mode := range []string{"complete", "kill"} { + t.Run(mode, func(t *testing.T) { + c := testConfig(t) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestMetricsWriterProcessHelper$", "--", c.Database) + out, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + in, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + waited := false + defer func() { + if !waited { + cancel() + _ = cmd.Wait() + } + }() + line, err := bufio.NewReader(out).ReadString('\n') + if err != nil || line != "collecting\n" { + t.Fatalf("child = %q, %v, %s", line, err, stderr.String()) + } + first, err := os.Stat(c.Database + ".writer.lock") + if err != nil { + t.Fatal(err) + } + collector := func(context.Context, Config, string) ([]Row, error) { + t.Fatal("overlapping collector reached provider") + return nil, nil + } + for _, command := range []string{"collect", "import"} { + result, err := Execute(ctx, command, c, collector, strings.NewReader("invalid input")) + if !errors.Is(err, errWriterBusy) || result.RowsWritten != 0 { + t.Fatalf("overlap %s = %+v, %v", command, result, err) + } + } + if result, err := Execute(ctx, "status", c, nil, nil); err != nil || !result.OK || result.Observations != 0 { + t.Fatalf("concurrent reader = %+v, %v", result, err) + } + // A different metrics database has independent ownership. + if _, err := Execute(ctx, "import", testConfig(t), nil, strings.NewReader("")); err != nil { + t.Fatal(err) + } + if mode == "kill" { + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + } else if _, err := in.Write([]byte("finish\n")); err != nil { + t.Fatal(err) + } + _ = in.Close() + err = cmd.Wait() + waited = true + if (mode == "complete") != (err == nil) { + t.Fatalf("child exit = %v: %s", err, stderr.String()) + } + if _, err := Execute(ctx, "import", c, nil, strings.NewReader("")); err != nil { + t.Fatalf("writer after %s = %v", mode, err) + } + last, err := os.Stat(c.Database + ".writer.lock") + if err != nil || !os.SameFile(first, last) || last.Mode().Perm() != 0600 { + t.Fatalf("persistent private lock = %v, %v", last, err) + } + result, err := Execute(ctx, "status", c, nil, nil) + want := 0 + if mode == "complete" { + want = 1 + } + if err != nil || result.Observations != want { + t.Fatalf("retained observations = %+v, %v", result, err) + } + }) + } +} + +func TestMetricsWriterProcessHelper(t *testing.T) { + i := slices.Index(os.Args, "--") + if i < 0 { + return + } + c := Config{Database: os.Args[i+1], Targets: []Target{{Entity: "OpenClaw", Target: "openclaw/openclaw"}}} + _, err := Execute(context.Background(), "collect", c, func(context.Context, Config, string) ([]Row, error) { + fmt.Println("collecting") + if _, err := bufio.NewReader(os.Stdin).ReadString('\n'); err != nil { + return nil, err + } + return []Row{testRow()}, nil + }, nil) + if err != nil { + t.Fatal(err) + } +} + +func TestMetricsWriterRejectsUnsafeLockPaths(t *testing.T) { + for _, kind := range []string{"directory", "symlink", "hardlink"} { + t.Run(kind, func(t *testing.T) { + c := testConfig(t) + path := c.Database + ".writer.lock" + target := filepath.Join(filepath.Dir(c.Database), "unrelated") + if err := os.WriteFile(target, []byte("retained"), 0644); err != nil { + t.Fatal(err) + } + var err error + switch kind { + case "directory": + err = os.Mkdir(path, 0700) + case "symlink": + err = os.Symlink(target, path) + case "hardlink": + err = os.Link(target, path) + } + if err != nil { + t.Fatal(err) + } + if _, err := Execute(context.Background(), "import", c, nil, strings.NewReader("")); err == nil { + t.Fatal("unsafe lock accepted") + } + if _, err := os.Stat(c.Database); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("database initialized before lock: %v", err) + } + if got, err := os.ReadFile(target); err != nil || string(got) != "retained" { + t.Fatalf("unrelated file = %q, %v", got, err) + } + }) + } +} diff --git a/internal/headlinemetrics/lock_unix.go b/internal/headlinemetrics/lock_unix.go new file mode 100644 index 00000000..b5d7d848 --- /dev/null +++ b/internal/headlinemetrics/lock_unix.go @@ -0,0 +1,33 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package headlinemetrics + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func checkSingleLink(f *os.File) error { + var info unix.Stat_t + if err := unix.Fstat(int(f.Fd()), &info); err != nil { + return err + } + if info.Nlink != 1 { + return errors.New("metrics files must not have hardlink aliases") + } + return nil +} + +func lockWriterFile(f *os.File) error { + if err := checkSingleLink(f); err != nil { + return err + } + + err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { + return errWriterBusy + } + return err +} diff --git a/internal/headlinemetrics/lock_windows.go b/internal/headlinemetrics/lock_windows.go new file mode 100644 index 00000000..4168dac8 --- /dev/null +++ b/internal/headlinemetrics/lock_windows.go @@ -0,0 +1,33 @@ +package headlinemetrics + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func checkSingleLink(f *os.File) error { + handle := windows.Handle(f.Fd()) + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + return err + } + if info.NumberOfLinks != 1 { + return errors.New("metrics files must not have hardlink aliases") + } + return nil +} + +func lockWriterFile(f *os.File) error { + if err := checkSingleLink(f); err != nil { + return err + } + handle := windows.Handle(f.Fd()) + + err := windows.LockFileEx(handle, windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &windows.Overlapped{}) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return errWriterBusy + } + return err +} diff --git a/internal/headlinemetrics/metrics.go b/internal/headlinemetrics/metrics.go new file mode 100644 index 00000000..815b7156 --- /dev/null +++ b/internal/headlinemetrics/metrics.go @@ -0,0 +1,562 @@ +// Package headlinemetrics owns a separate, append-only repository metrics store. +// It rejects thread archives before writes and never starts embedding/model work. +package headlinemetrics + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/openclaw/crawlkit/store" +) + +const Owner = "gitcrawl" + +var ErrPartialCollection = errors.New("one or more GitHub metrics unavailable; successful values and unknown observations were retained") + +type Target struct { + Entity string `json:"entity"` + Target string `json:"target"` +} +type Config struct { + Database string `json:"database"` + Targets []Target `json:"targets"` + CookieJar string `json:"cookieJar,omitempty"` + TokenEnv string `json:"tokenEnv,omitempty"` +} +type Row struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + Entity string `json:"entity"` + Target string `json:"target"` + Metric string `json:"metric,omitempty"` + Kind string `json:"kind"` + TS string `json:"ts"` + Value *float64 `json:"value"` + ObservedAt string `json:"observed_at"` + Provenance string `json:"provenance"` + Label string `json:"label,omitempty"` + URL string `json:"url,omitempty"` +} +type Collector func(context.Context, Config, string) ([]Row, error) + +type Result struct { + Source string `json:"source"` + Command string `json:"command"` + RowsWritten int `json:"rows_written"` + OK bool `json:"ok"` + Observations int `json:"observations,omitempty"` + Events int `json:"events,omitempty"` + LastObserved *string `json:"last_observed,omitempty"` +} + +var repositoryPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$`) +var envPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func ReadConfig(path string) (Config, error) { + var c Config + f, err := os.Open(path) + if err != nil { + return c, fmt.Errorf("read metrics config: %w", err) + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, 1024*1024+1)) + if err != nil { + return c, fmt.Errorf("read metrics config: %w", err) + } + if len(data) > 1024*1024 { + return c, errors.New("metrics config exceeds 1 MiB") + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + if err := d.Decode(&c); err != nil { + return c, errors.New("invalid metrics config JSON") + } + if err := d.Decode(new(any)); err != io.EOF { + return c, errors.New("metrics config must contain one JSON object") + } + return c, c.Validate() +} + +func (c Config) Validate() error { + if !filepath.IsAbs(c.Database) || strings.TrimSpace(c.Database) != c.Database || strings.ContainsAny(c.Database, "\x00?") { + return errors.New("metrics database must be an absolute filesystem path") + } + if len(c.Targets) == 0 { + return errors.New("metrics config requires targets") + } + if c.TokenEnv != "" && !envPattern.MatchString(c.TokenEnv) { + return errors.New("invalid tokenEnv name") + } + if c.CookieJar != "" { + return errors.New("GitHub metrics do not use cookieJar") + } + seen := map[string]bool{} + for _, t := range c.Targets { + if strings.TrimSpace(t.Entity) == "" || len(t.Entity) > 200 || len(t.Target) > 300 || !validRepository(t.Target) { + return errors.New("metrics target requires an entity and owner/repo") + } + key := strings.ToLower(t.Target) + if seen[key] { + return errors.New("duplicate metrics repository target") + } + seen[key] = true + } + return nil +} + +func validRepository(target string) bool { + if !repositoryPattern.MatchString(target) { + return false + } + _, name, _ := strings.Cut(target, "/") + return name != "." && name != ".." +} + +func Value(n float64) *float64 { + if math.IsNaN(n) || math.IsInf(n, 0) || n < 0 { + return nil + } + return &n +} +func Counter(t Target, metric string, value *float64, ts, basis string) Row { + return Row{Type: "metric", Entity: t.Entity, Target: t.Target, Metric: metric, Kind: "counter", TS: ts, Value: value, ObservedAt: ts, Provenance: basis} +} + +const Schema = ` +CREATE TABLE IF NOT EXISTS metric_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS metric_observations(sequence INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT NOT NULL UNIQUE,entity TEXT NOT NULL,target TEXT NOT NULL,metric TEXT NOT NULL,kind TEXT NOT NULL CHECK(kind IN ('counter','daily')),ts TEXT NOT NULL,value REAL,observed_at TEXT NOT NULL,provenance TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS metric_series ON metric_observations(target,metric,ts,sequence); +CREATE TABLE IF NOT EXISTS metric_events(sequence INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT NOT NULL UNIQUE,entity TEXT NOT NULL,target TEXT NOT NULL,kind TEXT NOT NULL,ts TEXT NOT NULL,label TEXT NOT NULL,url TEXT NOT NULL,observed_at TEXT NOT NULL,provenance TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS metric_runs(sequence INTEGER PRIMARY KEY AUTOINCREMENT,ts TEXT NOT NULL,status TEXT NOT NULL,rows_written INTEGER NOT NULL); +` + +// ownedReadOnly checks identity and schema version before any writable SQLite open. +func ownedReadOnly(ctx context.Context, path string) (*store.Store, error) { + if !filepath.IsAbs(path) || strings.TrimSpace(path) != path { + return nil, errors.New("metrics database must be an absolute path without surrounding whitespace") + } + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return nil, errors.New("metrics database must be a nonempty regular file") + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + err = checkSingleLink(f) + closeErr := f.Close() + if err != nil || closeErr != nil { + return nil, errors.Join(err, closeErr) + } + s, err := store.OpenReadOnly(ctx, path) + if err != nil { + return nil, err + } + if err := validateOwnership(ctx, s.DB()); err != nil { + s.Close() + return nil, err + } + return s, nil +} + +type ownershipReader interface { + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func validateOwnership(ctx context.Context, db ownershipReader) error { + var owner, version string + err := db.QueryRowContext(ctx, "SELECT value FROM metric_meta WHERE key='owner'").Scan(&owner) + if err == nil { + err = db.QueryRowContext(ctx, "SELECT value FROM metric_meta WHERE key='version'").Scan(&version) + } + var foreign int + if err == nil { + err = db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT IN ('metric_meta','metric_observations','metric_events','metric_runs','sqlite_sequence') AND name NOT LIKE 'sqlite_%'`).Scan(&foreign) + } + if err != nil || owner != Owner || version != "1" || foreign != 0 { + return errors.New("refusing a database not exclusively owned by gitcrawl metrics schema version 1") + } + return nil +} + +// beforeWritableOpen is nil except in tests that replace a checked path. +var beforeWritableOpen func() + +func Open(ctx context.Context, path string) (_ *store.Store, err error) { + if !filepath.IsAbs(path) || strings.TrimSpace(path) != path { + return nil, errors.New("metrics database path must be absolute") + } + info, err := os.Lstat(path) + newFile := errors.Is(err, os.ErrNotExist) + if err == nil { + if !info.Mode().IsRegular() { + return nil, errors.New("refusing a non-regular metrics database") + } + read, err := ownedReadOnly(ctx, path) + if err != nil { + return nil, err + } + if err = read.Close(); err != nil { + return nil, err + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } else { + if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, err + } + // Never initialize an existing empty archive or follow a database symlink. + var f *os.File + f, err = os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return nil, err + } + created, statErr := f.Stat() + if statErr != nil { + _ = f.Close() + return nil, statErr + } + info = created + // Remove only our newly created file on failure, never an existing + // database or a replacement installed at the same path. + defer func() { + if err != nil { + if current, e := os.Lstat(path); e == nil && os.SameFile(created, current) { + err = errors.Join(err, os.Remove(path)) + } + } + }() + if err = f.Close(); err != nil { + return nil, err + } + } + if beforeWritableOpen != nil { + beforeWritableOpen() + } + // Connecting must not apply schema before the opened database is checked. + s, err := store.Open(ctx, store.Options{Path: path, MaxOpenConns: 1, MaxIdleConns: 1}) + if err != nil { + return nil, err + } + if err = initialize(ctx, s.DB(), path, info, newFile); err != nil { + return nil, errors.Join(err, s.Close()) + } + return s, nil +} + +func initialize(ctx context.Context, db *sql.DB, path string, expected os.FileInfo, newFile bool) (err error) { + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer func() { err = errors.Join(err, conn.Close()) }() + if _, err = conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + return err + } + committed := false + defer func() { + if !committed { + rollbackCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, rollbackErr := conn.ExecContext(rollbackCtx, "ROLLBACK") + err = errors.Join(err, rollbackErr) + } + }() + // Validate through the pinned connection under the write lock, not through + // another pathname lookup that could inspect a different database. + if newFile { + var objects int + if err = conn.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master").Scan(&objects); err != nil { + return err + } + if objects != 0 { + return errors.New("refusing to initialize a nonempty metrics database") + } + } else if err = validateOwnership(ctx, conn); err != nil { + return err + } + current, err := os.Lstat(path) + if err != nil { + return err + } + if !current.Mode().IsRegular() || !os.SameFile(expected, current) { + return errors.New("metrics database path changed before initialization") + } + if _, err = conn.ExecContext(ctx, Schema); err != nil { + return err + } + if _, err = conn.ExecContext(ctx, "INSERT OR IGNORE INTO metric_meta VALUES('owner',?),('version','1')", Owner); err != nil { + return err + } + _, err = conn.ExecContext(ctx, "COMMIT") + committed = err == nil + return err +} + +func Validate(r Row) error { + if r.Type != "metric" && r.Type != "event" { + return errors.New("invalid observation type") + } + if strings.TrimSpace(r.Entity) == "" || !validRepository(r.Target) || len(r.Entity) > 200 || len(r.Target) > 300 || strings.TrimSpace(r.Provenance) == "" { + return errors.New("invalid observation identity") + } + for _, v := range []string{r.TS, r.ObservedAt} { + if _, err := time.Parse(time.RFC3339Nano, v); err != nil { + return errors.New("invalid observation time") + } + } + if r.Type == "metric" { + if r.Metric == "" || (r.Kind != "counter" && r.Kind != "daily") || (r.Value != nil && Value(*r.Value) == nil) { + return errors.New("invalid metric") + } + if r.Kind == "daily" { + at, _ := time.Parse(time.RFC3339Nano, r.TS) + observed, _ := time.Parse(time.RFC3339Nano, r.ObservedAt) + if !at.UTC().Truncate(24 * time.Hour).Before(observed.UTC().Truncate(24 * time.Hour)) { + return errors.New("daily observation must describe a completed UTC day") + } + } + } else if r.Kind == "" || r.Label == "" { + return errors.New("event kind and label are required") + } + return nil +} + +func insert(ctx context.Context, tx *sql.Tx, r Row) (int, error) { + if err := Validate(r); err != nil { + return 0, err + } + // Imported IDs preserve their exact history. Only freshly collected daily + // values suppress unchanged re-reads; later corrections append a new sequence. + if r.ID == "" && r.Type == "metric" && r.Kind == "daily" { + previous, provenance, err := latestDaily(ctx, tx, r) + if err == nil && provenance == r.Provenance && ((r.Value == nil && !previous.Valid) || (r.Value != nil && previous.Valid && previous.Float64 == *r.Value)) { + return 0, nil + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + } + if r.ID == "" { + b, _ := json.Marshal(r) + h := sha256.Sum256(b) + r.ID = hex.EncodeToString(h[:]) + } + var result sql.Result + var err error + if r.Type == "metric" { + result, err = tx.ExecContext(ctx, "INSERT INTO metric_observations(id,entity,target,metric,kind,ts,value,observed_at,provenance) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING", r.ID, r.Entity, r.Target, r.Metric, r.Kind, r.TS, r.Value, r.ObservedAt, r.Provenance) + } else { + result, err = tx.ExecContext(ctx, "INSERT INTO metric_events(id,entity,target,kind,ts,label,url,observed_at,provenance) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING", r.ID, r.Entity, r.Target, r.Kind, r.TS, r.Label, r.URL, r.ObservedAt, r.Provenance) + } + if err != nil { + return 0, err + } + n, err := result.RowsAffected() + return int(n), err +} + +// A daily import can spell the same UTC day with another offset or time of day. +// Compare parsed days without rewriting imported timestamps or delivery IDs. +func latestDaily(ctx context.Context, tx *sql.Tx, r Row) (sql.NullFloat64, string, error) { + rows, err := tx.QueryContext(ctx, `SELECT ts,value,provenance FROM metric_observations WHERE entity=? AND target=? AND metric=? AND kind='daily' ORDER BY sequence DESC`, r.Entity, r.Target, r.Metric) + if err != nil { + return sql.NullFloat64{}, "", err + } + defer rows.Close() + at, _ := time.Parse(time.RFC3339Nano, r.TS) + day := at.UTC().Truncate(24 * time.Hour) + for rows.Next() { + var raw, provenance string + var value sql.NullFloat64 + if err := rows.Scan(&raw, &value, &provenance); err != nil { + return value, "", err + } + previous, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + return value, "", errors.New("invalid stored daily observation time") + } + if previous.UTC().Truncate(24 * time.Hour).Equal(day) { + return value, provenance, nil + } + } + if err := rows.Err(); err != nil { + return sql.NullFloat64{}, "", err + } + return sql.NullFloat64{}, "", sql.ErrNoRows +} + +func Write(ctx context.Context, s *store.Store, rows []Row) (int, error) { + written := 0 + err := s.WithTx(ctx, func(tx *sql.Tx) error { + for _, r := range rows { + n, err := insert(ctx, tx, r) + if err != nil { + return err + } + written += n + } + return nil + }) + if err != nil { + return 0, err + } + return written, nil +} + +func inScope(c Config, r Row) bool { + for _, t := range c.Targets { + if t.Target == r.Target && t.Entity == r.Entity { + return true + } + } + return false +} + +// Import validates scope and streams the whole input in one transaction. A bad +// row (including one past a batch boundary) never leaves a partial history. +func Import(ctx context.Context, s *store.Store, c Config, in io.Reader) (int, error) { + written := 0 + err := s.WithTx(ctx, func(tx *sql.Tx) error { + scanner := bufio.NewScanner(in) + scanner.Buffer(make([]byte, 65536), 4*1024*1024) + line := 0 + for scanner.Scan() { + line++ + var r Row + d := json.NewDecoder(strings.NewReader(scanner.Text())) + d.DisallowUnknownFields() + if d.Decode(&r) != nil || d.Decode(new(any)) != io.EOF || !inScope(c, r) { + return fmt.Errorf("invalid import scope or JSON at line %d", line) + } + if r.ID == "" { + return fmt.Errorf("import ID required at line %d", line) + } + n, err := insert(ctx, tx, r) + if err != nil { + return fmt.Errorf("import line %d: %w", line, err) + } + written += n + } + return scanner.Err() + }) + if err != nil { + return 0, err + } + return written, nil +} + +func Execute(ctx context.Context, command string, c Config, collect Collector, in io.Reader) (Result, error) { + result := Result{Source: Owner, Command: command} + if err := c.Validate(); err != nil { + return result, err + } + if command != "status" && command != "import" && command != "collect" { + return result, errors.New("unknown metrics command") + } + if command == "status" { + s, err := ownedReadOnly(ctx, c.Database) + if err != nil { + return result, err + } + defer s.Close() + rows, err := s.DB().QueryContext(ctx, "SELECT observed_at FROM metric_observations") + if err != nil { + return result, err + } + var latest time.Time + for rows.Next() { + var raw string + if err = rows.Scan(&raw); err != nil { + break + } + var at time.Time + at, err = time.Parse(time.RFC3339Nano, raw) + if err != nil { + err = errors.New("invalid stored observation time") + break + } + result.Observations++ + // Imported timestamps retain their original offsets and precision. + if result.LastObserved == nil || at.After(latest) { + latest = at + result.LastObserved = &raw + } + } + err = errors.Join(err, rows.Err(), rows.Close()) + if err == nil { + err = s.DB().QueryRowContext(ctx, "SELECT count(*) FROM metric_events").Scan(&result.Events) + } + result.OK = err == nil + return result, err + } + // Own the database from before initialization through provider reads, commit, + // and close. SQLite transactions alone do not serialize collection attempts. + lock, err := acquireWriter(c.Database) + if err != nil { + return result, err + } + defer lock.Close() + s, err := Open(ctx, c.Database) + if err != nil { + return result, err + } + defer s.Close() + if command == "import" { + result.RowsWritten, err = Import(ctx, s, c, in) + result.OK = err == nil + return result, err + } + ts := time.Now().UTC().Format(time.RFC3339Nano) + rows, collectionErr := collect(ctx, c, ts) + for _, r := range rows { + if !inScope(c, r) { + return result, errors.New("collector returned invalid target") + } + } + // Preserve completed reads even when collection is canceled. This bounded, + // independent write does not start another request or retry collection. + writeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + err = s.WithTx(writeCtx, func(tx *sql.Tx) error { + for _, r := range rows { + n, err := insert(writeCtx, tx, r) + if err != nil { + return err + } + result.RowsWritten += n + } + status := "ok" + if collectionErr != nil { + status = "partial" + } + _, err := tx.ExecContext(writeCtx, "INSERT INTO metric_runs(ts,status,rows_written) VALUES(?,?,?)", ts, status, result.RowsWritten) + return err + }) + if err != nil { + result.RowsWritten = 0 + return result, err + } + result.OK = collectionErr == nil + if collectionErr != nil { + return result, ErrPartialCollection + } + return result, nil +} diff --git a/internal/headlinemetrics/metrics_test.go b/internal/headlinemetrics/metrics_test.go new file mode 100644 index 00000000..c4748835 --- /dev/null +++ b/internal/headlinemetrics/metrics_test.go @@ -0,0 +1,646 @@ +package headlinemetrics + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/openclaw/crawlkit/store" +) + +func testConfig(t *testing.T) Config { + t.Helper() + return Config{Database: filepath.Join(t.TempDir(), "metrics.sqlite"), Targets: []Target{{Entity: "OpenClaw", Target: "openclaw/openclaw"}, {Entity: "Example", Target: "example/project"}}} +} +func testRow() Row { + return Counter(Target{"OpenClaw", "openclaw/openclaw"}, "stars", Value(12), "2026-09-15T01:00:00Z", "github_rest") +} +func openTestStore(t *testing.T, c Config) *store.Store { + t.Helper() + s, err := Open(context.Background(), c.Database) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} +func ndjson(t *testing.T, rows ...Row) string { + t.Helper() + var out strings.Builder + for _, r := range rows { + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + out.Write(b) + out.WriteByte('\n') + } + return out.String() +} + +func TestStorePreservesZerosNullsDecreasesAndIdempotentImports(t *testing.T) { + ctx := context.Background() + c := testConfig(t) + s := openTestStore(t, c) + rows := []Row{} + for i, v := range []*float64{Value(12), Value(8), Value(0), nil} { + r := testRow() + r.ID = fmt.Sprint(i) + r.Value = v + rows = append(rows, r) + } + event := Row{Type: "event", ID: "import-release", Entity: "Example", Target: "example/project", Kind: "release", TS: "2026-09-14T00:00:00Z", ObservedAt: "2026-09-15T00:00:00Z", Provenance: "claw-track", Label: "v1", URL: "https://github.com/example/project/releases/tag/v1"} + rows = append(rows, event) + for _, want := range []int{5, 0} { + n, err := Import(ctx, s, c, strings.NewReader(ndjson(t, rows...))) + if err != nil || n != want { + t.Fatalf("import = %d,%v want %d", n, err, want) + } + } + cursor, err := s.DB().Query("SELECT value FROM metric_observations ORDER BY sequence") + if err != nil { + t.Fatal(err) + } + defer cursor.Close() + for _, want := range []*float64{Value(12), Value(8), Value(0), nil} { + if !cursor.Next() { + t.Fatal("missing observation") + } + var got sql.NullFloat64 + if err := cursor.Scan(&got); err != nil { + t.Fatal(err) + } + if got.Valid != (want != nil) || (want != nil && got.Float64 != *want) { + t.Fatalf("value = %+v want %v", got, want) + } + } + if cursor.Next() { + t.Fatal("duplicate observations") + } + result, err := Execute(ctx, "status", c, nil, nil) + if err != nil || result.Observations != 4 || result.Events != 1 || result.LastObserved == nil { + t.Fatalf("status = %+v %v", result, err) + } + info, err := os.Stat(c.Database) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0077 != 0 { + t.Fatalf("database permissions: %v", info.Mode()) + } +} + +func TestImportRollbackAfterBatchBoundaryAndInvalidInput(t *testing.T) { + for _, bad := range []string{"not json", `{"type":"metric"}`, "", strings.Repeat("x", 4*1024*1024+1)} { + t.Run(fmt.Sprint(len(bad)), func(t *testing.T) { + c := testConfig(t) + s := openTestStore(t, c) + var input strings.Builder + for i := 0; i < 501; i++ { + r := testRow() + r.ID = fmt.Sprint(i) + input.WriteString(ndjson(t, r)) + } + input.WriteString(bad + "\n") + n, err := Import(context.Background(), s, c, strings.NewReader(input.String())) + if err == nil || n != 0 { + t.Fatalf("import=%d,%v", n, err) + } + var count int + if err := s.DB().QueryRow("SELECT count(*) FROM metric_observations").Scan(&count); err != nil || count != 0 { + t.Fatalf("partial import committed: %d,%v", count, err) + } + }) + } + c := testConfig(t) + s := openTestStore(t, c) + for _, mutate := range []func(*Row){func(r *Row) { r.Entity = "wrong" }, func(r *Row) { r.Target = "other/repo" }, func(r *Row) { r.ID = "" }, func(r *Row) { r.Value = Value(-1); r.TS = "invalid" }} { + r := testRow() + r.ID = "id" + mutate(&r) + if _, err := Import(context.Background(), s, c, strings.NewReader(ndjson(t, r))); err == nil { + t.Fatalf("accepted invalid row %+v", r) + } + } +} + +func TestDailyRevisionsAppendAndImportedIDsRemainIndependent(t *testing.T) { + c := testConfig(t) + s := openTestStore(t, c) + ctx := context.Background() + r := testRow() + r.Kind = "daily" + r.Metric = "clones" + r.TS = "2026-09-14T23:59:59.999Z" + r.Provenance = "github_traffic" + for i, v := range []float64{10, 10, 9, 10} { + r.Value = Value(v) + r.ObservedAt = fmt.Sprintf("2026-09-15T%02d:00:00Z", i) + n, err := Write(ctx, s, []Row{r}) + want := 1 + if i == 1 { + want = 0 + } + if err != nil || n != want { + t.Fatalf("daily revision %d: %d,%v", i, n, err) + } + } + r.ID = "historical-1" + r.Provenance = "claw-track" + n, err := Import(ctx, s, c, strings.NewReader(ndjson(t, r))) + if err != nil || n != 1 { + t.Fatalf("history: %d,%v", n, err) + } + r.ID = "historical-2" + n, err = Import(ctx, s, c, strings.NewReader(ndjson(t, r))) + if err != nil || n != 1 { + t.Fatalf("distinct history ID: %d,%v", n, err) + } + var count int + var latest float64 + if err = s.DB().QueryRow("SELECT count(*) FROM metric_observations").Scan(&count); err != nil || count != 5 { + t.Fatalf("count %d %v", count, err) + } + if err = s.DB().QueryRow("SELECT value FROM metric_observations ORDER BY sequence DESC LIMIT 1").Scan(&latest); err != nil || latest != 10 { + t.Fatalf("latest %f %v", latest, err) + } +} + +func TestRefuseArchiveWrongOwnerVersionAndLinksWithoutChangingBytes(t *testing.T) { + for _, schema := range []string{ + "CREATE TABLE threads(id INTEGER PRIMARY KEY)", + "CREATE TABLE metric_meta(key TEXT PRIMARY KEY,value TEXT);INSERT INTO metric_meta VALUES('owner','redcrawl'),('version','1')", + "CREATE TABLE metric_meta(key TEXT PRIMARY KEY,value TEXT);INSERT INTO metric_meta VALUES('owner','gitcrawl'),('version','2')", + "CREATE TABLE metric_meta(key TEXT PRIMARY KEY,value TEXT);INSERT INTO metric_meta VALUES('owner','gitcrawl'),('version','1');CREATE TABLE threads(id INTEGER)", + } { + t.Run(schema, func(t *testing.T) { + c := testConfig(t) + s, err := store.Open(context.Background(), store.Options{Path: c.Database, Schema: schema}) + if err != nil { + t.Fatal(err) + } + s.Close() + before, err := os.ReadFile(c.Database) + if err != nil { + t.Fatal(err) + } + if s, err := Open(context.Background(), c.Database); err == nil { + s.Close() + t.Fatal("archive accepted") + } + if _, err := Execute(context.Background(), "status", c, nil, nil); err == nil { + t.Fatal("wrong schema status accepted") + } + after, _ := os.ReadFile(c.Database) + if string(before) != string(after) { + t.Fatal("archive bytes changed") + } + }) + } + c := testConfig(t) + s := openTestStore(t, c) + s.Close() + link := filepath.Join(t.TempDir(), "linked.db") + if err := os.Symlink(c.Database, link); err != nil { + t.Skip(err) + } + if s, err := Open(context.Background(), link); err == nil { + s.Close() + t.Fatal("database symlink accepted") + } + empty := filepath.Join(t.TempDir(), "empty.db") + if err := os.WriteFile(empty, nil, 0600); err != nil { + t.Fatal(err) + } + if s, err := Open(context.Background(), empty); err == nil { + s.Close() + t.Fatal("empty existing database accepted") + } +} + +func TestValidationAndReadOnlyStatus(t *testing.T) { + c := testConfig(t) + if _, err := Execute(context.Background(), "status", c, nil, nil); err == nil { + t.Fatal("missing status accepted") + } + if _, err := os.Stat(c.Database); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("status created database: %v", err) + } + for _, mutate := range []func(*Config){func(c *Config) { c.Database = "relative.db" }, func(c *Config) { c.Targets = nil }, func(c *Config) { c.Targets = append(c.Targets, c.Targets[0]) }, func(c *Config) { c.Targets[0].Target = "../archive" }, func(c *Config) { c.TokenEnv = "bad-name" }, func(c *Config) { c.CookieJar = "/unused" }} { + cfg := testConfig(t) + mutate(&cfg) + if err := cfg.Validate(); err == nil { + t.Fatalf("config accepted: %+v", cfg) + } + } + for _, mutate := range []func(*Row){func(r *Row) { r.Type = "unknown" }, func(r *Row) { r.Entity = "" }, func(r *Row) { r.Provenance = "" }, func(r *Row) { r.TS = "bad" }, func(r *Row) { r.Kind = "gauge" }, func(r *Row) { r.Value = new(float64); *r.Value = -1 }, func(r *Row) { r.Value = new(float64); *r.Value = math.Inf(1) }, func(r *Row) { r.Kind = "daily" }, func(r *Row) { r.Type = "event"; r.Label = "" }} { + r := testRow() + mutate(&r) + if err := Validate(r); err == nil { + t.Fatalf("row accepted: %+v", r) + } + } + for _, value := range []float64{-1, math.Inf(1), math.NaN()} { + if Value(value) != nil { + t.Fatal("invalid value accepted") + } + } + for _, body := range []string{`{"database":"/tmp/test","targets":[]} invalid`, `{"database":"/tmp/test","targets":[],"unknown":true}`} { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + if _, err := ReadConfig(path); err == nil { + t.Fatal("invalid JSON config accepted") + } + } +} + +func TestDatabaseWhitespaceCannotBypassArchiveOwnership(t *testing.T) { + c := testConfig(t) + if err := os.WriteFile(c.Database, []byte("archive bytes"), 0600); err != nil { + t.Fatal(err) + } + for _, suffix := range []string{" ", "\t", "\n"} { + path := c.Database + suffix + if s, err := Open(context.Background(), path); err == nil { + s.Close() + t.Fatal("whitespace path accepted") + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("created alias path: %v", err) + } + cfg := c + cfg.Database = path + if err := cfg.Validate(); err == nil { + t.Fatal("config accepted whitespace alias") + } + } + if b, _ := os.ReadFile(c.Database); string(b) != "archive bytes" { + t.Fatal("archive changed") + } + c.Targets[0].Target = "openclaw/.github" + if err := c.Validate(); err != nil { + t.Fatal(err) + } + c.Targets[0].Target = "openclaw/.." + if err := c.Validate(); err == nil { + t.Fatal("traversal target accepted") + } +} + +func TestCollectRetainsPartialAndCancelledResultsAtomically(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(fmt.Sprint(cancelled), func(t *testing.T) { + c := testConfig(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + collect := func(_ context.Context, _ Config, ts string) ([]Row, error) { + if cancelled { + cancel() + } + r := testRow() + r.TS = ts + r.ObservedAt = ts + missing := r + missing.Metric = "watchers" + missing.Value = nil + return []Row{r, missing}, errors.New("do not expose source credentials") + } + result, err := Execute(ctx, "collect", c, collect, nil) + if err == nil || result.OK || result.RowsWritten != 2 || strings.Contains(err.Error(), "credentials") { + t.Fatalf("result=%+v error=%v", result, err) + } + s, err := ownedReadOnly(context.Background(), c.Database) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var status string + var n int + if err := s.DB().QueryRow("SELECT status,rows_written FROM metric_runs").Scan(&status, &n); err != nil || status != "partial" || n != 2 { + t.Fatalf("run=%s,%d %v", status, n, err) + } + }) + } + c := testConfig(t) + result, err := Execute(context.Background(), "collect", c, func(context.Context, Config, string) ([]Row, error) { + r := testRow() + r.Target = "outside/scope" + return []Row{r}, nil + }, nil) + if err == nil || result.RowsWritten != 0 { + t.Fatalf("scope leak: %+v %v", result, err) + } + s := openTestStore(t, c) + r := testRow() + bad := r + bad.Type = "invalid" + n, err := Write(context.Background(), s, []Row{r, bad}) + if err == nil || n != 0 { + t.Fatalf("write rollback=%d %v", n, err) + } +} + +func TestStatusOrdersExistingTimestampsChronologically(t *testing.T) { + for _, times := range [][]string{ + {"2026-09-15T01:00:00+02:00", "2026-09-15T00:00:00Z"}, + {"2026-09-15T00:00:00Z", "2026-09-15T00:00:00.000000001Z"}, + } { + c := testConfig(t) + s := openTestStore(t, c) + for i, at := range times { + r := testRow() + r.ID = fmt.Sprint(i) + r.ObservedAt = at + if _, err := Import(context.Background(), s, c, strings.NewReader(ndjson(t, r))); err != nil { + t.Fatal(err) + } + } + result, err := Execute(context.Background(), "status", c, nil, nil) + if err != nil || result.LastObserved == nil || *result.LastObserved != times[1] { + t.Fatalf("status = %+v, %v; latest instant = %s", result, err, times[1]) + } + } +} + +func TestFailedInitializationCanRetryWithoutAdoptingForeignFiles(t *testing.T) { + c := testConfig(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if s, err := Open(ctx, c.Database); err == nil { + s.Close() + t.Fatal("canceled initialization succeeded") + } + if _, err := os.Stat(c.Database); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed initialization stranded a file: %v", err) + } + s := openTestStore(t, c) + if _, err := Write(context.Background(), s, []Row{testRow()}); err != nil { + t.Fatal(err) + } + s.Close() + if s, err := Open(ctx, c.Database); err == nil { + s.Close() + t.Fatal("canceled reopen succeeded") + } + result, err := Execute(context.Background(), "status", c, nil, nil) + if err != nil || result.Observations != 1 { + t.Fatalf("existing history lost after failed reopen: %+v, %v", result, err) + } +} + +func TestRefuseMetricsDatabaseHardlinkAliases(t *testing.T) { + c := testConfig(t) + s := openTestStore(t, c) + s.Close() + alias := filepath.Join(t.TempDir(), "alias.sqlite") + if err := os.Link(c.Database, alias); err != nil { + t.Skip(err) + } + before, err := os.ReadFile(c.Database) + if err != nil { + t.Fatal(err) + } + for _, path := range []string{c.Database, alias} { + cfg := c + cfg.Database = path + if _, err := Execute(context.Background(), "import", cfg, nil, strings.NewReader("")); err == nil { + t.Fatal("hardlinked database bypasses writer serialization") + } + } + after, err := os.ReadFile(c.Database) + if err != nil || string(after) != string(before) { + t.Fatal("hardlinked database changed") + } +} + +func TestDailyCollectionUsesLatestImportedUTCDay(t *testing.T) { + c := testConfig(t) + s := openTestStore(t, c) + ctx := context.Background() + r := testRow() + r.ID, r.Kind, r.Metric = "imported-day", "daily", "clones" + r.TS, r.Provenance = "2026-09-15T01:00:00+02:00", "github_traffic" + if _, err := Import(ctx, s, c, strings.NewReader(ndjson(t, r))); err != nil { + t.Fatal(err) + } + r.ID, r.TS = "", "2026-09-14T23:59:59.999Z" + for i, value := range []*float64{Value(12), nil, nil, Value(0), Value(12)} { + r.Value = value + r.ObservedAt = fmt.Sprintf("2026-09-15T%02d:00:00Z", i+2) + n, err := Write(ctx, s, []Row{r}) + want := 1 + if i == 0 || i == 2 { + want = 0 + } + if err != nil || n != want { + t.Fatalf("daily revision %d = %d, %v; want %d", i, n, err, want) + } + } +} + +func TestConfigRejectsContentBeyondSizeLimit(t *testing.T) { + c := testConfig(t) + data, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "metrics.json") + data = append(data, []byte(strings.Repeat(" ", 1024*1024)+`{"ignored":true}`)...) + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + if _, err := ReadConfig(path); err == nil { + t.Fatal("oversized config with trailing JSON was accepted") + } +} + +func TestOpenRejectsReplacedDatabaseBeforeSchemaWrites(t *testing.T) { + ctx := context.Background() + for _, existing := range []bool{false, true} { + for _, parent := range []bool{false, true} { + for _, metrics := range []bool{false, true} { + t.Run(fmt.Sprintf("existing=%t/parent=%t/metrics=%t", existing, parent, metrics), func(t *testing.T) { + root := t.TempDir() + active := filepath.Join(root, "active") + if err := os.Mkdir(active, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(active, "metrics.sqlite") + if existing { + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + } + replacementDir := filepath.Join(root, "replacement") + replacement := filepath.Join(replacementDir, "metrics.sqlite") + schema := "CREATE TABLE threads(id INTEGER PRIMARY KEY,body TEXT); INSERT INTO threads VALUES(1,'archive retained')" + if metrics { + schema = Schema + "INSERT INTO metric_meta VALUES('owner','gitcrawl'),('version','1'); INSERT INTO metric_runs(ts,status,rows_written) VALUES('retained','ok',7)" + } + s, err := store.Open(ctx, store.Options{Path: replacement, Schema: schema}) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + replacementInfo, err := os.Stat(replacement) + if err != nil { + t.Fatal(err) + } + called := false + beforeWritableOpen = func() { + called = true + from, to := replacement, path + if parent { + from, to = replacementDir, active + } + if err := os.Rename(to, to+".original"); err != nil { + t.Fatal(err) + } + if err := os.Rename(from, to); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { beforeWritableOpen = nil }) + opened, err := Open(ctx, path) + if opened != nil { + opened.Close() + } + if !called || err == nil || opened != nil { + t.Fatalf("replaced database accepted: called=%t store=%v error=%v", called, opened, err) + } + current, err := os.Stat(path) + if err != nil || !os.SameFile(replacementInfo, current) { + t.Fatalf("replacement removed or changed: %v", err) + } + read, err := store.OpenReadOnly(ctx, path) + if err != nil { + t.Fatal(err) + } + defer read.Close() + if metrics { + var retained int + if err := read.DB().QueryRow("SELECT rows_written FROM metric_runs WHERE ts='retained'").Scan(&retained); err != nil || retained != 7 { + t.Fatalf("replacement metrics history changed: %d, %v", retained, err) + } + } else { + var objects int + if err := read.DB().QueryRow("SELECT count(*) FROM sqlite_master WHERE name GLOB 'metric_*'").Scan(&objects); err != nil || objects != 0 { + t.Fatalf("metrics schema landed in archive: %d, %v", objects, err) + } + var body string + if err := read.DB().QueryRow("SELECT body FROM threads WHERE id=1").Scan(&body); err != nil || body != "archive retained" { + t.Fatalf("archive contents changed: %q, %v", body, err) + } + } + }) + } + } + } +} + +func TestOpenRevalidatesDatabaseChangedInPlace(t *testing.T) { + ctx := context.Background() + for _, existing := range []bool{false, true} { + t.Run(fmt.Sprint(existing), func(t *testing.T) { + c := testConfig(t) + if existing { + s := openTestStore(t, c) + if err := s.Close(); err != nil { + t.Fatal(err) + } + } + beforeWritableOpen = func() { + schema := "CREATE VIEW unrelated AS SELECT 1" + if existing { + schema = "UPDATE metric_meta SET value='another-owner' WHERE key='owner'" + } + s, err := store.Open(ctx, store.Options{Path: c.Database, Schema: schema}) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { beforeWritableOpen = nil }) + s, err := Open(ctx, c.Database) + if s != nil { + s.Close() + } + if err == nil || s != nil { + t.Fatalf("changed database accepted: %v, %v", s, err) + } + if existing { + read, err := store.OpenReadOnly(ctx, c.Database) + if err != nil { + t.Fatal(err) + } + defer read.Close() + var owner string + if err := read.DB().QueryRow("SELECT value FROM metric_meta WHERE key='owner'").Scan(&owner); err != nil || owner != "another-owner" { + t.Fatalf("replaced ownership overwritten: %q, %v", owner, err) + } + } + }) + } +} + +func TestOpenRollsBackSchemaWhenOwnershipInsertFails(t *testing.T) { + ctx := context.Background() + c := testConfig(t) + s, err := store.Open(ctx, store.Options{Path: c.Database, Schema: ` +CREATE TABLE metric_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL); +INSERT INTO metric_meta VALUES('owner','gitcrawl'),('version','1'); +CREATE TRIGGER reject_ownership BEFORE INSERT ON metric_meta BEGIN SELECT RAISE(ABORT,'fixture rejection'); END; +`}) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if s, err := Open(ctx, c.Database); err == nil { + s.Close() + t.Fatal("ownership insert unexpectedly succeeded") + } + read, err := store.OpenReadOnly(ctx, c.Database) + if err != nil { + t.Fatal(err) + } + var tables int + err = read.DB().QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table' AND name IN ('metric_observations','metric_events','metric_runs')").Scan(&tables) + closeErr := read.Close() + if err != nil || closeErr != nil || tables != 0 { + t.Fatalf("schema was not rolled back: tables=%d error=%v close=%v", tables, err, closeErr) + } + // The failed open must release its transaction and connection for a retry. + s, err = store.Open(ctx, store.Options{Path: c.Database, Schema: "DROP TRIGGER reject_ownership"}) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(ctx, c.Database) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/scripts/build-docs-site.mjs b/scripts/build-docs-site.mjs index bf09cb3f..12086d8a 100644 --- a/scripts/build-docs-site.mjs +++ b/scripts/build-docs-site.mjs @@ -16,7 +16,7 @@ const sections = [ ["Start", ["index.md", "installation.md", "quickstart.md", "concepts.md"]], ["Configure", ["configuration.md", "sync.md", "refresh-and-embed.md"]], ["Use", ["search.md", "clustering.md", "governance.md", "tui.md", "gh-shim.md"]], - ["Operate", ["portable-stores.md", "maintainer-archive.md", "automation.md", "releasing.md"]], + ["Operate", ["portable-stores.md", "maintainer-archive.md", "metrics.md", "automation.md", "releasing.md"]], ["Reference", ["commands.md", "reference.md"]], ]; From 56ece895c12c10355d4f4eb2d2760440932f77d8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 28 Sep 2026 04:23:56 -0700 Subject: [PATCH 18/21] fix: preserve literal args, reject inconsistent history pages, load closed-thread neighbors (#220) * fix(cli): preserve arguments after the end-of-options marker * fix(sync): reject inconsistent GraphQL continuation counts * fix(tui): load neighbors for selected closed threads --- docs/search.md | 3 +++ docs/sync.md | 2 +- docs/tui.md | 3 +++ internal/cli/args.go | 5 ++++ internal/cli/args_test.go | 24 ++++++++++++++++++ internal/cli/gh_search_test.go | 26 ++++++++++++++++++++ internal/cli/tui_neighbors.go | 9 ++++--- internal/cli/tui_test.go | 43 +++++++++++++++++++++++++++++++++ internal/github/history.go | 9 ++++++- internal/github/history_test.go | 38 +++++++++++++++++++++++++++++ 10 files changed, 156 insertions(+), 6 deletions(-) diff --git a/docs/search.md b/docs/search.md index cc89b8f1..8904d490 100644 --- a/docs/search.md +++ b/docs/search.md @@ -71,6 +71,9 @@ gitcrawl search prs "manifest cache" \ --limit 20 ``` +Put `--` before query terms that begin with a dash, with options before it: +`gitcrawl search issues -R owner/repo --json number,title -- --verbose`. + Recognized flags in this mode: | Flag | Description | diff --git a/docs/sync.md b/docs/sync.md index f3cc291d..155bf3bc 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -102,7 +102,7 @@ no REST requests or fallback. The regular sync path is unchanged. The profile requires the flags above; it rejects `--since`, `--limit` and full PR-detail hydration. It does not collect files, commit bodies, checks or Actions logs. An incomplete GraphQL response, unavailable parent, missing identity, -duplicate child within a connection, mismatched count or nonadvancing cursor +duplicate child within a connection, mismatched or changing count, or nonadvancing cursor fails the batch before archive writes. Supervisors should retry failed selections in isolation. Empty reviews remain retained, including approvals without bodies. Minimized comments and null diff --git a/docs/tui.md b/docs/tui.md index cef5fea4..b7632077 100644 --- a/docs/tui.md +++ b/docs/tui.md @@ -65,6 +65,9 @@ The view auto-refreshes from the local store every 15 seconds. There is no GitHu The action menu opened with `a` mirrors the right-click menu, so every mouse action has a keyboard equivalent. +Neighbor loading can use the saved embedding of a selected closed thread as +well as an open thread. Results contain open, locally active neighbors. + Jump input accepts the same thread references as the CLI: bare numbers, `#123`, `issues/123`, `pull/123`, `owner/repo#123`, and full GitHub issue or pull request URLs. diff --git a/internal/cli/args.go b/internal/cli/args.go index 6f1e1c24..889fdfe8 100644 --- a/internal/cli/args.go +++ b/internal/cli/args.go @@ -5,6 +5,11 @@ func normalizeCommandArgs(args []string, stringFlags map[string]bool) []string { var positionals []string for index := 0; index < len(args); index++ { arg := args[index] + if arg == "--" { + flags = append(flags, "--") + positionals = append(positionals, args[index+1:]...) + break + } name, ok := flagName(arg) if !ok { positionals = append(positionals, arg) diff --git a/internal/cli/args_test.go b/internal/cli/args_test.go index 6a156886..a6293a6b 100644 --- a/internal/cli/args_test.go +++ b/internal/cli/args_test.go @@ -1,10 +1,34 @@ package cli import ( + "flag" "reflect" "testing" ) +func TestNormalizeCommandArgsPreservesEndOfOptions(t *testing.T) { + for _, args := range [][]string{ + {"--limit", "5", "--", "--json"}, + {"first", "--limit", "5", "--", "--json", "-R", "last"}, + } { + t.Run(args[0], func(t *testing.T) { + fs := flag.NewFlagSet("search", flag.ContinueOnError) + limit := fs.String("limit", "", "") + jsonOut := fs.Bool("json", false, "") + if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"limit": true})); err != nil { + t.Fatal(err) + } + want := []string{"--json"} + if args[0] == "first" { + want = []string{"first", "--json", "-R", "last"} + } + if *limit != "5" || *jsonOut || !reflect.DeepEqual(fs.Args(), want) { + t.Fatalf("limit=%q json=%v args=%q; want limit=5 json=false args=%q", *limit, *jsonOut, fs.Args(), want) + } + }) + } +} + func TestNormalizeCommandArgsMovesFlagsBeforePositionals(t *testing.T) { got := normalizeCommandArgs([]string{"openclaw/openclaw", "--query", "download", "--json"}, map[string]bool{"query": true}) want := []string{"--query", "download", "--json", "openclaw/openclaw"} diff --git a/internal/cli/gh_search_test.go b/internal/cli/gh_search_test.go index aeb79dcc..9c9d7f80 100644 --- a/internal/cli/gh_search_test.go +++ b/internal/cli/gh_search_test.go @@ -42,6 +42,32 @@ func TestParseGHSearchDuration(t *testing.T) { } } +func TestGHSearchLiteralFlagQuery(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + seedPortableThread(t, dbPath, 42, "Broken --verbose flag") + seedPortableThread(t, dbPath, 43, "Unrelated issue") + app := New() + var stdout, stderr bytes.Buffer + app.Stdout, app.Stderr = &stdout, &stderr + if err := app.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatal(err) + } + stdout.Reset() + if err := app.Run(ctx, []string{"--config", configPath, "search", "issues", "-R", "openclaw/openclaw", "--json", "number", "--", "--verbose"}); err != nil { + t.Fatal(err) + } + var rows []struct{ Number int } + if err := json.Unmarshal(stdout.Bytes(), &rows); err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].Number != 42 { + t.Fatalf("search returned %s; want only issue #42", stdout.String()) + } +} + func TestGHSearchCacheStaleUsesRepoSyncRuns(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/cli/tui_neighbors.go b/internal/cli/tui_neighbors.go index c07d9a00..c3ced8a1 100644 --- a/internal/cli/tui_neighbors.go +++ b/internal/cli/tui_neighbors.go @@ -104,13 +104,14 @@ func loadThreadNeighbors(ctx context.Context, st *store.Store, repoID int64, pay threshold = 0.2 } targetThread, targetVector, err := st.ThreadVectorByNumber(ctx, store.ThreadVectorQuery{ - RepoID: repoID, - Model: payload.EmbedModel, - Basis: payload.EmbeddingBasis, + RepoID: repoID, + Model: payload.EmbedModel, + Basis: payload.EmbeddingBasis, + IncludeClosed: true, }, number) if err != nil { var fallbackErr error - targetThread, targetVector, fallbackErr = st.ThreadVectorByNumber(ctx, store.ThreadVectorQuery{RepoID: repoID}, number) + targetThread, targetVector, fallbackErr = st.ThreadVectorByNumber(ctx, store.ThreadVectorQuery{RepoID: repoID, IncludeClosed: true}, number) if fallbackErr != nil { return 0, 0, nil, err } diff --git a/internal/cli/tui_test.go b/internal/cli/tui_test.go index 97eb16b4..434d3a49 100644 --- a/internal/cli/tui_test.go +++ b/internal/cli/tui_test.go @@ -2712,6 +2712,49 @@ func TestTUILoadNeighborsFromStore(t *testing.T) { } } +func TestTUILoadNeighborsForClosedSelection(t *testing.T) { + for _, modelName := range []string{"test", "missing-configured-model"} { + t.Run(modelName, func(t *testing.T) { + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", RawJSON: "{}"}) + if err != nil { + t.Fatal(err) + } + targetID, err := seedTUIThreadVector(ctx, st, repoID, 1, "Closed target", []float64{1, 0}) + if err != nil { + t.Fatal(err) + } + neighborID, err := seedTUIThreadVector(ctx, st, repoID, 2, "Open neighbor", []float64{0.9, 0.1}) + if err != nil { + t.Fatal(err) + } + if _, err := st.DB().ExecContext(ctx, `update threads set state = 'closed' where id = ?`, targetID); err != nil { + t.Fatal(err) + } + model := newClusterBrowserModel(ctx, st, repoID, clusterBrowserPayload{Repository: "fixture/repo", EmbedModel: modelName, EmbeddingBasis: "title_original"}) + model.memberIndex = 0 + model.memberRows = []memberRow{{selectable: true, member: store.ClusterMemberDetail{Thread: store.Thread{ + ID: targetID, Number: 1, State: "closed", HTMLURL: "https://github.com/fixture/repo/issues/1", + }}}} + cmd := model.requestSelectedThreadNeighbors(10, 0.2) + if cmd == nil { + t.Fatal("neighbor command missing") + } + updated, _ := model.Update(cmd()) + model = updated.(clusterBrowserModel) + neighbors := model.neighborCache[targetID] + if len(neighbors) != 1 || neighbors[0].Thread.ID != neighborID { + t.Fatalf("neighbors=%+v status=%q; want open neighbor %d", neighbors, model.status, neighborID) + } + }) + } +} + func TestTUILoadNeighborsUsesConfiguredBackend(t *testing.T) { ctx := context.Background() st, err := store.Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) diff --git a/internal/github/history.go b/internal/github/history.go index b0012a27..0135e11b 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -241,6 +241,10 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error if conn == nil { return fmt.Errorf("missing %s connection", key) } + total, ok := historyInt(conn["totalCount"]) + if !ok || total < 0 { + return fmt.Errorf("invalid history %s count", key) + } fields, err := historyFields(typ, key) if err != nil { return err @@ -270,6 +274,9 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error return fmt.Errorf("history pagination identity mismatch") } nxt := historyMap(parent[key]) + if nextTotal, ok := historyInt(nxt["totalCount"]); !ok || nextTotal != total { + return fmt.Errorf("changed or missing history %s count", key) + } a, ok := conn["nodes"].([]any) if !ok { return fmt.Errorf("missing history nodes") @@ -285,7 +292,7 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error if !ok { return fmt.Errorf("missing history nodes") } - if total, ok := historyInt(conn["totalCount"]); !ok || total != len(children) { + if total != len(children) { return fmt.Errorf("incomplete history %s count", key) } ids := map[string]bool{} diff --git a/internal/github/history_test.go b/internal/github/history_test.go index 3c3a88c8..5668e7d5 100644 --- a/internal/github/history_test.go +++ b/internal/github/history_test.go @@ -138,6 +138,43 @@ func TestGraphQLHistoryRejectsShortConnection(t *testing.T) { } } +func TestGraphQLHistoryRejectsChangedContinuationCount(t *testing.T) { + for _, total := range []any{1, 3, nil} { + t.Run(fmt.Sprint(total), func(t *testing.T) { + comment := func(id string) map[string]any { + return map[string]any{"id": id, "__typename": "IssueComment", "fullDatabaseId": id, "body": id} + } + node := historyTestNode() + first := historyTestConnection(comment("1")) + first["totalCount"] = 2 + first["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "first"} + node["comments"] = first + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": "2099-01-01T01:00:00Z"}} + if strings.Contains(req.Query, "issueOrPullRequest") { + data["repository"] = map[string]any{"databaseId": 42, "nameWithOwner": "fixture/repo", "n0": node} + } + if strings.Contains(req.Query, "node(id:") { + last := historyTestConnection(comment("2")) + last["totalCount"] = total + data["node"] = map[string]any{"id": node["id"], "comments": last} + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + batch, err := New(Options{BaseURL: server.URL}).FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err == nil || len(batch.Items) != 0 { + t.Fatalf("accepted inconsistent continuation count %v: items=%d err=%v", total, len(batch.Items), err) + } + }) + } +} + func TestGraphQLHistoryIssueDiscussion(t *testing.T) { node := historyTestNode() node["__typename"] = "Issue" @@ -212,6 +249,7 @@ func TestGraphQLHistoryReviewThreadCompleteness(t *testing.T) { switch req.Variables["id"] { case "PR_fixture": conn := historyTestConnection(thread2) + conn["totalCount"] = 2 if mode == "repeated-thread" { conn["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "thread-first"} } From 58e5d6299e3640ea42a74808c427cc6239ebbe3a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 28 Sep 2026 04:41:14 -0700 Subject: [PATCH 19/21] chore: prepare 0.13.0 release notes (#221) --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a72f533..354d8112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,14 @@ # Changelog -## Unreleased +## 0.13.0 - 2026-09-28 +**Highlights:** New `gitcrawl metrics` commands for repository headline history, and much faster clustering and exact neighbors. + +- Add `gitcrawl metrics collect|import|status --config metrics.json` to keep stars, forks, subscribers, open issue/PR counts, completed-day clones, and stable releases in a separate private SQLite store, with atomic imports, preserved unknown values, daily corrections, partial-result reporting when quota runs out, and JSON output. Thanks @hannesrudolph. - Prepare vectors once for 2.6× faster cluster scoring and 2.0× faster exact neighbors; enable `GOEXPERIMENT=simd` in release builds for a further 2.6×/1.7× on Apple M3 Ultra (1,024 dimensions). +- Reject GraphQL history pages whose continuation reports a different total count, so a comment deleted mid-pagination can no longer leave a stale conversation marked complete. +- Keep arguments after `--` literal (for example `gitcrawl search -- --flag-like-text`) instead of treating them as options. +- Load TUI neighbors for a selected closed thread that has a stored embedding instead of reporting it missing. - Disable automatic Git maintenance during portable-refresh fixture setup so temporary repositories do not launch detached cleanup work. ## 0.12.0 - 2026-09-24 From 6ccd9334ccb0d2303ba1e8457d7569e643c64720 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 05:01:33 -0700 Subject: [PATCH 20/21] chore: open next unreleased section (#222) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 354d8112..f05b2fb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## Unreleased + ## 0.13.0 - 2026-09-28 **Highlights:** New `gitcrawl metrics` commands for repository headline history, and much faster clustering and exact neighbors. From d8d19effd37d78528d13b4cee169e13d02f16edd Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 29 Sep 2026 18:53:05 -0700 Subject: [PATCH 21/21] build(deps): update crawlkit to v0.16.6 (#223) --- CHANGELOG.md | 2 ++ README.md | 2 +- docs/installation.md | 2 +- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f05b2fb2..8f862514 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Update CrawlKit to v0.16.6 for faster vector validation in the TurboVec search backend. + ## 0.13.0 - 2026-09-28 **Highlights:** New `gitcrawl metrics` commands for repository headline history, and much faster clustering and exact neighbors. diff --git a/README.md b/README.md index 7488818e..1c98a4e1 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Homebrew is the smallest install on macOS and Linux: brew install openclaw/tap/gitcrawl ``` -Prebuilt archives for macOS, Linux, and Windows are available from [GitHub Releases](https://github.com/openclaw/gitcrawl/releases/latest). The [installation guide](docs/installation.md) also covers source builds and update checks; source builds require Go 1.27.1 or newer for CrawlKit v0.16.5. Binaries built with this toolchain require macOS 13 Ventura or newer. A [Docker source build](docs/docker.md) keeps its runtime state under one mounted directory. +Prebuilt archives for macOS, Linux, and Windows are available from [GitHub Releases](https://github.com/openclaw/gitcrawl/releases/latest). The [installation guide](docs/installation.md) also covers source builds and update checks; source builds require Go 1.27.1 or newer for CrawlKit v0.16.6. Binaries built with this toolchain require macOS 13 Ventura or newer. A [Docker source build](docs/docker.md) keeps its runtime state under one mounted directory. Sync needs a GitHub token from `GITHUB_TOKEN` or `gh auth token`. Generating summaries and embeddings also needs `OPENAI_API_KEY`; semantic search and clustering use those stored embeddings, while ordinary sync and keyword search do not. diff --git a/docs/installation.md b/docs/installation.md index 23bb9b75..074b485e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -20,7 +20,7 @@ permalink: /installation/ gitcrawl runs on macOS 13 Ventura or newer and Linux. Windows is not actively tested. -CrawlKit v0.16.5 requires Go 1.27, so source and Docker builds use Go 1.27.1 +CrawlKit v0.16.6 requires Go 1.27, so source and Docker builds use Go 1.27.1 or newer. Go 1.27 also raises the minimum macOS version for newly built binaries to macOS 13; the previous Go 1.26 build baseline no longer applies. diff --git a/go.mod b/go.mod index ddf141d3..1f28c063 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/charmbracelet/x/ansi v0.11.8 github.com/mattn/go-isatty v0.0.24 github.com/muesli/termenv v0.16.0 - github.com/openclaw/crawlkit v0.16.5 + github.com/openclaw/crawlkit v0.16.6 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.48.0 modernc.org/sqlite v1.59.0 diff --git a/go.sum b/go.sum index f70845b4..4fe717d5 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.16.5 h1:4hUiPo6lLEwqLPtgBKXVeHhVVCu4hYD5R+dMN8nAgbU= -github.com/openclaw/crawlkit v0.16.5/go.mod h1:XTGhBPNiMqKzPuKNwAH9ufbUU0Vel0pEQAZFqS/b7GQ= +github.com/openclaw/crawlkit v0.16.6 h1:Jq2hvcy0ugIkxd+GfNnMPryvYbIsONyg05rVMNbTLfU= +github.com/openclaw/crawlkit v0.16.6/go.mod h1:XTGhBPNiMqKzPuKNwAH9ufbUU0Vel0pEQAZFqS/b7GQ= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=