diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb5bd21..17d142a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,6 +76,13 @@ jobs: echo "total coverage: ${total}%" awk -v total="$total" 'BEGIN { if (total + 0 < 85.0) { printf("coverage %.1f%% is below 85.0%%\n", total); exit 1 } }' + - name: Test portable SIMD and scalar fallback + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + # Release binaries are built with this experiment, so test everything they ship. + GOEXPERIMENT=simd go test ./... + GOEXPERIMENT=simd GODEBUG=simd=0 go test ./internal/vector ./internal/cli + - name: Build run: go build -ldflags "-X github.com/openclaw/gitcrawl/internal/cli.version=${GITHUB_SHA:0:7}" -o bin/gitcrawl ./cmd/gitcrawl diff --git a/.gitignore b/.gitignore index 90906075..997f246c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ tmp/ cache/ logs/ vectors/ + +.crabbox/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 676f374e..c4ed3986 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -11,6 +11,7 @@ builds: binary: gitcrawl env: - CGO_ENABLED=0 + - GOEXPERIMENT=simd ldflags: - -s -w -X github.com/openclaw/gitcrawl/internal/cli.version={{ .Version }} targets: @@ -23,6 +24,7 @@ builds: binary: gitcrawl env: - CGO_ENABLED=0 + - GOEXPERIMENT=simd ldflags: - -s -w -X github.com/openclaw/gitcrawl/internal/cli.version={{ .Version }} targets: diff --git a/CHANGELOG.md b/CHANGELOG.md index d4714a60..c79c8660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +- Add `gitcrawl analytics` for publication repair, actor evidence, continuous collection, and durable review-state recovery with separate core/enrichment coverage and private failure receipts. Thanks @hannesrudolph. + +- Update CrawlKit to v0.16.6 for faster vector validation in the TurboVec search backend. + +## 0.13.0 - 2026-09-28 + +**Highlights:** New `gitcrawl metrics` commands for repository headline history, and much faster clustering and exact neighbors. + +- Add `gitcrawl metrics collect|import|status --config metrics.json` to keep stars, forks, subscribers, open issue/PR counts, completed-day clones, and stable releases in a separate private SQLite store, with atomic imports, preserved unknown values, daily corrections, partial-result reporting when quota runs out, and JSON output. Thanks @hannesrudolph. +- Prepare vectors once for 2.6× faster cluster scoring and 2.0× faster exact neighbors; enable `GOEXPERIMENT=simd` in release builds for a further 2.6×/1.7× on Apple M3 Ultra (1,024 dimensions). +- Reject GraphQL history pages whose continuation reports a different total count, so a comment deleted mid-pagination can no longer leave a stale conversation marked complete. +- Keep arguments after `--` literal (for example `gitcrawl search -- --flag-like-text`) instead of treating them as options. +- Load TUI neighbors for a selected closed thread that has a stored embedding instead of reporting it missing. - Disable automatic Git maintenance during portable-refresh fixture setup so temporary repositories do not launch detached cleanup work. ## 0.12.0 - 2026-09-24 diff --git a/Dockerfile b/Dockerfile index 2c30ff53..53f1fce5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . ARG VERSION=dev -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath \ +RUN CGO_ENABLED=0 GOEXPERIMENT=simd GOOS=linux go build -trimpath \ -ldflags="-s -w -X github.com/openclaw/gitcrawl/internal/cli.version=${VERSION}" \ -o /out/gitcrawl ./cmd/gitcrawl diff --git a/README.md b/README.md index 9fd7ae0c..1c98a4e1 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Homebrew is the smallest install on macOS and Linux: brew install openclaw/tap/gitcrawl ``` -Prebuilt archives for macOS, Linux, and Windows are available from [GitHub Releases](https://github.com/openclaw/gitcrawl/releases/latest). The [installation guide](docs/installation.md) also covers source builds and update checks; source builds require Go 1.27.1 or newer for CrawlKit v0.16.5. Binaries built with this toolchain require macOS 13 Ventura or newer. A [Docker source build](docs/docker.md) keeps its runtime state under one mounted directory. +Prebuilt archives for macOS, Linux, and Windows are available from [GitHub Releases](https://github.com/openclaw/gitcrawl/releases/latest). The [installation guide](docs/installation.md) also covers source builds and update checks; source builds require Go 1.27.1 or newer for CrawlKit v0.16.6. Binaries built with this toolchain require macOS 13 Ventura or newer. A [Docker source build](docs/docker.md) keeps its runtime state under one mounted directory. Sync needs a GitHub token from `GITHUB_TOKEN` or `gh auth token`. Generating summaries and embeddings also needs `OPENAI_API_KEY`; semantic search and clustering use those stored embeddings, while ordinary sync and keyword search do not. @@ -64,6 +64,7 @@ Octopool owns pooled live `gh` reads. Gitcrawl keeps local mirror, search, clust | Check archive health | `gitcrawl status` / `gitcrawl doctor` | [Configuration](docs/configuration.md) | | Refresh a portable subscriber | `gitcrawl portable refresh --expected-remote URL` | [Portable stores](docs/portable-stores.md#routine-subscriber-refresh) | | Mirror GitHub threads | `gitcrawl sync owner/repo` | [Sync](docs/sync.md) | +| Collect repository counters and releases | `gitcrawl metrics collect\|import\|status --config metrics.json` | [Repository metrics](docs/metrics.md) | | Search threads or indexed code | `gitcrawl search ...` | [Search](docs/search.md) | | Build and inspect clusters | `gitcrawl refresh`, `clusters`, `tui` | [Clustering](docs/clustering.md) | | Export a code-free conversation snapshot | `gitcrawl capture owner/repo` | [Capture](docs/capture.md) | diff --git a/docs/analytics-source.md b/docs/analytics-source.md new file mode 100644 index 00000000..bdbe6df0 --- /dev/null +++ b/docs/analytics-source.md @@ -0,0 +1,231 @@ +--- +title: Analytics source preparation +nav_order: 8 +permalink: /analytics-source/ +--- + +# Analytics source preparation + +`gitcrawl analytics owner/repo` supports collector-owned publication repair, +identity enrichment and ongoing GraphQL collection for a full-history archive. +It uses the configured native source database and the existing token helper. +It does not run embeddings or classification models. + +```sh +gitcrawl --config SOURCE_CONFIG analytics owner/repo --json +gitcrawl --config SOURCE_CONFIG analytics owner/repo --status --json +gitcrawl --config SOURCE_CONFIG analytics owner/repo --apply --json +gitcrawl --config SOURCE_CONFIG --github-token-command TOKEN_HELPER \ + analytics owner/repo --enrich --watch --json +``` + +Without apply/enrich/watch/once, the command audits retained publication evidence +read-only. `--apply` performs an idempotent, bounded repair after taking an +operator-managed consistent backup. It stores `submitted_at_gh` and +`publication_at_gh` for current comments and source revisions. Review submission +is distinct from draft creation; pending reviews and comments with an explicit +null provider publication time have no published event. Legacy REST review +comments without publication evidence remain unknown rather than using draft +creation as publication. Raw +payloads, existing IDs, genuine creation times and original observation times +remain unchanged, and normalization creates no fictional provider edit. + +Source schema 14 added these fields and the actor/coverage evidence tables. +Historical normalization completion is recorded in +`analytics_collection_state.publication_repair_generation`; downstream consumers +must reconcile the affected datasets when this generation changes rather than +relying solely on append-only revision IDs. + +GraphQL history now retains actor node IDs. Profile collection keeps the shared +login and URL fields for all GitHub Actor types, including organizations and +enterprise user accounts. `--enrich` recovers missing identities +through original content nodes, not login guessing, and stores public actor +profiles. Unavailable nodes are explicit unknowns; permission or transport +failures are not converted to successful missing-data evidence. New source actors +enter the profile queue, and profiles become eligible for refresh after 24 hours. +Operational queue tables are separate from publishable evidence. Each enrichment +pass visits each queued identity in order and refetches healthy peers separately +when a provider rejects particular nodes. A one-shot `--enrich` returns an +incomplete result after finishing eligible work; watch mode retries failed nodes +on its next two-minute poll, without blocking later identities. + +`--watch` polls updated issues and PRs every two minutes, overlapping the prior +verified watermark by five minutes. It paginates without GitHub search's hit cap +and hydrates relevant updated threads. Issue and PR lanes have independent durable +checkpoints and process at most two discovery pages per cycle. Eight two-thread +requests progress independently per page. A rejected batch splits into individual +requests; an item may be passed only after its failure is durably queued. Unrelated +items and the other discovery lane continue. Core retries process at most eight +due items before discovery, with a two-minute request deadline and bounded +exponential backoff. After persisting core coverage, targeted review recovery +uses the time until the next nominal two-minute core poll; it does not add a +two-minute idle wait after recovery. Each wave rechecks actual GraphQL quota and +admits up to 256 items through 32 recovery workers, with up to eight PRs per +request. Waves are also limited to a 16 MiB estimated response budget, using a +conservative 64 KiB/item initial estimate that grows with measured bytes. Individual +recovery responses are capped at 32 MiB; rejected batches are isolated normally. +Ordinary capture retains its existing eight two-item workers. Admission +uses the authoritative GraphQL `rateLimit` response rather than REST resource +counters, which can differ. Recovery and its quota probe omit redundant REST +`/rate_limit` preflights: an explicit GraphQL probe binds actual quota to the +selected credential, and every content/page request checks that credential and +unexpired balance against the reserve. Rotation requires a new probe. Ordinary +core transport retains its REST preflight. If that fresh response carries an +expired GraphQL snapshot (including crossing reset during preflight), it performs +a bounded quota-only GraphQL refresh under the existing request lock. The refresh +uses the selected credential and API origin; rotation before content dispatch +fails closed. A stale, invalid, failed, or below-reserve refresh cannot authorize +content. The refreshed balance is checked against the pending page's estimate, +without changing its identity, cursor, or completeness validation. Each history +session also enforces its configured floor against observed GraphQL balances +before pagination. A +client retains the lowest observed GraphQL balance until the reset boundary +passes. An upward sample or a shifted future reset cannot increase admission; +REST snapshot refreshes do not overwrite this evidence. Logs distinguish the +raw provider balance/reset from the conservative effective admission values. A +32-point-per-item admission margin and per-request native quota checks preserve +3,000 points for recovery, leaving 1,500 points above ordinary capture's floor. +Missing or expired quota stops provider recovery; local discovery can continue. +The request window yields before core polling and cancellation retains retry +receipts and the last committed scan. Quota and numeric per-query cost logs make +the actual spending observable without credentials or content bodies. +No partial connection is accepted as complete evidence. Non-nested +continuation pages use 100 nodes to avoid repeated small round trips. Identity/profile enrichment uses eight disjoint 100-node requests with normal +quota guards and can run concurrently under +the same source owner. `--once` performs one resumable update cycle. + +Discovery skips unresolved core retry items. A review-only recovery queue does +not defer a newly discovered core edit: that edit is collected as core work. If +it fails, a core retry obligation prevents repeated discovery attempts until the +bounded scheduler selects it. This prevents enrichment backoff from hiding fresh +core data while its verified coverage watermark advances. +When both queues contain an item, its core retry owns the backoff; the review +retry pass cannot dispatch the same item. Recovery resolves only after an +accepted membership observation exists, including a proven empty set. + +Targeted recovery fetches only PR identity/update metadata and fully paginated +review threads with their complete inline comments and reply-to identities. It +does not request or project the PR body/title, issue comments, or review history. +An independent native child observation reserves only the review-thread family; +existing review state, revisions and exact membership publish in one transaction +after repository/node/number binding. Canonical threads, comments, revisions and +vectors remain untouched. Current content changes remain ordinary capture's job. +A review-only success cannot resolve a core traversal failure. + +Up to one quarter of a recovery wave is reserved for already-attempted due retries; +the remaining slots serve first-pass work, with unused capacity shared. This avoids +waiting behind the entire seeded census. Explicit retained `NOT_FOUND` evidence +has at least a 15-minute backoff; it never implies deletion or empty membership. +`review_state_wave` logs actual peak busy workers, worker/provider/database time, +items, response bytes and reported query points. Private success receipts identify +review-only work and its fetch/persistence timings. + +Hydration workers reuse the watch owner's open store instead of repeating +full-archive migration checks for each batch. Independent guarded clients overlap +network work; native transactions still coordinate source writes. Enrichment +continues checking its queues every two minutes after the initial drain. New +observations enqueue unresolved content identities or known actor profiles +directly, avoiding repeated whole-archive scans during watch operation. + +A completed historical discovery receipt must name the matching repository in +its `repository` field, with explicit nonnegative totals and valid timestamps +for both lanes. It supplies the initial discovery baseline; either lane may +contain zero items. Missing or mismatched repository identity cannot establish +coverage. Keep each receipt with the source archive it describes; when an older +receipt lacks identity, establish its repository provenance before adding that +field. Do not use another repository's completed receipt to skip historical work. +The pre-upgrade checkpoint is retained; migration copies its in-flight cursor into +the corresponding independent lane. `through:owner/repo:issues` and +`through:owner/repo:pullRequests` record discovery progress; the aggregate watermark +is their minimum. These are not proof that queued failures have been repaired. +`analytics_coverage.complete` continues to describe verified core issue/PR/comment +traversal. Core failures clear it until reconciled; the previously verified +watermark remains available while another page is in progress. Historical +review-state enrichment does not invalidate core contributor/response coverage. + +## Review-state and failure contracts (schema 15) + +GraphQL collection requires explicit `isResolved` and `isOutdated` values and +retains review-thread IDs, source state, source comments and immediate reply IDs. +It writes the existing tables: + +- `pull_request_review_threads`: current observation, keyed by + `(thread_id, review_thread_id)`, including `is_resolved`, `is_outdated`, + `comments_json`, retained `raw_json`, and actual `fetched_at`. +- `pull_request_review_thread_revisions`: append-only changes, with integer `id` + and actual `recorded_at`; unchanged observations do not manufacture revisions. +- `pull_request_review_thread_syncs`: last complete observation per `thread_id`. + New nullable `review_thread_ids_json` contains its exact native ID membership. + NULL means not acquired under this contract; `[]` means a complete empty set. +- `analytics_review_state_coverage`: keyed by `repository`, with `cursor`, + `ceiling`, `scanned`, `queued`, `pending_items`, `scan_complete`, `complete`, + and `observed_at`. This separate completion contract requires a finished + targeted scan and no outstanding review-state recovery items. + +Absence from a complete membership set never invents a provider deletion or +removes retained history. Existing comment IDs/replies and +`thread_child_observation_memberships` keep their existing contract. Consumers +must distinguish historical rows from the latest provider membership. + +The watch inspects bounded chunks of 5,000 primary-key rows within each recovery +window up to a captured +ceiling, queuing only PRs with retained review threads or unknown connection data. +Complete retained zero-count GraphQL connections materialize `[]` using their +actual retained observation time. Conditional writes preserve a newer live +observation; incomplete or undated evidence instead enters provider recovery. +`review_state_recovery:owner/repo` records cursor, ceiling, scanned/queued counts +and scan completion. Queue completion is separately required for review-state +coverage, never for the existing core coverage flag. This +targeted recovery does not restart historical discovery or enable remote syncing; +missing historical resolution states cannot be reconstructed from old payloads. + +Portable exports omit analytics source tables and actor evidence; those datasets +remain available in the native source archive. Cloud snapshots omit collector +queues, checkpoints, repair receipts and fetch/retry diagnostics, and strip raw +actor payloads while retaining public actor fields. Exports preserve the source. + +Operational tables are **not public conversation datasets**: + +- `analytics_fetch_attempts(id, repository, number, operation, started_at, + finished_at, status, error_class, error_text, evidence_json)` retains successful + and rejected GraphQL work. `number=0` identifies a discovery-lane request. + Rejection evidence is bounded structural metadata, IDs, counts, pagination and + body lengths/hashes; it contains no credentials, response headers or prose bodies. + Partial-response receipts also retain `graphql_errors`: total count, up to eight + allowlisted provider codes and query paths of at most eight components, with + explicit truncation markers. Unknown codes/path components are null. Messages, + arbitrary field values and identities are excluded from this error metadata. + Every rejection also has private `cause` metadata: allowlisted category/type, + guard or validation code, HTTP status when available, up to eight request or + pagination stages, and numeric quota/reset evidence for quota guards. The + primary error chain is bounded to sixteen links with explicit truncation. + Wrapping a transport or guard error as a validation failure retains this cause + without changing the existing error class, retry or acceptance behavior. No + error messages, URLs, tokens, headers, bodies, identities or certificate subjects + are copied into this diagnostic metadata. Unknown types remain `unclassified`. + Older receipts are not rewritten to infer a cause from later provider reads. +- `analytics_retries(repository, number, operation, first_seen_at, last_seen_at, + next_attempt_at, attempts, last_attempt_id, resolved_at)` is keyed by + `(repository, number, operation)`. Resolved entries and attempt history remain. + Operations include `graphql_history` (core traversal), `review_state` + (targeted enrichment), `discover_issues`, and + `discover_pullRequests`. Recovery queue rows start with zero attempts. + +`analytics --status --json` reads bounded recent receipts, outstanding retry count, +discovery coverage and review-state scan progress without credentials or collection. +One-shot collection errors and watch logs omit provider prose. Timestamped logs distinguish `github_update_complete`, `github_update_failed`, +`github_coverage_pending` (core), and `review_state_progress` (enrichment). +`review_state_quota` records fresh limit/remaining/reset/reserve and admitted wave +size; `review_state_cost` records actual provider points per recovery query. +The older successful-only `sync_runs` table is not a +complete failure ledger. Existing embeddings are reused; this command does not +generate embeddings or reinterpret empty review bodies as missing replies. + +The permanent `runner.lock` coordinates ownership with the full-history backfill +supervisor. Do not unlink it or start a second supervisor against the same archive. +A finished backfill should be disabled as an automatic startup job when ongoing +collection takes ownership. The token-command result is held only in memory and +refreshed before its expected expiry, preserving one credential across quota +reservation and dispatch. Existing provider-rate protections remain active. + +See [sync](/sync/), [configuration](/configuration/) and the [command reference](/commands/). diff --git a/docs/commands.md b/docs/commands.md index aa4f11ef..08734baf 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -38,11 +38,23 @@ These work on every command. | `gitcrawl configure [--summary-model --embed-model --embedding-basis --json]` | Update model fields in `config.toml` | [Configuration](/configuration/#gitcrawl-configure) | | `gitcrawl version` | Print version | — | +## Repository metrics + +| Command | Purpose | Docs | +| --- | --- | --- | +| `gitcrawl metrics collect --config metrics.json [--json]` | Observe stars, forks, actual watchers, open PRs/issues, optional daily clones, and stable releases in a separate database | [Repository metrics](/metrics/) | +| `gitcrawl metrics import --config metrics.json [--json]` | Atomically import scoped NDJSON history from stdin, preserving NULLs and IDs | [Repository metrics](/metrics/#storage-imports-and-failures) | +| `gitcrawl metrics status --config metrics.json [--json]` | Inspect the metrics database without writes or network calls | [Repository metrics](/metrics/) | + +For `metrics`, `--config` selects an independent JSON config; it never selects or +initializes the normal thread archive. No embedding or model calls are made. + ## Sync | Command | Purpose | Docs | | --- | --- | --- | | `gitcrawl sync owner/repo [--state --since --numbers --limit --include-comments --include-pr-details --with pr-details --graphql-history --force --progress-file --json]` | Sync issues and PRs from GitHub into local SQLite | [Sync](/sync/) | +| `gitcrawl analytics owner/repo [--apply --enrich --watch --once --json]` | Audit/repair source publication dates, enrich stable identities, and maintain GraphQL updates | [Analytics source preparation](/analytics-source/) | | `gitcrawl sync-failures owner/repo [--include-resolved --limit N --json]` | List failed issue, comment, and PR hydration attempts and optional resolved history | [Sync](/sync/#hydration-depth) | | `gitcrawl coverage [owner/repo \| --repos owner/a,owner/b] [--min-missing-pr-details N --json]` | Report archive, PR-detail, and enrichment coverage/freshness | — | | `gitcrawl fill-pr-details owner/repo [--limit --order --batch-size --reserve-rate-limit --include-comments --json-progress --json]` | Hydrate locally missing pull request detail rows in bounded batches | — | diff --git a/docs/installation.md b/docs/installation.md index 6ef6fa5e..074b485e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -20,7 +20,7 @@ permalink: /installation/ gitcrawl runs on macOS 13 Ventura or newer and Linux. Windows is not actively tested. -CrawlKit v0.16.5 requires Go 1.27, so source and Docker builds use Go 1.27.1 +CrawlKit v0.16.6 requires Go 1.27, so source and Docker builds use Go 1.27.1 or newer. Go 1.27 also raises the minimum macOS version for newly built binaries to macOS 13; the previous Go 1.26 build baseline no longer applies. @@ -76,6 +76,17 @@ go build \ Symlink or copy `bin/gitcrawl` somewhere on your `PATH` (`~/bin`, `/usr/local/bin`, `~/.local/bin`). +For a metrics-only source deployment alongside an existing portable subscriber, +use an [isolated versioned metrics runtime](/metrics/#isolated-source-built-installation) +and verify it with `metrics status` and its separate config. Keep the subscriber's +binary selection and refresh job unchanged. + +Source builds use scalar vector scoring by default. Set `GOEXPERIMENT=simd` on +`go build` to enable Go 1.27's experimental portable SIMD kernels for clustering +and exact neighbors. The API may change in Go 1.28; unsupported hardware and +`GODEBUG=simd=0` use the scalar kernels. Release archives and Docker builds enable +this SIMD path. + ## GitHub CLI shim migration `gitcrawl gh` moved to Octopool: diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 00000000..9b803c12 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,303 @@ +--- +title: Repository metrics +nav_order: 16 +permalink: /metrics/ +--- + +# Repository metrics + +`gitcrawl metrics` collects repository headline counters and stable release events +into a separate, private SQLite database. It does not sync the thread archive, +refresh a portable store, generate embeddings, call a model, or start a scheduler. + +## Configuration and commands + +Create a metrics JSON config. `database` must be an absolute filesystem path to a +new file or an existing Gitcrawl metrics database, outside your archive and portable +store. The example path is illustrative; choose a private directory on your host. + +```json +{ + "database": "/private/metrics/gitcrawl/metrics.sqlite", + "targets": [ + {"entity": "OpenClaw", "target": "openclaw/openclaw"}, + {"entity": "Example", "target": "example/project"} + ] +} +``` + +```sh +gitcrawl metrics collect --config /private/metrics/github.json --json +gitcrawl metrics import --config /private/metrics/github.json --json < history.ndjson +gitcrawl metrics status --config /private/metrics/github.json --json +gitcrawl help metrics +``` + +The config is independent of `config.toml`, `GITCRAWL_CONFIG`, and `GITCRAWL_DB_PATH`. For these commands, +`--config` selects the metrics JSON file; it can appear before `metrics` or after +the subcommand. Global output flags and command-local `--json` work normally. + +Public counters and releases can be read without authentication. Authenticated +collection uses `GITHUB_TOKEN`, then the normal native `gh auth token` resolver. +An optional `tokenEnv` selects another environment variable. A global +`--github-token-command /absolute/executable` selects the normal managed credential +provider exclusively on supported platforms. Tokens never appear in results. +The metrics config contains no token values; cookie authentication is not used. + +## Isolated source-built installation + +A source-built metrics runtime can coexist with an official Gitcrawl installation +that refreshes a portable mirror. Give the metrics binary a private, versioned +directory and invoke that exact executable. The installed metrics CLI path is: + +```text +$HOME/.local/share/gitcrawl/metrics-runtimes//gitcrawl +``` + +Copy an already validated artifact into a new directory; do not overwrite an +existing version. Record its full source commit and expected SHA-256 from the +validation handoff, verify the hash before and after copying, and use directory +mode `0700` and executable mode `0500`. This installation does not replace a +`current` symlink, the command on `PATH`, an archive config, or a refresh job. + +Local source builds do not require official release signing credentials under the +[installation policy](/installation/#install-from-source). On macOS, verify the +source artifact's signature with `codesign --verify --strict` before and after +copying. This is source-build verification, not official release notarization; +official release signing and notarization remain the [release workflow's](/releasing/) +responsibility. Do not change signing policies or remove quarantine to force an +untrusted artifact to run. + +The installation check is read-only and uses the separately provided metrics config: + +```sh +metrics_revision=SOURCE_COMMIT +metrics_binary="$HOME/.local/share/gitcrawl/metrics-runtimes/$metrics_revision/gitcrawl" +"$metrics_binary" --version +"$metrics_binary" metrics status \ + --config "$HOME/.local/share/gitcrawl/metrics.json" --json +``` + +Keep a machine-local installation receipt at the path returned by +`git rev-parse --git-path metrics-runtime-installation.json`. Record the exact +resolved CLI/config/database paths, source commit, artifact hash, signature result, +read-only status, and preservation checks there. Git metadata keeps these private +host details out of the public documentation and PR. The coordinator's handoff +should contain the same exact CLI path. Installation alone does not authorize +collection, imports, scheduling, or a final cutover. + +### Local signing for background collection + +A directly launched macOS job has its own file-access identity; permission granted +to a terminal does not prove the background executable has access. When the owner +authorizes an existing local signing certificate, sign a copy of the verified +runtime with a stable identifier and an explicit requirement bound to that +certificate. Use the already selected public SHA-1 fingerprint, without searching +for another identity or changing keychain trust or access controls. + +```sh +metrics_revision=SOURCE_COMMIT +metrics_identity=SELECTED_PUBLIC_CERTIFICATE_SHA1 +metrics_identifier=com.example.gitcrawl.metrics +metrics_source="$HOME/.local/share/gitcrawl/metrics-runtimes/$metrics_revision/gitcrawl" +metrics_install="$HOME/.local/libexec/gitcrawl-metrics/$metrics_revision" +metrics_requirement="identifier \"$metrics_identifier\" and certificate leaf = H\"$metrics_identity\"" +codesign --verify --strict "$metrics_source" +# Check the source SHA-256 against the validation receipt before copying. +umask 077 +mkdir -p "$(dirname "$metrics_install")" +mkdir "$metrics_install" +cp "$metrics_source" "$metrics_install/gitcrawl" +chmod 0700 "$metrics_install/gitcrawl" +codesign --force --sign "$metrics_identity" --identifier "$metrics_identifier" \ + --requirements "=designated => $metrics_requirement" --timestamp=none \ + "$metrics_install/gitcrawl" +codesign --verify --strict --test-requirement "=$metrics_requirement" \ + "$metrics_install/gitcrawl" +codesign --display --requirements - "$metrics_install/gitcrawl" +chmod 0500 "$metrics_install/gitcrawl" +shasum -a 256 "$metrics_source" "$metrics_install/gitcrawl" +``` + +Record both hashes, the source commit, certificate fingerprint, identifier and +requirement in the private receipt. Preserve the original runtime. Point only the +metrics LaunchAgent's first argument at the signed copy, preserving its other +settings. If this job was disabled, enable its exact label before bootstrapping. +Request one collection and let the user approve normal macOS file-access prompts. +Follow the specific permission request through its prompt and decision: a default +`SystemPolicyAllFiles` probe denial can precede a normal +`SystemPolicyRemovableVolumes` prompt. That probe alone does not establish a Full +Disk Access requirement. Keep the same authorized attempt running while the user +answers its normal prompt, then verify the terminal result and metrics database. +If permission remains blocked, stop and disable that job and report the evidence; +do not retry repeatedly, grant Full Disk Access, export keys, add privileged +wrappers, or relocate data. Local signing is not official release notarization. + +## Hourly collection on macOS + +After authorizing local collection, create a separate user LaunchAgent that calls +the pinned native binary directly. Use absolute paths; launchd does not expand +`~`, `$HOME`, or shell variables in a plist. Keep the config and logs outside Git +and shared publication. Give their directories mode `0700` and files mode `0600`. +Create both log files before bootstrapping the job. + +The following is a template for `~/Library/LaunchAgents/org.openclaw.gitcrawl.metrics.plist`. +Replace `/Users/you` and `SOURCE_COMMIT` with your actual installation paths: + +```xml + + + + + Labelorg.openclaw.gitcrawl.metrics + ProgramArguments + + /Users/you/.local/share/gitcrawl/metrics-runtimes/SOURCE_COMMIT/gitcrawl + metricscollect + --config/Users/you/.local/share/gitcrawl/metrics.json + --json + + WorkingDirectory/Users/you/.local/share/gitcrawl + EnvironmentVariables + + HOME/Users/you + PATH/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + GITCRAWL_NO_UPDATE_CHECK1 + + StartCalendarIntervalMinute6 + KeepAlive + Umask63 + StandardOutPath/Users/you/.local/share/gitcrawl/metrics-logs/stdout.log + StandardErrorPath/Users/you/.local/share/gitcrawl/metrics-logs/stderr.log + + +``` + +The job runs hourly at minute 6 in the host's local time, while observations use +UTC. `63` is the decimal representation of umask `0077`. The explicit minimal +`PATH` supports the existing native `gh` credential resolver without interactive +shell initialization. Do not put credentials in the plist. The versioned runtime +is not auto-updated; `GITCRAWL_NO_UPDATE_CHECK` also disables release notices. + +Validate and bootstrap this new job once, then request one immediate collection: + +```sh +metrics_plist="$HOME/Library/LaunchAgents/org.openclaw.gitcrawl.metrics.plist" +plutil -lint "$metrics_plist" +launchctl bootstrap "gui/$(id -u)" "$metrics_plist" +launchctl kickstart "gui/$(id -u)/org.openclaw.gitcrawl.metrics" +launchctl print "gui/$(id -u)/org.openclaw.gitcrawl.metrics" +``` + +Wait for the job to exit and check its last exit code, private logs, and the latest +`metric_runs` row using read-only SQLite. Confirm all five required counters for +every configured target have non-NULL values at that run's timestamp. A successful +bootstrap or a PID alone is not collection evidence. An unavailable optional clone +report is not a required-counter failure. On a provider failure, inspect the +recorded result before retrying; `KeepAlive` is disabled to avoid rapid restarts. +Existing archive refresh jobs are independent and need no changes. + +`collect` and `import` hold a nonblocking native OS lock on +`.writer.lock` from before database initialization through database +close. A second writer exits nonzero before collection or import; `status` remains +read-only and available. Ownership is released even if the process is killed. +The private lock file remains in place: never delete or replace it while writers +can run. Use the locking runtime for every writer; older binaries do not honor +this lock. + +## What is collected + +Each invocation observes all configured targets at a single UTC timestamp. Run +`collect` hourly with an external scheduler when hourly history is required. + +| Metric | Meaning | +| --- | --- | +| `stars` | Repository `stargazers_count` | +| `forks` | Repository `forks_count`; changes between snapshots are net changes | +| `watchers` | Actual subscribers, `subscribers_count`; **not** `watchers_count`, which aliases stars | +| `open_prs` | Open pull-request search count, only when results are complete | +| `open_issues` | Combined repository issue/PR count minus the valid open-PR count | +| `clones` | Optional daily clone counts from the authenticated traffic endpoint | + +Missing, invalid, or incomplete required counts remain SQL `NULL`; they never +become zero. If the PR count is unavailable or exceeds the combined count, +`open_issues` remains unknown. Real zeroes and decreases are retained. Individual +fork creation events are not crawled. + +Clone traffic requires repository push access (or a fine-grained token with repository +administration read permission). Permission-denied HTTP 403/404 means optional +unavailability and does not fail otherwise healthy collection. A rate-limit +response is a collection failure, including on the traffic endpoint. Only completed UTC +days are recorded: `ts` is that day's final millisecond and `observed_at` is the +actual read time. Unchanged daily values are not re-appended; corrected values +receive a new sequence. Imported daily timestamps retain their original spelling; +group them by UTC day. Sum only the latest observation for each day, never all +revisions. GitHub's traffic window limits how far a missed day can be backfilled. + +Stable releases exclude drafts and prereleases. All release pages are read; +events use stable GitHub release IDs so repeated collection is idempotent. +`published_at` is preferred, with `created_at` as the fallback for older records. + +Each target costs one repository request, one search request for the open-PR +count, optionally one traffic request, and one request per 100-release page +(including a final empty page when the total is a multiple of 100). Every +collection rereads the release history. Search has its own GitHub quota; there +is no metrics-specific quota reserve or incremental release checkpoint. The +shared client retries a rate-limited request once with a wait capped at five +minutes. An exhausted rate limit stops collection, retaining completed reads and +leaving later targets unattempted. Choose the schedule and target count accordingly. + +## Storage, imports, and failures + +New database files are private (`0600`). Existing files must identify themselves +with `metric_meta.owner = gitcrawl` and `metric_meta.version = 1`. Databases with +foreign tables, another owner/version, database symlinks or hard-link aliases, and +pre-existing empty files are rejected before a writable open. Existing databases +are inspected read-only for this check; no archive runtime or config is loaded. +Before applying the metrics schema or ownership metadata, the writable connection +rechecks ownership under a write transaction. Newly created databases must still +be empty, and changed file identities are rejected. Schema and ownership metadata +commit together on that same connection. +A failed first initialization removes only the newly created file so it can be +retried; pre-existing files are never removed. A process killed during that first +initialization can still leave an unowned file requiring operator inspection. `status` checks identity read-only and +does not create a missing database. Never point this config at the thread archive. + +The delivery tables are: + +- `metric_observations(sequence, id, entity, target, metric, kind, ts, value, + observed_at, provenance)` — counters and daily observations; `value` is nullable. +- `metric_events(sequence, id, entity, target, kind, ts, label, url, observed_at, + provenance)` — release history. +- `metric_runs(sequence, ts, status, rows_written)` — completed collection attempts. + +`status` reports total observations and events. Its `last_observed` is the latest +observation instant, comparing parsed timestamps even when imported offsets or +fractional precision differ; it is absent until an observation exists. + +Observation and event sequences advance independently. Read each table using its +own delivery cursor. Daily revisions supersede by latest sequence. Counter values +are snapshots, not increments; derive net change from consecutive observations. + +Import accepts one JSON object per line on stdin: + +```json +{"type":"metric","id":"history:github:watchers:1","entity":"OpenClaw","target":"openclaw/openclaw","metric":"watchers","kind":"counter","ts":"2026-09-14T00:00:00Z","value":null,"observed_at":"2026-09-15T00:00:00Z","provenance":"historical-import"} +{"type":"event","id":"history:github:release:1","entity":"Example","target":"example/project","kind":"release","ts":"2026-09-14T00:00:00Z","label":"v1","url":"https://github.com/example/project/releases/tag/v1","observed_at":"2026-09-15T00:00:00Z","provenance":"historical-import"} +``` + +IDs are required and idempotent within each destination table. Imported explicit +IDs preserve distinct observations even when values match. `entity` and `target` +must match a configured pair. Config files are limited to 1 MiB. Import validates every row and commits the whole +input atomically; a malformed or out-of-scope late row rolls everything back. +Daily imports must identify a day completed before `observed_at`'s UTC day. + +On a partial source failure, successful reads and explicit unknown counter values +are committed together with a `partial` run; stdout contains the result and the +command exits nonzero. A canceled collection gets a bounded opportunity to retain +already completed reads, without starting another network request. Diagnostics +never include GitHub response bodies or credential output. + +This command does not install schedules, migrate another application's history, +change existing refresh jobs, or publish the private database. Those are explicit +operator/integration responsibilities. diff --git a/docs/search.md b/docs/search.md index cc89b8f1..8904d490 100644 --- a/docs/search.md +++ b/docs/search.md @@ -71,6 +71,9 @@ gitcrawl search prs "manifest cache" \ --limit 20 ``` +Put `--` before query terms that begin with a dash, with options before it: +`gitcrawl search issues -R owner/repo --json number,title -- --verbose`. + Recognized flags in this mode: | Flag | Description | diff --git a/docs/sync.md b/docs/sync.md index f3cc291d..0e367c3f 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -102,7 +102,7 @@ no REST requests or fallback. The regular sync path is unchanged. The profile requires the flags above; it rejects `--since`, `--limit` and full PR-detail hydration. It does not collect files, commit bodies, checks or Actions logs. An incomplete GraphQL response, unavailable parent, missing identity, -duplicate child within a connection, mismatched count or nonadvancing cursor +duplicate child within a connection, mismatched or changing count, or nonadvancing cursor fails the batch before archive writes. Supervisors should retry failed selections in isolation. Empty reviews remain retained, including approvals without bodies. Minimized comments and null @@ -369,3 +369,10 @@ gitcrawl sync owner/repo --numbers "$NUMS" --with pr-details - [Refresh and embed](/refresh-and-embed/) — the wrapper that runs sync, embed, and cluster end to end - [gh shim migration](/gh-shim/) — Octopool owns pooled `gh` reads now - [Portable stores](/portable-stores/) — sharing the synced cache across machines + +## Analytics source preparation + +The [analytics source command](/analytics-source/) repairs retained review publication +timestamps without rewriting raw evidence and maintains a full-history archive +through incremental GraphQL collection. Its actor IDs and coverage receipts are +source evidence for downstream analytics; account classifications remain derived. diff --git a/docs/tui.md b/docs/tui.md index cef5fea4..b7632077 100644 --- a/docs/tui.md +++ b/docs/tui.md @@ -65,6 +65,9 @@ The view auto-refreshes from the local store every 15 seconds. There is no GitHu The action menu opened with `a` mirrors the right-click menu, so every mouse action has a keyboard equivalent. +Neighbor loading can use the saved embedding of a selected closed thread as +well as an open thread. Results contain open, locally active neighbors. + Jump input accepts the same thread references as the CLI: bare numbers, `#123`, `issues/123`, `pull/123`, `owner/repo#123`, and full GitHub issue or pull request URLs. diff --git a/go.mod b/go.mod index ddf141d3..1f28c063 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/charmbracelet/x/ansi v0.11.8 github.com/mattn/go-isatty v0.0.24 github.com/muesli/termenv v0.16.0 - github.com/openclaw/crawlkit v0.16.5 + github.com/openclaw/crawlkit v0.16.6 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.48.0 modernc.org/sqlite v1.59.0 diff --git a/go.sum b/go.sum index f70845b4..4fe717d5 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.16.5 h1:4hUiPo6lLEwqLPtgBKXVeHhVVCu4hYD5R+dMN8nAgbU= -github.com/openclaw/crawlkit v0.16.5/go.mod h1:XTGhBPNiMqKzPuKNwAH9ufbUU0Vel0pEQAZFqS/b7GQ= +github.com/openclaw/crawlkit v0.16.6 h1:Jq2hvcy0ugIkxd+GfNnMPryvYbIsONyg05rVMNbTLfU= +github.com/openclaw/crawlkit v0.16.6/go.mod h1:XTGhBPNiMqKzPuKNwAH9ufbUU0Vel0pEQAZFqS/b7GQ= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= diff --git a/internal/cli/analytics.go b/internal/cli/analytics.go new file mode 100644 index 00000000..7e6d024d --- /dev/null +++ b/internal/cli/analytics.go @@ -0,0 +1,421 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "github.com/openclaw/gitcrawl/internal/config" + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" + "github.com/openclaw/gitcrawl/internal/syncer" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +func (a *App) analyticsClient(ctx context.Context, cfg config.Config) (*gh.Client, error) { + token := a.resolveGitHubToken(ctx, cfg) + var provider func(context.Context) (string, error) + var e error + if a.githubTokenCommand != nil { + provider, e = githubTokenProvider(*a.githubTokenCommand) + if e != nil { + return nil, e + } + } + if provider != nil { + fetch := provider + var mu sync.Mutex + var cached string + var expires time.Time + provider = func(ctx context.Context) (string, error) { + mu.Lock() + defer mu.Unlock() + if cached != "" && time.Now().Before(expires) { + return cached, nil + } + value, e := fetch(ctx) + if e != nil { + return "", e + } + cached = value + expires = time.Now().Add(45 * time.Minute) + return value, nil + } + a.analyticsTokenProvider = provider + } + if provider == nil && token.Value == "" { + return nil, fmt.Errorf("missing GitHub credential") + } + return gh.New(gh.Options{Token: token.Value, TokenProvider: provider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: analyticsCoreReserve}), nil +} +func (a *App) runAnalytics(ctx context.Context, args []string) error { + for _, arg := range args { + if arg == "--help" || arg == "-h" { + _, err := fmt.Fprintln(a.Stdout, "Usage: gitcrawl [--config SOURCE_CONFIG] [--github-token-command TOKEN_HELPER] analytics owner/repo [--status|--apply|--enrich] [--watch|--once] [--json]\nWithout action flags: read-only publication audit. --status reads bounded failure/recovery status; --apply repairs retained timestamps; --enrich collects actor evidence; --watch maintains GraphQL updates.") + return err + } + } + + fs := flag.NewFlagSet("analytics", flag.ContinueOnError) + fs.SetOutput(io.Discard) + apply := fs.Bool("apply", false, "apply source publication corrections") + enrich := fs.Bool("enrich", false, "collect missing provider identities and actor profiles") + watch := fs.Bool("watch", false, "maintain GraphQL updates every two minutes") + once := fs.Bool("once", false, "run one GraphQL update cycle") + status := fs.Bool("status", false, "read bounded collection, failure and recovery status") + fs.Bool("json", false, "JSON output") + if e := fs.Parse(normalizeCommandArgs(args, nil)); e != nil { + return e + } + if fs.NArg() != 1 { + return fmt.Errorf("analytics requires owner/repo") + } + owner, repo, e := parseOwnerRepo(fs.Arg(0)) + if e != nil { + return e + } + a.format = FormatJSON + cfg, e := config.LoadRuntime(a.configPath) + if e != nil { + return e + } + if *status { + if *apply || *enrich || *watch || *once { + return fmt.Errorf("--status cannot be combined with collection actions") + } + rt, err := a.openLocalRuntimeReadOnly(ctx) + if err != nil { + return err + } + defer rt.Store.Close() + result, err := rt.Store.AnalyticsIntegrityStatus(ctx, owner+"/"+repo) + if err != nil { + return err + } + return a.writeOutput("analytics_status", result, false) + } + if !*apply && !*enrich && !*watch && !*once { + rt, e := a.openLocalRuntimeReadOnly(ctx) + if e != nil { + return e + } + defer rt.Store.Close() + r, e := rt.Store.RepairPublication(ctx, false) + if e != nil { + return e + } + return a.writeOutput("analytics_repair_dry_run", r, false) + } + lock, e := os.OpenFile(filepath.Join(filepath.Dir(cfg.DBPath), "runner.lock"), os.O_CREATE|os.O_RDWR, 0600) + if e != nil { + return e + } + defer lock.Close() + if e = lockPortableFile(lock); e != nil { + return fmt.Errorf("collector ownership lock busy: %w", e) + } + rt, e := a.openLocalRuntime(ctx) + if e != nil { + return e + } + defer rt.Store.Close() + if *apply { + r, e := rt.Store.RepairPublication(ctx, true) + if e != nil { + return e + } + if e = a.writeOutput("analytics_repair", r, false); e != nil { + return e + } + } + if !*enrich && !*watch && !*once { + return nil + } + client, e := a.analyticsClient(ctx, cfg) + if e != nil { + return e + } + if *watch || *once { + if e = rt.Store.SeedAnalyticsNodes(ctx, true); e != nil { + return e + } + var existing int + if e = rt.Store.DB().QueryRowContext(ctx, "SELECT count(*) FROM analytics_coverage WHERE repository=?", owner+"/"+repo).Scan(&existing); e != nil { + return e + } + if existing == 0 { + b, err := os.ReadFile(filepath.Join(filepath.Dir(a.configPath), "status.json")) + if err == nil { + var status struct { + Repository string `json:"repository"` + Phase string `json:"phase"` + Discovery []struct { + Kind string `json:"kind"` + Done int `json:"done"` + Total *int `json:"total"` + Updated string `json:"updated_at"` + } `json:"discovery"` + } + if json.Unmarshal(b, &status) == nil && strings.EqualFold(status.Repository, owner+"/"+repo) && status.Phase == "complete" && len(status.Discovery) == 2 { + var through time.Time + issues, prs := -1, -1 + valid := true + for _, d := range status.Discovery { + at, timeErr := time.Parse(time.RFC3339Nano, d.Updated) + if d.Done != 1 || d.Total == nil || *d.Total < 0 || timeErr != nil { + valid = false + continue + } + if through.IsZero() || at.Before(through) { + through = at + } + + if d.Kind == "issues" { + issues = *d.Total + } else if d.Kind == "pullRequests" { + prs = *d.Total + } else { + valid = false + } + } + if valid && issues >= 0 && prs >= 0 { + if e = rt.Store.SaveAnalyticsCoverage(ctx, owner+"/"+repo, through.UTC().Format(time.RFC3339Nano), issues, prs); e != nil { + return e + } + } + } + } + } + } + // Independent guarded clients permit disjoint evidence batches to overlap; + // each preserves the normal quota reservation and uses the same cached token. + actorClients := make([]*gh.Client, 8) + for i := range actorClients { + token := a.resolveGitHubToken(ctx, cfg) + actorClients[i] = gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: analyticsCoreReserve}) + } + parallelWatch := *watch && *enrich + if parallelWatch { + pollCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + defer func() { cancel(); <-done }() + go func() { + defer close(done) + for { + nextPoll := time.Now().Add(analyticsPollInterval) + pollErr := a.analyticsCycle(pollCtx, rt.Store, client, owner, repo) + a.analyticsUpdateLog(pollErr) + select { + case <-pollCtx.Done(): + return + case <-time.After(time.Until(nextPoll)): + } + } + }() + } + if *enrich { + for _, profiles := range []bool{true, false} { + if e = rt.Store.SeedAnalyticsNodes(ctx, profiles); e != nil { + return e + } + } + e = maintainAnalyticsEnrichment(ctx, parallelWatch, 2*time.Minute, func() error { + failedBatches := 0 + for _, profiles := range []bool{true, false, true} { + cursor := "" + for { + var ids []string + if profiles { + ids, e = rt.Store.AnalyticsProfileNodes(ctx, 800, cursor) + } else { + ids, e = rt.Store.AnalyticsIdentityNodes(ctx, 800, cursor) + } + if e != nil { + return e + } + if len(ids) == 0 { + break + } + // Advance past failures for this pass; watch retries on its next poll. + cursor = ids[len(ids)-1] + var group sync.WaitGroup + var failures []error + var failureMu sync.Mutex + for offset := 0; offset < len(ids); offset += 100 { + part := append([]string(nil), ids[offset:min(offset+100, len(ids))]...) + worker := actorClients[offset/100] + group.Add(1) + go func() { + defer group.Done() + err := collectAnalyticsActors(ctx, rt.Store, worker, part, profiles) + if err != nil { + failureMu.Lock() + failures = append(failures, err) + failureMu.Unlock() + } + }() + } + group.Wait() + if len(failures) > 0 { + fmt.Fprintf(a.Stderr, "{\"event\":\"actor_enrichment_retry\",\"failed_batches\":%d}\n", len(failures)) + failedBatches += len(failures) + } + + fmt.Fprintf(a.Stderr, "{\"event\":\"actor_enrichment\",\"profiles\":%t,\"nodes\":%d}\n", profiles, len(ids)) + + } + } + if failedBatches > 0 && !parallelWatch { + return fmt.Errorf("actor enrichment incomplete: %d failed batches", failedBatches) + } + return nil + }) + if e != nil { + return e + } + } + + if parallelWatch { + <-ctx.Done() + return ctx.Err() + } + for *watch || *once { + nextPoll := time.Now().Add(analyticsPollInterval) + e = a.analyticsCycle(ctx, rt.Store, client, owner, repo) + if e != nil { + if !*watch { + return &analyticsCommandError{cause: e} + } + } + a.analyticsUpdateLog(e) + if !*watch { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Until(nextPoll)): + } + } + return nil +} + +type updateCheckpoint struct { + Started string `json:"started"` + Since string `json:"since"` + Kind int `json:"kind"` + Cursor string `json:"cursor"` + Issues int `json:"issues"` + PRs int `json:"prs"` +} + +// Smaller requests prevent high-fanout conversation queries exhausting GitHub's +// execution deadline. Split a persistently failing transient batch without +// accepting partial conversation evidence or changing transports. +func (a *App) syncAnalyticsBatch(ctx context.Context, s *store.Store, owner, repo string, numbers []int, operation string) error { + cfg, err := config.LoadRuntime(a.configPath) + if err != nil { + return err + } + token := a.resolveGitHubToken(ctx, cfg) + reserve := analyticsCoreReserve + var responseLimit int64 + var responseBytes, points, providerMillis int64 + var reporter gh.Reporter + if operation == "review_state" { + reserve = analyticsReviewReserve + responseLimit = 32 << 20 + // This call owns its client, history session and reporter. Synchronous + // callbacks are never shared with the other analyticsNumbers workers. + var effectiveRemaining, effectiveReset int + reporter = func(message string) { + var bytes, callNumber, millis int64 + if _, err := fmt.Sscanf(message, "[github] graphql bytes %d", &bytes); err == nil { + responseBytes += bytes + return + } + if _, err := fmt.Sscanf(message, "[github] graphql timing %d %d", &callNumber, &millis); err == nil { + providerMillis += millis + return + } + var providerRemaining, providerReset int + if _, err := fmt.Sscanf(message, "[github] graphql quota provider_remaining %d provider_reset %d effective_remaining %d effective_reset %d", &providerRemaining, &providerReset, &effectiveRemaining, &effectiveReset); err == nil { + return + } + var call, cost, remaining, reset int + if _, err := fmt.Sscanf(message, "[github] graphql cost %d %d remaining %d reset %d", &call, &cost, &remaining, &reset); err == nil { + points += int64(cost) + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_cost\",\"at\":%q,\"points\":%d,\"remaining\":%d,\"reset_unix\":%d,\"provider_remaining\":%d,\"provider_reset_unix\":%d}\n", time.Now().UTC().Format(time.RFC3339Nano), cost, effectiveRemaining, effectiveReset, remaining, reset) + } + } + } + client := gh.New(gh.Options{Token: token.Value, TokenProvider: a.analyticsTokenProvider, BaseURL: githubBaseURL(), RateLimit: a.observeGitHubRateLimit(ctx), RateLimitReserve: reserve, GraphQLResponseLimit: responseLimit, GraphQLQuotaGuard: operation == "review_state"}) + // The watch owner has already validated and opened this store. Reopening it + // for every two threads repeats full-archive migration audits and serializes + // otherwise independent network work. Native transactions still own writes. + stats, err := syncer.New(client, s).Sync(ctx, syncer.Options{Owner: owner, Repo: repo, GraphQLHistory: true, ReviewStateOnly: operation == "review_state", ReceiptOperation: operation, State: "all", Numbers: numbers, IncludeComments: true, IncludePRMetadata: true, Reporter: reporter}) + if metrics, ok := ctx.Value(analyticsMetricsKey{}).(*analyticsWorkMetrics); ok { + metrics.bytes.Add(responseBytes) + metrics.points.Add(points) + metrics.providerMillis.Add(providerMillis) + metrics.dbMillis.Add(stats.PersistMillis) + metrics.attempted.Add(int64(len(numbers))) + if err == nil { + metrics.recovered.Add(int64(stats.ThreadsSynced)) + } + } + return err +} + +// A drained queue is not the end of a watch: new actors and stale profiles +// become eligible later. Initial source scanning happens outside this loop. +func maintainAnalyticsEnrichment(ctx context.Context, watch bool, interval time.Duration, drain func() error) error { + for { + if err := drain(); err != nil { + return err + } + if !watch { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } +} + +// Reject partial data, then refetch healthy peers without provider-rejected IDs. +func collectAnalyticsActors(ctx context.Context, s *store.Store, client *gh.Client, ids []string, profiles bool) error { + nodes, err := client.AnalyticsNodes(ctx, ids, profiles) + if err != nil { + var rejected *gh.AnalyticsNodesError + if errors.As(err, &rejected) { + blocked := make(map[string]bool, len(rejected.IDs)) + for _, id := range rejected.IDs { + blocked[id] = true + } + healthy := make([]string, 0, len(ids)) + for _, id := range ids { + if !blocked[id] { + healthy = append(healthy, id) + } + } + if len(healthy) > 0 && len(healthy) < len(ids) { + return errors.Join(err, collectAnalyticsActors(ctx, s, client, healthy, profiles)) + } + } + return err + } + at := time.Now().UTC().Format(time.RFC3339Nano) + if profiles { + return s.SaveActorProfiles(ctx, nodes, at) + } + return s.SaveActorEvidence(ctx, nodes, at) +} diff --git a/internal/cli/analytics_actor_failure_test.go b/internal/cli/analytics_actor_failure_test.go new file mode 100644 index 00000000..7e03ee16 --- /dev/null +++ b/internal/cli/analytics_actor_failure_test.go @@ -0,0 +1,82 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsEnrichIsolatesForbiddenActorAndTerminates(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + dir := t.TempDir() + path := filepath.Join(dir, "source.db") + st, err := store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, err = st.DB().Exec(`INSERT INTO analytics_pending_nodes(node_id,kind) VALUES('blocked','profile'),('healthy','profile')`); err != nil { + t.Fatal(err) + } + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":4102444800},"core":{"limit":20000,"remaining":19000,"reset":4102444800}}}`) + return + } + requests.Add(1) + var request struct { + Variables struct { + IDs []string `json:"ids"` + } `json:"variables"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + nodes, failures := []any{}, []any{} + for i, id := range request.Variables.IDs { + if id == "blocked" { + nodes = append(nodes, nil) + failures = append(failures, map[string]any{"type": "FORBIDDEN", "path": []any{"nodes", i}, "message": "synthetic-private-actor-failure"}) + } else { + nodes = append(nodes, map[string]any{"id": id, "__typename": "User", "login": "fixture"}) + } + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"nodes": nodes}, "errors": failures}) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + app := New() + app.Stdout, app.Stderr = io.Discard, io.Discard + err = app.Run(ctx, []string{"--config", writeDoctorTestConfig(t, dir, path), "analytics", "fixture/repo", "--enrich"}) + if err == nil || errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), "synthetic-private-actor-failure") { + t.Errorf("expected bounded, safe incomplete-enrichment error, got %v", err) + } + var healthy, blocked int + if err = st.DB().QueryRow(`SELECT count(*) FROM actor_profiles WHERE node_id='healthy'`).Scan(&healthy); err != nil { + t.Fatal(err) + } + if err = st.DB().QueryRow(`SELECT count(*) FROM actor_profiles WHERE node_id='blocked'`).Scan(&blocked); err != nil { + t.Fatal(err) + } + if healthy != 1 || blocked != 0 { + t.Errorf("healthy=%d blocked=%d; forbidden evidence must stay unknown without blocking its peer", healthy, blocked) + } + if requests.Load() > 4 { + t.Errorf("unbounded actor retry: %d", requests.Load()) + } +} diff --git a/internal/cli/analytics_empty_baseline_test.go b/internal/cli/analytics_empty_baseline_test.go new file mode 100644 index 00000000..08de1893 --- /dev/null +++ b/internal/cli/analytics_empty_baseline_test.go @@ -0,0 +1,72 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsAcceptsEmptyHistoricalBaseline(t *testing.T) { + for _, tc := range []struct { + repository string + reject bool + }{ + {"fixture/repo", false}, {"FIXTURE/REPO", false}, {"other/repo", true}, {"", true}, + } { + t.Run(tc.repository, func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, "source.db") + st, err := store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + cfg := writeDoctorTestConfig(t, dir, path) + receipt := fmt.Sprintf(`{"repository":%q,"phase":"complete","discovery":[{"kind":"issues","done":1,"total":0,"updated_at":"2026-01-01T00:00:00Z"},{"kind":"pullRequests","done":1,"total":0,"updated_at":"2026-01-01T00:00:00Z"}]}`, tc.repository) + if err := os.WriteFile(filepath.Join(dir, "status.json"), []byte(receipt), 0600); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":4102444800},"core":{"limit":20000,"remaining":19000,"reset":4102444800}}}`) + return + } + conn := map[string]any{"totalCount": 0, "nodes": []any{}, "pageInfo": map[string]any{"hasNextPage": false, "endCursor": nil}} + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": "2099-01-01T00:00:00Z"}, "repository": map[string]any{"issues": conn, "pullRequests": conn}}}) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + app := New() + app.Stdout, app.Stderr = io.Discard, io.Discard + err = app.Run(ctx, []string{"--config", cfg, "analytics", "fixture/repo", "--once"}) + if tc.reject { + if err == nil { + t.Fatal("unbound historical receipt established coverage") + } + var count int + if err := st.DB().QueryRow(`SELECT count(*) FROM analytics_coverage WHERE repository='fixture/repo'`).Scan(&count); err != nil || count != 0 { + t.Fatalf("unbound baseline persisted: count=%d err=%v", count, err) + } + return + } + if err != nil { + t.Fatal(err) + } + + var complete, issues, prs int + if err := st.DB().QueryRow(`SELECT complete,issues,pull_requests FROM analytics_coverage WHERE repository='fixture/repo'`).Scan(&complete, &issues, &prs); err != nil || complete != 1 || issues != 0 || prs != 0 { + t.Fatalf("empty baseline coverage: %d %d %d %v", complete, issues, prs, err) + } + }) + } +} diff --git a/internal/cli/analytics_export_privacy_test.go b/internal/cli/analytics_export_privacy_test.go new file mode 100644 index 00000000..9c7ad5e4 --- /dev/null +++ b/internal/cli/analytics_export_privacy_test.go @@ -0,0 +1,117 @@ +package cli + +import ( + "bytes" + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsExportsExcludePrivateCollectorState(t *testing.T) { + for _, mode := range []string{"cloud", "portable", "portable-compatibility"} { + t.Run(mode, func(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "source.db") + st, err := store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + _, err = st.DB().ExecContext(ctx, ` + INSERT INTO analytics_fetch_attempts(repository,number,operation,started_at,finished_at,status,error_text,evidence_json) + VALUES('private/repo',1,'graphql_history','2026-01-01T00:00:00Z','2026-01-01T00:00:01Z','failed','synthetic-private-analytics-attempt','{"id":"synthetic-private-analytics-evidence"}'); + INSERT INTO analytics_retries(repository,number,operation,first_seen_at,last_seen_at,next_attempt_at) + VALUES('synthetic-private-analytics-retry',1,'graphql_history','','',''); + INSERT INTO analytics_pending_nodes(node_id,kind) VALUES('synthetic-private-analytics-queue','profile'); + INSERT INTO analytics_collection_state(name,value,updated_at) VALUES('updates:private/repo','synthetic-private-analytics-cursor',''); + INSERT INTO analytics_repair_receipts(name,cursor,updated_at) VALUES('synthetic-private-analytics-repair',1,''); + INSERT INTO actor_profiles(node_id,login,actor_type,observed_at,raw_json) + VALUES('actor','fixture','User','','{"diagnostic":"synthetic-private-analytics-profile"}'); + INSERT INTO actor_identity_evidence(node_id,actor_node_id,observed_at,raw_json) + VALUES('comment','actor','','{"diagnostic":"synthetic-private-analytics-identity"}'); + `) + if err != nil { + t.Fatal(err) + } + if err = st.Close(); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + source, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer source.Close() + var snapshot string + if mode == "cloud" { + var cleanup func() + snapshot, _, cleanup, err = cloudSQLiteSnapshotPath(ctx, source, path, gitcrawlCloudPublishOptions{}) + if err != nil { + t.Fatal(err) + } + defer cleanup() + } else { + var cleanup func() + snapshot, cleanup, err = sqliteSnapshotPath(ctx, source, path) + if err != nil { + t.Fatal(err) + } + defer cleanup() + derived, err := store.Open(ctx, snapshot) + if err != nil { + t.Fatal(err) + } + // Even --no-vacuum pruning must remove discarded diagnostic bytes. + _, err = derived.PrunePortablePayloads(ctx, store.PortablePruneOptions{BodyChars: 100, RetainSanitizedPayloadColumns: mode == "portable-compatibility"}) + closeErr := derived.Close() + if err != nil { + t.Fatal(err) + } + if closeErr != nil { + t.Fatal(closeErr) + } + } + published, err := sql.Open("sqlite", snapshot) + if err != nil { + t.Fatal(err) + } + defer published.Close() + for _, table := range []string{"analytics_fetch_attempts", "analytics_retries", "analytics_pending_nodes", "analytics_collection_state", "analytics_repair_receipts"} { + exists, err := sqliteTableExists(ctx, published, table) + if err != nil { + t.Fatal(err) + } + if exists { + t.Errorf("export retained private collector table %s", table) + } + } + if err = published.Close(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(snapshot) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte("synthetic-private-analytics-")) { + t.Error("export retained private analytics bytes") + } + if err = source.Close(); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(before, after) { + t.Fatal("export changed source archive") + } + }) + } +} diff --git a/internal/cli/analytics_integrity.go b/internal/cli/analytics_integrity.go new file mode 100644 index 00000000..03a7adaf --- /dev/null +++ b/internal/cli/analytics_integrity.go @@ -0,0 +1,546 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" +) + +var errAnalyticsIncomplete = errors.New("analytics coverage remains incomplete") + +// Preserve cancellation and completeness identity without returning provider prose. +type analyticsCommandError struct{ cause error } + +func (e *analyticsCommandError) Error() string { + if errors.Is(e.cause, errAnalyticsIncomplete) { + return errAnalyticsIncomplete.Error() + } + class, message, _ := gh.HistoryFailureDetails(e.cause) + return fmt.Sprintf("analytics collection failed (%s): %s", class, message) +} + +func (e *analyticsCommandError) Unwrap() error { return e.cause } + +const ( + analyticsPollInterval = 2 * time.Minute + analyticsCoreReserve = 1500 + // Keep another 1500 points available to ordinary capture above its floor. + analyticsReviewReserve = 3000 + analyticsReviewWorkers = 32 + analyticsReviewBatch = 8 + analyticsReviewWave = analyticsReviewWorkers * analyticsReviewBatch + analyticsReviewWaveBytes = 16 << 20 +) + +type analyticsMetricsKey struct{} +type analyticsWorkMetrics struct { + busy, maxBusy, bytes, points, attempted, recovered, providerMillis, dbMillis, workerMillis atomic.Int64 +} + +func (m *analyticsWorkMetrics) enter() func() { + start := time.Now() + busy := m.busy.Add(1) + for old := m.maxBusy.Load(); busy > old; old = m.maxBusy.Load() { + if m.maxBusy.CompareAndSwap(old, busy) { + break + } + } + return func() { m.workerMillis.Add(time.Since(start).Milliseconds()); m.busy.Add(-1) } +} + +func (a *App) analyticsUpdateLog(err error) { + event := "github_update_complete" + fields := map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano)} + if err != nil { + event = "github_update_failed" + if errors.Is(err, errAnalyticsIncomplete) { + event = "github_coverage_pending" + fields["error"] = errAnalyticsIncomplete.Error() + } else { + class, message, _ := gh.HistoryFailureDetails(err) + fields["error_class"] = class + fields["error"] = message + } + } + fields["event"] = event + encoded, _ := json.Marshal(fields) + fmt.Fprintln(a.Stderr, string(encoded)) +} + +func migrateAnalyticsLanes(ctx context.Context, s *store.Store, repository string) error { + // runAnalytics initializes coverage from a verified phase=complete discovery + // receipt before any cycle. A partial checkpoint alone cannot certify a + // historical baseline; missing baseline evidence must remain an error. + marker := "independent_lanes:" + repository + value, err := s.AnalyticsState(ctx, marker) + if err != nil || value != "" { + return err + } + value, err = s.AnalyticsState(ctx, "updates:"+repository) + if err != nil { + return err + } + var legacy updateCheckpoint + if value != "" { + if err = json.Unmarshal([]byte(value), &legacy); err != nil { + return err + } + } + through, err := s.AnalyticsState(ctx, "through:"+repository) + if err != nil { + return err + } + if through == "" { + if err = s.DB().QueryRowContext(ctx, "SELECT through FROM analytics_coverage WHERE repository=?", repository).Scan(&through); err != nil { + return fmt.Errorf("verified historical coverage watermark required: %w", err) + } + } + var verifiedThrough string + if err = s.DB().QueryRowContext(ctx, "SELECT through FROM analytics_coverage WHERE repository=?", repository).Scan(&verifiedThrough); err != nil { + return err + } + if _, err = time.Parse(time.RFC3339Nano, verifiedThrough); err != nil { + return fmt.Errorf("invalid verified historical watermark: %w", err) + } + return s.WithTx(ctx, func(tx *store.Store) error { + // through retains the previous verified baseline even when a transient + // post-upgrade failure temporarily clears complete before migration. + if err := tx.SetAnalyticsState(ctx, "core_baseline_verified:"+repository, "1"); err != nil { + return err + } + for i, kind := range []string{"issues", "pullRequests"} { + cp := updateCheckpoint{} + laneThrough := through + if legacy.Started != "" { + if legacy.Kind > i { + laneThrough = legacy.Started + total := legacy.Issues + if i == 1 { + total = legacy.PRs + } + encodedTotal, _ := json.Marshal(total) + if err := tx.SetAnalyticsState(ctx, "total:"+repository+":"+kind, string(encodedTotal)); err != nil { + return err + } + } else { + cp = legacy + cp.Kind = i + if legacy.Kind < i { + cp.Cursor = "" + } + } + } + encoded := "" + if cp.Started != "" { + b, _ := json.Marshal(cp) + encoded = string(b) + } + if err := tx.SetAnalyticsState(ctx, "updates:"+repository+":"+kind, encoded); err != nil { + return err + } + if err := tx.SetAnalyticsState(ctx, "through:"+repository+":"+kind, laneThrough); err != nil { + return err + } + } + // The original receipt/checkpoint is retained for historical evidence. + return tx.SetAnalyticsState(ctx, marker, time.Now().UTC().Format(time.RFC3339Nano)) + }) +} + +func (a *App) analyticsCycle(ctx context.Context, s *store.Store, c *gh.Client, owner, repo string) error { + nextCore := time.Now().Add(analyticsPollInterval) + repository := owner + "/" + repo + if err := migrateAnalyticsLanes(ctx, s, repository); err != nil { + return err + } + // Ordinary capture and its retry obligations always go first. Historical + // review enrichment uses only the remaining time before the next core poll. + due, err := s.DueAnalyticsRetries(ctx, repository, time.Now().UTC().Format(time.RFC3339Nano), 8, "graphql_history") + if err != nil { + return err + } + if err = a.analyticsNumbers(ctx, s, owner, repo, due, false, "graphql_history"); err != nil { + return err + } + var failures []error + for i, kind := range []string{"issues", "pullRequests"} { + if err = a.analyticsLane(ctx, s, c, owner, repo, kind, i); err != nil { + failures = append(failures, err) + } + if ctx.Err() != nil { + return errors.Join(append(failures, ctx.Err())...) + } + } + var through string + baseline, err := s.AnalyticsState(ctx, "core_baseline_verified:"+repository) + if err != nil { + return err + } + complete := baseline == "1" && len(failures) == 0 + var totals [2]int + // Preserve last verified totals until this lane obtains a new provider count. + _ = s.DB().QueryRowContext(ctx, "SELECT issues,pull_requests FROM analytics_coverage WHERE repository=?", repository).Scan(&totals[0], &totals[1]) + for i, kind := range []string{"issues", "pullRequests"} { + at, e := s.AnalyticsState(ctx, "through:"+repository+":"+kind) + if e != nil { + return e + } + if through == "" || at < through { + through = at + } + value, e := s.AnalyticsState(ctx, "total:"+repository+":"+kind) + if e != nil { + return e + } + if value != "" { + if e = json.Unmarshal([]byte(value), &totals[i]); e != nil { + return e + } + } + } + outstanding, err := s.AnalyticsCoreOutstanding(ctx, repository) + if err != nil { + return err + } + complete = complete && outstanding == 0 + if err = s.WithTx(ctx, func(tx *store.Store) error { + if e := tx.SaveAnalyticsCoverage(ctx, repository, through, totals[0], totals[1]); e != nil { + return e + } + if e := tx.SetAnalyticsCoverageComplete(ctx, repository, complete); e != nil { + return e + } + return tx.SetAnalyticsState(ctx, "through:"+repository, through) + }); err != nil { + return err + } + reviewErr := a.analyticsReviewRecovery(ctx, s, c, owner, repo, nextCore.Add(-5*time.Second)) + if len(failures) > 0 { + return errors.Join(append(failures, reviewErr)...) + } + if !complete { + return errors.Join(fmt.Errorf("%w: core_unresolved=%d", errAnalyticsIncomplete, outstanding), reviewErr) + } + return reviewErr +} + +// Fill the time formerly spent idle with bounded, quota-checked waves through +// the existing executor. Each scan chunk and item receipt remains durable. +func (a *App) analyticsReviewRecovery(ctx context.Context, s *store.Store, c *gh.Client, owner, repo string, deadline time.Time) (resultErr error) { + window, cancel := context.WithDeadline(ctx, deadline) + defer cancel() + yield := func(err error) error { + if ctx.Err() == nil && window.Err() == context.DeadlineExceeded && analyticsCancellationOnly(err) { + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_yield\",\"at\":%q}\n", time.Now().UTC().Format(time.RFC3339Nano)) + return nil + } + return err + } + repository := owner + "/" + repo + var progress store.ReviewStateRecovery + bytesPerItem := int64(64 << 10) + haveProgress, quotaBlocked := false, false + defer func() { + if !haveProgress { + return + } + // Cancellation cannot erase the last committed scan or completed items. + receiptCtx, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + if err := s.SaveReviewStateCoverage(receiptCtx, repository, progress); err != nil { + resultErr = errors.Join(resultErr, err) + return + } + var pending int + if err := s.DB().QueryRowContext(receiptCtx, "SELECT pending_items FROM analytics_review_state_coverage WHERE repository=?", repository).Scan(&pending); err != nil { + resultErr = errors.Join(resultErr, err) + return + } + encoded, _ := json.Marshal(map[string]any{"event": "review_state_progress", "at": time.Now().UTC().Format(time.RFC3339Nano), "scanned": progress.Scanned, "ceiling": progress.Ceiling, "queued": progress.Queued, "pending_items": pending, "scan_complete": progress.Done, "complete": progress.Done && pending == 0}) + fmt.Fprintln(a.Stderr, string(encoded)) + }() + for time.Until(deadline) > 5*time.Second { + if err := window.Err(); err != nil { + return yield(err) + } + next, err := s.SeedReviewStateRecovery(window, repository, 5000) + if err != nil { + return yield(err) + } + progress, haveProgress = next, true + // Persist progress even if quota is exhausted or the process is stopped. + if err = s.SaveReviewStateCoverage(window, repository, progress); err != nil { + return yield(err) + } + if quotaBlocked { + if progress.Done { + return nil + } + continue + } + due, err := s.DueReviewStateWork(window, repository, time.Now().UTC().Format(time.RFC3339Nano), analyticsReviewWave) + if err != nil { + return yield(err) + } + if len(due) == 0 { + if progress.Done { + return nil + } + continue + } + observedQuota, rawQuota, err := c.AnalyticsRateLimit(window) + if err != nil { + if window.Err() != nil { + return yield(err) + } + class, message, _ := gh.HistoryFailureDetails(err) + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_quota_deferred\",\"at\":%q,\"error_class\":%q,\"error\":%q}\n", time.Now().UTC().Format(time.RFC3339Nano), class, message) + quotaBlocked = true + continue + } + budget, quota, err := analyticsReviewBudget([]gh.RateLimitSnapshot{observedQuota}, time.Now()) + if err != nil { + fmt.Fprintf(a.Stderr, "{\"event\":\"review_state_quota_deferred\",\"at\":%q,\"error\":%q}\n", time.Now().UTC().Format(time.RFC3339Nano), err.Error()) + quotaBlocked = true + continue + } + budget = min(budget, int(analyticsReviewWaveBytes/bytesPerItem)) + if budget > 0 && budget < len(due) { + // Retry fairness is relative to actual admitted work, including + // a smaller point/byte budget, not the maximum wave size. + due, err = s.DueReviewStateWork(window, repository, time.Now().UTC().Format(time.RFC3339Nano), budget) + if err != nil { + return yield(err) + } + } + encoded, _ := json.Marshal(map[string]any{"event": "review_state_quota", "at": time.Now().UTC().Format(time.RFC3339Nano), "limit": quota.Limit, "remaining": quota.Remaining, "reset_at": quota.ResetAt, "provider_remaining": rawQuota.Remaining, "provider_reset_at": rawQuota.ResetAt, "reserve": analyticsReviewReserve, "wave_items": min(len(due), budget)}) + fmt.Fprintln(a.Stderr, string(encoded)) + if budget == 0 { + quotaBlocked = true + continue + } + // Item-level reserve failures are durably queued and absorbed by + // analyticsIsolatedBatch; the next wave reprobes quota and keeps scanning. + // Remaining errors include unrecorded storage failures, not safe deferrals. + metrics := &analyticsWorkMetrics{} + waveStarted := time.Now() + err = a.analyticsNumbers(context.WithValue(window, analyticsMetricsKey{}, metrics), s, owner, repo, due[:min(len(due), budget)], false, "review_state") + if metrics.attempted.Load() > 0 { + bytesPerItem = max(bytesPerItem, metrics.bytes.Load()/metrics.attempted.Load()) + } + measured, _ := json.Marshal(map[string]any{"event": "review_state_wave", "at": time.Now().UTC().Format(time.RFC3339Nano), "elapsed_ms": time.Since(waveStarted).Milliseconds(), "workers": analyticsReviewWorkers, "max_busy_workers": metrics.maxBusy.Load(), "busy_worker_ms": metrics.workerMillis.Load(), "attempted_items": metrics.attempted.Load(), "recovered_items": metrics.recovered.Load(), "response_bytes": metrics.bytes.Load(), "reported_points": metrics.points.Load(), "provider_ms": metrics.providerMillis.Load(), "db_ms": metrics.dbMillis.Load(), "byte_budget": analyticsReviewWaveBytes}) + fmt.Fprintln(a.Stderr, string(measured)) + if err != nil { + return yield(err) + } + } + return ctx.Err() +} + +// Joined storage/receipt errors must never disappear behind a window timeout. +func analyticsCancellationOnly(err error) bool { + if err == nil { + return false + } + if joined, ok := err.(interface{ Unwrap() []error }); ok { + for _, child := range joined.Unwrap() { + if !analyticsCancellationOnly(child) { + return false + } + } + return true + } + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + return analyticsCancellationOnly(wrapped.Unwrap()) + } + return err == context.Canceled || err == context.DeadlineExceeded +} + +func analyticsReviewBudget(limits []gh.RateLimitSnapshot, now time.Time) (int, gh.RateLimitSnapshot, error) { + for _, quota := range limits { + if quota.Resource != "graphql" { + continue + } + if quota.Limit <= 0 || quota.Remaining < 0 || !quota.ResetAt.After(now) { + return 0, quota, fmt.Errorf("fresh GraphQL quota required for review recovery") + } + // A conservative admission margin limits in-flight overshoot. Native + // request guards recheck actual remaining quota before every request. + return min(analyticsReviewWave, max(0, quota.Remaining-analyticsReviewReserve)/32), quota, nil + } + return 0, gh.RateLimitSnapshot{}, fmt.Errorf("GraphQL quota unavailable for review recovery") +} + +func (a *App) analyticsNumbers(ctx context.Context, s *store.Store, owner, repo string, numbers []int, discovery bool, operation string) (resultErr error) { + var wg sync.WaitGroup + var failures []error + // Every return, including cancelled admission, waits for durable receipts. + defer func() { + wg.Wait() + resultErr = errors.Join(append(failures, resultErr)...) + }() + workers, batchSize := 8, 2 + if operation == "review_state" { + workers, batchSize = analyticsReviewWorkers, analyticsReviewBatch + } + slots := make(chan struct{}, workers) + var mu sync.Mutex + for i := 0; i < len(numbers); i += batchSize { + if err := ctx.Err(); err != nil { + return err + } + var part []int + for _, n := range numbers[i:min(i+batchSize, len(numbers))] { + if discovery { + // Review-only recovery must not defer a newly discovered core edit. + // If this core attempt fails, it creates graphql_history retry state + // and every later overlap skips it until the bounded scheduler retries. + deferred, e := s.AnalyticsItemQueued(ctx, owner+"/"+repo, n) + if e != nil { + return e + } + if deferred { + continue + } + } + part = append(part, n) + } + if len(part) == 0 { + continue + } + select { + case slots <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + if err := ctx.Err(); err != nil { + <-slots + return err + } + wg.Add(1) + go func(part []int) { + defer wg.Done() + defer func() { <-slots }() + if metrics, ok := ctx.Value(analyticsMetricsKey{}).(*analyticsWorkMetrics); ok { + defer metrics.enter()() + } + if e := a.analyticsIsolatedBatch(ctx, s, owner, repo, part, operation); e != nil { + mu.Lock() + failures = append(failures, e) + mu.Unlock() + } + }(part) + } + return nil +} + +func (a *App) analyticsIsolatedBatch(ctx context.Context, s *store.Store, owner, repo string, numbers []int, operation string) error { + bounded, cancel := context.WithTimeout(ctx, 2*time.Minute) + err := a.syncAnalyticsBatch(bounded, s, owner, repo, numbers, operation) + cancel() + if err == nil { + return nil + } + if ctx.Err() != nil { + return err + } + // Persisted failure receipts allow splitting all rejected batches, including + // partial/invalid connections. A poison item cannot block its healthy peer. + if len(numbers) > 1 { + var failures []error + for _, n := range numbers { + if e := a.analyticsIsolatedBatch(ctx, s, owner, repo, []int{n}, operation); e != nil { + failures = append(failures, e) + } + } + return errors.Join(failures...) + } + queued, e := s.AnalyticsItemQueued(ctx, owner+"/"+repo, numbers[0], operation) + if e != nil { + return errors.Join(err, e) + } + if !queued { + return err + } // Never advance past an unrecorded failure. + return nil +} + +func (a *App) analyticsLane(ctx context.Context, s *store.Store, c *gh.Client, owner, repo, kind string, index int) error { + repository := owner + "/" + repo + key := "updates:" + repository + ":" + kind + value, err := s.AnalyticsState(ctx, key) + if err != nil { + return err + } + var cp updateCheckpoint + if value != "" { + if err = json.Unmarshal([]byte(value), &cp); err != nil { + return err + } + } + if cp.Started == "" { + through, e := s.AnalyticsState(ctx, "through:"+repository+":"+kind) + if e != nil { + return e + } + at, e := time.Parse(time.RFC3339Nano, through) + if e != nil { + return e + } + cp = updateCheckpoint{Started: time.Now().UTC().Format(time.RFC3339Nano), Since: at.Add(-5 * time.Minute).Format(time.RFC3339Nano), Kind: index} + } + since, err := time.Parse(time.RFC3339Nano, cp.Since) + if err != nil { + return err + } + for pages := 0; pages < 2; pages++ { + started := time.Now().UTC().Format(time.RFC3339Nano) + page, e := c.UpdatedNumbers(ctx, owner, repo, kind, cp.Cursor, since) + attempt := store.AnalyticsAttempt{Repository: repository, Operation: "discover_" + kind, StartedAt: started, FinishedAt: time.Now().UTC().Format(time.RFC3339Nano), Status: "success", Evidence: json.RawMessage(`{}`)} + if e != nil { + attempt.Status = "failed" + attempt.ErrorClass, attempt.ErrorText, attempt.Evidence = gh.HistoryFailureDetails(e) + } + receiptCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + writeErr := s.RecordAnalyticsAttempt(receiptCtx, attempt) + cancel() + if e != nil || writeErr != nil { + return errors.Join(e, writeErr) + } + if e = a.analyticsNumbers(ctx, s, owner, repo, page.Numbers, true, "graphql_history"); e != nil { + return e + } + done := !page.More || (!page.Oldest.IsZero() && page.Oldest.Before(since)) + if !done { + cp.Cursor = page.Cursor + } + if e = s.WithTx(ctx, func(tx *store.Store) error { + count, _ := json.Marshal(page.Total) + if e := tx.SetAnalyticsState(ctx, "total:"+repository+":"+kind, string(count)); e != nil { + return e + } + if done { + if e := tx.SetAnalyticsState(ctx, "through:"+repository+":"+kind, cp.Started); e != nil { + return e + } + return tx.SetAnalyticsState(ctx, key, "") + } + encoded, _ := json.Marshal(cp) + return tx.SetAnalyticsState(ctx, key, string(encoded)) + }); e != nil { + return e + } + fmt.Fprintf(a.Stderr, "{\"event\":\"github_update_page\",\"at\":%q,\"kind\":%q,\"threads\":%d}\n", time.Now().UTC().Format(time.RFC3339Nano), kind, len(page.Numbers)) + if done { + return nil + } + } + return nil +} diff --git a/internal/cli/analytics_integrity_test.go b/internal/cli/analytics_integrity_test.go new file mode 100644 index 00000000..8423d9ac --- /dev/null +++ b/internal/cli/analytics_integrity_test.go @@ -0,0 +1,284 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsPoisonItemAndDiscoveryLaneDoNotStarvePeers(t *testing.T) { + for _, brokenDiscovery := range []bool{false, true} { + t.Run(fmt.Sprint(brokenDiscovery), func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, "archive.db") + s, err := store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer func() { s.Close() }() + at := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano) + baseline := time.Now().UTC().Add(-10 * time.Minute).Format(time.RFC3339Nano) + if err = s.SetAnalyticsState(ctx, "through:fixture/repo", baseline); err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", baseline, 1, 1); err != nil { + t.Fatal(err) + } + var broken atomic.Bool + broken.Store(true) + var revised atomic.Bool + conn := func(nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"nodes": nodes, "totalCount": len(nodes), "pageInfo": map[string]any{"hasNextPage": false, "endCursor": "end"}} + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":50000,"remaining":49000,"reset":4102444800},"core":{"limit":50000,"remaining":49000,"reset":4102444800}}}`) + return + } + if r.URL.Path != "/graphql" { + t.Errorf("unexpected fallback: %s", r.URL.Path) + http.Error(w, "no fallback", 400) + return + } + var req struct{ Query string } + if decodeErr := json.NewDecoder(r.Body).Decode(&req); decodeErr != nil { + t.Error(decodeErr) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 48000, "resetAt": "2099-01-01T00:00:00Z"}} + if strings.Contains(req.Query, "orderBy") { + kind, n := "issues", 1 + if strings.Contains(req.Query, "pullRequests(first:") { + kind, n = "pullRequests", 2 + } + page := conn(map[string]any{"number": n, "updatedAt": at}) + if brokenDiscovery && broken.Load() && n == 1 { + delete(page, "pageInfo") + } + data["repository"] = map[string]any{kind: page} + } else if strings.Contains(req.Query, "issueOrPullRequest") { + n, typ := 1, "Issue" + if strings.Contains(req.Query, "number:2)") { + n, typ = 2, "PullRequest" + } + node := map[string]any{"id": fmt.Sprint("node-", n), "fullDatabaseId": fmt.Sprint(n), "__typename": typ, "number": n, "title": "fixture", "body": "retained body", "state": "OPEN", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": at, "url": fmt.Sprintf("https://github.com/fixture/repo/issues/%d", n), "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "labels": conn(), "assignees": conn(), "comments": conn()} + if revised.Load() && n == 2 { + node["body"] = "fresh core body" + } + if n == 2 { + node["reviews"] = conn() + node["reviewThreads"] = conn() + } + if !brokenDiscovery && broken.Load() && n == 1 { + node["comments"].(map[string]any)["totalCount"] = 1 + } + data["repository"] = map[string]any{"id": "repo", "databaseId": 1, "nameWithOwner": "fixture/repo", "n0": node} + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + a := New() + a.Stderr = io.Discard + a.configPath = writeDoctorTestConfig(t, dir, path) + client := gh.New(gh.Options{BaseURL: server.URL, Token: "test-token-placeholder"}) + reviewRetry := store.AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "review_state", StartedAt: "2099-01-01T00:00:00Z", FinishedAt: "2099-01-01T00:00:01Z", Status: "failed", Evidence: json.RawMessage(`{}`)} + if !brokenDiscovery { + if err = s.RecordAnalyticsAttempt(ctx, reviewRetry); err != nil { + t.Fatal(err) + } + } else { + // A transient core failure can precede first lane migration. Its + // incomplete flag must not erase the verified historical baseline. + failure := store.AnalyticsAttempt{Repository: "fixture/repo", Operation: "discover_issues", StartedAt: baseline, FinishedAt: baseline, Status: "failed", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, failure); err != nil { + t.Fatal(err) + } + var complete int + if err = s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&complete); err != nil || complete != 0 { + t.Fatalf("failure did not clear core coverage: complete=%d err=%v", complete, err) + } + } + if !brokenDiscovery { + // Exercise the real first-watch entry point with no coverage row or + // completed-update watermark: the completed discovery receipt owns it. + if _, err = s.DB().Exec("DELETE FROM analytics_coverage"); err != nil { + t.Fatal(err) + } + s.SetAnalyticsState(ctx, "through:fixture/repo", "") + receipt, _ := json.Marshal(map[string]any{"repository": "fixture/repo", "phase": "complete", "discovery": []map[string]any{{"kind": "issues", "done": 1, "total": 1, "updated_at": baseline}, {"kind": "pullRequests", "done": 1, "total": 1, "updated_at": baseline}}}) + if err = os.WriteFile(filepath.Join(dir, "status.json"), receipt, 0600); err != nil { + t.Fatal(err) + } + a.Stdout = io.Discard + err = a.runAnalytics(ctx, []string{"fixture/repo", "--once", "--json"}) + } else { + err = a.analyticsCycle(ctx, s, client, "fixture", "repo") + } + if err == nil { + t.Fatal("incomplete coverage reported complete") + } + if !brokenDiscovery && !errors.Is(err, errAnalyticsIncomplete) { + t.Fatal(err) + } + var count int + s.DB().QueryRow("SELECT count(*) FROM threads WHERE number=2").Scan(&count) + if count != 1 { + t.Fatal("independent PR never persisted") + } + s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&count) + if count != 0 { + t.Fatal("poison item hidden by complete=true") + } + if !brokenDiscovery { + var before, after int + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE number=1 AND operation='graphql_history'").Scan(&before) + if err = a.analyticsCycle(ctx, s, client, "fixture", "repo"); !errors.Is(err, errAnalyticsIncomplete) { + t.Fatalf("unexpected repeat outcome %v", err) + } + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE number=1 AND operation='graphql_history'").Scan(&after) + if before != after { + t.Fatal("review-only failure bypassed core retry backoff repeatedly") + } + } + broken.Store(false) + // Simulate a due retry after a process restart, without dropping its history. + if _, err = s.DB().Exec("UPDATE analytics_retries SET next_attempt_at='2000-01-01T00:00:00Z'"); err != nil { + t.Fatal(err) + } + s.Close() + s, err = store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if err = a.analyticsCycle(ctx, s, client, "fixture", "repo"); err != nil { + t.Fatal(err) + } + if outstanding, err := s.AnalyticsOutstanding(ctx, "fixture/repo"); err != nil || outstanding != 0 { + t.Fatalf("not recovered: %d %v", outstanding, err) + } + if err = s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&count); err != nil || count != 1 { + t.Fatalf("verified core baseline did not recover: complete=%d err=%v", count, err) + } + s.DB().QueryRow("SELECT count(*) FROM threads").Scan(&count) + if count != 2 { + t.Fatalf("wrong recovered content count %d", count) + } + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE status='failed'").Scan(&count) + if count < 1 { + t.Fatal("failure history erased") + } + reviewRetry.Number = 2 + if err = s.RecordAnalyticsAttempt(ctx, reviewRetry); err != nil { + t.Fatal(err) + } + revised.Store(true) + if err = a.analyticsCycle(ctx, s, client, "fixture", "repo"); err != nil { + t.Fatal(err) + } + var body string + s.DB().QueryRow("SELECT body FROM threads WHERE number=2").Scan(&body) + if body != "fresh core body" { + t.Fatal("review recovery backoff hid a newly discovered core update") + } + }) + } +} + +func TestAnalyticsLaneMigrationRetainsInflightCursorAndLegacyReceipt(t *testing.T) { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + legacy := `{"started":"2026-01-01T12:00:00Z","since":"2026-01-01T10:55:00Z","kind":1,"cursor":"opaque-provider-cursor","issues":10,"prs":20}` + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T11:00:00Z", 10, 20); err != nil { + t.Fatal(err) + } + s.SetAnalyticsState(ctx, "through:fixture/repo", "2026-01-01T11:00:00Z") + s.SetAnalyticsState(ctx, "updates:fixture/repo", legacy) + if err = migrateAnalyticsLanes(ctx, s, "fixture/repo"); err != nil { + t.Fatal(err) + } + value, err := s.AnalyticsState(ctx, "updates:fixture/repo:pullRequests") + if err != nil { + t.Fatal(err) + } + var cp updateCheckpoint + if err = json.Unmarshal([]byte(value), &cp); err != nil { + t.Fatal(err) + } + if cp.Cursor != "opaque-provider-cursor" || cp.Since != "2026-01-01T10:55:00Z" { + t.Fatal("in-flight provider cursor reset") + } + total, _ := s.AnalyticsState(ctx, "total:fixture/repo:issues") + if total != "10" { + t.Fatalf("completed lane total lost: %s", total) + } + old, _ := s.AnalyticsState(ctx, "updates:fixture/repo") + if old != legacy { + t.Fatal("legacy evidence overwritten") + } + s.SetAnalyticsState(ctx, "updates:fixture/repo:pullRequests", "new-progress") + if err = migrateAnalyticsLanes(ctx, s, "fixture/repo"); err != nil { + t.Fatal(err) + } + value, _ = s.AnalyticsState(ctx, "updates:fixture/repo:pullRequests") + if value != "new-progress" { + t.Fatal("restart reseeded an existing checkpoint") + } +} + +func TestAnalyticsDiscoveryLeavesDueItemsToBoundedRetryScheduler(t *testing.T) { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for i := 1; i <= 24; i++ { + a := store.AnalyticsAttempt{Repository: "fixture/repo", Number: i, Operation: "graphql_history", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + } + due, err := s.DueAnalyticsRetries(ctx, "fixture/repo", "2099-01-01T00:00:00Z", 8, "graphql_history") + if err != nil || len(due) != 8 { + t.Fatalf("retry bound: %v %v", due, err) + } + // This app has no usable configuration. Any attempted collection would fail; + // discovery must skip even due items not selected by the retry budget. + a := New() + numbers := []int{} + for i := 1; i <= 24; i++ { + numbers = append(numbers, i) + } + if err = a.analyticsNumbers(ctx, s, "fixture", "repo", numbers, true, "graphql_history"); err != nil { + t.Fatal(err) + } + var attempts int + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts").Scan(&attempts) + if attempts != 24 { + t.Fatal("discovery bypassed the retry scheduler") + } +} diff --git a/internal/cli/analytics_partial_response_test.go b/internal/cli/analytics_partial_response_test.go new file mode 100644 index 00000000..853f52e2 --- /dev/null +++ b/internal/cli/analytics_partial_response_test.go @@ -0,0 +1,167 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsPairedPartialRejectionIsolatesPeerAndPreservesUnavailableHistory(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dbpath := filepath.Join(dir, "archive.db") + s, err := store.Open(ctx, dbpath) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var reject atomic.Bool + beforeAt := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano) + afterAt := time.Now().UTC().Format(time.RFC3339Nano) + conn := func(nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"totalCount": len(nodes), "nodes": nodes, "pageInfo": map[string]any{"hasNextPage": false, "endCursor": "end"}} + } + aliases := regexp.MustCompile(`n([0-9]+): issueOrPullRequest\(number:([0-9]+)\)`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":4102444800},"core":{"limit":20000,"remaining":19000,"reset":4102444800}}}`) + return + } + var req struct{ Query string } + if e := json.NewDecoder(r.Body).Decode(&req); e != nil { + t.Error(e) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": "2099-01-01T00:00:00Z"}} + response := map[string]any{"data": data} + repo := map[string]any{"id": "repo", "databaseId": 1, "nameWithOwner": "fixture/repo"} + var failures []any + matches := aliases.FindAllStringSubmatch(req.Query, -1) + for _, m := range matches { + n, _ := strconv.Atoi(m[2]) + alias := "n" + m[1] + if reject.Load() && n == 16945 && len(matches) == 1 { + var retainedBody string + if e := s.DB().QueryRow("SELECT body FROM threads WHERE number=16945").Scan(&retainedBody); e != nil || retainedBody != "retained original" { + t.Errorf("paired partial response was applied before isolated success: %q %v", retainedBody, e) + } + } + if reject.Load() && n == 16944 { + repo[alias] = nil + failures = append(failures, map[string]any{"type": "NOT_FOUND", "path": []any{"repository", alias}, "message": "private provider prose"}) + continue + } + at, body := beforeAt, "retained original" + if reject.Load() { + at, body = afterAt, "isolated success" + } + comment := map[string]any{"id": fmt.Sprint("C", n), "__typename": "IssueComment", "fullDatabaseId": fmt.Sprint(n + 1000000), "body": "retained comment", "createdAt": beforeAt, "updatedAt": beforeAt, "publishedAt": beforeAt, "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "url": fmt.Sprintf("https://github.com/fixture/repo/pull/%d#comment", n)} + repo[alias] = map[string]any{"id": fmt.Sprint("PR", n), "fullDatabaseId": fmt.Sprint(n), "__typename": "PullRequest", "number": n, "title": "fixture", "body": body, "state": "CLOSED", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": at, "url": fmt.Sprintf("https://github.com/fixture/repo/pull/%d", n), "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "labels": conn(), "assignees": conn(), "comments": conn(comment), "reviews": conn(), "reviewThreads": conn()} + } + if len(repo) > 3 { + data["repository"] = repo + } + if len(failures) > 0 { + response["errors"] = failures + } + json.NewEncoder(w).Encode(response) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + a := New() + a.Stderr = io.Discard + a.configPath = writeDoctorTestConfig(t, dir, dbpath) + if err = a.syncAnalyticsBatch(ctx, s, "fixture", "repo", []int{16944, 16945}, "graphql_history"); err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", beforeAt, 0, 2); err != nil { + t.Fatal(err) + } + // Reproduce a retained legacy PR with unknown review membership. + if _, err = s.DB().Exec("DELETE FROM pull_request_review_thread_syncs WHERE thread_id=(SELECT id FROM threads WHERE number=16944)"); err != nil { + t.Fatal(err) + } + snapshot := func() string { + t.Helper() + tables := []string{"threads", "thread_revisions", "comments", "comment_revisions"} + out := map[string][][]any{} + for _, table := range tables { + where := "thread_id=(SELECT id FROM threads WHERE number=16944)" + if table == "threads" { + where = "number=16944" + } else if table == "comment_revisions" { + where = "comment_id IN (SELECT id FROM comments WHERE thread_id=(SELECT id FROM threads WHERE number=16944))" + } + rows, e := s.DB().Query("SELECT * FROM " + table + " WHERE " + where + " ORDER BY id") + if e != nil { + t.Fatal(e) + } + cols, _ := rows.Columns() + values := [][]any{} + for rows.Next() { + row := make([]any, len(cols)) + dest := make([]any, len(cols)) + for i := range row { + dest[i] = &row[i] + } + if e = rows.Scan(dest...); e != nil { + t.Fatal(e) + } + values = append(values, row) + } + if e = rows.Err(); e != nil { + t.Fatal(e) + } + rows.Close() + out[table] = values + } + b, _ := json.Marshal(out) + return string(b) + } + retained := snapshot() + reject.Store(true) + if err = a.analyticsIsolatedBatch(ctx, s, "fixture", "repo", []int{16944, 16945}, "review_state"); err != nil { + t.Fatal(err) + } + if snapshot() != retained { + t.Fatal("unavailable PR history changed") + } + var unknown, complete int + s.DB().QueryRow("SELECT count(*) FROM pull_request_review_thread_syncs WHERE thread_id=(SELECT id FROM threads WHERE number=16944)").Scan(&unknown) + s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&complete) + if unknown != 0 || complete != 1 { + t.Fatalf("fabricated membership or invalidated core: %d %d", unknown, complete) + } + var evidence string + if err = s.DB().QueryRow("SELECT evidence_json FROM analytics_fetch_attempts WHERE number=16944 AND operation='review_state' ORDER BY id DESC LIMIT 1").Scan(&evidence); err != nil { + t.Fatal(err) + } + if !strings.Contains(evidence, `"type":"NOT_FOUND"`) || !strings.Contains(evidence, `"path":["repository","n0"]`) || strings.Contains(evidence, "private provider prose") { + t.Fatal("missing or unsafe durable cause", evidence) + } + var pending, peerResolved, success int + s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE number=16944 AND operation='review_state' AND resolved_at IS NULL AND attempts=2").Scan(&pending) + s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE number=16945 AND operation='review_state' AND resolved_at IS NOT NULL").Scan(&peerResolved) + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE number=16945 AND operation='review_state' AND status='success'").Scan(&success) + var membership, body string + s.DB().QueryRow("SELECT x.review_thread_ids_json,t.body FROM threads t JOIN pull_request_review_thread_syncs x ON x.thread_id=t.id WHERE t.number=16945").Scan(&membership, &body) + if pending != 1 || peerResolved != 1 || success != 1 || membership != "[]" || body != "retained original" { + t.Fatalf("isolation/retry proof failed: %d %d %d %s %s", pending, peerResolved, success, membership, body) + } +} diff --git a/internal/cli/analytics_readonly_test.go b/internal/cli/analytics_readonly_test.go new file mode 100644 index 00000000..d1ecffba --- /dev/null +++ b/internal/cli/analytics_readonly_test.go @@ -0,0 +1,148 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsCommandsRespectOwnershipAndExplicitRepair(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, "source.db") + s, err := store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + repoID, err := s.UpsertRepository(ctx, store.Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", RawJSON: `{}`, UpdatedAt: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + tid, err := s.UpsertThread(ctx, store.Thread{RepoID: repoID, GitHubID: "1", Number: 1, Kind: "pull_request", State: "open", Title: "retained", HTMLURL: "https://github.com/fixture/repo/pull/1", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{}`, ContentHash: "h", UpdatedAt: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + _, err = s.UpsertComment(ctx, store.Comment{ThreadID: tid, GitHubID: "review", CommentType: "pull_review", RawJSON: `{"submitted_at":"2026-01-02T00:00:00Z"}`, CreatedAtGitHub: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + if _, err = s.DB().ExecContext(ctx, "UPDATE comments SET submitted_at_gh=NULL,publication_at_gh=NULL; UPDATE comment_revisions SET submitted_at_gh=NULL,publication_at_gh=NULL"); err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 1); err != nil { + t.Fatal(err) + } + if err = s.RecordAnalyticsAttempt(ctx, store.AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "review_state", Status: "failed", ErrorClass: "validation", ErrorText: "private-rejection-sentinel", Evidence: json.RawMessage(`{"private":"private-rejection-sentinel"}`), StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z"}); err != nil { + t.Fatal(err) + } + cfg := writeDoctorTestConfig(t, dir, path) + lock, err := os.OpenFile(filepath.Join(dir, "runner.lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + if err = lockPortableFile(lock); err != nil { + t.Fatal(err) + } + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.Error(w, "unexpected provider request", 500) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + run := func(args ...string) (map[string]any, error) { + t.Helper() + a := New() + var out bytes.Buffer + a.Stdout = &out + a.Stderr = io.Discard + err := a.Run(ctx, append([]string{"--config", cfg, "analytics", "fixture/repo", "--json"}, args...)) + if err != nil { + return nil, err + } + if strings.Contains(out.String(), "private-rejection-sentinel") { + t.Fatal("status exposed private rejection evidence") + } + var payload map[string]any + if err = json.Unmarshal(out.Bytes(), &payload); err != nil { + t.Fatal(err) + } + return payload, nil + } + status, err := run("--status") + if err != nil { + t.Fatal(err) + } + coverage, ok := status["coverage"].(map[string]any) + if !ok || coverage["complete"] != true || status["unresolved_retries"] != float64(1) || status["core_unresolved_retries"] != float64(0) { + t.Fatalf("review failure hid core coverage: %+v", status) + } + audit, err := run() + if err != nil { + t.Fatal(err) + } + if audit["would_change"] != float64(2) || audit["changed"] != float64(0) { + t.Fatalf("audit mutated or missed repair: %+v", audit) + } + if _, err = run("--apply"); err == nil || !strings.Contains(err.Error(), "ownership lock busy") { + t.Fatalf("mutation bypassed collector owner: %v", err) + } + if _, err = run("--status", "--once"); err == nil || !strings.Contains(err.Error(), "cannot be combined") { + t.Fatalf("mixed read/write mode accepted: %v", err) + } + var modified, attempts int + if err = s.DB().QueryRowContext(ctx, "SELECT (SELECT count(*) FROM comments WHERE publication_at_gh IS NOT NULL)+(SELECT count(*) FROM comment_revisions WHERE publication_at_gh IS NOT NULL)").Scan(&modified); err != nil || modified != 0 { + t.Fatalf("audit applied publication repair: %d %v", modified, err) + } + if err = s.DB().QueryRowContext(ctx, "SELECT count(*) FROM analytics_fetch_attempts").Scan(&attempts); err != nil || attempts != 1 { + t.Fatalf("read created a collection receipt: %d %v", attempts, err) + } + // An explicit repair succeeds only after the fixture collector releases its + // lock. It normalizes retained evidence without inventing a new revision. + var rawBefore, recordedBefore string + if err = s.DB().QueryRowContext(ctx, "SELECT raw_json,recorded_at FROM comment_revisions").Scan(&rawBefore, &recordedBefore); err != nil { + t.Fatal(err) + } + if err = lock.Close(); err != nil { + t.Fatal(err) + } + repaired, err := run("--apply") + if err != nil || repaired["changed"] != float64(2) { + t.Fatalf("explicit repair=%+v err=%v", repaired, err) + } + generation, err := s.AnalyticsState(ctx, "publication_repair_generation") + if err != nil || generation == "" { + t.Fatalf("missing repair generation: %q %v", generation, err) + } + var rawAfter, recordedAfter, published string + var revisions int + if err = s.DB().QueryRowContext(ctx, "SELECT count(*),raw_json,recorded_at,publication_at_gh FROM comment_revisions").Scan(&revisions, &rawAfter, &recordedAfter, &published); err != nil { + t.Fatal(err) + } + if revisions != 1 || rawBefore != rawAfter || recordedBefore != recordedAfter || published != "2026-01-02T00:00:00Z" { + t.Fatal("repair changed retained source history or publication time") + } + replay, err := run("--apply") + if err != nil || replay["changed"] != float64(0) { + t.Fatalf("repair replay=%+v err=%v", replay, err) + } + again, err := s.AnalyticsState(ctx, "publication_repair_generation") + if err != nil || generation != again { + t.Fatalf("idempotent replay changed repair generation: %q %q %v", generation, again, err) + } + if requests.Load() != 0 { + t.Fatalf("read-only modes contacted provider %d times", requests.Load()) + } +} diff --git a/internal/cli/analytics_test.go b/internal/cli/analytics_test.go new file mode 100644 index 00000000..5fd55cf4 --- /dev/null +++ b/internal/cli/analytics_test.go @@ -0,0 +1,28 @@ +package cli + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestAnalyticsEnrichmentContinuesAfterDraining(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + calls := 0 + err := maintainAnalyticsEnrichment(ctx, true, time.Millisecond, func() error { + calls++ + if calls == 2 { + cancel() + } + return nil + }) + if calls != 2 || !errors.Is(err, context.Canceled) { + t.Fatalf("calls=%d err=%v", calls, err) + } + calls = 0 + if err := maintainAnalyticsEnrichment(context.Background(), false, time.Millisecond, func() error { calls++; return nil }); err != nil || calls != 1 { + t.Fatalf("one-shot calls=%d err=%v", calls, err) + } +} diff --git a/internal/cli/analytics_throughput_test.go b/internal/cli/analytics_throughput_test.go new file mode 100644 index 00000000..8b0f2976 --- /dev/null +++ b/internal/cli/analytics_throughput_test.go @@ -0,0 +1,340 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestAnalyticsRecoveryUsesQuotaAfterCoreAndResumesCancellation(t *testing.T) { + for _, mode := range []string{"available", "reserved", "quota_drops", "quota_races", "concurrent", "cancel", "narrow_fair"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + s, err := store.Open(ctx, filepath.Join(dir, "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + _, err = s.DB().Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo',1,'{}','2026-01-01')`) + if err != nil { + t.Fatal(err) + } + oldItems := 48 + if mode == "quota_drops" || mode == "concurrent" { + oldItems = 300 + } + if mode == "quota_races" { + oldItems = 6001 + } + tx, err := s.DB().BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + for n := 1; n <= oldItems; n++ { + _, err = tx.Exec(`INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at,last_pulled_at) VALUES(?,1,?,?,'pull_request','open','','','[]','[]','{}','fixture','2026-01-01','2026-01-01T00:00:00Z')`, n, fmt.Sprint(n), n) + if err != nil { + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + if mode == "narrow_fair" { + for n := 1; n <= 16; n++ { + if _, err = s.DB().Exec("INSERT INTO analytics_retries(repository,number,operation,first_seen_at,last_seen_at,next_attempt_at,attempts) VALUES('fixture/repo',?,'review_state','2000-01-01T00:00:00Z','2000-01-01T00:00:00Z','2000-01-01T00:00:00Z',1)", n); err != nil { + t.Fatal(err) + } + } + } + baseline := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano) + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", baseline, 0, oldItems); err != nil { + t.Fatal(err) + } + ready := make(chan struct{}) + var concurrentBatches atomic.Int64 + var coreSeen, cancelled atomic.Bool + var recovered, probes, recoveryProbes atomic.Int64 + quota := func() int { + if mode == "narrow_fair" { + if recovered.Load() >= 16 { + return 3020 + } + return 3530 + } + + if mode == "quota_races" && recoveryProbes.Load() > 1 { + return 2999 + } + if mode == "reserved" || (mode == "quota_drops" && recovered.Load() >= 16) { + return 3020 + } + return 19000 + } + conn := func(nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"totalCount": len(nodes), "nodes": nodes, "pageInfo": map[string]any{"hasNextPage": false, "endCursor": "end"}} + } + numbers := regexp.MustCompile(`n([0-9]+): issueOrPullRequest\(number:([0-9]+)\)`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/rate_limit" { + probes.Add(1) + if coreSeen.Load() { + recoveryProbes.Add(1) + } + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":%d,"reset":4102444800},"core":{"limit":20000,"remaining":19999,"reset":4102444800}}}`, quota()) + return + } + if r.URL.Path != "/graphql" { + t.Errorf("unexpected endpoint %s", r.URL.Path) + http.Error(w, "unexpected", 400) + return + } + var req struct{ Query string } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + if coreSeen.Load() && !strings.Contains(req.Query, "orderBy") && !strings.Contains(req.Query, "issueOrPullRequest") { + recoveryProbes.Add(1) + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": quota(), "limit": 20000, "resetAt": "2099-01-01T00:00:00Z"}} + if strings.Contains(req.Query, "orderBy") { + kind, page := "issues", conn() + if strings.Contains(req.Query, "pullRequests(first:") { + kind, page = "pullRequests", conn(map[string]any{"number": 10000, "updatedAt": time.Now().UTC().Format(time.RFC3339Nano)}) + } + data["repository"] = map[string]any{kind: page} + } else if matches := numbers.FindAllStringSubmatch(req.Query, -1); len(matches) > 0 { + if mode == "concurrent" && !strings.Contains(req.Query, "number:10000)") { + if concurrentBatches.Add(1) == 32 { + close(ready) + } + select { + case <-ready: + case <-time.After(10 * time.Second): + t.Error("32 workers were not active") + return + } + } + repo := map[string]any{"id": "repo", "databaseId": 1, "nameWithOwner": "fixture/repo"} + for _, m := range matches { + n, _ := strconv.Atoi(m[2]) + if n == 10000 { + coreSeen.Store(true) + } else { + if !coreSeen.Load() { + t.Error("recovery ran before ordinary capture") + } + var complete int + if e := s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&complete); e != nil || complete != 1 { + t.Errorf("core coverage unavailable during review: %d %v", complete, e) + } + if mode == "cancel" && cancelled.CompareAndSwap(false, true) { + cancel() + <-r.Context().Done() + return + } + recovered.Add(1) + } + node := map[string]any{"id": fmt.Sprint("PR-", n), "fullDatabaseId": fmt.Sprint(n), "__typename": "PullRequest", "number": n, "title": "fixture", "body": "retained", "state": "OPEN", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": baseline, "url": fmt.Sprintf("https://github.com/fixture/repo/pull/%d", n), "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "labels": conn(), "assignees": conn(), "comments": conn(), "reviews": conn(), "reviewThreads": conn()} + repo["n"+m[1]] = node + } + data["repository"] = repo + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + a := New() + a.Stderr = io.Discard + a.configPath = writeDoctorTestConfig(t, dir, filepath.Join(dir, "archive.db")) + client := gh.New(gh.Options{BaseURL: server.URL, Token: "test-token-placeholder"}) + err = a.analyticsCycle(ctx, s, client, "fixture", "repo") + if mode == "cancel" { + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + var failed int + if e := s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE operation='review_state' AND status='failed'").Scan(&failed); e != nil || failed == 0 { + t.Fatalf("missing cancellation receipts: %d %v", failed, e) + } + if _, err = s.DB().Exec("UPDATE analytics_retries SET next_attempt_at='2000-01-01T00:00:00Z' WHERE resolved_at IS NULL"); err != nil { + t.Fatal(err) + } + // Resume from the real committed cursor and retry rows; no reset/reseed. + err = a.analyticsCycle(context.Background(), s, client, "fixture", "repo") + } + if err != nil { + t.Fatal(err) + } + var resolved, pending, scanned, complete int + if err = s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE operation='review_state' AND resolved_at IS NOT NULL").Scan(&resolved); err != nil { + t.Fatal(err) + } + if err = s.DB().QueryRow("SELECT pending_items,scanned,complete FROM analytics_review_state_coverage").Scan(&pending, &scanned, &complete); err != nil { + t.Fatal(err) + } + want := oldItems + if mode == "reserved" || mode == "quota_races" { + want = 0 + } + if mode == "quota_drops" { + want = analyticsReviewWave + } + if mode == "narrow_fair" { + want = 16 + var fresh int + if err = s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE operation='review_state' AND resolved_at IS NOT NULL AND attempts=0").Scan(&fresh); err != nil || fresh != 12 { + t.Fatalf("shrunken wave starved fresh work: %d %v", fresh, err) + } + } + if resolved != want || pending != oldItems-want || scanned != oldItems+1 || (complete == 1) != (want == oldItems) { + t.Fatalf("resolved=%d pending=%d scanned=%d complete=%d, want recovered%d", resolved, pending, scanned, complete, want) + } + if mode == "quota_races" { + var deferred int + if e := s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE error_class='rate_limit'").Scan(&deferred); e != nil || deferred == 0 { + t.Fatalf("native reserve guard not exercised: %d %v", deferred, e) + } + } + if probes.Load() == 0 { + t.Fatal("never obtained actual quota") + } + if err = s.DB().QueryRow("SELECT complete FROM analytics_coverage").Scan(&complete); err != nil || complete != 1 { + t.Fatalf("core coverage affected by recovery: %d %v", complete, err) + } + }) + } +} + +func TestAnalyticsReviewBudgetRejectsMissingAndExpiredQuota(t *testing.T) { + now := time.Now() + for _, limits := range [][]gh.RateLimitSnapshot{nil, {{Resource: "core", Limit: 20000, Remaining: 19000, ResetAt: now.Add(time.Hour)}}, {{Resource: "graphql", Limit: 20000, Remaining: 19000, ResetAt: now.Add(-time.Second)}}} { + if _, _, err := analyticsReviewBudget(limits, now); err == nil { + t.Fatal("invalid quota admitted recovery") + } + } +} + +func TestAnalyticsRecoveryScansPastOneChunkWhenQuotaIsReserved(t *testing.T) { + for _, quotaJSON := range []string{`{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":3020,"resetAt":"2099-01-01T00:00:00Z"}}}`, `{"data":{}}`, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":19000,"resetAt":"2000-01-01T00:00:00Z"}}}`} { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + _, err = s.DB().Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo',1,'{}','2026-01-01'); + WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<6001) + INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at,last_pulled_at) + SELECT x,1,printf('%d',x),x,'pull_request','open','','','[]','[]','{}','fixture','2026-01-01','2026-01-01T00:00:00Z' FROM n`) + if err != nil { + t.Fatal(err) + } + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.Path != "/graphql" { + t.Errorf("provider work admitted inside reserve: %s", r.URL.Path) + } + fmt.Fprint(w, quotaJSON) + })) + defer server.Close() + a := New() + a.Stderr = io.Discard + c := gh.New(gh.Options{BaseURL: server.URL}) + if err = a.analyticsReviewRecovery(ctx, s, c, "fixture", "repo", time.Now().Add(time.Minute)); err != nil { + t.Fatal(err) + } + var scanned, pending, done, complete int + if err = s.DB().QueryRow("SELECT scanned,pending_items,scan_complete,complete FROM analytics_review_state_coverage").Scan(&scanned, &pending, &done, &complete); err != nil { + t.Fatal(err) + } + if scanned != 6001 || pending != 6001 || done != 1 || complete != 0 || requests.Load() != 1 { + t.Fatalf("scan stalled or reserve breached: %d %d %d %d requests=%d", scanned, pending, done, complete, requests.Load()) + } + } +} + +func TestAnalyticsRecoveryDeadlineYieldsWithDurableReceipt(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + s, err := store.Open(ctx, filepath.Join(dir, "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + _, err = s.DB().Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo',1,'{}','2026-01-01'); + INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at,last_pulled_at) VALUES(1,1,'1',1,'pull_request','open','','','[]','[]','{}','fixture','2026-01-01','2026-01-01T00:00:00Z')`) + if err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 0, 1); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":4102444800}}}`) + return + } + var req struct{ Query string } + if e := json.NewDecoder(r.Body).Decode(&req); e != nil { + t.Error(e) + return + } + if !strings.Contains(req.Query, "issueOrPullRequest") { + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":19000,"resetAt":"2099-01-01T00:00:00Z"}}}`) + return + } + <-r.Context().Done() + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + a := New() + a.Stderr = io.Discard + a.configPath = writeDoctorTestConfig(t, dir, filepath.Join(dir, "archive.db")) + c := gh.New(gh.Options{BaseURL: server.URL, Token: "test-token-placeholder"}) + if err = a.analyticsReviewRecovery(ctx, s, c, "fixture", "repo", time.Now().Add(6*time.Second)); err != nil { + t.Fatal("window yield failed core cycle:", err) + } + var failed, pending, core int + if err = s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE status='failed' AND error_class='cancelled'").Scan(&failed); err != nil { + t.Fatal(err) + } + if err = s.DB().QueryRow("SELECT pending_items FROM analytics_review_state_coverage").Scan(&pending); err != nil { + t.Fatal(err) + } + if err = s.DB().QueryRow("SELECT complete FROM analytics_coverage").Scan(&core); err != nil { + t.Fatal(err) + } + if failed != 1 || pending != 1 || core != 1 { + t.Fatalf("lost cancellation state: failed=%d pending=%d core=%d", failed, pending, core) + } + if analyticsCancellationOnly(errors.Join(context.DeadlineExceeded, errors.New("receipt failed"))) { + t.Fatal("storage error hidden as yield") + } + if !analyticsCancellationOnly(errors.Join(fmt.Errorf("request: %w", context.DeadlineExceeded), context.Canceled)) { + t.Fatal("normal cancellation not recognized") + } +} diff --git a/internal/cli/analytics_watch_privacy_test.go b/internal/cli/analytics_watch_privacy_test.go new file mode 100644 index 00000000..c432d3f7 --- /dev/null +++ b/internal/cli/analytics_watch_privacy_test.go @@ -0,0 +1,92 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/openclaw/gitcrawl/internal/store" +) + +type analyticsCancelLog struct { + bytes.Buffer + cancel context.CancelFunc +} + +func (w *analyticsCancelLog) Write(p []byte) (int, error) { + n, err := w.Buffer.Write(p) + if bytes.Contains(p, []byte(`"event":"github_update_failed"`)) { + w.cancel() + } + return n, err +} + +func TestAnalyticsWatchRedactsProviderFailures(t *testing.T) { + for _, mode := range []string{"--watch", "--once"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + dir := t.TempDir() + path := filepath.Join(dir, "source.db") + st, err := store.Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if err = st.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 1); err != nil { + t.Fatal(err) + } + if err = st.Close(); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":4102444800},"core":{"limit":20000,"remaining":19000,"reset":4102444800}}}`) + return + } + fmt.Fprint(w, `{"errors":[{"type":"FORBIDDEN","message":"synthetic-private-provider-message"}]}`) + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITHUB_TOKEN", "test-token-placeholder") + app := New() + log := &analyticsCancelLog{cancel: cancel} + app.Stdout, app.Stderr = io.Discard, log + err = app.Run(ctx, []string{"--config", writeDoctorTestConfig(t, dir, path), "analytics", "fixture/repo", mode}) + if mode == "--watch" && !errors.Is(err, context.Canceled) { + t.Fatalf("watch error: %v", err) + } + if bytes.Contains(log.Bytes(), []byte("synthetic-private-provider-message")) { + t.Fatal("watch exposed provider prose") + } + if mode == "--once" && (err == nil || strings.Contains(err.Error(), "synthetic-private-provider-message")) { + t.Fatalf("one-shot returned unsafe failure: %v", err) + } + if mode == "--watch" && !bytes.Contains(log.Bytes(), []byte(`"error_class"`)) { + t.Fatalf("missing safe failure classification: %s", log.String()) + } + }) + } +} + +func TestAnalyticsIncompleteErrorsPreserveIdentityWithoutJoinedProse(t *testing.T) { + cause := errors.Join(errAnalyticsIncomplete, fmt.Errorf("synthetic-private-joined-failure")) + wrapped := &analyticsCommandError{cause: cause} + if !errors.Is(wrapped, errAnalyticsIncomplete) || strings.Contains(wrapped.Error(), "synthetic-private") { + t.Fatal("unsafe or unidentifiable incomplete error") + } + var log bytes.Buffer + app := New() + app.Stderr = &log + app.analyticsUpdateLog(cause) + if strings.Contains(log.String(), "synthetic-private") { + t.Fatal("incomplete log exposed joined provider prose") + } +} diff --git a/internal/cli/app.go b/internal/cli/app.go index 0a390fc7..0886bfa3 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -25,11 +25,13 @@ const ( ) type App struct { - githubTokenCommand *string - githubTokenMu sync.Mutex - observedGitHubToken string - Stdout io.Writer - Stderr io.Writer + analyticsTokenProvider func(context.Context) (string, error) + githubTokenCommand *string + githubTokenMu sync.Mutex + observedGitHubToken string + Stdout io.Writer + Stderr io.Writer + Stdin io.Reader configPath string format OutputFormat @@ -53,6 +55,7 @@ func New() *App { return &App{ Stdout: os.Stdout, Stderr: os.Stderr, + Stdin: os.Stdin, format: FormatText, getWorkingDirectory: os.Getwd, } @@ -126,6 +129,10 @@ func (a *App) Run(ctx context.Context, args []string) error { return a.runDoctor(ctx, rest[1:]) case "status": return a.runStatus(ctx, rest[1:]) + case "metrics": + return a.runMetrics(ctx, rest[1:]) + case "analytics": + return a.runAnalytics(ctx, rest[1:]) case "sync": return a.runSync(ctx, rest[1:]) case "fill-pr-details": diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 1a3c3fdd..f2f9ba39 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -4344,10 +4344,10 @@ func TestDoctorJSONReportsCurrentSchemaDiagnosticsWithoutMutation(t *testing.T) if got := schema["state"]; got != "current" { t.Fatalf("db_schema.state = %#v, payload=%#v", got, schema) } - if got := schema["current_version"]; got != float64(13) { + if got := schema["current_version"]; got != float64(15) { t.Fatalf("db_schema.current_version = %#v, payload=%#v", got, schema) } - if got := schema["supported_version"]; got != float64(13) { + if got := schema["supported_version"]; got != float64(15) { t.Fatalf("db_schema.supported_version = %#v, payload=%#v", got, schema) } if got := schema["child_observation_reservations"]; got != true { @@ -4609,7 +4609,7 @@ func TestDoctorJSONReportsLegacyPendingSchemaWithoutMutation(t *testing.T) { t.Fatalf("pr_details.duplicate_path_files_supported = %#v, payload=%#v", got, prDetails) } pending := doctorStringList(t, schema, "pending_migrations") - if !doctorListContains(pending, "schema_version_3_to_13") || + if !doctorListContains(pending, "schema_version_3_to_15") || !doctorListContains(pending, "pull_request_files_position_key") || !doctorListContains(pending, "thread_child_observation_reservations_table") { t.Fatalf("pending_migrations = %#v", pending) diff --git a/internal/cli/args.go b/internal/cli/args.go index 6f1e1c24..889fdfe8 100644 --- a/internal/cli/args.go +++ b/internal/cli/args.go @@ -5,6 +5,11 @@ func normalizeCommandArgs(args []string, stringFlags map[string]bool) []string { var positionals []string for index := 0; index < len(args); index++ { arg := args[index] + if arg == "--" { + flags = append(flags, "--") + positionals = append(positionals, args[index+1:]...) + break + } name, ok := flagName(arg) if !ok { positionals = append(positionals, arg) diff --git a/internal/cli/args_test.go b/internal/cli/args_test.go index 6a156886..a6293a6b 100644 --- a/internal/cli/args_test.go +++ b/internal/cli/args_test.go @@ -1,10 +1,34 @@ package cli import ( + "flag" "reflect" "testing" ) +func TestNormalizeCommandArgsPreservesEndOfOptions(t *testing.T) { + for _, args := range [][]string{ + {"--limit", "5", "--", "--json"}, + {"first", "--limit", "5", "--", "--json", "-R", "last"}, + } { + t.Run(args[0], func(t *testing.T) { + fs := flag.NewFlagSet("search", flag.ContinueOnError) + limit := fs.String("limit", "", "") + jsonOut := fs.Bool("json", false, "") + if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"limit": true})); err != nil { + t.Fatal(err) + } + want := []string{"--json"} + if args[0] == "first" { + want = []string{"first", "--json", "-R", "last"} + } + if *limit != "5" || *jsonOut || !reflect.DeepEqual(fs.Args(), want) { + t.Fatalf("limit=%q json=%v args=%q; want limit=5 json=false args=%q", *limit, *jsonOut, fs.Args(), want) + } + }) + } +} + func TestNormalizeCommandArgsMovesFlagsBeforePositionals(t *testing.T) { got := normalizeCommandArgs([]string{"openclaw/openclaw", "--query", "download", "--json"}, map[string]bool{"query": true}) want := []string{"--query", "download", "--json", "openclaw/openclaw"} diff --git a/internal/cli/cloud_sqlite.go b/internal/cli/cloud_sqlite.go index 258bf1f8..c4503751 100644 --- a/internal/cli/cloud_sqlite.go +++ b/internal/cli/cloud_sqlite.go @@ -99,6 +99,8 @@ func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error { table string name string }{ + {table: "actor_identity_evidence", name: "raw_json"}, + {table: "actor_profiles", name: "raw_json"}, {table: "repositories", name: "raw_json"}, {table: "threads", name: "raw_json"}, {table: "comments", name: "raw_json"}, @@ -158,6 +160,8 @@ func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error { } } for _, table := range []string{ + "analytics_fetch_attempts", "analytics_retries", "analytics_pending_nodes", + "analytics_collection_state", "analytics_repair_receipts", "thread_changed_files", "thread_hunk_signatures", "thread_code_snapshots", diff --git a/internal/cli/cluster_graph.go b/internal/cli/cluster_graph.go index 688ad413..72d59eb6 100644 --- a/internal/cli/cluster_graph.go +++ b/internal/cli/cluster_graph.go @@ -81,24 +81,8 @@ func buildDurableClusterInputs(ctx context.Context, st *store.Store, repoID int6 } nodes = append(nodes, clusterer.Node{ThreadID: stored.ThreadID, Number: thread.Number, Title: thread.Title}) } - candidateByPair := map[string]clusterer.Edge{} - for left := 0; left < len(nodes); left++ { - for right := left + 1; right < len(nodes); right++ { - leftID := nodes[left].ThreadID - rightID := nodes[right].ThreadID - score := vector.Cosine(vectorByThreadID[leftID], vectorByThreadID[rightID]) - if score < options.Threshold { - continue - } - if score < highConfidenceEdgeScore && titleTokenOverlap(threads[leftID].Title, threads[rightID].Title) < weakEdgeMinTitleOverlap { - continue - } - if threads[leftID].Kind != threads[rightID].Kind && score < options.CrossKindThreshold { - continue - } - upsertClusterEdge(candidateByPair, leftID, rightID, score) - } - } + candidateByPair := scoreClusterEdges(nodes, threads, vectorByThreadID, options) + repoFullName, err := repositoryFullNameByID(ctx, st, repoID) if err != nil { return nil, 0, err @@ -155,6 +139,32 @@ func buildDurableClusterInputs(ctx context.Context, st *store.Store, repoID int6 return inputs, len(edges), nil } +func scoreClusterEdges(nodes []clusterer.Node, threads map[int64]store.Thread, vectorByThreadID map[int64][]float64, options clusterBuildOptions) map[string]clusterer.Edge { + prepared := make([]vector.Prepared, len(nodes)) + for i, node := range nodes { + prepared[i] = vector.Prepare(vectorByThreadID[node.ThreadID]) + } + candidateByPair := map[string]clusterer.Edge{} + for left := 0; left < len(nodes); left++ { + for right := left + 1; right < len(nodes); right++ { + leftID := nodes[left].ThreadID + rightID := nodes[right].ThreadID + score := prepared[left].Cosine(prepared[right]) + if score < options.Threshold { + continue + } + if score < highConfidenceEdgeScore && titleTokenOverlap(threads[leftID].Title, threads[rightID].Title) < weakEdgeMinTitleOverlap { + continue + } + if threads[leftID].Kind != threads[rightID].Kind && score < options.CrossKindThreshold { + continue + } + upsertClusterEdge(candidateByPair, leftID, rightID, score) + } + } + return candidateByPair +} + func upsertClusterEdge(edges map[string]clusterer.Edge, leftID, rightID int64, score float64) { if leftID == rightID { return diff --git a/internal/cli/cluster_graph_bench_test.go b/internal/cli/cluster_graph_bench_test.go new file mode 100644 index 00000000..b7a6924c --- /dev/null +++ b/internal/cli/cluster_graph_bench_test.go @@ -0,0 +1,133 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "math/rand/v2" + "path/filepath" + "testing" + "time" + + clusterer "github.com/openclaw/gitcrawl/internal/cluster" + "github.com/openclaw/gitcrawl/internal/config" + "github.com/openclaw/gitcrawl/internal/store" +) + +func clusterBenchmarkData(count int) ([]clusterer.Node, map[int64]store.Thread, map[int64][]float64) { + rng := rand.New(rand.NewPCG(7, 11)) + dims := config.Default().OpenAI.EmbedDimensions + nodes := make([]clusterer.Node, count) + threads := make(map[int64]store.Thread, count) + vectors := make(map[int64][]float64, count) + var center []float64 + for i := range nodes { + if i%8 == 0 { + center = make([]float64, dims) + for j := range center { + center[j] = rng.NormFloat64() + } + } + values := make([]float64, dims) + for j := range values { + values[j] = center[j] + 0.15*rng.NormFloat64() + } + id := int64(i + 1) + title := fmt.Sprintf("Synthetic embedding group %d", i/8) + nodes[i] = clusterer.Node{ThreadID: id, Number: i + 1, Title: title} + kind := "issue" + if i%3 == 0 { + kind = "pull_request" + } + threads[id] = store.Thread{Number: i + 1, Kind: kind, State: "open", Title: title} + vectors[id] = values + } + return nodes, threads, vectors +} + +var benchmarkEdges map[string]clusterer.Edge + +func BenchmarkClusterEdges2000(b *testing.B) { + nodes, threads, vectors := clusterBenchmarkData(2000) + opts := clusterBuildOptions{Threshold: 0.90, CrossKindThreshold: defaultCrossKindMinScore} + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + benchmarkEdges = scoreClusterEdges(nodes, threads, vectors, opts) + } +} + +func BenchmarkClusterCommand2000(b *testing.B) { + ctx := context.Background() + b.Setenv("GITCRAWL_NO_UPDATE_CHECK", "1") + dir := b.TempDir() + configPath, dbPath := filepath.Join(dir, "config.toml"), filepath.Join(dir, "gitcrawl.db") + app := New() + app.Stdout, app.Stderr = io.Discard, io.Discard + if err := app.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + b.Fatal(err) + } + st, err := store.Open(ctx, dbPath) + if err != nil { + b.Fatal(err) + } + defer st.Close() + now := time.Now().UTC().Format(time.RFC3339Nano) + repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "synthetic", Name: "bench", FullName: "synthetic/bench", UpdatedAt: now}) + if err != nil { + b.Fatal(err) + } + nodes, threads, vectors := clusterBenchmarkData(2000) + if err := st.WithTx(ctx, func(st *store.Store) error { + for _, node := range nodes { + thread := threads[node.ThreadID] + thread.RepoID, thread.GitHubID = repoID, fmt.Sprint(node.ThreadID) + thread.LabelsJSON, thread.AssigneesJSON, thread.RawJSON = "[]", "[]", "{}" + thread.ContentHash, thread.UpdatedAt = fmt.Sprint(node.ThreadID), now + if _, err := st.UpsertThread(ctx, thread); err != nil { + return err + } + } + return nil + }); err != nil { + b.Fatal(err) + } + tasks, err := st.ListEmbeddingTasks(ctx, store.EmbeddingTaskOptions{RepoID: repoID, Basis: "title_original", Model: "text-embedding-3-small", Force: true}) + if err != nil { + b.Fatal(err) + } + if len(tasks) != len(nodes) { + b.Fatalf("tasks: %d want %d", len(tasks), len(nodes)) + } + // Bulk fixture insertion stays outside the timed command and avoids per-row fsync. + tx, err := st.DB().BeginTx(ctx, nil) + if err != nil { + b.Fatal(err) + } + defer tx.Rollback() + for _, task := range tasks { + values := vectors[task.ThreadID] + data, err := json.Marshal(values) + if err != nil { + b.Fatal(err) + } + if _, err := tx.ExecContext(ctx, `insert into thread_vectors(thread_id,basis,model,dimensions,content_hash,vector_json,vector_backend,created_at,updated_at) values(?,?,?,?,?,?,?,?,?)`, task.ThreadID, "title_original", "text-embedding-3-small", len(values), task.ContentHash, string(data), "exact", now, now); err != nil { + b.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + + if err := st.Close(); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if err := app.Run(ctx, []string{"--config", configPath, "cluster", "synthetic/bench", "--threshold", "0.90", "--json"}); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/cli/cluster_graph_scoring_test.go b/internal/cli/cluster_graph_scoring_test.go new file mode 100644 index 00000000..a37030d8 --- /dev/null +++ b/internal/cli/cluster_graph_scoring_test.go @@ -0,0 +1,58 @@ +package cli + +import ( + clusterer "github.com/openclaw/gitcrawl/internal/cluster" + "github.com/openclaw/gitcrawl/internal/store" + "github.com/openclaw/gitcrawl/internal/vector" + "math" + "testing" +) + +func TestPreparedClusterEdgesMatchOriginal(t *testing.T) { + nodes, threads, vectors := clusterBenchmarkData(96) + vectors[1] = []float64{math.NaN()} + vectors[2] = nil + vectors[3] = []float64{0} + vectors[4] = []float64{1, 2} + vectors[5] = []float64{math.MaxFloat64, math.MaxFloat64} + vectors[6] = []float64{math.SmallestNonzeroFloat64, math.SmallestNonzeroFloat64} + var maxDeviation float64 + for _, threshold := range []float64{-1, 0, 0.3, 0.9, 1} { + opts := clusterBuildOptions{Threshold: threshold, CrossKindThreshold: defaultCrossKindMinScore} + got, want := scoreClusterEdges(nodes, threads, vectors, opts), originalClusterEdges(nodes, threads, vectors, opts) + if len(got) != len(want) { + t.Fatalf("threshold %g: %d edges want %d", threshold, len(got), len(want)) + } + for pair, expected := range want { + actual, ok := got[pair] + deviation := math.Abs(actual.Score - expected.Score) + maxDeviation = max(maxDeviation, deviation) + if !ok || actual.LeftThreadID != expected.LeftThreadID || actual.RightThreadID != expected.RightThreadID || deviation > 1e-12 { + t.Fatalf("threshold %g pair %s: %+v want %+v", threshold, pair, actual, expected) + } + } + } + t.Logf("max cluster-edge score deviation: %.17g", maxDeviation) +} + +func originalClusterEdges(nodes []clusterer.Node, threads map[int64]store.Thread, vectorByThreadID map[int64][]float64, options clusterBuildOptions) map[string]clusterer.Edge { + candidateByPair := map[string]clusterer.Edge{} + for left := 0; left < len(nodes); left++ { + for right := left + 1; right < len(nodes); right++ { + leftID := nodes[left].ThreadID + rightID := nodes[right].ThreadID + score := vector.Cosine(vectorByThreadID[leftID], vectorByThreadID[rightID]) + if score < options.Threshold { + continue + } + if score < highConfidenceEdgeScore && titleTokenOverlap(threads[leftID].Title, threads[rightID].Title) < weakEdgeMinTitleOverlap { + continue + } + if threads[leftID].Kind != threads[rightID].Kind && score < options.CrossKindThreshold { + continue + } + upsertClusterEdge(candidateByPair, leftID, rightID, score) + } + } + return candidateByPair +} diff --git a/internal/cli/control.go b/internal/cli/control.go index 90863dad..af94af7a 100644 --- a/internal/cli/control.go +++ b/internal/cli/control.go @@ -37,9 +37,12 @@ func (a *App) runMetadata(args []string) error { DefaultCache: cfg.CacheDir, DefaultLogs: cfg.LogDir, } - manifest.Capabilities = []string{"metadata", "status", "doctor", "sync", "capture", "coverage", "search", "code-index", "tui", "portable", "remote", "cloud-publish", "clusters", "summaries", "embeddings"} + manifest.Capabilities = []string{"metadata", "status", "metrics", "doctor", "sync", "capture", "coverage", "search", "code-index", "tui", "portable", "remote", "cloud-publish", "clusters", "summaries", "embeddings"} manifest.Privacy = control.Privacy{ContainsPrivateMessages: true, ExportsSecrets: false, LocalOnlyScopes: []string{"github", "git", "sqlite", "portable"}} manifest.Commands = map[string]control.Command{ + "metrics-collect": {Title: "Collect repository metrics", Argv: []string{"gitcrawl", "metrics", "collect", "--config", "METRICS_CONFIG", "--json"}, JSON: true, Mutates: true}, + "metrics-import": {Title: "Import metric history from stdin", Argv: []string{"gitcrawl", "metrics", "import", "--config", "METRICS_CONFIG", "--json"}, JSON: true, Mutates: true}, + "metrics-status": {Title: "Metrics database status", Argv: []string{"gitcrawl", "metrics", "status", "--config", "METRICS_CONFIG", "--json"}, JSON: true}, "status": {Title: "Status", Argv: []string{"gitcrawl", "status", "--json"}, JSON: true}, "remote-status": {Title: "Remote archive status", Argv: []string{"gitcrawl", "remote", "status", "--json"}, JSON: true}, "remote-archives": {Title: "Remote archive list", Argv: []string{"gitcrawl", "remote", "archives", "--json"}, JSON: true}, diff --git a/internal/cli/gh_search_test.go b/internal/cli/gh_search_test.go index aeb79dcc..174a0711 100644 --- a/internal/cli/gh_search_test.go +++ b/internal/cli/gh_search_test.go @@ -42,6 +42,32 @@ func TestParseGHSearchDuration(t *testing.T) { } } +func TestGHSearchLiteralFlagQuery(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + seedPortableThread(t, dbPath, 42, "Broken --verbose flag") + seedPortableThread(t, dbPath, 43, "Unrelated issue") + app := New() + var stdout, stderr bytes.Buffer + app.Stdout, app.Stderr = &stdout, &stderr + if err := app.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatal(err) + } + stdout.Reset() + if err := app.Run(ctx, []string{"--config", configPath, "search", "issues", "-R", "openclaw/openclaw", "--json", "number", "--", "--verbose"}); err != nil { + t.Fatal(err) + } + var rows []struct{ Number int } + if err := json.Unmarshal(stdout.Bytes(), &rows); err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].Number != 42 { + t.Fatalf("search returned %s; want only issue #42", stdout.String()) + } +} + func TestGHSearchCacheStaleUsesRepoSyncRuns(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -265,8 +291,8 @@ func TestGHSearchSyncIfStaleMigratesFreshPortableRuntime(t *testing.T) { if err := rt.Store.DB().QueryRowContext(ctx, `pragma user_version`).Scan(&schemaVersion); err != nil { t.Fatalf("read runtime schema version: %v", err) } - if schemaVersion != 13 { - t.Fatalf("runtime schema version = %d, want 13", schemaVersion) + if schemaVersion != 15 { + t.Fatalf("runtime schema version = %d, want 15", schemaVersion) } var tableName string if err := rt.Store.DB().QueryRowContext(ctx, `select name from sqlite_schema where type = 'table' and name = 'sync_runs'`).Scan(&tableName); err != nil { diff --git a/internal/cli/help.go b/internal/cli/help.go index a9d9c4d1..8474dd4f 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -45,6 +45,7 @@ Core commands: metadata print crawlkit control metadata check-update check for a newer gitcrawl release status print fast read-only archive status + metrics collect, import, or inspect an independent metrics database remote status print remote archive status remote archives list remote archives visible to the current identity remote login authenticate with GitHub org access for a remote archive @@ -53,6 +54,7 @@ Core commands: init create config, optionally from a portable store doctor check config, token, and database readiness sync sync GitHub issue and pull request metadata + analytics repair publication dates, enrich identities, or watch GraphQL updates sync-failures list failed sync hydration attempts coverage report local archive PR-detail completeness fill-pr-details hydrate locally missing pull request detail rows @@ -90,6 +92,19 @@ No API server is provided. There is intentionally no serve command. ` var commandUsageTexts = map[string]string{ + "metrics": `gitcrawl metrics collects repository headline metrics in a separate SQLite database. + +Usage: + gitcrawl metrics collect --config /absolute/metrics.json [--json] + gitcrawl metrics import --config /absolute/metrics.json [--json] < history.ndjson + gitcrawl metrics status --config /absolute/metrics.json [--json] + +The JSON config requires database (absolute path) and targets [{entity,target}]. +Optional tokenEnv overrides GITHUB_TOKEN; native gh auth and --github-token-command +are supported. No archive, portable-store, embedding, or model operations occur. +Import validates target scope and IDs atomically. Unknown values remain SQL NULL. +Status is read-only. Clone traffic 403/404 is optional unavailability. +`, "metadata": `gitcrawl metadata prints crawlkit control metadata. Usage: diff --git a/internal/cli/metrics.go b/internal/cli/metrics.go new file mode 100644 index 00000000..bc940913 --- /dev/null +++ b/internal/cli/metrics.go @@ -0,0 +1,84 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "net/http" + "time" + + "github.com/openclaw/gitcrawl/internal/config" + "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/headlinemetrics" +) + +func (a *App) runMetrics(ctx context.Context, args []string) error { + if len(args) == 0 || args[0] == "help" || args[0] == "--help" || args[0] == "-h" { + return a.printCommandUsage("metrics") + } + command := args[0] + if command != "collect" && command != "import" && command != "status" { + return usageErr(fmt.Errorf("unknown metrics command %q", command)) + } + fs := flag.NewFlagSet("metrics "+command, flag.ContinueOnError) + fs.SetOutput(io.Discard) + path := fs.String("config", a.configPath, "metrics JSON config path") + jsonOut := fs.Bool("json", false, "write JSON output") + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return a.printCommandUsage("metrics") + } + return usageErr(err) + } + if *path == "" || fs.NArg() != 0 { + return usageErr(errors.New("metrics requires --config and no positional arguments")) + } + c, err := headlinemetrics.ReadConfig(*path) + if err != nil { + return usageErr(err) + } + a.applyCommandJSON(*jsonOut) + var collect headlinemetrics.Collector + if command == "collect" { + cfg := config.Default() + if c.TokenEnv != "" { + cfg.GitHub.TokenEnv = c.TokenEnv + } + token := a.resolveGitHubToken(ctx, cfg) + var provider func(context.Context) (string, error) + if a.githubTokenCommand != nil { + provider, err = githubTokenProvider(*a.githubTokenCommand) + if err != nil { + return usageErr(err) + } + } + client := github.New(github.Options{Token: token.Value, TokenProvider: provider, BaseURL: githubBaseURL(), HTTPClient: &http.Client{ + Timeout: 30 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }}) + collect = headlinemetrics.GitHubCollector(client, token.Value != "" || provider != nil) + } + in := a.Stdin + if in == nil { + in = http.NoBody + } + result, err := headlinemetrics.Execute(ctx, command, c, collect, in) + // Partial collection is still a useful structured result; diagnostics stay + // on stderr and a nonzero exit communicates unavailable required metrics. + if err == nil || errors.Is(err, headlinemetrics.ErrPartialCollection) { + var output any = result + if command == "status" { + output = struct { + headlinemetrics.Result + Observations int `json:"observations"` + Events int `json:"events"` + }{result, result.Observations, result.Events} + } + if writeErr := a.writeOutput("metrics "+command, output, false); writeErr != nil { + return writeErr + } + } + return err +} diff --git a/internal/cli/metrics_test.go b/internal/cli/metrics_test.go new file mode 100644 index 00000000..4892996a --- /dev/null +++ b/internal/cli/metrics_test.go @@ -0,0 +1,224 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/openclaw/gitcrawl/internal/headlinemetrics" +) + +func metricsConfigFixture(t *testing.T) (string, headlinemetrics.Config) { + t.Helper() + dir := t.TempDir() + c := headlinemetrics.Config{Database: filepath.Join(dir, "metrics.sqlite"), Targets: []headlinemetrics.Target{{Entity: "OpenClaw", Target: "openclaw/openclaw"}}} + raw, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "metrics.json") + if err := os.WriteFile(path, raw, 0600); err != nil { + t.Fatal(err) + } + return path, c +} +func metricsApp(t *testing.T) (*App, *bytes.Buffer) { + t.Helper() + app := New() + out := new(bytes.Buffer) + app.Stdout = out + app.Stderr = new(bytes.Buffer) + app.githubAuthTokenLookup = func(context.Context) (string, error) { + t.Error("unexpected credential lookup") + return "", errors.New("no credential") + } + return app, out +} + +func TestMetricsNativeHelpMetadataAndUsage(t *testing.T) { + for _, args := range [][]string{{"--help"}, {"help", "metrics"}, {"metrics"}, {"metrics", "help"}, {"metrics", "-h"}, {"metrics", "collect", "--help"}, {"--json", "metrics", "status", "-h"}, {"metrics", "import", "--help"}} { + app, out := metricsApp(t) + if err := app.Run(context.Background(), args); err != nil || !strings.Contains(out.String(), "metrics") { + t.Fatalf("%v: %v %s", args, err, out) + } + } + app, out := metricsApp(t) + if err := app.Run(context.Background(), []string{"metadata", "--json"}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"metrics-collect", "metrics-import", "metrics-status"} { + if !strings.Contains(out.String(), name) { + t.Fatalf("metadata missing %s", name) + } + } + for _, args := range [][]string{{"metrics", "bad"}, {"metrics", "collect"}, {"metrics", "status", "--unknown"}, {"metrics", "import", "--config", "/missing", "extra"}, {"metrics", "collect", "--config", "/missing"}} { + app, _ := metricsApp(t) + if err := app.Run(context.Background(), args); err == nil || ExitCode(err) != 2 { + t.Fatalf("%v: usage error %v", args, err) + } + } + if releaseNotificationAllowed([]string{"metrics", "status"}) { + t.Fatal("metrics triggers release side effects") + } +} + +func TestMetricsImportStatusNativeJSONAndArchiveIsolation(t *testing.T) { + path, c := metricsConfigFixture(t) + archive := filepath.Join(t.TempDir(), "archive.db") + if err := os.WriteFile(archive, []byte("untouched archive"), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("GITCRAWL_DB_PATH", archive) + t.Setenv("GITCRAWL_CONFIG", filepath.Join(t.TempDir(), "does-not-exist.toml")) + row := `{"type":"metric","id":"history:1","entity":"OpenClaw","target":"openclaw/openclaw","metric":"watchers","kind":"counter","ts":"2026-09-14T00:00:00Z","value":null,"observed_at":"2026-09-15T00:00:00Z","provenance":"claw-track"}` + for i, args := range [][]string{{"--json", "metrics", "import", "--config", path}, {"metrics", "import", "--config", path, "--json"}} { + app, out := metricsApp(t) + app.Stdin = strings.NewReader(row + "\n") + if err := app.Run(context.Background(), args); err != nil { + t.Fatal(err) + } + var r headlinemetrics.Result + if err := json.Unmarshal(out.Bytes(), &r); err != nil { + t.Fatal(err) + } + if r.RowsWritten != 1-i || !r.OK { + t.Fatalf("import=%+v", r) + } + } + for _, args := range [][]string{{"metrics", "status", "--config", path, "--json"}, {"--config", path, "--json", "metrics", "status"}, {"--format", "json", "metrics", "status", "--config", path}} { + app, out := metricsApp(t) + if err := app.Run(context.Background(), args); err != nil { + t.Fatal(err) + } + var r headlinemetrics.Result + if err := json.Unmarshal(out.Bytes(), &r); err != nil || r.Observations != 1 { + t.Fatalf("status=%s %v", out, err) + } + } + if b, _ := os.ReadFile(archive); string(b) != "untouched archive" { + t.Fatal("archive changed") + } + info, err := os.Stat(c.Database) + if err != nil || info.Size() == 0 { + t.Fatalf("metrics DB absent: %v", err) + } + app, _ := metricsApp(t) + app.Stdin = strings.NewReader("bad") + if err := app.Run(context.Background(), []string{"metrics", "import", "--config", path}); err == nil || ExitCode(err) != 1 { + t.Fatal("invalid history accepted") + } +} + +func TestMetricsCollectUsesNativeCredentialsAndKeepsPartialOutput(t *testing.T) { + for _, mode := range []string{"environment", "gh-fallback", "managed", "partial", "quota"} { + t.Run(mode, func(t *testing.T) { + if mode == "managed" && runtime.GOOS == "windows" { + t.Skip("managed helper is Unix-only") + } + path, c := metricsConfigFixture(t) + c.TokenEnv = "GITCRAWL_METRICS_TEST_TOKEN" + b, _ := json.Marshal(c) + if err := os.WriteFile(path, b, 0600); err != nil { + t.Fatal(err) + } + t.Setenv("GITHUB_TOKEN", "ambient-must-not-win") + t.Setenv(c.TokenEnv, "") + token := "fixture-token" + if mode == "environment" || mode == "partial" || mode == "quota" { + t.Setenv(c.TokenEnv, token) + } + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.Header.Get("Authorization") != "Bearer "+token { + t.Error("wrong credential selected") + } + switch r.URL.Path { + case "/repos/openclaw/openclaw": + if mode == "quota" { + w.WriteHeader(http.StatusTooManyRequests) + return + } + fmt.Fprint(w, `{"stargazers_count":4,"forks_count":2,"subscribers_count":1,"open_issues_count":8}`) + case "/search/issues": + if mode == "partial" { + fmt.Fprint(w, `{"incomplete_results":true,"total_count":3}`) + } else { + fmt.Fprint(w, `{"total_count":3}`) + } + case "/repos/openclaw/openclaw/traffic/clones": + w.WriteHeader(403) + case "/repos/openclaw/openclaw/releases": + fmt.Fprint(w, `[]`) + default: + t.Errorf("unexpected API/model/archive request %s", r.URL.Path) + w.WriteHeader(404) + } + })) + defer server.Close() + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITCRAWL_OPENAI_BASE_URL", server.URL) + app, out := metricsApp(t) + args := []string{"metrics", "collect", "--config", path, "--json"} + if mode == "gh-fallback" { + app.githubAuthTokenLookup = func(context.Context) (string, error) { return token, nil } + } + if mode == "managed" { + helper := filepath.Join(t.TempDir(), "credential-helper") + if err := os.WriteFile(helper, []byte("#!/bin/sh\nprintf '%s\\n' 'fixture-token'\n"), 0700); err != nil { + t.Fatal(err) + } + args = append([]string{"--github-token-command", helper}, args...) + } + err := app.Run(context.Background(), args) + if (err != nil) != (mode == "partial" || mode == "quota") { + t.Fatalf("error=%v", err) + } + var result headlinemetrics.Result + if e := json.Unmarshal(out.Bytes(), &result); e != nil { + t.Fatal(e) + } + wantRows, wantRequests := 5, 4 + if mode == "quota" { + wantRows, wantRequests = 0, 1 + } + if result.RowsWritten != wantRows || result.OK == (mode == "partial" || mode == "quota") || requests != wantRequests { + t.Fatalf("result=%+v requests=%d", result, requests) + } + if strings.Contains(out.String(), token) { + t.Fatal("token printed") + } + }) + } +} + +func TestMetricsEmptyStatusReportsZeroTotals(t *testing.T) { + path, _ := metricsConfigFixture(t) + app, _ := metricsApp(t) + app.Stdin = strings.NewReader("") + if err := app.Run(context.Background(), []string{"metrics", "import", "--config", path, "--json"}); err != nil { + t.Fatal(err) + } + app, out := metricsApp(t) + if err := app.Run(context.Background(), []string{"metrics", "status", "--config", path, "--json"}); err != nil { + t.Fatal(err) + } + var result map[string]any + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatal(err) + } + for _, key := range []string{"observations", "events"} { + if value, present := result[key]; !present || value != float64(0) { + t.Fatalf("status must report %s=0: %s", key, out) + } + } +} diff --git a/internal/cli/releasecheck.go b/internal/cli/releasecheck.go index 67d285a1..73b93d15 100644 --- a/internal/cli/releasecheck.go +++ b/internal/cli/releasecheck.go @@ -37,6 +37,11 @@ func (a *App) maybeNotifyRelease(ctx context.Context, args []string) { } func releaseNotificationAllowed(args []string) bool { + // Metrics commands must not touch archive/runtime state or make unrelated + // network requests, including during read-only imports and status checks. + if len(args) > 0 && args[0] == "metrics" { + return false + } if len(args) == 0 || args[0] != "fill-pr-details" { return true } diff --git a/internal/cli/sync.go b/internal/cli/sync.go index 16a8b1d7..ee53ccfe 100644 --- a/internal/cli/sync.go +++ b/internal/cli/sync.go @@ -400,6 +400,9 @@ func (a *App) syncRepository(ctx context.Context, owner, repo string, options sy return command(ctx) } } + if a.analyticsTokenProvider != nil { + provider = a.analyticsTokenProvider + } if provider == nil && token.Value == "" { return syncer.Stats{}, dbTargetInfo{}, fmt.Errorf("missing GitHub token: set %s or authenticate gh", cfg.GitHub.TokenEnv) } diff --git a/internal/cli/tui_neighbors.go b/internal/cli/tui_neighbors.go index c07d9a00..c3ced8a1 100644 --- a/internal/cli/tui_neighbors.go +++ b/internal/cli/tui_neighbors.go @@ -104,13 +104,14 @@ func loadThreadNeighbors(ctx context.Context, st *store.Store, repoID int64, pay threshold = 0.2 } targetThread, targetVector, err := st.ThreadVectorByNumber(ctx, store.ThreadVectorQuery{ - RepoID: repoID, - Model: payload.EmbedModel, - Basis: payload.EmbeddingBasis, + RepoID: repoID, + Model: payload.EmbedModel, + Basis: payload.EmbeddingBasis, + IncludeClosed: true, }, number) if err != nil { var fallbackErr error - targetThread, targetVector, fallbackErr = st.ThreadVectorByNumber(ctx, store.ThreadVectorQuery{RepoID: repoID}, number) + targetThread, targetVector, fallbackErr = st.ThreadVectorByNumber(ctx, store.ThreadVectorQuery{RepoID: repoID, IncludeClosed: true}, number) if fallbackErr != nil { return 0, 0, nil, err } diff --git a/internal/cli/tui_test.go b/internal/cli/tui_test.go index 97eb16b4..434d3a49 100644 --- a/internal/cli/tui_test.go +++ b/internal/cli/tui_test.go @@ -2712,6 +2712,49 @@ func TestTUILoadNeighborsFromStore(t *testing.T) { } } +func TestTUILoadNeighborsForClosedSelection(t *testing.T) { + for _, modelName := range []string{"test", "missing-configured-model"} { + t.Run(modelName, func(t *testing.T) { + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", RawJSON: "{}"}) + if err != nil { + t.Fatal(err) + } + targetID, err := seedTUIThreadVector(ctx, st, repoID, 1, "Closed target", []float64{1, 0}) + if err != nil { + t.Fatal(err) + } + neighborID, err := seedTUIThreadVector(ctx, st, repoID, 2, "Open neighbor", []float64{0.9, 0.1}) + if err != nil { + t.Fatal(err) + } + if _, err := st.DB().ExecContext(ctx, `update threads set state = 'closed' where id = ?`, targetID); err != nil { + t.Fatal(err) + } + model := newClusterBrowserModel(ctx, st, repoID, clusterBrowserPayload{Repository: "fixture/repo", EmbedModel: modelName, EmbeddingBasis: "title_original"}) + model.memberIndex = 0 + model.memberRows = []memberRow{{selectable: true, member: store.ClusterMemberDetail{Thread: store.Thread{ + ID: targetID, Number: 1, State: "closed", HTMLURL: "https://github.com/fixture/repo/issues/1", + }}}} + cmd := model.requestSelectedThreadNeighbors(10, 0.2) + if cmd == nil { + t.Fatal("neighbor command missing") + } + updated, _ := model.Update(cmd()) + model = updated.(clusterBrowserModel) + neighbors := model.neighborCache[targetID] + if len(neighbors) != 1 || neighbors[0].Thread.ID != neighborID { + t.Fatalf("neighbors=%+v status=%q; want open neighbor %d", neighbors, model.status, neighborID) + } + }) + } +} + func TestTUILoadNeighborsUsesConfiguredBackend(t *testing.T) { ctx := context.Background() st, err := store.Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) diff --git a/internal/github/analytics.go b/internal/github/analytics.go new file mode 100644 index 00000000..dcba5347 --- /dev/null +++ b/internal/github/analytics.go @@ -0,0 +1,159 @@ +package github + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" +) + +// AnalyticsNodesError identifies node-scoped rejections without provider prose. +type AnalyticsNodesError struct{ IDs []string } + +func (*AnalyticsNodesError) Error() string { return "actor evidence rejected for requested nodes" } + +// AnalyticsNodes reads public provider evidence for the requested native node IDs. +func (c *Client) AnalyticsNodes(ctx context.Context, ids []string, profiles bool) ([]map[string]any, error) { + fields := `... on Issue {author{login __typename ... on Node{id}}} ... on PullRequest {author{login __typename ... on Node{id}}} ... on IssueComment {author{login __typename ... on Node{id}}} ... on PullRequestReview {author{login __typename ... on Node{id}}} ... on PullRequestReviewComment {author{login __typename ... on Node{id}}}` + if profiles { + fields = `... on Actor {login url} ... on User {name bio createdAt} ... on Bot {createdAt} ... on Mannequin {createdAt}` + } + payload, e := json.Marshal(graphqlEnvelope{Query: `query($ids:[ID!]!){rateLimit{cost remaining limit used resetAt} nodes(ids:$ids){id __typename ` + fields + `}}`, Variables: map[string]any{"ids": ids}}) + if e != nil { + return nil, e + } + var envelope struct { + Data json.RawMessage `json:"data"` + Errors []struct { + Message string `json:"message"` + Type string `json:"type"` + Path []any `json:"path"` + } `json:"errors"` + } + if e = c.doJSON(ctx, http.MethodPost, c.graphQLURL, bytes.NewReader(payload), nil, &envelope); e != nil { + return nil, e + } + var rejected []string + for _, failure := range envelope.Errors { + if len(failure.Path) < 2 || failure.Path[0] != "nodes" { + return nil, fmt.Errorf("actor evidence GraphQL response rejected") + } + index, ok := historyInt(failure.Path[1]) + if !ok || index < 0 || index >= len(ids) { + return nil, fmt.Errorf("actor evidence GraphQL response rejected") + } + if failure.Type != "NOT_FOUND" { + rejected = append(rejected, ids[index]) + } + } + if len(rejected) > 0 { + return nil, &AnalyticsNodesError{IDs: rejected} + } + + var data map[string]any + if e = json.Unmarshal(envelope.Data, &data); e != nil { + return nil, e + } + + nodes, ok := data["nodes"].([]any) + if !ok || len(nodes) != len(ids) { + return nil, fmt.Errorf("incomplete actor identity response") + } + out := make([]map[string]any, 0, len(ids)) + for i, v := range nodes { + n, ok := v.(map[string]any) + if !ok { + n = map[string]any{"id": ids[i], "__typename": "Unavailable", "unavailable": true} + } + if n["id"] != ids[i] { + return nil, fmt.Errorf("actor node identity mismatch") + } + out = append(out, n) + } + return out, nil +} + +type UpdatedPage struct { + Numbers []int + Cursor string + More bool + Total int + Oldest time.Time +} + +// AnalyticsRateLimit reads the same GraphQL balance charged by history queries. +// REST resource counters can differ and must not admit recovery on that basis. +func (c *Client) AnalyticsRateLimit(ctx context.Context) (effective, observed RateLimitSnapshot, err error) { + // Reuse the same credential-bound reserve state without changing ordinary + // content clients' existing transport policy. + quotaClient := *c + quotaClient.graphQLQuotaGuard = true + h := historySession{client: "aClient, remaining: 20000} + data, err := h.quota(ctx) + if err != nil { + return RateLimitSnapshot{}, RateLimitSnapshot{}, err + } + rate := historyMap(data["rateLimit"]) + limit, ok := historyInt(rate["limit"]) + if !ok || limit <= 0 { + return RateLimitSnapshot{}, RateLimitSnapshot{}, fmt.Errorf("GraphQL quota limit unavailable") + } + remaining, _ := historyInt(rate["remaining"]) + reset, _ := time.Parse(time.RFC3339, historyString(rate["resetAt"])) + observed = RateLimitSnapshot{Resource: "graphql", Limit: limit, Remaining: remaining, ResetAt: reset} + return c.reserve.observeGraphQL(observed, time.Now()), observed, nil +} + +func (c *Client) UpdatedNumbers(ctx context.Context, owner, repo, kind, after string, since time.Time) (UpdatedPage, error) { + if kind != "issues" && kind != "pullRequests" { + return UpdatedPage{}, fmt.Errorf("invalid discovery kind") + } + h := historySession{client: c, remaining: 20000} + var cursor any + if after != "" { + cursor = after + } + data, e := h.request(ctx, `query($owner:String!,$repo:String!,$after:String){rateLimit{cost remaining limit used resetAt} repository(owner:$owner,name:$repo){`+kind+`(first:100,after:$after,orderBy:{field:UPDATED_AT,direction:DESC}){totalCount pageInfo{hasNextPage endCursor} nodes{number updatedAt}}}}`, map[string]any{"owner": owner, "repo": repo, "after": cursor}, 1) + if e != nil { + return UpdatedPage{}, e + } + r := historyMap(historyMap(data["repository"])[kind]) + p := UpdatedPage{} + var valid bool + p.Total, valid = historyInt(r["totalCount"]) + if !valid || p.Total < 0 { + return p, historyFailure("discovery_validation", 0, r, fmt.Errorf("missing update-discovery count")) + } + info := historyMap(r["pageInfo"]) + var hasPageFlag bool + p.More, hasPageFlag = info["hasNextPage"].(bool) + nodes, hasNodes := r["nodes"].([]any) + // totalCount covers the whole connection. A resumed final page can become + // empty after deletion/reordering; it must not pin its cursor forever. + if !hasPageFlag || !hasNodes || (after == "" && ((!p.More && len(nodes) != p.Total) || (len(nodes) == 0 && p.Total > 0))) || len(nodes) > p.Total { + return p, historyFailure("discovery_validation", 0, r, fmt.Errorf("incomplete update-discovery page")) + } + p.Cursor = historyString(info["endCursor"]) + seen := make(map[int]bool, len(nodes)) + for _, raw := range nodes { + n := historyMap(raw) + number, ok := historyInt(n["number"]) + at, e := time.Parse(time.RFC3339Nano, historyString(n["updatedAt"])) + if !ok || number < 1 || e != nil || seen[number] { + return p, historyFailure("discovery_validation", 0, r, fmt.Errorf("invalid update-discovery evidence")) + } + seen[number] = true + if !at.Before(since) { + p.Numbers = append(p.Numbers, number) + } + if p.Oldest.IsZero() || at.Before(p.Oldest) { + p.Oldest = at + } + } + if p.More && (p.Cursor == "" || p.Cursor == after || p.Oldest.IsZero()) { + return p, historyFailure("discovery_validation", 0, r, fmt.Errorf("update cursor did not advance")) + } + return p, nil +} diff --git a/internal/github/analytics_test.go b/internal/github/analytics_test.go new file mode 100644 index 00000000..3697be67 --- /dev/null +++ b/internal/github/analytics_test.go @@ -0,0 +1,223 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestAnalyticsDiscoveryFiltersOldRowsAndRejectsBadCursors(t *testing.T) { + more := false + cursor := "end" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "limit": 20000, "used": 1000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}, "repository": map[string]any{"issues": map[string]any{"totalCount": 2, "pageInfo": map[string]any{"hasNextPage": more, "endCursor": cursor}, "nodes": []any{map[string]any{"number": 2, "updatedAt": "2026-09-24T00:00:00Z"}, map[string]any{"number": 1, "updatedAt": "2026-09-01T00:00:00Z"}}}}}}) + })) + defer server.Close() + c := New(Options{Token: "fixture", BaseURL: server.URL}) + since, _ := time.Parse(time.RFC3339, "2026-09-23T00:00:00Z") + p, e := c.UpdatedNumbers(context.Background(), "fixture", "repo", "issues", "", since) + if e != nil || len(p.Numbers) != 1 || p.Numbers[0] != 2 || p.Total != 2 { + t.Fatalf("%+v %v", p, e) + } + more = true + cursor = "same" + if _, e = c.UpdatedNumbers(context.Background(), "fixture", "repo", "issues", "same", since); e == nil { + t.Fatal("non-advancing cursor accepted") + } +} +func TestAnalyticsDiscoveryEmptyContinuation(t *testing.T) { + for _, tc := range []struct { + name, after string + more, wantError bool + }{ + {"final continuation", "previous", false, false}, + {"incomplete initial page", "", false, true}, + {"empty advancing page", "previous", true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": "2099-01-01T00:00:00Z"}, "repository": map[string]any{"issues": map[string]any{"totalCount": 100, "pageInfo": map[string]any{"hasNextPage": tc.more, "endCursor": "next"}, "nodes": []any{}}}}}) + })) + defer server.Close() + c := New(Options{Token: "test-token-placeholder", BaseURL: server.URL}) + p, err := c.UpdatedNumbers(context.Background(), "fixture", "repo", "issues", tc.after, time.Time{}) + if (err != nil) != tc.wantError { + t.Fatalf("page=%+v error=%v, wantError=%v", p, err, tc.wantError) + } + if err == nil && (p.More || len(p.Numbers) != 0 || p.Total != 100) { + t.Fatalf("incorrect final page: %+v", p) + } + }) + } +} + +func TestAnalyticsUnavailableNodesAreNotAuthorizationSuccess(t *testing.T) { + typ := "NOT_FOUND" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"nodes": []any{map[string]any{"id": "one", "__typename": "User", "login": "fixture"}, nil}}, "errors": []any{map[string]any{"type": typ, "message": "fixture unavailable", "path": []any{"nodes", 1}}}}) + })) + defer server.Close() + c := New(Options{Token: "fixture", BaseURL: server.URL}) + nodes, e := c.AnalyticsNodes(context.Background(), []string{"one", "two"}, true) + if e != nil || len(nodes) != 2 || nodes[1]["id"] != "two" || nodes[1]["__typename"] != "Unavailable" { + t.Fatalf("%+v %v", nodes, e) + } + typ = "FORBIDDEN" + if _, e = c.AnalyticsNodes(context.Background(), []string{"one", "two"}, true); e == nil { + t.Fatal("authorization failure became missing-data evidence") + } +} + +func TestAnalyticsQuotaAndHistoryReserveUseActualGraphQLBalance(t *testing.T) { + var contentRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprint(w, `{"resources":{"graphql":{"limit":20000,"remaining":19999,"reset":4102444800}}}`) + return + } + var req struct{ Query string } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + if strings.Contains(req.Query, "issueOrPullRequest") { + contentRequests++ + } + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":2990,"resetAt":"2099-01-01T00:00:00Z"}}}`) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, Token: "test-token-placeholder", RateLimitReserve: 3000}) + quota, _, err := c.AnalyticsRateLimit(context.Background()) + if err != nil || quota.Remaining != 2990 || quota.Limit != 20000 { + t.Fatalf("REST counter admitted work: %+v %v", quota, err) + } + if _, err = c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil); err == nil || !strings.Contains(err.Error(), "quota reserve reached") { + t.Fatalf("actual GraphQL reserve ignored: %v", err) + } + if contentRequests != 0 { + t.Fatal("content dispatched inside actual GraphQL reserve") + } +} + +func TestAnalyticsQuotaDoesNotIncreaseOnAnomalousGraphQLSample(t *testing.T) { + reset := time.Now().UTC().Add(time.Hour).Truncate(time.Second) + balance := 3070 + shifted := reset + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":19999,"reset":%d}}}`, reset.Unix()) + return + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": balance, "resetAt": shifted.Format(time.RFC3339)}}}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, Token: "test-token-placeholder", RateLimitReserve: 1500}) + first, _, err := c.AnalyticsRateLimit(context.Background()) + if err != nil { + t.Fatal(err) + } + balance = 19985 + shifted = reset.Add(4 * time.Second) + high, raw, err := c.AnalyticsRateLimit(context.Background()) + if err != nil || first.Remaining != 3070 || high.Remaining != 3070 || raw.Remaining != 19985 || high.ResetAt != shifted { + t.Fatalf("anomaly increased admission: first=%+v effective=%+v raw=%+v err=%v", first, high, raw, err) + } + balance = 3050 + shifted = reset + low, _, err := c.AnalyticsRateLimit(context.Background()) + if err != nil || low.Remaining != 3050 || !low.ResetAt.Equal(reset.Add(4*time.Second)) { + t.Fatalf("lower balance/reset boundary lost: %+v %v", low, err) + } + // REST refreshes cannot replace this GraphQL evidence. Only a real rollover + // after the later observed reset boundary can replenish the balance. + c.reserve.replace([]RateLimitSnapshot{{Resource: "graphql", Remaining: 19999, ResetAt: reset.Add(time.Hour)}}) + same := c.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 19900, ResetAt: reset.Add(time.Hour)}, reset.Add(2*time.Second)) + if same.Remaining != 3050 || !same.ResetAt.Equal(reset.Add(4*time.Second)) { + t.Fatal("early rollover increased balance") + } + next := c.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 19900, ResetAt: reset.Add(time.Hour)}, reset.Add(5*time.Second)) + if next.Remaining != 19900 { + t.Fatal("actual rollover could not refill budget") + } +} + +func TestGraphQLQuotaJitterAfterPreviousBoundaryCannotRefill(t *testing.T) { + now := time.Now().UTC() + r := newRateLimitReserve(3000, nil) + r.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 3050, ResetAt: now.Add(time.Minute)}, now) + held := r.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 19985, ResetAt: now.Add(time.Minute + 4*time.Second)}, now.Add(time.Minute+time.Second)) + if held.Remaining != 3050 || !held.ResetAt.Equal(now.Add(time.Minute+4*time.Second)) { + t.Fatalf("post-boundary jitter refilled: %+v", held) + } + renewed := r.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 19980, ResetAt: now.Add(time.Hour)}, now.Add(time.Minute+5*time.Second)) + if renewed.Remaining != 19980 { + t.Fatal("genuine rollover remained clamped") + } +} + +func TestGraphQLHistoryQuotaLogLabelsProviderAndEffectiveBalance(t *testing.T) { + reset := time.Now().UTC().Add(time.Hour).Truncate(time.Second) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":19999,"reset":%d}}}`, reset.Unix()) + return + } + fmt.Fprintf(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":19985,"resetAt":%q}}}`, reset.Add(4*time.Second).Format(time.RFC3339)) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 3000}) + c.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 3070, ResetAt: reset}, time.Now()) + var messages []string + h := historySession{client: c, remaining: 20000, reporter: func(message string) { messages = append(messages, message) }} + data, err := h.request(context.Background(), "query { rateLimit { cost limit remaining resetAt } }", nil, 1) + if err != nil { + t.Fatal(err) + } + if raw, ok := historyInt(historyMap(data["rateLimit"])["remaining"]); !ok || raw != 19985 { + t.Fatal("raw provider evidence rewritten") + } + log := strings.Join(messages, "\n") + if !strings.Contains(log, "provider_remaining 19985") || !strings.Contains(log, "effective_remaining 3070") { + t.Fatal("raw and effective quota not labeled", log) + } +} + +func TestAnalyticsProfilesCoverActorInterface(t *testing.T) { + for _, kind := range []string{"Organization", "EnterpriseUserAccount"} { + t.Run(kind, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + node := map[string]any{"id": "actor", "__typename": kind} + if strings.Contains(req.Query, "... on Actor {") || strings.Contains(req.Query, "... on "+kind+" {") { + node["login"], node["url"] = "fixture", "https://github.com/fixture" + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"nodes": []any{node}}}) + })) + defer server.Close() + nodes, err := New(Options{Token: "test-token-placeholder", BaseURL: server.URL}).AnalyticsNodes(context.Background(), []string{"actor"}, true) + if err != nil || len(nodes) != 1 || nodes[0]["login"] != "fixture" || nodes[0]["url"] != "https://github.com/fixture" { + t.Fatalf("actor fields lost: %+v %v", nodes, err) + } + }) + } +} + +func TestAnalyticsRejectsShortFinalDiscoveryPage(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":1,"remaining":19000,"resetAt":"2099-01-01T00:00:00Z"},"repository":{"issues":{"totalCount":2,"nodes":[{"number":1,"updatedAt":"2026-01-01T00:00:00Z"}],"pageInfo":{"hasNextPage":false,"endCursor":"end"}}}}}`) + })) + defer server.Close() + client := New(Options{Token: "test-token-placeholder", BaseURL: server.URL}) + if _, err := client.UpdatedNumbers(context.Background(), "fixture", "repo", "issues", "", time.Time{}); err == nil { + t.Fatal("short final discovery page was accepted") + } +} diff --git a/internal/github/client.go b/internal/github/client.go index 612e0e92..310f924e 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -20,19 +20,23 @@ import ( type Reporter func(message string) type Client struct { - httpClient *http.Client - baseURL string - graphQLURL string - token string - tokenProvider func(context.Context) (string, error) - userAgent string - pageDelay time.Duration - rateLimit RateLimitObserver - reserve *rateLimitReserve + httpClient *http.Client + baseURL string + graphQLURL string + token string + tokenProvider func(context.Context) (string, error) + userAgent string + pageDelay time.Duration + rateLimit RateLimitObserver + reserve *rateLimitReserve + graphQLResponseLimit int64 + graphQLQuotaGuard bool } type Options struct { - Token string + GraphQLResponseLimit int64 + GraphQLQuotaGuard bool + Token string // TokenProvider exclusively selects credentials immediately before dispatch. TokenProvider func(context.Context) (string, error) BaseURL string @@ -41,7 +45,8 @@ type Options struct { PageDelay time.Duration RateLimit RateLimitObserver // RateLimitReserve preserves a best-effort observed floor for the shared - // token. Guarded requests refresh /rate_limit before dispatch so other token + // token. Unless GraphQLQuotaGuard uses explicit observed GraphQL quota, + // guarded requests refresh /rate_limit before dispatch so other token // consumers are observed, but unrelated consumers cannot be locked between // that probe and dispatch. RateLimitReserve int @@ -79,6 +84,45 @@ type rateLimitReserve struct { mu sync.Mutex reserve int snapshots map[string]RateLimitSnapshot + // GraphQL response evidence must survive REST snapshot replacement and + // upward provider anomalies until the observed reset boundary has passed. + graphqlObserved RateLimitSnapshot + graphqlToken string +} + +type graphQLQuotaProbeKey struct{} +type graphQLRequestEstimateKey struct{} + +func (r *rateLimitReserve) bindGraphQLToken(token string) { + if r == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.graphqlToken != token { + r.graphqlObserved = RateLimitSnapshot{} + r.graphqlToken = token + } +} + +func (r *rateLimitReserve) beforeObservedGraphQL(token string, cost int) error { + if r == nil { + return requestFailureAt("dispatch_guard", "quota_guard_missing", fmt.Errorf("observed GraphQL quota guard required")) + } + r.mu.Lock() + defer r.mu.Unlock() + q := r.graphqlObserved + if token != r.graphqlToken { + return requestFailureAt("dispatch_guard", "credential_changed", fmt.Errorf("GraphQL credential changed; quota probe required")) + } + if q.Resource != "graphql" || !q.ResetAt.After(time.Now()) { + return requestFailureAt("dispatch_guard", "quota_observation_stale", fmt.Errorf("fresh observed GraphQL quota required")) + } + if q.Remaining-cost < r.reserve { + return &RateLimitReserveError{RateLimit: q, Reserve: r.reserve} + } + r.graphqlObserved.Remaining -= cost + return nil } type rateLimitRequestLockKey struct{} @@ -133,14 +177,16 @@ func New(options Options) *Client { userAgent = "gitcrawl" } client := &Client{ - httpClient: httpClient, - baseURL: baseURL, - graphQLURL: graphQLURLForBaseURL(baseURL), - token: options.Token, - tokenProvider: options.TokenProvider, - userAgent: userAgent, - pageDelay: options.PageDelay, - rateLimit: options.RateLimit, + httpClient: httpClient, + baseURL: baseURL, + graphQLURL: graphQLURLForBaseURL(baseURL), + token: options.Token, + tokenProvider: options.TokenProvider, + userAgent: userAgent, + pageDelay: options.PageDelay, + graphQLResponseLimit: options.GraphQLResponseLimit, + graphQLQuotaGuard: options.GraphQLQuotaGuard, + rateLimit: options.RateLimit, } if options.RateLimitReserve > 0 { client.reserve = newRateLimitReserve(options.RateLimitReserve, options.InitialRateLimits) @@ -165,7 +211,7 @@ func (r *rateLimitReserve) beforeRequest(resource string, cost int) error { defer r.mu.Unlock() snapshot, ok := r.snapshots[resource] if !ok { - return fmt.Errorf("github %s rate limit status unavailable; cannot preserve reserve %d", resource, r.reserve) + return requestFailureAt("dispatch_guard", "quota_snapshot_missing", fmt.Errorf("github %s rate limit status unavailable; cannot preserve reserve %d", resource, r.reserve)) } if !snapshot.ResetAt.IsZero() && !time.Now().UTC().Before(snapshot.ResetAt) { return &rateLimitStatusExpiredError{RateLimit: snapshot} @@ -187,6 +233,28 @@ func (r *rateLimitReserve) observe(snapshot RateLimitSnapshot) { r.snapshots[snapshot.Resource] = snapshot } +func (r *rateLimitReserve) observeGraphQL(snapshot RateLimitSnapshot, now time.Time) RateLimitSnapshot { + if r == nil || snapshot.Remaining < 0 || !snapshot.ResetAt.After(now) { + return snapshot + } + r.mu.Lock() + defer r.mu.Unlock() + previous := r.graphqlObserved + nearbyLaterReset := !previous.ResetAt.IsZero() && snapshot.ResetAt.After(previous.ResetAt) && snapshot.ResetAt.Sub(previous.ResetAt) <= time.Minute + if previous.ResetAt.After(now) || nearbyLaterReset { + snapshot.Remaining = min(snapshot.Remaining, previous.Remaining) + // A shifted reset timestamp before the prior boundary is not a new + // window. Wait through both boundaries before accepting a refill. + // An early sample of the next hourly window must not postpone the + // current boundary by another hour. Only nearby reset jitter extends it. + if previous.ResetAt.After(snapshot.ResetAt) || snapshot.ResetAt.Sub(previous.ResetAt) > time.Minute { + snapshot.ResetAt = previous.ResetAt + } + } + r.graphqlObserved = snapshot + return snapshot +} + func (r *rateLimitReserve) replace(snapshots []RateLimitSnapshot) { if r == nil { return @@ -246,7 +314,7 @@ func (c *Client) getRateLimits(ctx context.Context, reporter Reporter, selectedT } defer resp.Body.Close() if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - return nil, "", fmt.Errorf("decode github response: %w", err) + return nil, "", requestFailureAt("rest_quota_decode", "", fmt.Errorf("decode github response: %w", err)) } token := strings.TrimPrefix(resp.Request.Header.Get("Authorization"), "Bearer ") host := rateLimitHostForBaseURL(c.baseURL) @@ -525,7 +593,7 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader if targetErr != nil || originErr != nil || target.User != nil || origin.Host == "" || (target.Scheme != "https" && target.Scheme != "http") || !strings.EqualFold(target.Scheme, origin.Scheme) || !strings.EqualFold(target.Host, origin.Host) { - return nil, errors.New("GitHub token provider requires the configured API origin") + return nil, requestFailureAt("dispatch_guard", "origin_mismatch", errors.New("GitHub token provider requires the configured API origin")) } } resource, cost := c.requestRateLimit(method, fullURL) @@ -538,11 +606,12 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader } } var probeToken string - if c.reserve != nil && cost > 0 { + observedGraphQL := c.graphQLQuotaGuard && resource == "graphql" + if c.reserve != nil && cost > 0 && !observedGraphQL { var err error _, probeToken, err = c.getRateLimits(ctx, reporter, nil) if err != nil { - return nil, fmt.Errorf("refresh GitHub rate limit status: %w", err) + return nil, requestFailureAt("rest_preflight", "", fmt.Errorf("refresh GitHub rate limit status: %w", err)) } } token := c.token @@ -555,34 +624,50 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader return nil, err } } - if c.tokenProvider != nil && c.reserve != nil && cost > 0 && token != probeToken { + if c.tokenProvider != nil && c.reserve != nil && cost > 0 && !observedGraphQL && token != probeToken { // A new token cannot spend the previous token's quota. Allow one new // probe, then reject further rotation before the protected request. _, probeToken, err := c.getRateLimits(ctx, reporter, &token) if err != nil { - return nil, fmt.Errorf("refresh GitHub rate limit status: %w", err) + return nil, requestFailureAt("rest_preflight", "", fmt.Errorf("refresh GitHub rate limit status: %w", err)) } token, err = c.requestToken(ctx) if err != nil { return nil, err } if token != probeToken { - return nil, errors.New("GitHub token changed during rate limit reservation") + return nil, requestFailureAt("dispatch_guard", "credential_changed", errors.New("GitHub token changed during rate limit reservation")) } } - if err := c.reserve.beforeRequest(resource, cost); err != nil { - var expired *rateLimitStatusExpiredError - if c.tokenProvider != nil || !errors.As(err, &expired) { + if observedGraphQL { + probe, _ := ctx.Value(graphQLQuotaProbeKey{}).(bool) + if probe { + c.reserve.bindGraphQLToken(token) + } else if err := c.reserve.beforeObservedGraphQL(token, cost); err != nil { return nil, err } - _, refreshErr := c.GetRateLimits(ctx, reporter) - if refreshErr != nil { - return nil, fmt.Errorf("refresh GitHub rate limit status: %w", refreshErr) - } - if err := c.reserve.beforeRequest(resource, cost); err != nil { - return nil, err + } else if err := c.reserve.beforeRequest(resource, cost); err != nil { + var expired *rateLimitStatusExpiredError + if errors.As(err, &expired) && resource == "graphql" { + if err := c.refreshExpiredGraphQLQuota(ctx, token, cost); err != nil { + return nil, requestFailureAt("graphql_quota_refresh", "", err) + } + } else { + if c.tokenProvider != nil || !errors.As(err, &expired) { + return nil, requestFailureAt("dispatch_guard", "", err) + } + _, refreshErr := c.GetRateLimits(ctx, reporter) + if refreshErr != nil { + return nil, requestFailureAt("rest_preflight", "", fmt.Errorf("refresh GitHub rate limit status: %w", refreshErr)) + } + if err := c.reserve.beforeRequest(resource, cost); err != nil { + return nil, err + } } } + if resource == "graphql" && !observedGraphQL { + c.reserve.bindGraphQLToken(token) + } req, err := http.NewRequestWithContext(ctx, method, fullURL, body) if err != nil { return nil, err @@ -599,7 +684,7 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader reporter.Printf("[github] request %s %s", method, path) resp, err := c.guardedHTTPClient().Do(req) if err != nil { - return nil, fmt.Errorf("github request: %w", err) + return nil, requestFailureAt("transport", "", fmt.Errorf("github request: %w", err)) } responseResource, responseCost := c.requestRateLimit(resp.Request.Method, resp.Request.URL.String()) responseToken := strings.TrimPrefix(resp.Request.Header.Get("Authorization"), "Bearer ") @@ -620,6 +705,33 @@ func (c *Client) doOnce(ctx context.Context, method, path string, body io.Reader } } +// refreshExpiredGraphQLQuota runs only after an expired REST GraphQL snapshot, +// with requestMu already held. A quota-only probe may cross that stale boundary; +// content may not. Reuse the authoritative probe and conservative same-window +// accounting, pinning its credential until the protected dispatch is rechecked. +func (c *Client) refreshExpiredGraphQLQuota(ctx context.Context, token string, cost int) error { + quotaClient := *c + quotaClient.tokenProvider = func(context.Context) (string, error) { return token, nil } + _, observed, err := quotaClient.AnalyticsRateLimit(ctx) + if err != nil { + return err + } + if !observed.ResetAt.After(time.Now()) { + return &rateLimitStatusExpiredError{RateLimit: observed} + } + current, err := c.requestToken(ctx) + if err != nil { + return err + } + if current != token { + return requestFailureAt("dispatch_guard", "credential_changed", errors.New("GitHub token changed during quota refresh")) + } + // A history session checked its estimate before entering transport, using + // the prior observation. Recheck against the newly refreshed balance too. + estimate, _ := ctx.Value(graphQLRequestEstimateKey{}).(int) + return c.reserve.beforeObservedGraphQL(token, max(cost, estimate)) +} + func (c *Client) guardedHTTPClient() *http.Client { if c.reserve == nil && c.tokenProvider == nil { return c.httpClient @@ -649,7 +761,7 @@ func (c *Client) requestToken(ctx context.Context) (string, error) { if ctx.Err() != nil { return "", ctx.Err() } - return "", errors.New("GitHub token provider failed") + return "", requestFailureAt("credential", "credential_provider_failed", errors.New("GitHub token provider failed")) } return token, nil } diff --git a/internal/github/headline_metrics.go b/internal/github/headline_metrics.go new file mode 100644 index 00000000..626dd569 --- /dev/null +++ b/internal/github/headline_metrics.go @@ -0,0 +1,70 @@ +package github + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" +) + +// Pointer counts distinguish an unavailable field from an actual zero. +type RepositoryHeadline struct { + Stars *float64 `json:"stargazers_count"` + Forks *float64 `json:"forks_count"` + Watchers *float64 `json:"subscribers_count"` + OpenIssuesAndPulls *float64 `json:"open_issues_count"` +} + +type PullCount struct { + Count *float64 `json:"total_count"` + Incomplete bool `json:"incomplete_results"` +} + +type CloneTraffic struct { + Clones []struct { + Timestamp string `json:"timestamp"` + Count *float64 `json:"count"` + } `json:"clones"` +} + +type Release struct { + ID int64 `json:"id"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + Published string `json:"published_at"` + Created string `json:"created_at"` + Name string `json:"name"` + Tag string `json:"tag_name"` + URL string `json:"html_url"` +} + +func (c *Client) RepositoryHeadline(ctx context.Context, repository string) (RepositoryHeadline, error) { + var out RepositoryHeadline + err := c.doJSON(ctx, http.MethodGet, "/repos/"+repository, nil, nil, &out) + return out, err +} + +func (c *Client) OpenPullCount(ctx context.Context, repository string) (PullCount, error) { + var out PullCount + err := c.doJSON(ctx, http.MethodGet, "/search/issues?q="+url.QueryEscape("repo:"+repository+" is:pr is:open")+"&per_page=1", nil, nil, &out) + return out, err +} + +func (c *Client) CloneTraffic(ctx context.Context, repository string) (CloneTraffic, error) { + var out CloneTraffic + err := c.doJSON(ctx, http.MethodGet, "/repos/"+repository+"/traffic/clones?per=day", nil, nil, &out) + if err == nil && out.Clones == nil { + err = errors.New("missing GitHub clone traffic list") + } + return out, err +} + +func (c *Client) ReleasePage(ctx context.Context, repository string, page int) ([]Release, error) { + var out []Release + err := c.doJSON(ctx, http.MethodGet, fmt.Sprintf("/repos/%s/releases?per_page=100&page=%d", repository, page), nil, nil, &out) + if err == nil && out == nil { + err = errors.New("missing GitHub release list") + } + return out, err +} diff --git a/internal/github/history.go b/internal/github/history.go index b0012a27..89d047fe 100644 --- a/internal/github/history.go +++ b/internal/github/history.go @@ -16,26 +16,30 @@ import ( ) type HistoryItem struct { - Thread, Pull map[string]any - Comments, Reviews, ReviewComments []map[string]any + Thread, Pull map[string]any + Comments, Reviews, ReviewComments, ReviewThreads []map[string]any } type HistoryBatch struct { Repository map[string]any Items []HistoryItem } -const historyActor = `author { login __typename url }` +const historyActor = `author { login __typename url ... on Node { id } }` const historyComment = `id __typename fullDatabaseId body ` + historyActor + ` authorAssociation createdAt updatedAt publishedAt url isMinimized minimizedReason` const historyInline = historyComment + ` path diffHunk line startLine originalLine originalStartLine position originalPosition state subjectType outdated commit { oid } originalCommit { oid } replyTo { id fullDatabaseId } pullRequestReview { id fullDatabaseId }` var historyReview = historyComment + ` state submittedAt commit { oid } ` + historyConnection("comments", historyInline, "") -var historyReviewThread = `id __typename ` + historyConnection("comments", historyInline, "") +var historyReviewThread = `id __typename path line startLine isResolved isOutdated viewerCanResolve viewerCanUnresolve viewerCanReply ` + historyConnection("comments", historyInline, "") var historyCommon = `id __typename fullDatabaseId number title body ` + historyActor + ` authorAssociation createdAt updatedAt closedAt url state locked activeLockReason repository { nameWithOwner } milestone { number title state dueOn createdAt updatedAt url } ` + historyConnection("labels", `id name color description`, "") + " " + historyConnection("assignees", `id login __typename url`, "") + " " + historyConnection("comments", historyComment, "") var historyIssue = historyCommon + ` stateReason` var historyPull = historyCommon + ` isDraft merged mergedAt mergedBy { login __typename url } mergeCommit { oid } mergeable mergeStateStatus maintainerCanModify additions deletions changedFiles headRefName headRefOid baseRefName baseRefOid headRepository { nameWithOwner } baseRepository { nameWithOwner } commits { totalCount } ` + historyConnection("reviews", historyReview, "") + " " + historyConnection("reviewThreads", historyReviewThread, "") func historyConnection(name, fields, after string) string { - return name + `(first:20` + after + `) { totalCount pageInfo { hasNextPage endCursor } nodes { ` + fields + ` } }` + size := "20" + if after != "" && name != "reviews" && name != "reviewThreads" { + size = "100" + } + return name + `(first:` + size + after + `) { totalCount pageInfo { hasNextPage endCursor } nodes { ` + fields + ` } }` } type historySession struct { @@ -46,6 +50,10 @@ type historySession struct { retrySleep func(context.Context, time.Duration) error } +func (h *historySession) quota(ctx context.Context) (map[string]any, error) { + return h.request(context.WithValue(ctx, graphQLQuotaProbeKey{}, true), `query { rateLimit {cost remaining limit used resetAt} }`, nil, 1) +} + func (h *historySession) request(ctx context.Context, query string, variables map[string]any, estimate int) (map[string]any, error) { for attempt := 0; ; attempt++ { data, err := h.requestOnce(ctx, query, variables, estimate) @@ -103,17 +111,23 @@ func sleepHistoryRetry(ctx context.Context, duration time.Duration) error { func (h *historySession) requestOnce(ctx context.Context, query string, variables map[string]any, estimate int) (map[string]any, error) { if h.calls >= 1000 { - return nil, fmt.Errorf("GraphQL history pagination budget exceeded") + return nil, requestFailureAt("graphql_response", "pagination_budget", fmt.Errorf("GraphQL history pagination budget exceeded")) + } + reserve := 500 + if h.client.reserve != nil { + reserve = max(reserve, h.client.reserve.reserve) } - if h.remaining < 500+estimate { - return nil, fmt.Errorf("GraphQL history quota reserve reached") + // Retain the configured floor against actual GraphQL responses as well as + // the existing REST quota guard, including every pagination request. + if h.remaining < reserve+estimate { + return nil, fmt.Errorf("GraphQL history quota reserve reached: %w", &RateLimitReserveError{RateLimit: RateLimitSnapshot{Resource: "graphql", Remaining: h.remaining}, Reserve: reserve}) } h.calls++ // An unanswered request is charged conservatively by external supervisors. h.reporter.Printf("[github] graphql budget %d %d", h.calls, estimate) var data map[string]any started := time.Now() - err := h.client.doGraphQL(ctx, query, variables, h.reporter, &data) + err := h.client.doGraphQL(context.WithValue(ctx, graphQLRequestEstimateKey{}, estimate), query, variables, h.reporter, &data) h.reporter.Printf("[github] graphql timing %d %d", h.calls, time.Since(started).Milliseconds()) if err != nil { return nil, err @@ -121,17 +135,19 @@ func (h *historySession) requestOnce(ctx context.Context, query string, variable rate := historyMap(data["rateLimit"]) cost, ok := historyInt(rate["cost"]) if !ok || cost < 0 { - return nil, fmt.Errorf("GraphQL history missing cost") + return nil, requestFailureAt("graphql_response", "quota_cost_missing", fmt.Errorf("GraphQL history missing cost")) } remaining, ok := historyInt(rate["remaining"]) if !ok || remaining < 0 { - return nil, fmt.Errorf("GraphQL history missing remaining quota") + return nil, requestFailureAt("graphql_response", "quota_remaining_missing", fmt.Errorf("GraphQL history missing remaining quota")) } reset, err := time.Parse(time.RFC3339, historyString(rate["resetAt"])) if err != nil { - return nil, fmt.Errorf("GraphQL history invalid reset") + return nil, requestFailureAt("graphql_response", "quota_reset_invalid", fmt.Errorf("GraphQL history invalid reset")) } - h.remaining = min(h.remaining-cost, remaining) + effective := h.client.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: remaining, ResetAt: reset}, time.Now()) + h.remaining = min(h.remaining-cost, effective.Remaining) + h.reporter.Printf("[github] graphql quota provider_remaining %d provider_reset %d effective_remaining %d effective_reset %d", remaining, reset.Unix(), h.remaining, effective.ResetAt.Unix()) h.reporter.Printf("[github] graphql cost %d %d remaining %d reset %d", h.calls, cost, remaining, reset.Unix()) return data, nil } @@ -178,10 +194,10 @@ func (c *Client) FetchGraphQLHistory(ctx context.Context, owner, repo string, nu node := historyMap(r[fmt.Sprintf("n%d", i)]) got, _ := historyInt(node["number"]) if got != n || !strings.EqualFold(historyString(historyMap(node["repository"])["nameWithOwner"]), owner+"/"+repo) || historyString(node["id"]) == "" { - return result, fmt.Errorf("GraphQL history item #%d unavailable or moved", n) + return result, historyFailure("identity", n, node, fmt.Errorf("GraphQL history item #%d unavailable or moved", n)) } if err := h.hydrate(ctx, node); err != nil { - return result, fmt.Errorf("GraphQL history #%d: %w", n, err) + return result, historyFailure("validation", n, node, fmt.Errorf("GraphQL history #%d: %w", n, err)) } item, err := historyItem(node) if err != nil { @@ -213,6 +229,13 @@ func historyFields(typ, key string) (string, error) { func (h *historySession) hydrate(ctx context.Context, node map[string]any) error { typ := historyString(node["__typename"]) + if typ == "PullRequestReviewThread" { + for _, field := range []string{"isResolved", "isOutdated", "viewerCanResolve", "viewerCanUnresolve", "viewerCanReply"} { + if _, ok := node[field].(bool); !ok { + return fmt.Errorf("missing or invalid review-thread %s", field) + } + } + } var required []string switch typ { case "Issue": @@ -232,6 +255,11 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error return fmt.Errorf("missing provider identity") } } + return h.hydrateConnections(ctx, node) +} + +func (h *historySession) hydrateConnections(ctx context.Context, node map[string]any) error { + typ := historyString(node["__typename"]) for _, key := range []string{"labels", "assignees", "comments", "reviews", "reviewThreads"} { connection, exists := node[key] if !exists { @@ -239,7 +267,11 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error } conn := historyMap(connection) if conn == nil { - return fmt.Errorf("missing %s connection", key) + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("missing %s connection", key)) + } + total, ok := historyInt(conn["totalCount"]) + if !ok || total < 0 { + return fmt.Errorf("invalid history %s count", key) } fields, err := historyFields(typ, key) if err != nil { @@ -250,50 +282,53 @@ func (h *historySession) hydrate(ctx context.Context, node map[string]any) error page := historyMap(conn["pageInfo"]) next, ok := page["hasNextPage"].(bool) if !ok { - return fmt.Errorf("missing %s pageInfo", key) + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("missing %s pageInfo", key)) } if !next { break } cursor := historyString(page["endCursor"]) if cursor == "" || seen[cursor] { - return fmt.Errorf("nonadvancing %s cursor", key) + return requestFailureAt("pagination_"+key, "connection_cursor", fmt.Errorf("nonadvancing %s cursor", key)) } seen[cursor] = true q := `query($id:ID!,$after:String!){node(id:$id){id ... on ` + typ + `{` + historyConnection(key, fields, `,after:$after`) + `}} rateLimit{cost remaining limit used resetAt}}` data, err := h.request(ctx, q, map[string]any{"id": node["id"], "after": cursor}, 2) if err != nil { - return err + return requestFailureAt("pagination_"+key, "", err) } parent := historyMap(data["node"]) if parent["id"] != node["id"] { - return fmt.Errorf("history pagination identity mismatch") + return requestFailureAt("pagination_"+key, "connection_identity", fmt.Errorf("history pagination identity mismatch")) } nxt := historyMap(parent[key]) + if nextTotal, ok := historyInt(nxt["totalCount"]); !ok || nextTotal != total { + return fmt.Errorf("changed or missing history %s count", key) + } a, ok := conn["nodes"].([]any) if !ok { - return fmt.Errorf("missing history nodes") + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("missing history nodes")) } b, ok := nxt["nodes"].([]any) if !ok || len(b) == 0 { - return fmt.Errorf("empty history continuation") + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("empty history continuation")) } conn["nodes"] = append(a, b...) conn["pageInfo"] = nxt["pageInfo"] } children, ok := conn["nodes"].([]any) if !ok { - return fmt.Errorf("missing history nodes") + return requestFailureAt("pagination_"+key, "connection_shape", fmt.Errorf("missing history nodes")) } - if total, ok := historyInt(conn["totalCount"]); !ok || total != len(children) { - return fmt.Errorf("incomplete history %s count", key) + if total != len(children) { + return requestFailureAt("pagination_"+key, "connection_count", fmt.Errorf("incomplete history %s count", key)) } ids := map[string]bool{} for _, child := range children { m := historyMap(child) id := historyString(m["id"]) if id == "" || ids[id] { - return fmt.Errorf("missing or duplicate history child identity") + return requestFailureAt("pagination_"+key, "connection_identity", fmt.Errorf("missing or duplicate history child identity")) } ids[id] = true if err := h.hydrate(ctx, m); err != nil { @@ -374,6 +409,7 @@ func historyItem(node map[string]any) (HistoryItem, error) { // A comment's review association is nullable. Threads independently // supply standalone comments; review bodies and their metadata stay above. for _, thread := range historyNodes(node, "reviewThreads") { + item.ReviewThreads = append(item.ReviewThreads, thread) for _, comment := range historyNodes(thread, "comments") { id := historyString(comment["id"]) if _, exists := inlineByID[id]; exists { diff --git a/internal/github/history_cause.go b/internal/github/history_cause.go new file mode 100644 index 00000000..4257108c --- /dev/null +++ b/internal/github/history_cause.go @@ -0,0 +1,145 @@ +package github + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "io" + "net" + "net/url" +) + +// requestFailure adds source-owned diagnostic labels without changing the error +// text, unwrap chain, retry policy or acceptance decision. +type requestFailure struct { + cause error + stage, code string +} + +func (e *requestFailure) Error() string { return e.cause.Error() } +func (e *requestFailure) Unwrap() error { return e.cause } +func requestFailureAt(stage, code string, err error) error { + if err == nil { + return nil + } + return &requestFailure{cause: err, stage: stage, code: code} +} + +// All emitted strings are fixed labels. Never serialize Error(), URL, address, +// field/type names from JSON errors, certificate subjects, headers or tokens. +func safeHistoryCause(err error) map[string]any { + out := map[string]any{"category": "unknown", "type": "unclassified"} + stages := []string{} + stage := func(value string) { + switch value { + case "graphql_quota_refresh", "rest_preflight", "rest_quota_decode", "dispatch_guard", "credential", "transport", "graphql_request", "graphql_response", "validation", "identity", "partial_response", "response_decode", "response_size", "missing_data", "discovery_validation", "pagination_labels", "pagination_assignees", "pagination_comments", "pagination_reviews", "pagination_reviewThreads": + if len(stages) < 8 && (len(stages) == 0 || stages[len(stages)-1] != value) { + stages = append(stages, value) + } + } + } + set := func(category, typ, code string) { out["category"] = category; out["type"] = typ; out["code"] = code } + quota := func(q RateLimitSnapshot, reserve int) { + v := map[string]any{"remaining": q.Remaining, "limit": q.Limit} + switch q.Resource { + case "graphql", "core", "search": + v["resource"] = q.Resource + } + if !q.ResetAt.IsZero() { + v["reset_at"] = q.ResetAt.UTC().Format("2006-01-02T15:04:05Z07:00") + } + if reserve > 0 { + v["reserve"] = reserve + } + out["quota"] = v + } + for depth := 0; err != nil && depth < 16; depth++ { + switch e := err.(type) { + case *requestFailure: + stage(e.stage) + switch e.code { + case "quota_guard_missing", "quota_snapshot_missing", "quota_observation_stale", "credential_changed", "origin_mismatch": + set("guard", "native_guard", e.code) + case "credential_provider_failed": + set("credential", "native_provider", e.code) + case "quota_cost_missing", "quota_remaining_missing", "quota_reset_invalid", "pagination_budget": + set("validation", "native_validation", e.code) + case "connection_shape", "connection_cursor", "connection_identity", "connection_count": + set("validation", "native_validation", e.code) + } + case *HistoryFailure: + stage(e.Stage) + case *rateLimitStatusExpiredError: + set("guard", "quota_snapshot", "quota_snapshot_expired") + quota(e.RateLimit, 0) + case *RateLimitReserveError: + set("guard", "quota_reserve", "quota_reserve_reached") + quota(e.RateLimit, e.Reserve) + case *RequestError: + set("http", "github_http", "http_status") + if e.Status >= 100 && e.Status <= 599 { + out["http_status"] = e.Status + } + case *json.SyntaxError: + set("decode", "json_syntax", "invalid_json") + case *json.UnmarshalTypeError: + set("decode", "json_type", "invalid_json_type") + case *net.DNSError: + set("network", "dns", "dns_error") + out["timeout"] = e.Timeout() + case *net.OpError: + set("network", "net_operation", "network_error") + out["timeout"] = e.Timeout() + case *url.Error: + set("network", "url_request", "network_error") + out["timeout"] = e.Timeout() + case *tls.CertificateVerificationError: + set("tls", "certificate_verification", "certificate_invalid") + case x509.UnknownAuthorityError: + set("tls", "unknown_authority", "certificate_invalid") + case x509.HostnameError: + set("tls", "hostname", "certificate_invalid") + case x509.CertificateInvalidError: + set("tls", "certificate", "certificate_invalid") + } + switch err { + case context.Canceled: + set("cancelled", "context", "cancelled") + case context.DeadlineExceeded: + set("cancelled", "context", "deadline") + case io.EOF: + set("transport", "io", "eof") + case io.ErrUnexpectedEOF: + set("transport", "io", "unexpected_eof") + } + // A receipt describes its primary cause, not an unbounded joined error tree. + switch e := err.(type) { + case interface{ Unwrap() error }: + err = e.Unwrap() + case interface{ Unwrap() []error }: + children := e.Unwrap() + err = nil + if len(children) > 0 { + err = children[0] + } + default: + err = nil + } + } + if err != nil { + out["chain_truncated"] = true + } + out["stages"] = stages + return out +} + +func historyEvidenceWithCause(evidence json.RawMessage, err error) json.RawMessage { + var value map[string]json.RawMessage + if json.Unmarshal(evidence, &value) != nil || value == nil { + value = map[string]json.RawMessage{} + } + value["cause"], _ = json.Marshal(safeHistoryCause(err)) + out, _ := json.Marshal(value) + return out +} diff --git a/internal/github/history_cause_test.go b/internal/github/history_cause_test.go new file mode 100644 index 00000000..0ef3c3f4 --- /dev/null +++ b/internal/github/history_cause_test.go @@ -0,0 +1,245 @@ +package github + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "testing" + "time" +) + +func TestHistoryCauseKeepsOnlyBoundedTypedMetadata(t *testing.T) { + canary := "private-token-host-body-identity" + cases := []struct { + name string + err error + category, typ, code string + }{ + {"unknown", errors.New(canary), "unknown", "unclassified", ""}, + {"cancel", context.Canceled, "cancelled", "context", "cancelled"}, + {"deadline", context.DeadlineExceeded, "cancelled", "context", "deadline"}, + {"eof", io.EOF, "transport", "io", "eof"}, + {"short", io.ErrUnexpectedEOF, "transport", "io", "unexpected_eof"}, + {"json syntax", &json.SyntaxError{}, "decode", "json_syntax", "invalid_json"}, + {"json type", &json.UnmarshalTypeError{Value: canary, Field: canary, Struct: canary, Type: reflect.TypeOf(0)}, "decode", "json_type", "invalid_json_type"}, + {"dns", &net.DNSError{Name: canary, Server: canary, Err: canary, IsTimeout: true}, "network", "dns", "dns_error"}, + {"operation", &net.OpError{Op: canary, Net: canary, Err: errors.New(canary)}, "network", "net_operation", "network_error"}, + {"url", &url.Error{Op: canary, URL: "https://" + canary, Err: errors.New(canary)}, "network", "url_request", "network_error"}, + {"tls", &tls.CertificateVerificationError{Err: errors.New(canary)}, "tls", "certificate_verification", "certificate_invalid"}, + {"authority", x509.UnknownAuthorityError{}, "tls", "unknown_authority", "certificate_invalid"}, + {"hostname", x509.HostnameError{Host: canary}, "tls", "hostname", "certificate_invalid"}, + {"certificate", x509.CertificateInvalidError{Detail: canary}, "tls", "certificate", "certificate_invalid"}, + {"http", &RequestError{Method: canary, URL: canary, Status: 401, Body: canary, Headers: http.Header{"Authorization": []string{canary}}}, "http", "github_http", "http_status"}, + {"expired", &rateLimitStatusExpiredError{RateLimit: RateLimitSnapshot{Resource: "graphql", Remaining: 19999, ResetAt: time.Unix(100, 0)}}, "guard", "quota_snapshot", "quota_snapshot_expired"}, + {"reserve", &RateLimitReserveError{RateLimit: RateLimitSnapshot{Resource: "graphql", Remaining: 1500}, Reserve: 1500}, "guard", "quota_reserve", "quota_reserve_reached"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := safeHistoryCause(fmt.Errorf("%s: %w", canary, tc.err)) + if got["category"] != tc.category || got["type"] != tc.typ { + t.Fatalf("%+v", got) + } + if tc.code != "" && got["code"] != tc.code { + t.Fatalf("code=%v", got["code"]) + } + b, _ := json.Marshal(got) + if strings.Contains(string(b), canary) || len(b) > 2048 { + t.Fatal("unsafe/unbounded diagnostic", string(b)) + } + }) + } + for code, category := range map[string]string{"quota_guard_missing": "guard", "quota_snapshot_missing": "guard", "quota_observation_stale": "guard", "credential_changed": "guard", "origin_mismatch": "guard", "credential_provider_failed": "credential", "quota_cost_missing": "validation", "quota_remaining_missing": "validation", "quota_reset_invalid": "validation", "pagination_budget": "validation", "connection_shape": "validation", "connection_cursor": "validation", "connection_identity": "validation", "connection_count": "validation"} { + base := errors.New(canary) + wrapped := requestFailureAt("dispatch_guard", code, base) + got := safeHistoryCause(wrapped) + if got["code"] != code || got["category"] != category || !errors.Is(wrapped, base) || wrapped.Error() != base.Error() { + t.Fatalf("diagnostic changed behavior: %s %+v", code, got) + } + } + var long error = errors.New(canary) + for i := 0; i < 30; i++ { + stage := "rest_preflight" + if i%2 == 0 { + stage = "graphql_request" + } + long = requestFailureAt(stage, "", long) + } + got := safeHistoryCause(long) + if got["chain_truncated"] != true || len(got["stages"].([]string)) != 8 { + t.Fatal("unbounded cause chain", got) + } + unknown := safeHistoryCause(requestFailureAt(canary, canary, errors.New(canary))) + b, _ := json.Marshal(unknown) + if strings.Contains(string(b), canary) { + t.Fatal("untrusted label leaked") + } + joined := safeHistoryCause(errors.Join(requestFailureAt("rest_preflight", "", io.ErrUnexpectedEOF), errors.New(canary))) + if joined["code"] != "unexpected_eof" { + t.Fatal("primary joined cause lost") + } + if requestFailureAt("transport", "", nil) != nil { + t.Fatal("nil error manufactured") + } + // Retry eligibility must stay identical after diagnostic wrapping. + for _, e := range []error{io.EOF, io.ErrUnexpectedEOF, &RequestError{Status: 503}, &RequestError{Status: 401}, context.Canceled} { + if transientHistoryError(e) != transientHistoryError(requestFailureAt("graphql_request", "", e)) { + t.Fatal("retry semantics changed") + } + } +} + +func TestHistoryCauseDiagnosesNativePreflightWithoutBypassingGuards(t *testing.T) { + for _, mode := range []string{"expired", "missing", "decode", "rotation", "http", "missing_cost", "missing_remaining", "invalid_reset"} { + t.Run(mode, func(t *testing.T) { + gql, credentials := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + if mode == "decode" { + fmt.Fprint(w, `{"resources":private-response`) + return + } + if mode == "missing" { + fmt.Fprint(w, `{"resources":{}}`) + return + } + if mode == "http" { + http.Error(w, "private-response", 401) + return + } + reset := time.Now().Add(time.Hour).Unix() + if mode == "expired" { + reset = time.Now().Add(-time.Minute).Unix() + } + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":%d}}}`, reset) + return + } + gql++ + rate := map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)} + if mode == "expired" { + rate["resetAt"] = time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + } + if mode == "missing_cost" { + delete(rate, "cost") + } + if mode == "missing_remaining" { + delete(rate, "remaining") + } + if mode == "invalid_reset" { + rate["resetAt"] = "private-response" + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": rate}}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { + credentials++ + if mode == "rotation" { + return fmt.Sprint("test-token-", credentials), nil + } + return "test-token-placeholder", nil + }}) + batch, err := c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err == nil || len(batch.Items) != 0 { + t.Fatal("failed evidence accepted") + } + class, _, evidence := HistoryFailureDetails(err) + var record struct { + Cause struct { + Code string `json:"code"` + Status int `json:"http_status"` + Stages []string `json:"stages"` + } `json:"cause"` + } + if e := json.Unmarshal(evidence, &record); e != nil { + t.Fatal(e) + } + want := map[string]string{"expired": "quota_snapshot_expired", "missing": "quota_snapshot_missing", "decode": "invalid_json", "rotation": "credential_changed", "http": "http_status", "missing_cost": "quota_cost_missing", "missing_remaining": "quota_remaining_missing", "invalid_reset": "quota_reset_invalid"}[mode] + if record.Cause.Code != want { + t.Fatalf("mode=%s class=%s evidence=%s", mode, class, evidence) + } + if mode == "http" { + if class != "http" || record.Cause.Status != 401 { + t.Fatal("HTTP status lost") + } + } else if class != "fetch" { + t.Fatal("existing error class changed", class) + } + if strings.Contains(string(evidence), "private-response") || strings.Contains(string(evidence), "test-token") { + t.Fatal("private value leaked") + } + if mode == "expired" && gql != 1 { + t.Fatal("expected only the authoritative quota refresh") + } + if mode == "missing" || mode == "decode" || mode == "rotation" || mode == "http" { + if gql != 0 { + t.Fatal("guard dispatched GraphQL content") + } + } + }) + } +} + +func TestHistoryCauseRetainsPaginationGuardAndAllowsNormalRetry(t *testing.T) { + expired := true + restCalls, gqlCalls := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + restCalls++ + reset := time.Now().Add(time.Hour).Unix() + if expired && restCalls == 3 { + reset = time.Now().Add(-time.Minute).Unix() + } + fmt.Fprintf(w, `{"resources":{"graphql":{"limit":20000,"remaining":19000,"reset":%d}}}`, reset) + return + } + gqlCalls++ + var req graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}} + if expired && restCalls == 3 { + historyMap(data["rateLimit"])["resetAt"] = time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + } + if strings.Contains(req.Query, "issueOrPullRequest") { + node := historyTestNode() + node["labels"] = map[string]any{"totalCount": 2, "nodes": []any{map[string]any{"id": "L1", "name": "one"}}, "pageInfo": map[string]any{"hasNextPage": true, "endCursor": "first"}} + data["repository"] = map[string]any{"databaseId": 42, "nameWithOwner": "fixture/repo", "n0": node} + } + if req.Variables["after"] != nil { + page := historyTestConnection(map[string]any{"id": "L2", "name": "two"}) + page["totalCount"] = 2 + data["node"] = map[string]any{"id": "PR_fixture", "labels": page} + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "test-token-placeholder", nil }}) + batch, err := c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err == nil || len(batch.Items) != 0 || gqlCalls != 3 { + t.Fatal("partial pagination accepted") + } + class, _, evidence := HistoryFailureDetails(err) + var record map[string]json.RawMessage + if err = json.Unmarshal(evidence, &record); err != nil { + t.Fatal(err) + } + if class != "validation" || record["structure"] == nil || !strings.Contains(string(record["cause"]), "pagination_labels") || !strings.Contains(string(record["cause"]), "quota_snapshot_expired") { + t.Fatalf("lost wrapper or cause: %s", evidence) + } + expired = false + batch, err = c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err != nil || len(batch.Items) != 1 || len(historyNodes(historyMap(batch.Items[0].Thread["_graphql"]), "labels")) != 2 { + t.Fatalf("normal retry: items=%d err=%v", len(batch.Items), err) + } +} diff --git a/internal/github/history_evidence.go b/internal/github/history_evidence.go new file mode 100644 index 00000000..89e9cd6a --- /dev/null +++ b/internal/github/history_evidence.go @@ -0,0 +1,185 @@ +package github + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "strings" +) + +// HistoryFailure retains a bounded structural receipt, never credentials, HTTP +// headers, prose bodies or actor/profile text. Existing accepted raw evidence +// remains in the archive. A rejected body is represented by its length/hash. +type HistoryFailure struct { + Cause error + Stage string + Number int + Evidence json.RawMessage +} + +type historyResponseReader struct { + reader io.Reader + hash hash.Hash + read, hashed int64 +} + +func (r *historyResponseReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.read += int64(n) + keep := min(int64(n), 128*1024-r.hashed) + if keep > 0 { + _, _ = r.hash.Write(p[:keep]) + r.hashed += keep + } + return n, err +} + +func (r *historyResponseReader) failure(err error) *HistoryFailure { + evidence, _ := json.Marshal(map[string]any{"version": 1, "response_bytes_read": r.read, "hashed_prefix_bytes": r.hashed, "prefix_sha256": hex.EncodeToString(r.hash.Sum(nil)), "complete_response": false}) + return &HistoryFailure{Cause: err, Stage: "response_decode", Evidence: evidence} +} + +func (e *HistoryFailure) Error() string { return e.Cause.Error() } +func (e *HistoryFailure) Unwrap() error { return e.Cause } + +// Error messages can contain prose or identities. Retain only bounded provider +// codes and known query path components, separately from the existing receipt. +func graphQLRejection(data any, failures []graphqlResponseError, cause error) error { + var evidence map[string]json.RawMessage + // SafeHistoryEvidence always wraps data in an object, including nil data. + _ = json.Unmarshal(SafeHistoryEvidence(data), &evidence) + items := make([]map[string]any, 0, min(len(failures), 8)) + for _, failure := range failures[:min(len(failures), 8)] { + var code any + switch failure.Type { + case "NOT_FOUND", "FORBIDDEN", "UNAUTHORIZED", "UNPROCESSABLE", "RATE_LIMITED", "INTERNAL", "INTERNAL_SERVER_ERROR", "SERVICE_UNAVAILABLE", "MAX_NODE_LIMIT_EXCEEDED", "EXCESSIVE_PAGINATION", "RESOURCE_LIMITS_EXCEEDED": + code = failure.Type + } + path := make([]any, 0, min(len(failure.Path), 8)) + for _, component := range failure.Path[:min(len(failure.Path), 8)] { + var safe any + switch value := component.(type) { + case string: + switch value { + case "query", "repository", "node", "nodes", "issue", "pullRequest", "issueOrPullRequest", "issues", "pullRequests", "comments", "reviews", "reviewThreads", "labels", "assignees", "edges", "pageInfo", "totalCount", "hasNextPage", "endCursor", "rateLimit", "id", "__typename", "body", "author", "state", "isResolved", "isOutdated": + safe = value + } + if len(value) >= 2 && len(value) <= 3 && value[0] == 'n' && strings.Trim(value[1:], "0123456789") == "" { + safe = value // Native generated aliases, never entity IDs. + } + case json.Number: + if index, err := value.Int64(); err == nil && index >= 0 && index <= 10000 { + safe = index + } + } + path = append(path, safe) + } + items = append(items, map[string]any{"type": code, "path": path, "path_truncated": len(failure.Path) > 8}) + } + evidence["graphql_errors"], _ = json.Marshal(map[string]any{"count": len(failures), "items": items, "truncated": len(failures) > 8}) + encoded, _ := json.Marshal(evidence) + return &HistoryFailure{Cause: cause, Stage: "partial_response", Evidence: encoded} +} + +func historyFailure(stage string, number int, data any, err error) error { + evidence := SafeHistoryEvidence(data) + var upstream *HistoryFailure + if errors.As(err, &upstream) { + stage = upstream.Stage + evidence, _ = json.Marshal(map[string]json.RawMessage{"context": evidence, "upstream_rejection": upstream.Evidence}) + } + return &HistoryFailure{Cause: err, Stage: stage, Number: number, Evidence: evidence} +} + +func SafeHistoryEvidence(data any) json.RawMessage { + raw, _ := json.Marshal(data) + sum := sha256.Sum256(raw) + var project func(any, int) any + project = func(value any, depth int) any { + if depth > 9 { + return map[string]any{"truncated": true} + } + switch v := value.(type) { + case map[string]any: + out := map[string]any{} + for k, child := range v { + if len(k) > 1 && k[0] == 'n' && strings.Trim(k[1:], "0123456789") == "" { + out[k] = project(child, depth+1) + continue + } + switch k { + case "id", "node_id", "fullDatabaseId", "number", "__typename", "totalCount", "hasNextPage", "endCursor", "createdAt", "updatedAt", "publishedAt", "submittedAt", "state", "isResolved", "isOutdated", "type": + switch scalar := child.(type) { + case string: + if len(scalar) <= 512 { + out[k] = scalar + } else { + out[k] = map[string]any{"truncated": true} + } + case bool, float64, int, int64, json.Number, nil: + out[k] = scalar + default: + out[k] = map[string]any{"invalid_type": true} + } + case "repository", "node", "data", "nodes", "pageInfo", "comments", "reviews", "reviewThreads", "labels", "assignees", "errors", "path": + out[k] = project(child, depth+1) + case "body": + if text, ok := child.(string); ok { + h := sha256.Sum256([]byte(text)) + out["body_evidence"] = map[string]any{"bytes": len(text), "sha256": hex.EncodeToString(h[:])} + } + } + } + return out + case []any: + items := make([]any, 0, min(len(v), 12)) + for _, item := range v[:min(len(v), 12)] { + items = append(items, project(item, depth+1)) + } + return map[string]any{"count": len(v), "sample": items, "truncated": len(v) > 12} + default: + // Unknown strings may be provider error messages or private prose. + return nil + } + } + out, _ := json.Marshal(map[string]any{"version": 1, "response_bytes": len(raw), "response_sha256": hex.EncodeToString(sum[:]), "structure": project(data, 0)}) + if len(out) > 128*1024 { + out, _ = json.Marshal(map[string]any{"version": 1, "response_bytes": len(raw), "response_sha256": hex.EncodeToString(sum[:]), "structure_truncated": true}) + } + return out +} + +// HistoryFailureDetails is safe to persist or log even if the original error +// included an HTTP body. It deliberately does not return that body/message. +func HistoryFailureDetails(err error) (string, string, json.RawMessage) { + diagnostic := historyEvidenceWithCause(json.RawMessage(`{}`), err) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "cancelled", "GraphQL attempt cancelled or timed out", diagnostic + } + var quota *RateLimitReserveError + if errors.As(err, "a) { + return "rate_limit", quota.Error(), diagnostic + } + var failure *HistoryFailure + if errors.As(err, &failure) { + message := fmt.Sprintf("GraphQL %s rejected for item %d", failure.Stage, failure.Number) + for _, connection := range []string{"comments", "reviews", "reviewThreads", "labels", "assignees"} { + for _, reason := range []string{"incomplete history " + connection + " count", "nonadvancing " + connection + " cursor", "missing history " + connection} { + if strings.HasSuffix(failure.Cause.Error(), reason) { + message += ": " + reason + } + } + } + return failure.Stage, message, historyEvidenceWithCause(failure.Evidence, err) + } + var response *RequestError + if errors.As(err, &response) { + return "http", fmt.Sprintf("GitHub HTTP %d", response.Status), diagnostic + } + return "fetch", "GraphQL collection failed", diagnostic +} diff --git a/internal/github/history_evidence_test.go b/internal/github/history_evidence_test.go new file mode 100644 index 00000000..c4a43226 --- /dev/null +++ b/internal/github/history_evidence_test.go @@ -0,0 +1,142 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHistoryFailureEvidenceRetainsStructureWithoutSecretsOrBodies(t *testing.T) { + data := map[string]any{"id": "PR_fixture", "body": "private prose", "Authorization": "Bearer secret", "author": map[string]any{"login": "private-login"}, "comments": map[string]any{"totalCount": 2, "nodes": []any{map[string]any{"id": "C1", "body": "retained-by-hash"}}, "pageInfo": map[string]any{"hasNextPage": false}}} + evidence := SafeHistoryEvidence(data) + for _, secret := range []string{"private prose", "Bearer secret", "private-login", "retained-by-hash"} { + if strings.Contains(string(evidence), secret) { + t.Fatal("private data in safe receipt") + } + } + if !json.Valid(evidence) || !strings.Contains(string(evidence), `"totalCount":2`) || !strings.Contains(string(evidence), `"sha256"`) { + t.Fatal("missing structural evidence") + } + err := historyFailure("validation", 7, data, errors.New("GraphQL history #7: incomplete history comments count")) + class, message, stored := HistoryFailureDetails(err) + var preserved map[string]json.RawMessage + if err := json.Unmarshal(stored, &preserved); err != nil { + t.Fatal(err) + } + if preserved["cause"] == nil { + t.Fatal("missing safe cause metadata") + } + delete(preserved, "cause") + originalFields, _ := json.Marshal(preserved) + if class != "validation" || !strings.Contains(message, "incomplete history comments count") || string(originalFields) != string(evidence) { + t.Fatalf("receipt %s %s", class, message) + } +} + +func TestMalformedGraphQLResponseHasPrivateBoundedReceipt(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"data":{"body":"private malformed response"`)) + })) + defer server.Close() + var out map[string]any + err := New(Options{BaseURL: server.URL}).doGraphQL(context.Background(), "query { rateLimit { cost } }", nil, nil, &out) + if err == nil { + t.Fatal("malformed JSON accepted") + } + class, _, evidence := HistoryFailureDetails(err) + if class != "response_decode" || !strings.Contains(string(evidence), "prefix_sha256") || strings.Contains(string(evidence), "private malformed response") { + t.Fatalf("bad rejection receipt: %s %s", class, evidence) + } +} + +func TestReviewThreadMissingStateFailsClosed(t *testing.T) { + node := map[string]any{"id": "T1", "__typename": "PullRequestReviewThread", "comments": historyTestConnection()} + h := historySession{} + if err := h.hydrate(context.Background(), node); err == nil { + t.Fatal("unknown resolution became false") + } + for _, field := range []string{"isResolved", "isOutdated", "viewerCanResolve", "viewerCanUnresolve", "viewerCanReply"} { + node[field] = false + } + node["isResolved"] = true + if err := h.hydrate(context.Background(), node); err != nil { + t.Fatal(err) + } +} + +func TestGraphQLRejectedResponseKeepsOnlyBoundedSafeErrorMetadata(t *testing.T) { + failures := []any{} + for i := 0; i < 12; i++ { + typ := "NOT_FOUND" + if i == 1 { + typ = "private-identity" + } + failures = append(failures, map[string]any{"type": typ, "message": "private provider prose", "path": []any{"repository", "n0", "comments", 0, "body", "private-identity", "IC_private_identity", 1000000000, "omitted"}}) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"repository": map[string]any{"n0": nil, "n1": map[string]any{"body": "private peer prose"}}}, "errors": failures}) + })) + defer server.Close() + out := map[string]any{"unchanged": true} + err := New(Options{BaseURL: server.URL}).doGraphQL(context.Background(), "query { repository { id } }", nil, nil, &out) + if err == nil || len(out) != 1 || out["unchanged"] != true { + t.Fatal("partial data accepted") + } + class, _, evidence := HistoryFailureDetails(err) + if class != "partial_response" { + t.Fatal(class) + } + for _, private := range []string{"private provider prose", "private peer prose", "private-identity", "IC_private_identity", "omitted"} { + if strings.Contains(string(evidence), private) { + t.Fatal("unsafe rejection metadata retained", private) + } + } + var receipt struct { + Errors struct { + Count int `json:"count"` + Items []struct { + Type *string `json:"type"` + Path []any `json:"path"` + Truncated bool `json:"path_truncated"` + } `json:"items"` + Truncated bool `json:"truncated"` + } `json:"graphql_errors"` + } + if err = json.Unmarshal(evidence, &receipt); err != nil { + t.Fatal(err) + } + if receipt.Errors.Count != 12 || len(receipt.Errors.Items) != 8 || !receipt.Errors.Truncated { + t.Fatal("error bound lost") + } + first := receipt.Errors.Items[0] + if first.Type == nil || *first.Type != "NOT_FOUND" || len(first.Path) != 8 || !first.Truncated || first.Path[0] != "repository" || first.Path[1] != "n0" || first.Path[3] != float64(0) || first.Path[5] != nil || first.Path[6] != nil || first.Path[7] != nil || receipt.Errors.Items[1].Type != nil { + t.Fatalf("incorrect safe metadata: %+v", receipt.Errors) + } +} + +func TestGraphQLRejectedNullOrAbsentDataRetainsSafeEnvelope(t *testing.T) { + for _, payload := range []string{ + `{"errors":[{"type":"NOT_FOUND","path":["repository","n0"],"message":"private prose"}]}`, + `{"data":null,"errors":[{"type":"NOT_FOUND","path":["repository","n0"],"message":"private prose"}]}`, + } { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(payload)) })) + var out map[string]any + err := New(Options{BaseURL: server.URL}).doGraphQL(context.Background(), "query { repository { id } }", nil, nil, &out) + server.Close() + if err == nil { + t.Fatal("error envelope accepted") + } + class, _, evidence := HistoryFailureDetails(err) + var record map[string]any + if e := json.Unmarshal(evidence, &record); e != nil { + t.Fatal(e) + } + if class != "partial_response" || record["version"] != float64(1) || record["graphql_errors"] == nil || record["structure"] != nil || strings.Contains(string(evidence), "private prose") { + t.Fatalf("missing/redaction-invalid envelope: %s", evidence) + } + } +} diff --git a/internal/github/history_test.go b/internal/github/history_test.go index 3c3a88c8..857aac52 100644 --- a/internal/github/history_test.go +++ b/internal/github/history_test.go @@ -138,6 +138,43 @@ func TestGraphQLHistoryRejectsShortConnection(t *testing.T) { } } +func TestGraphQLHistoryRejectsChangedContinuationCount(t *testing.T) { + for _, total := range []any{1, 3, nil} { + t.Run(fmt.Sprint(total), func(t *testing.T) { + comment := func(id string) map[string]any { + return map[string]any{"id": id, "__typename": "IssueComment", "fullDatabaseId": id, "body": id} + } + node := historyTestNode() + first := historyTestConnection(comment("1")) + first["totalCount"] = 2 + first["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "first"} + node["comments"] = first + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "remaining": 19000, "resetAt": "2099-01-01T01:00:00Z"}} + if strings.Contains(req.Query, "issueOrPullRequest") { + data["repository"] = map[string]any{"databaseId": 42, "nameWithOwner": "fixture/repo", "n0": node} + } + if strings.Contains(req.Query, "node(id:") { + last := historyTestConnection(comment("2")) + last["totalCount"] = total + data["node"] = map[string]any{"id": node["id"], "comments": last} + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + batch, err := New(Options{BaseURL: server.URL}).FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if err == nil || len(batch.Items) != 0 { + t.Fatalf("accepted inconsistent continuation count %v: items=%d err=%v", total, len(batch.Items), err) + } + }) + } +} + func TestGraphQLHistoryIssueDiscussion(t *testing.T) { node := historyTestNode() node["__typename"] = "Issue" @@ -179,6 +216,13 @@ func TestGraphQLHistoryReviewThreadCompleteness(t *testing.T) { node["reviews"] = historyTestConnection(review) thread1 := map[string]any{"id": "T1", "__typename": "PullRequestReviewThread", "comments": historyTestConnection(associated)} thread2 := map[string]any{"id": "T2", "__typename": "PullRequestReviewThread", "comments": historyTestConnection()} + for _, thread := range []map[string]any{thread1, thread2} { + for _, field := range []string{"isResolved", "isOutdated", "viewerCanResolve", "viewerCanUnresolve", "viewerCanReply"} { + thread[field] = false + } + } + thread1["isResolved"] = true + thread2["isOutdated"] = true thread2["comments"].(map[string]any)["totalCount"] = 1 thread2["comments"].(map[string]any)["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "comment-first"} node["reviewThreads"] = historyTestConnection(thread1) @@ -212,6 +256,7 @@ func TestGraphQLHistoryReviewThreadCompleteness(t *testing.T) { switch req.Variables["id"] { case "PR_fixture": conn := historyTestConnection(thread2) + conn["totalCount"] = 2 if mode == "repeated-thread" { conn["pageInfo"] = map[string]any{"hasNextPage": true, "endCursor": "thread-first"} } @@ -247,9 +292,12 @@ func TestGraphQLHistoryReviewThreadCompleteness(t *testing.T) { t.Fatal(err) } item := batch.Items[0] - if pages != 2 || len(item.ReviewComments) != 2 || len(item.Reviews) != 1 || len(item.Comments) != 1 { + if pages != 2 || len(item.ReviewComments) != 2 || len(item.Reviews) != 1 || len(item.Comments) != 1 || len(item.ReviewThreads) != 2 { t.Fatalf("incomplete or duplicate conversation: pages=%d item=%+v", pages, item) } + if item.ReviewThreads[0]["isResolved"] != true || item.ReviewThreads[1]["isOutdated"] != true { + t.Fatal("review state lost") + } if item.Reviews[0]["state"] != "APPROVED" || item.Reviews[0]["body"] != "" || item.Comments[0]["body"] != "discussion" { t.Fatal("review metadata or discussion lost") } diff --git a/internal/github/quota_refresh_test.go b/internal/github/quota_refresh_test.go new file mode 100644 index 00000000..7a7d20e8 --- /dev/null +++ b/internal/github/quota_refresh_test.go @@ -0,0 +1,353 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" +) + +// A fresh REST reply can still contain an expired GraphQL resource. The +// credential-bound GraphQL observation must be refreshed, never invented. +func TestExpiredGraphQLRESTSnapshotRefreshesFromProvider(t *testing.T) { + rest, probes, content := 0, 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + rest++ + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + var request graphqlEnvelope + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + if request.Query == "query { viewer { id } }" { + content++ + } else { + probes++ + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"viewer": map[string]any{"id": "fixture"}, "rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}}}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "fixture", nil }}) + var result map[string]any + if err := c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &result); err != nil { + t.Fatal(err) + } + if rest != 1 || probes != 1 || content != 1 || historyString(historyMap(result["viewer"])["id"]) != "fixture" { + t.Fatalf("rest=%d probes=%d content=%d result=%v", rest, probes, content, result) + } +} + +func TestExpiredGraphQLQuotaRefreshFailsClosed(t *testing.T) { + for _, mode := range []string{"low", "stale", "missing_cost", "missing_remaining", "bad_reset", "missing_limit", "http", "partial", "decode", "rotation", "credential_failure", "cancel"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + probes, content, credentials := 0, 0, 0 + canary := "fixture-private-value" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + if strings.Contains(q.Query, "viewer") { + content++ + t.Error("content dispatched after failed refresh") + return + } + probes++ + if r.Header.Get("Authorization") != "Bearer fixture" { + t.Error("refresh credential changed") + } + switch mode { + case "http": + http.Error(w, canary, 401) + return + case "decode": + fmt.Fprint(w, "{") + return + case "cancel": + cancel() + // Wait until the client closes this request before the handler + // can return an empty 200 and race cancellation with EOF. + <-r.Context().Done() + return + } + rate := map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)} + switch mode { + case "low": + rate["remaining"] = 1500 + case "stale": + rate["resetAt"] = time.Now().UTC().Add(-time.Second).Format(time.RFC3339) + case "missing_cost": + delete(rate, "cost") + case "missing_remaining": + delete(rate, "remaining") + case "bad_reset": + rate["resetAt"] = canary + case "missing_limit": + delete(rate, "limit") + } + envelope := map[string]any{"data": map[string]any{"rateLimit": rate}} + if mode == "partial" { + envelope["errors"] = []any{map[string]any{"type": "FORBIDDEN", "message": canary, "path": []any{"rateLimit"}}} + } + json.NewEncoder(w).Encode(envelope) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { + credentials++ + if credentials == 3 { + if mode == "rotation" { + return "replacement", nil + } + if mode == "credential_failure" { + return "", errors.New(canary) + } + } + return "fixture", nil + }}) + var out map[string]any + err := c.doGraphQL(ctx, "query { viewer { id } }", nil, nil, &out) + wantProbes := 1 + if mode == "decode" { + wantProbes = 3 + } + if err == nil || probes != wantProbes || content != 0 || out != nil { + t.Fatalf("err=%v probes=%d content=%d out=%v", err, probes, content, out) + } + cause := safeHistoryCause(err) + code, _ := cause["code"].(string) + wants := map[string]string{"low": "quota_reserve_reached", "stale": "quota_snapshot_expired", "missing_cost": "quota_cost_missing", "missing_remaining": "quota_remaining_missing", "bad_reset": "quota_reset_invalid", "http": "http_status", "decode": "unexpected_eof", "rotation": "credential_changed", "credential_failure": "credential_provider_failed", "cancel": "cancelled"} + if want := wants[mode]; want != "" && code != want { + t.Fatalf("cause=%v want=%s", cause, want) + } + _, _, evidence := HistoryFailureDetails(err) + if strings.Contains(string(evidence), canary) || !strings.Contains(string(evidence), "graphql_quota_refresh") { + t.Fatalf("unsafe or missing refresh evidence: %s", evidence) + } + }) + } +} + +func TestExpiredGraphQLQuotaRefreshBindsRotatedCredential(t *testing.T) { + var probes, content []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + credential := r.Header.Get("Authorization") + if r.URL.Path == "/rate_limit" { + probes = append(probes, credential) + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + if strings.Contains(q.Query, "viewer") { + content = append(content, credential) + } else { + probes = append(probes, credential) + } + fmt.Fprint(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":19000,"resetAt":"2099-01-01T00:00:00Z"}}}`) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: sequenceTokenProvider(t, "first", "second", "second", "second")}) + // Observations from the old credential must not constrain or admit the new one. + c.reserve.bindGraphQLToken("first") + c.reserve.observeGraphQL(RateLimitSnapshot{Resource: "graphql", Remaining: 0, ResetAt: time.Now().Add(time.Hour)}, time.Now()) + var out map[string]any + if err := c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &out); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(probes, []string{"Bearer first", "Bearer second", "Bearer second"}) || !reflect.DeepEqual(content, []string{"Bearer second"}) { + t.Fatalf("wrong binding: probes=%v content=%v", probes, content) + } +} + +func TestExpiredGraphQLQuotaRefreshConcurrentReserve(t *testing.T) { + var active, peak, probes, content atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := active.Add(1) + defer active.Add(-1) + for p := peak.Load(); n > p; p = peak.Load() { + if peak.CompareAndSwap(p, n) { + break + } + } + if r.URL.Path == "/rate_limit" { + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + balance := 19999 + if strings.Contains(q.Query, "viewer") { + content.Add(1) + } else if probes.Add(1) == 1 { + balance = 1501 + } + fmt.Fprintf(w, `{"data":{"rateLimit":{"cost":1,"limit":20000,"remaining":%d,"resetAt":"2099-01-01T00:00:00Z"}}}`, balance) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "fixture", nil }}) + results := make(chan error, 8) + for range 8 { + go func() { + var out map[string]any + results <- c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &out) + }() + } + success, blocked := 0, 0 + for range 8 { + err := <-results + var floor *RateLimitReserveError + if err == nil { + success++ + } else if errors.As(err, &floor) { + blocked++ + } else { + t.Fatal(err) + } + } + if success != 1 || blocked != 7 || content.Load() != 1 || probes.Load() != 8 || peak.Load() != 1 { + t.Fatalf("success=%d blocked=%d content=%d probes=%d simultaneous=%d", success, blocked, content.Load(), probes.Load(), peak.Load()) + } +} + +func TestExpiredGraphQLQuotaRefreshRefusesRedirect(t *testing.T) { + var leaked atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { leaked.Add(1) })) + defer target.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + writeRateLimits(t, w, time.Now().Add(-time.Second), 19983, 19983) + return + } + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "fixture", nil }}) + var out map[string]any + err := c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &out) + var req *RequestError + if !errors.As(err, &req) || req.Status != 302 || leaked.Load() != 0 { + t.Fatalf("redirect: err=%v target=%d", err, leaked.Load()) + } +} + +func TestExpiredGraphQLQuotaRefreshPreservesPaginationValidation(t *testing.T) { + for _, mode := range []string{"complete", "short_page", "low_page", "stale_refresh"} { + t.Run(mode, func(t *testing.T) { + rest, probes, pages := 0, 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + rest++ + reset := time.Now().Add(time.Hour) + if rest == 3 { + reset = time.Now().Add(-time.Second) + } + writeRateLimits(t, w, reset, 19983, 19983) + return + } + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + rate := map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": "2099-01-01T00:00:00Z"} + data := map[string]any{"rateLimit": rate} + if strings.Contains(q.Query, "issueOrPullRequest") { + n := historyTestNode() + n["labels"] = map[string]any{"totalCount": 2, "nodes": []any{map[string]any{"id": "L1", "name": "one"}}, "pageInfo": map[string]any{"hasNextPage": true, "endCursor": "first"}} + data["repository"] = map[string]any{"databaseId": 42, "nameWithOwner": "fixture/repo", "n0": n} + } else if q.Variables["after"] != nil { + pages++ + if q.Variables["after"] != "first" || q.Variables["id"] != "PR_fixture" { + t.Error("continuation identity/cursor changed") + } + page := historyTestConnection(map[string]any{"id": "L2", "name": "two"}) + if mode == "short_page" { + page = historyTestConnection() + } + page["totalCount"] = 2 + data["node"] = map[string]any{"id": "PR_fixture", "labels": page} + } else if rest == 3 { + probes++ + if mode == "low_page" { + rate["remaining"] = 1501 + } + if mode == "stale_refresh" { + rate["resetAt"] = time.Now().UTC().Add(-time.Second).Format(time.RFC3339) + } + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, RateLimitReserve: 1500, TokenProvider: func(context.Context) (string, error) { return "fixture", nil }}) + batch, err := c.FetchGraphQLHistory(context.Background(), "fixture", "repo", []int{1}, nil) + if probes != 1 || rest != 3 { + t.Fatalf("unbounded refresh: probes=%d rest=%d", probes, rest) + } + if mode == "complete" { + if err != nil || pages != 1 || len(batch.Items) != 1 { + t.Fatalf("items=%d pages=%d err=%v", len(batch.Items), pages, err) + } + item := batch.Items[0] + if item.Thread["id"] != "PR_fixture" || item.Thread["body"] != "body" || item.Pull["id"] != "9007199254740993" || len(historyNodes(historyMap(item.Thread["_graphql"]), "labels")) != 2 { + t.Fatal("content/identity/membership changed") + } + } else { + if err == nil || len(batch.Items) != 0 { + t.Fatal("incomplete membership accepted") + } + if mode == "low_page" { + var floor *RateLimitReserveError + if !errors.As(err, &floor) || pages != 0 { + t.Fatalf("refreshed quota ignored page estimate: pages=%d err=%v", pages, err) + } + } + if mode == "stale_refresh" && pages != 0 { + t.Fatal("old valid observation masked invalid refresh") + } + } + }) + } +} + +func TestFreshGraphQLQuotaDoesNotTriggerRefresh(t *testing.T) { + for _, provider := range []bool{false, true} { + t.Run(fmt.Sprint(provider), func(t *testing.T) { + rest, content := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rate_limit" { + rest++ + writeRateLimits(t, w, time.Now().Add(time.Hour), 19000, 19000) + return + } + content++ + var q graphqlEnvelope + json.NewDecoder(r.Body).Decode(&q) + if q.Query != "query { viewer { id } }" { + t.Error("unexpected refresh") + } + fmt.Fprint(w, `{"data":{"viewer":{"id":"fixture"}}}`) + })) + defer server.Close() + opts := Options{BaseURL: server.URL, RateLimitReserve: 1500, Token: "fixture"} + if provider { + opts.TokenProvider = func(context.Context) (string, error) { return "fixture", nil } + } + c := New(opts) + var out map[string]any + if err := c.doGraphQL(context.Background(), "query { viewer { id } }", nil, nil, &out); err != nil || rest != 1 || content != 1 { + t.Fatalf("rest=%d content=%d err=%v", rest, content, err) + } + }) + } +} diff --git a/internal/github/review_state.go b/internal/github/review_state.go new file mode 100644 index 00000000..0de13454 --- /dev/null +++ b/internal/github/review_state.go @@ -0,0 +1,68 @@ +package github + +import ( + "context" + "fmt" + "strings" + "time" +) + +// ReviewStateItem contains only review evidence. It must never enter the +// full-thread projection, since canonical bodies/comments were not requested. +type ReviewStateItem struct { + Number int + NodeID string + RepositoryID string + RepositoryNodeID string + UpdatedAt string + Threads []map[string]any +} + +func (c *Client) FetchGraphQLReviewState(ctx context.Context, owner, repo string, numbers []int, reporter Reporter) ([]ReviewStateItem, error) { + if len(numbers) == 0 || len(numbers) > 8 { + return nil, fmt.Errorf("review-state query requires 1..8 numbers") + } + h := historySession{client: c, reporter: reporter, remaining: 20000} + if _, err := h.quota(ctx); err != nil { + return nil, err + } + var fields strings.Builder + for i, n := range numbers { + if n < 1 { + return nil, fmt.Errorf("invalid review-state number") + } + fmt.Fprintf(&fields, `n%d: issueOrPullRequest(number:%d) {__typename ... on PullRequest{id number updatedAt repository{nameWithOwner} %s}} `, i, n, historyConnection("reviewThreads", historyReviewThread, "")) + } + data, err := h.request(ctx, `query($owner:String!,$repo:String!){repository(owner:$owner,name:$repo){id databaseId nameWithOwner `+fields.String()+`} rateLimit{cost remaining limit used resetAt}}`, map[string]any{"owner": owner, "repo": repo}, 16) + if err != nil { + return nil, err + } + r := historyMap(data["repository"]) + if !strings.EqualFold(historyString(r["nameWithOwner"]), owner+"/"+repo) || historyString(r["id"]) == "" { + return nil, historyFailure("identity", 0, r, fmt.Errorf("review-state repository identity mismatch")) + } + repoID, validRepoID := historyInt(r["databaseId"]) + if !validRepoID || repoID <= 0 { + return nil, historyFailure("identity", 0, r, fmt.Errorf("missing repository database identity")) + } + out := make([]ReviewStateItem, 0, len(numbers)) + for i, n := range numbers { + node := historyMap(r[fmt.Sprint("n", i)]) + got, valid := historyInt(node["number"]) + if !valid || got != n || historyString(node["__typename"]) != "PullRequest" || historyString(node["id"]) == "" || !strings.EqualFold(historyString(historyMap(node["repository"])["nameWithOwner"]), owner+"/"+repo) { + return nil, historyFailure("identity", n, node, fmt.Errorf("review-state PR identity mismatch")) + } + updated := historyString(node["updatedAt"]) + if _, err = time.Parse(time.RFC3339Nano, updated); err != nil { + return nil, historyFailure("validation", n, node, fmt.Errorf("invalid review-state source time")) + } + if historyMap(node["reviewThreads"]) == nil { + return nil, historyFailure("validation", n, node, fmt.Errorf("missing history reviewThreads")) + } + if err = h.hydrateConnections(ctx, map[string]any{"__typename": "PullRequest", "id": node["id"], "reviewThreads": node["reviewThreads"]}); err != nil { + return nil, historyFailure("validation", n, node, err) + } + out = append(out, ReviewStateItem{Number: n, RepositoryID: fmt.Sprint(repoID), RepositoryNodeID: historyString(r["id"]), NodeID: historyString(node["id"]), UpdatedAt: updated, Threads: historyNodes(node, "reviewThreads")}) + } + return out, nil +} diff --git a/internal/github/review_state_test.go b/internal/github/review_state_test.go new file mode 100644 index 00000000..a94101a2 --- /dev/null +++ b/internal/github/review_state_test.go @@ -0,0 +1,166 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestReviewStateOnlyStrictPaginationAndReplyFields(t *testing.T) { + for _, mode := range []string{"complete", "null", "short", "cursor", "wrong_parent", "missing_state"} { + t.Run(mode, func(t *testing.T) { + conn := func(total int, more bool, cursor string, nodes ...any) map[string]any { + if nodes == nil { + nodes = []any{} + } + return map[string]any{"totalCount": total, "nodes": nodes, "pageInfo": map[string]any{"hasNextPage": more, "endCursor": cursor}} + } + comment := func(id string, reply any) map[string]any { + return map[string]any{"id": id, "__typename": "PullRequestReviewComment", "fullDatabaseId": id, "body": "inline retained", "author": map[string]any{"id": "actor", "login": "fixture", "__typename": "User"}, "url": "https://github.com/fixture/repo/pull/7#comment", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-02T00:00:00Z", "replyTo": reply} + } + thread := func(id string, comments any) map[string]any { + return map[string]any{"id": id, "__typename": "PullRequestReviewThread", "isResolved": true, "isOutdated": false, "viewerCanResolve": false, "viewerCanUnresolve": true, "viewerCanReply": true, "path": "file.go", "line": 9, "comments": comments} + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Query string + Variables map[string]any + } + json.NewDecoder(r.Body).Decode(&req) + data := map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": 19000, "resetAt": time.Now().UTC().Add(time.Hour).Format(time.RFC3339)}} + if strings.Contains(req.Query, "issueOrPullRequest") { + if strings.Contains(req.Query, "title") || strings.Contains(req.Query, "labels") || strings.Contains(req.Query, "reviews(") { + t.Error("unrelated history requested") + } + first := thread("RT1", conn(2, true, "c1", comment("C1", nil))) + if mode == "missing_state" { + delete(first, "isResolved") + } + node := map[string]any{"__typename": "PullRequest", "id": "PR7", "number": 7, "updatedAt": "2026-01-02T00:00:00Z", "repository": map[string]any{"nameWithOwner": "fixture/repo"}, "reviewThreads": conn(2, true, "rt1", first)} + if mode == "short" { + node["reviewThreads"] = conn(2, false, "end", first) + } + var selected any = node + if mode == "null" { + selected = nil + } + data["repository"] = map[string]any{"id": "R1", "databaseId": 1, "nameWithOwner": "fixture/repo", "n0": selected} + } else if req.Variables["id"] == "PR7" { + parent := "PR7" + if mode == "wrong_parent" { + parent = "OTHER" + } + data["node"] = map[string]any{"id": parent, "reviewThreads": conn(2, mode == "cursor", "rt1", thread("RT2", conn(0, false, "")))} + } else if req.Variables["id"] == "RT1" { + data["node"] = map[string]any{"id": "RT1", "comments": conn(2, false, "c2", comment("C2", map[string]any{"id": "C1", "fullDatabaseId": "C1"}))} + } + json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer server.Close() + items, err := New(Options{BaseURL: server.URL}).FetchGraphQLReviewState(context.Background(), "fixture", "repo", []int{7}, nil) + if mode != "complete" { + if err == nil { + t.Fatal("incomplete evidence accepted", mode) + } + return + } + if err != nil || len(items) != 1 || len(items[0].Threads) != 2 { + t.Fatalf("%+v %v", items, err) + } + comments := historyNodes(items[0].Threads[0], "comments") + if len(comments) != 2 || historyMap(comments[1]["replyTo"])["id"] != "C1" || comments[0]["body"] != "inline retained" || historyMap(comments[0]["author"])["login"] != "fixture" { + t.Fatal("reply/body/author evidence lost") + } + }) + } +} +func TestReviewStateResponseByteLimitRejectsWithoutPartialResult(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"data":{"padding":"` + strings.Repeat("x", 8192) + `"}}`)) + })) + defer server.Close() + var out map[string]any + err := New(Options{BaseURL: server.URL, GraphQLResponseLimit: 1024}).doGraphQL(context.Background(), "query { rateLimit {cost} }", nil, nil, &out) + class, _, _ := HistoryFailureDetails(err) + if err == nil || class != "response_size" || out != nil { + t.Fatalf("oversize response accepted: %v %s", err, class) + } +} + +func TestReviewStateGraphQLGuardRequiresObservedCredentialQuota(t *testing.T) { + for _, mode := range []string{"normal", "low_probe", "low_page", "rotation", "expired", "unprobed"} { + t.Run(mode, func(t *testing.T) { + calls, rest, tokenCalls := 0, 0, 0 + reset := time.Now().UTC().Add(time.Hour) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/graphql" { + rest++ + t.Error("redundant REST quota request") + http.Error(w, "unexpected", 500) + return + } + calls++ + remaining := 19000 + if mode == "low_probe" || mode == "low_page" && calls == 2 { + remaining = 3000 + } + if mode == "expired" { + reset = time.Now().Add(-time.Second) + } + json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"rateLimit": map[string]any{"cost": 1, "limit": 20000, "remaining": remaining, "resetAt": reset.Format(time.RFC3339)}}}) + })) + defer server.Close() + c := New(Options{BaseURL: server.URL, GraphQLQuotaGuard: true, RateLimitReserve: 3000, TokenProvider: func(context.Context) (string, error) { + tokenCalls++ + if mode == "rotation" && tokenCalls > 1 { + return "changed-fixture", nil + } + return "fixture", nil + }}) + h := historySession{client: c, remaining: 20000} + if mode != "unprobed" { + if _, err := h.quota(context.Background()); err != nil { + t.Fatal(err) + } + } + _, err := h.request(context.Background(), `query { node(id:"fixture"){id} rateLimit{cost remaining limit resetAt}}`, nil, 16) + if mode == "normal" || mode == "low_page" { + if err != nil { + t.Fatal(err) + } + _, err = h.request(context.Background(), `query { node(id:"page"){id} rateLimit{cost remaining limit resetAt}}`, nil, 16) + } + if mode == "normal" { + if err != nil || calls != 3 { + t.Fatalf("calls=%d err=%v", calls, err) + } + } else if err == nil { + t.Fatal("unguarded content accepted") + } + expected := 1 + if mode == "normal" { + expected = 3 + } + if mode == "low_page" { + expected = 2 + } + if mode == "unprobed" { + expected = 0 + } + if calls != expected || rest != 0 { + t.Fatalf("calls=%d rest=%d", calls, rest) + } + if mode == "low_probe" || mode == "low_page" { + var reserve *RateLimitReserveError + if !errors.As(err, &reserve) { + t.Fatalf("not reserve error: %v", err) + } + } + }) + } +} diff --git a/internal/github/review_threads.go b/internal/github/review_threads.go index 1d8cf1f2..390f3ab5 100644 --- a/internal/github/review_threads.go +++ b/internal/github/review_threads.go @@ -3,8 +3,10 @@ package github import ( "bytes" "context" + "crypto/sha256" "encoding/json" "fmt" + "io" "net/http" "strings" ) @@ -112,10 +114,14 @@ type graphqlEnvelope struct { } type graphqlResponseEnvelope struct { - Data json.RawMessage `json:"data"` - Errors []struct { - Message string `json:"message"` - } `json:"errors"` + Data json.RawMessage `json:"data"` + Errors []graphqlResponseError `json:"errors"` +} + +type graphqlResponseError struct { + Message string `json:"message"` + Type string `json:"type"` + Path []any `json:"path"` } // ListPullReviewThreads fetches GitHub's review-thread graph for a pull request. @@ -235,21 +241,40 @@ func (c *Client) doGraphQL(ctx context.Context, query string, variables map[stri return fmt.Errorf("encode graphql request: %w", err) } var envelope graphqlResponseEnvelope - if err := c.doJSON(ctx, http.MethodPost, c.graphQLURL, bytes.NewReader(payload), reporter, &envelope); err != nil { - return err + response, err := c.do(ctx, http.MethodPost, c.graphQLURL, bytes.NewReader(payload), reporter) + if err != nil { + return requestFailureAt("graphql_request", "", err) + } + defer response.Body.Close() + reader := &historyResponseReader{reader: response.Body, hash: sha256.New()} + var input io.Reader = reader + if c.graphQLResponseLimit > 0 { + input = io.LimitReader(reader, c.graphQLResponseLimit+1) + } + decodeErr := decodeJSON(input, &envelope) + reporter.Printf("[github] graphql bytes %d", reader.read) + if c.graphQLResponseLimit > 0 && reader.read > c.graphQLResponseLimit { + failure := reader.failure(fmt.Errorf("review-state response exceeded byte limit")) + failure.Stage = "response_size" + return failure + } + if err := decodeErr; err != nil { + return reader.failure(fmt.Errorf("decode github response: %w", err)) } if len(envelope.Errors) > 0 { messages := make([]string, 0, len(envelope.Errors)) for _, graphqlErr := range envelope.Errors { messages = append(messages, graphqlErr.Message) } - return fmt.Errorf("github graphql: %s", strings.Join(messages, "; ")) + var rejected any + _ = decodeJSON(bytes.NewReader(envelope.Data), &rejected) + return graphQLRejection(rejected, envelope.Errors, fmt.Errorf("github graphql: %s", strings.Join(messages, "; "))) } if len(envelope.Data) == 0 || string(envelope.Data) == "null" { - return fmt.Errorf("github graphql response missing data") + return historyFailure("missing_data", 0, nil, fmt.Errorf("github graphql response missing data")) } if err := decodeJSON(bytes.NewReader(envelope.Data), out); err != nil { - return fmt.Errorf("decode github graphql data: %w", err) + return historyFailure("response_decode", 0, nil, fmt.Errorf("decode github graphql data: %w", err)) } return nil } diff --git a/internal/headlinemetrics/github.go b/internal/headlinemetrics/github.go new file mode 100644 index 00000000..6c2c67a8 --- /dev/null +++ b/internal/headlinemetrics/github.go @@ -0,0 +1,171 @@ +package headlinemetrics + +import ( + "context" + "errors" + "fmt" + "math" + "net/http" + "strings" + "time" + + "github.com/openclaw/gitcrawl/internal/github" +) + +// GitHubCollector uses the same HTTP client and credential provider as native +// Gitcrawl commands. Clone traffic is optional and requires authentication. +func GitHubCollector(client *github.Client, trafficEnabled bool) Collector { + return func(ctx context.Context, c Config, ts string) ([]Row, error) { + if err := c.Validate(); err != nil { + return nil, err + } + observed, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + return nil, errors.New("invalid collection timestamp") + } + rows := []Row{} + failed := false + for _, t := range c.Targets { + if err := ctx.Err(); err != nil { + return rows, err + } + repo, err := client.RepositoryHeadline(ctx, t.Target) + if stopCollection(ctx, err) { + return rows, errors.Join(err, ctx.Err()) + } + if err != nil { + failed = true + repo = github.RepositoryHeadline{} + } + prs, err := client.OpenPullCount(ctx, t.Target) + if err != nil || prs.Incomplete { + failed = true + prs = github.PullCount{} + } + var issues *float64 + if validCount(repo.OpenIssuesAndPulls) != nil && validCount(prs.Count) != nil { + issues = Value(*repo.OpenIssuesAndPulls - *prs.Count) + } + for _, m := range []struct { + Name string + Value *float64 + }{ + {"stars", repo.Stars}, {"forks", repo.Forks}, {"watchers", repo.Watchers}, {"open_prs", prs.Count}, {"open_issues", issues}, + } { + value := validCount(m.Value) + if value == nil { + failed = true + } + rows = append(rows, Counter(t, m.Name, value, ts, "github_rest")) + } + if stopCollection(ctx, err) { + return rows, errors.Join(err, ctx.Err()) + } + if trafficEnabled { + traffic, err := client.CloneTraffic(ctx, t.Target) + if stopCollection(ctx, err) { + return rows, errors.Join(err, ctx.Err()) + } + var requestErr *github.RequestError + optional := errors.As(err, &requestErr) && (requestErr.Status == 403 || requestErr.Status == 404) + if err != nil && !optional { + failed = true + } + if err == nil { + for _, v := range traffic.Clones { + day, err := time.Parse(time.RFC3339Nano, v.Timestamp) + if err != nil { + failed = true + continue + } + day = day.UTC().Truncate(24 * time.Hour) + if !day.Before(observed.UTC().Truncate(24 * time.Hour)) { + continue + } + value := validCount(v.Count) + if value == nil { + failed = true + } + r := Counter(t, "clones", value, day.Add(24*time.Hour-time.Millisecond).Format(time.RFC3339Nano), "github_traffic") + r.Kind = "daily" + r.ObservedAt = ts + rows = append(rows, r) + } + } + } + // Follow all release pages. Do not silently truncate stable release history. + for page := 1; ; page++ { + releases, err := client.ReleasePage(ctx, t.Target, page) + if stopCollection(ctx, err) { + return rows, errors.Join(err, ctx.Err()) + } + if err != nil { + failed = true + break + } + for _, v := range releases { + if v.Draft || v.Prerelease { + continue + } + at := v.Published + if at == "" { + at = v.Created + } + when, err := time.Parse(time.RFC3339Nano, at) + label := v.Name + if label == "" { + label = v.Tag + } + if err != nil || v.ID <= 0 || label == "" { + failed = true + continue + } + rows = append(rows, Row{Type: "event", ID: fmt.Sprintf("github-release:%d", v.ID), Entity: t.Entity, Target: t.Target, Kind: "release", TS: when.UTC().Format(time.RFC3339Nano), ObservedAt: ts, Provenance: "github_releases", Label: label, URL: v.URL}) + } + if len(releases) < 100 { + break + } + } + } + if err := ctx.Err(); err != nil { + return rows, err + } + if failed { + return rows, errors.New("one or more GitHub metrics unavailable") + } + return rows, nil + } +} + +func validCount(value *float64) *float64 { + if value == nil || math.Trunc(*value) != *value { + return nil + } + return Value(*value) +} + +// Permission-denied traffic is optional; exhausted quota and cancellation stop +// all acquisition after the shared HTTP client's bounded retry. +func stopCollection(ctx context.Context, err error) bool { + // Retain a successfully decoded response even if cancellation arrived as + // it finished; the next request uses the canceled context. + if err == nil { + return false + } + if ctx.Err() != nil { + return true + } + var reserve *github.RateLimitReserveError + if errors.As(err, &reserve) { + return true + } + var response *github.RequestError + if !errors.As(err, &response) { + return false + } + return response.Status == http.StatusTooManyRequests || + (response.Status == http.StatusForbidden && + (response.Headers.Get("X-RateLimit-Remaining") == "0" || + response.Headers.Get("Retry-After") != "" || + strings.Contains(strings.ToLower(response.Body), "rate limit"))) +} diff --git a/internal/headlinemetrics/github_test.go b/internal/headlinemetrics/github_test.go new file mode 100644 index 00000000..60920f13 --- /dev/null +++ b/internal/headlinemetrics/github_test.go @@ -0,0 +1,310 @@ +package headlinemetrics + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/openclaw/gitcrawl/internal/github" +) + +func fixtureCollector(t *testing.T, override func(http.ResponseWriter, *http.Request) bool, traffic bool) Collector { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + t.Errorf("unexpected mutation %s", r.Method) + } + if r.Header.Get("Authorization") != "Bearer fixture-token" { + t.Error("missing normal client authorization") + } + if override != nil && override(w, r) { + return + } + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/search/issues": + if !strings.Contains(r.URL.Query().Get("q"), "is:pr is:open") { + t.Error("incorrect pull query") + } + fmt.Fprint(w, `{"total_count":3,"incomplete_results":false}`) + case strings.HasSuffix(r.URL.Path, "/traffic/clones"): + fmt.Fprint(w, `{"clones":[{"timestamp":"2026-09-14T00:00:00Z","count":0},{"timestamp":"2026-09-15T00:00:00Z","count":7}]}`) + case strings.HasSuffix(r.URL.Path, "/releases"): + fmt.Fprint(w, `[{"id":42,"published_at":"2026-09-14T15:00:00Z","name":"Stable","tag_name":"v1","html_url":"https://example.test/v1"},{"id":43,"draft":true},{"id":44,"prerelease":true}]`) + case strings.HasPrefix(r.URL.Path, "/repos/"): + fmt.Fprint(w, `{"stargazers_count":100,"forks_count":5,"watchers_count":100,"subscribers_count":7,"open_issues_count":11}`) + default: + t.Errorf("unexpected API %s", r.URL.String()) + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + return GitHubCollector(github.New(github.Options{BaseURL: server.URL, Token: "fixture-token", HTTPClient: server.Client()}), traffic) +} + +func TestGitHubMetricsUseActualWatchersSeparateIssuesAndCompletedDays(t *testing.T) { + c := testConfig(t) + rows, err := fixtureCollector(t, nil, true)(context.Background(), c, "2026-09-15T01:00:00Z") + if err != nil { + t.Fatal(err) + } + if len(rows) != 14 { + t.Fatalf("rows=%d", len(rows)) + } + for _, target := range c.Targets { + values := map[string]float64{} + events := 0 + for _, r := range rows { + if r.Target != target.Target { + continue + } + if r.Type == "event" { + events++ + if r.Label != "Stable" { + t.Fatal(r) + } + continue + } + if r.Value == nil { + t.Fatal("unexpected unknown", r) + } + values[r.Metric] = *r.Value + if r.Kind == "daily" && (r.TS != "2026-09-14T23:59:59.999Z" || r.ObservedAt != "2026-09-15T01:00:00Z") { + t.Fatal("incorrect UTC day", r) + } + } + if values["stars"] != 100 || values["forks"] != 5 || values["watchers"] != 7 || values["open_prs"] != 3 || values["open_issues"] != 8 || values["clones"] != 0 || events != 1 { + t.Fatalf("%s values=%v events=%d", target.Target, values, events) + } + } +} + +func TestMissingOrIncompletePRCountNeverFallsBackToCombinedIssues(t *testing.T) { + for _, body := range []string{`{}`, `{"total_count":3,"incomplete_results":true}`, `{"total_count":-1}`, `{"total_count":30}`, `{"total_count":3,"incomplete_results":"invalid"}`} { + t.Run(body, func(t *testing.T) { + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path == "/search/issues" { + fmt.Fprint(w, body) + return true + } + return false + }, false) + rows, err := collect(context.Background(), testConfig(t), "2026-09-15T01:00:00Z") + if err == nil { + t.Fatal("missing count reported healthy") + } + for _, r := range rows { + if r.Metric == "open_issues" && r.Value != nil { + t.Fatal("combined count used", r) + } + } + }) + } +} + +func TestOptionalCloneTrafficAndOtherPartialFailures(t *testing.T) { + for _, status := range []int{403, 404} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if strings.HasSuffix(r.URL.Path, "/traffic/clones") { + w.WriteHeader(status) + fmt.Fprint(w, "private response") + return true + } + return false + }, true) + rows, err := collect(context.Background(), testConfig(t), "2026-09-15T01:00:00Z") + if err != nil { + t.Fatal(err) + } + if len(rows) != 12 { + t.Fatalf("unexpected optional traffic rows %d", len(rows)) + } + }) + } + cases := []struct { + name string + match func(*http.Request) bool + body string + missing string + }{ + {"missing watchers", func(r *http.Request) bool { return r.URL.Path == "/repos/openclaw/openclaw" }, `{"stargazers_count":1,"forks_count":1,"watchers_count":900,"open_issues_count":10}`, "watchers"}, + {"malformed repo", func(r *http.Request) bool { return r.URL.Path == "/repos/openclaw/openclaw" }, `{"stargazers_count":12,`, "stars"}, + {"malformed traffic", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/traffic/clones") }, `{"clones":[{"timestamp":"bad","count":3}]}`, ""}, + {"missing clone count", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/traffic/clones") }, `{"clones":[{"timestamp":"2026-09-14T00:00:00Z"}]}`, "clones"}, + {"invalid stable release", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/releases") }, `[{"id":9,"published_at":"bad","name":"v1"}]`, ""}, + {"malformed releases", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/releases") }, `{`, ""}, + {"null releases", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/releases") }, `null`, ""}, + {"missing traffic", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/traffic/clones") }, `{}`, ""}, + {"null traffic", func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/traffic/clones") }, `{"clones":null}`, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if tc.match(r) { + fmt.Fprint(w, tc.body) + return true + } + return false + }, true) + rows, err := collect(context.Background(), testConfig(t), "2026-09-15T01:00:00Z") + if err == nil || len(rows) < 10 { + t.Fatalf("partial=%d,%v", len(rows), err) + } + if tc.missing != "" { + found := false + for _, r := range rows { + if r.Target == "openclaw/openclaw" && r.Metric == tc.missing { + found = true + if r.Value != nil { + t.Fatal("invalid value retained", r) + } + } + } + if !found { + t.Fatal("missing unknown observation") + } + } + }) + } +} + +func TestReleasePaginationBeyondFivePagesAndNoUnauthenticatedTraffic(t *testing.T) { + pages := 0 + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if strings.HasSuffix(r.URL.Path, "/traffic/clones") { + t.Error("unauthenticated traffic request") + return true + } + if !strings.HasSuffix(r.URL.Path, "/releases") { + return false + } + pages++ + size := 100 + if pages == 6 { + size = 1 + } + releases := make([]github.Release, size) + for i := range releases { + releases[i] = github.Release{ID: int64(pages*100 + i), Created: "2026-09-14T00:00:00Z", Tag: "v1"} + } + if err := json.NewEncoder(w).Encode(releases); err != nil { + t.Error(err) + } + return true + }, false) + c := testConfig(t) + c.Targets = c.Targets[:1] + rows, err := collect(context.Background(), c, "2026-09-15T01:00:00Z") + if err != nil { + t.Fatal(err) + } + if pages != 6 || len(rows) != 506 { + t.Fatalf("pages=%d rows=%d", pages, len(rows)) + } + if _, err := collect(context.Background(), c, "bad"); err == nil { + t.Fatal("invalid clock accepted") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := collect(ctx, c, "2026-09-15T01:00:00Z"); err == nil { + t.Fatal("cancellation accepted") + } +} + +func TestQuotaFailureStopsCollectionAndRetainsCompletedReads(t *testing.T) { + for _, endpoint := range []string{"/repos/openclaw/openclaw", "/search/issues", "/repos/openclaw/openclaw/traffic/clones", "/repos/openclaw/openclaw/releases"} { + t.Run(endpoint, func(t *testing.T) { + limited := false + requestsAfter := 0 + collect := fixtureCollector(t, func(w http.ResponseWriter, r *http.Request) bool { + if limited { + requestsAfter++ + } + if r.URL.Path == endpoint { + limited = true + w.Header().Set("X-RateLimit-Remaining", "0") + w.WriteHeader(http.StatusForbidden) + return true + } + return false + }, true) + c := testConfig(t) + result, err := Execute(context.Background(), "collect", c, func(ctx context.Context, cfg Config, _ string) ([]Row, error) { + return collect(ctx, cfg, "2026-09-15T01:00:00Z") + }, nil) + if err == nil || result.OK || requestsAfter != 0 { + t.Fatalf("quota stop = %+v, %v; extra requests = %d", result, err, requestsAfter) + } + want := 5 + if endpoint == "/repos/openclaw/openclaw" { + want = 0 + } else if strings.HasSuffix(endpoint, "/releases") { + want = 6 + } + if result.RowsWritten != want { + t.Fatalf("retained %d rows; want %d", result.RowsWritten, want) + } + }) + } +} + +type metricsRoundTripper func(*http.Request) (*http.Response, error) + +func (f metricsRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type cancelMetricsBody struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (b cancelMetricsBody) Close() error { + err := b.ReadCloser.Close() + b.cancel() + return err +} + +func TestCancellationRetainsTheResponseJustRead(t *testing.T) { + for _, endpoint := range []string{"/repos/openclaw/openclaw", "/repos/openclaw/openclaw/traffic/clones", "/repos/openclaw/openclaw/releases"} { + t.Run(endpoint, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := github.New(github.Options{BaseURL: "https://example.test", HTTPClient: &http.Client{Transport: metricsRoundTripper(func(r *http.Request) (*http.Response, error) { + if err := r.Context().Err(); err != nil { + return nil, err + } + payload := `{"stargazers_count":4,"forks_count":0,"subscribers_count":2,"open_issues_count":1}` + switch r.URL.Path { + case "/search/issues": + payload = `{"total_count":0}` + case "/repos/openclaw/openclaw/traffic/clones": + payload = `{"clones":[{"timestamp":"2026-09-14T00:00:00Z","count":3}]}` + case "/repos/openclaw/openclaw/releases": + payload = `[{"id":1,"published_at":"2026-09-14T00:00:00Z","tag_name":"v1"}]` + } + var body io.ReadCloser = io.NopCloser(strings.NewReader(payload)) + if r.URL.Path == endpoint { + body = cancelMetricsBody{body, cancel} + } + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body, Request: r}, nil + })}}) + c := testConfig(t) + c.Targets = c.Targets[:1] + rows, err := GitHubCollector(client, true)(ctx, c, "2026-09-15T01:00:00Z") + want := 5 + if strings.HasSuffix(endpoint, "/clones") { + want = 6 + } else if strings.HasSuffix(endpoint, "/releases") { + want = 7 + } + if err == nil || len(rows) != want || rows[0].Value == nil || *rows[0].Value != 4 { + t.Fatalf("completed reads = %+v, %v; want %d rows", rows, err, want) + } + }) + } +} diff --git a/internal/headlinemetrics/lock.go b/internal/headlinemetrics/lock.go new file mode 100644 index 00000000..4a7ea3af --- /dev/null +++ b/internal/headlinemetrics/lock.go @@ -0,0 +1,55 @@ +package headlinemetrics + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +var errWriterBusy = errors.New("another metrics writer is running") + +// The persistent sidecar must never be unlinked: all writers lock the same inode. +// Closing the file (including process exit) releases ownership without stale PIDs. +func acquireWriter(database string) (*os.File, error) { + if err := os.MkdirAll(filepath.Dir(database), 0700); err != nil { + return nil, err + } + path := filepath.Clean(database) + ".writer.lock" + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if errors.Is(err, os.ErrExist) { + info, statErr := os.Lstat(path) + if statErr != nil { + return nil, statErr + } + if !info.Mode().IsRegular() { + return nil, errors.New("metrics writer lock must be a regular file") + } + f, err = os.OpenFile(path, os.O_RDWR, 0600) + } + if err != nil { + return nil, err + } + fail := func(err error) (*os.File, error) { + _ = f.Close() + return nil, err + } + info, err := f.Stat() + if err != nil { + return fail(err) + } + entry, err := os.Lstat(path) + if err != nil { + return fail(err) + } + if !info.Mode().IsRegular() || !entry.Mode().IsRegular() || !os.SameFile(info, entry) { + return fail(errors.New("metrics writer lock path changed or is not regular")) + } + if err := lockWriterFile(f); err != nil { + return fail(fmt.Errorf("lock metrics database: %w", err)) + } + if err := f.Chmod(0600); err != nil { + return fail(err) + } + return f, nil +} diff --git a/internal/headlinemetrics/lock_other.go b/internal/headlinemetrics/lock_other.go new file mode 100644 index 00000000..3f6ca899 --- /dev/null +++ b/internal/headlinemetrics/lock_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows + +package headlinemetrics + +import ( + "errors" + "os" +) + +func lockWriterFile(*os.File) error { + return errors.New("metrics writer locking is unsupported on this platform") +} + +func checkSingleLink(*os.File) error { + return errors.New("metrics file identity checks are unsupported on this platform") +} diff --git a/internal/headlinemetrics/lock_test.go b/internal/headlinemetrics/lock_test.go new file mode 100644 index 00000000..caf87cd0 --- /dev/null +++ b/internal/headlinemetrics/lock_test.go @@ -0,0 +1,152 @@ +package headlinemetrics + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +func TestExecuteSerializesWriterProcesses(t *testing.T) { + for _, mode := range []string{"complete", "kill"} { + t.Run(mode, func(t *testing.T) { + c := testConfig(t) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestMetricsWriterProcessHelper$", "--", c.Database) + out, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + in, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + waited := false + defer func() { + if !waited { + cancel() + _ = cmd.Wait() + } + }() + line, err := bufio.NewReader(out).ReadString('\n') + if err != nil || line != "collecting\n" { + t.Fatalf("child = %q, %v, %s", line, err, stderr.String()) + } + first, err := os.Stat(c.Database + ".writer.lock") + if err != nil { + t.Fatal(err) + } + collector := func(context.Context, Config, string) ([]Row, error) { + t.Fatal("overlapping collector reached provider") + return nil, nil + } + for _, command := range []string{"collect", "import"} { + result, err := Execute(ctx, command, c, collector, strings.NewReader("invalid input")) + if !errors.Is(err, errWriterBusy) || result.RowsWritten != 0 { + t.Fatalf("overlap %s = %+v, %v", command, result, err) + } + } + if result, err := Execute(ctx, "status", c, nil, nil); err != nil || !result.OK || result.Observations != 0 { + t.Fatalf("concurrent reader = %+v, %v", result, err) + } + // A different metrics database has independent ownership. + if _, err := Execute(ctx, "import", testConfig(t), nil, strings.NewReader("")); err != nil { + t.Fatal(err) + } + if mode == "kill" { + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + } else if _, err := in.Write([]byte("finish\n")); err != nil { + t.Fatal(err) + } + _ = in.Close() + err = cmd.Wait() + waited = true + if (mode == "complete") != (err == nil) { + t.Fatalf("child exit = %v: %s", err, stderr.String()) + } + if _, err := Execute(ctx, "import", c, nil, strings.NewReader("")); err != nil { + t.Fatalf("writer after %s = %v", mode, err) + } + last, err := os.Stat(c.Database + ".writer.lock") + if err != nil || !os.SameFile(first, last) || last.Mode().Perm() != 0600 { + t.Fatalf("persistent private lock = %v, %v", last, err) + } + result, err := Execute(ctx, "status", c, nil, nil) + want := 0 + if mode == "complete" { + want = 1 + } + if err != nil || result.Observations != want { + t.Fatalf("retained observations = %+v, %v", result, err) + } + }) + } +} + +func TestMetricsWriterProcessHelper(t *testing.T) { + i := slices.Index(os.Args, "--") + if i < 0 { + return + } + c := Config{Database: os.Args[i+1], Targets: []Target{{Entity: "OpenClaw", Target: "openclaw/openclaw"}}} + _, err := Execute(context.Background(), "collect", c, func(context.Context, Config, string) ([]Row, error) { + fmt.Println("collecting") + if _, err := bufio.NewReader(os.Stdin).ReadString('\n'); err != nil { + return nil, err + } + return []Row{testRow()}, nil + }, nil) + if err != nil { + t.Fatal(err) + } +} + +func TestMetricsWriterRejectsUnsafeLockPaths(t *testing.T) { + for _, kind := range []string{"directory", "symlink", "hardlink"} { + t.Run(kind, func(t *testing.T) { + c := testConfig(t) + path := c.Database + ".writer.lock" + target := filepath.Join(filepath.Dir(c.Database), "unrelated") + if err := os.WriteFile(target, []byte("retained"), 0644); err != nil { + t.Fatal(err) + } + var err error + switch kind { + case "directory": + err = os.Mkdir(path, 0700) + case "symlink": + err = os.Symlink(target, path) + case "hardlink": + err = os.Link(target, path) + } + if err != nil { + t.Fatal(err) + } + if _, err := Execute(context.Background(), "import", c, nil, strings.NewReader("")); err == nil { + t.Fatal("unsafe lock accepted") + } + if _, err := os.Stat(c.Database); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("database initialized before lock: %v", err) + } + if got, err := os.ReadFile(target); err != nil || string(got) != "retained" { + t.Fatalf("unrelated file = %q, %v", got, err) + } + }) + } +} diff --git a/internal/headlinemetrics/lock_unix.go b/internal/headlinemetrics/lock_unix.go new file mode 100644 index 00000000..b5d7d848 --- /dev/null +++ b/internal/headlinemetrics/lock_unix.go @@ -0,0 +1,33 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package headlinemetrics + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func checkSingleLink(f *os.File) error { + var info unix.Stat_t + if err := unix.Fstat(int(f.Fd()), &info); err != nil { + return err + } + if info.Nlink != 1 { + return errors.New("metrics files must not have hardlink aliases") + } + return nil +} + +func lockWriterFile(f *os.File) error { + if err := checkSingleLink(f); err != nil { + return err + } + + err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { + return errWriterBusy + } + return err +} diff --git a/internal/headlinemetrics/lock_windows.go b/internal/headlinemetrics/lock_windows.go new file mode 100644 index 00000000..4168dac8 --- /dev/null +++ b/internal/headlinemetrics/lock_windows.go @@ -0,0 +1,33 @@ +package headlinemetrics + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func checkSingleLink(f *os.File) error { + handle := windows.Handle(f.Fd()) + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + return err + } + if info.NumberOfLinks != 1 { + return errors.New("metrics files must not have hardlink aliases") + } + return nil +} + +func lockWriterFile(f *os.File) error { + if err := checkSingleLink(f); err != nil { + return err + } + handle := windows.Handle(f.Fd()) + + err := windows.LockFileEx(handle, windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &windows.Overlapped{}) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return errWriterBusy + } + return err +} diff --git a/internal/headlinemetrics/metrics.go b/internal/headlinemetrics/metrics.go new file mode 100644 index 00000000..815b7156 --- /dev/null +++ b/internal/headlinemetrics/metrics.go @@ -0,0 +1,562 @@ +// Package headlinemetrics owns a separate, append-only repository metrics store. +// It rejects thread archives before writes and never starts embedding/model work. +package headlinemetrics + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/openclaw/crawlkit/store" +) + +const Owner = "gitcrawl" + +var ErrPartialCollection = errors.New("one or more GitHub metrics unavailable; successful values and unknown observations were retained") + +type Target struct { + Entity string `json:"entity"` + Target string `json:"target"` +} +type Config struct { + Database string `json:"database"` + Targets []Target `json:"targets"` + CookieJar string `json:"cookieJar,omitempty"` + TokenEnv string `json:"tokenEnv,omitempty"` +} +type Row struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + Entity string `json:"entity"` + Target string `json:"target"` + Metric string `json:"metric,omitempty"` + Kind string `json:"kind"` + TS string `json:"ts"` + Value *float64 `json:"value"` + ObservedAt string `json:"observed_at"` + Provenance string `json:"provenance"` + Label string `json:"label,omitempty"` + URL string `json:"url,omitempty"` +} +type Collector func(context.Context, Config, string) ([]Row, error) + +type Result struct { + Source string `json:"source"` + Command string `json:"command"` + RowsWritten int `json:"rows_written"` + OK bool `json:"ok"` + Observations int `json:"observations,omitempty"` + Events int `json:"events,omitempty"` + LastObserved *string `json:"last_observed,omitempty"` +} + +var repositoryPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$`) +var envPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func ReadConfig(path string) (Config, error) { + var c Config + f, err := os.Open(path) + if err != nil { + return c, fmt.Errorf("read metrics config: %w", err) + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, 1024*1024+1)) + if err != nil { + return c, fmt.Errorf("read metrics config: %w", err) + } + if len(data) > 1024*1024 { + return c, errors.New("metrics config exceeds 1 MiB") + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + if err := d.Decode(&c); err != nil { + return c, errors.New("invalid metrics config JSON") + } + if err := d.Decode(new(any)); err != io.EOF { + return c, errors.New("metrics config must contain one JSON object") + } + return c, c.Validate() +} + +func (c Config) Validate() error { + if !filepath.IsAbs(c.Database) || strings.TrimSpace(c.Database) != c.Database || strings.ContainsAny(c.Database, "\x00?") { + return errors.New("metrics database must be an absolute filesystem path") + } + if len(c.Targets) == 0 { + return errors.New("metrics config requires targets") + } + if c.TokenEnv != "" && !envPattern.MatchString(c.TokenEnv) { + return errors.New("invalid tokenEnv name") + } + if c.CookieJar != "" { + return errors.New("GitHub metrics do not use cookieJar") + } + seen := map[string]bool{} + for _, t := range c.Targets { + if strings.TrimSpace(t.Entity) == "" || len(t.Entity) > 200 || len(t.Target) > 300 || !validRepository(t.Target) { + return errors.New("metrics target requires an entity and owner/repo") + } + key := strings.ToLower(t.Target) + if seen[key] { + return errors.New("duplicate metrics repository target") + } + seen[key] = true + } + return nil +} + +func validRepository(target string) bool { + if !repositoryPattern.MatchString(target) { + return false + } + _, name, _ := strings.Cut(target, "/") + return name != "." && name != ".." +} + +func Value(n float64) *float64 { + if math.IsNaN(n) || math.IsInf(n, 0) || n < 0 { + return nil + } + return &n +} +func Counter(t Target, metric string, value *float64, ts, basis string) Row { + return Row{Type: "metric", Entity: t.Entity, Target: t.Target, Metric: metric, Kind: "counter", TS: ts, Value: value, ObservedAt: ts, Provenance: basis} +} + +const Schema = ` +CREATE TABLE IF NOT EXISTS metric_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS metric_observations(sequence INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT NOT NULL UNIQUE,entity TEXT NOT NULL,target TEXT NOT NULL,metric TEXT NOT NULL,kind TEXT NOT NULL CHECK(kind IN ('counter','daily')),ts TEXT NOT NULL,value REAL,observed_at TEXT NOT NULL,provenance TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS metric_series ON metric_observations(target,metric,ts,sequence); +CREATE TABLE IF NOT EXISTS metric_events(sequence INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT NOT NULL UNIQUE,entity TEXT NOT NULL,target TEXT NOT NULL,kind TEXT NOT NULL,ts TEXT NOT NULL,label TEXT NOT NULL,url TEXT NOT NULL,observed_at TEXT NOT NULL,provenance TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS metric_runs(sequence INTEGER PRIMARY KEY AUTOINCREMENT,ts TEXT NOT NULL,status TEXT NOT NULL,rows_written INTEGER NOT NULL); +` + +// ownedReadOnly checks identity and schema version before any writable SQLite open. +func ownedReadOnly(ctx context.Context, path string) (*store.Store, error) { + if !filepath.IsAbs(path) || strings.TrimSpace(path) != path { + return nil, errors.New("metrics database must be an absolute path without surrounding whitespace") + } + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return nil, errors.New("metrics database must be a nonempty regular file") + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + err = checkSingleLink(f) + closeErr := f.Close() + if err != nil || closeErr != nil { + return nil, errors.Join(err, closeErr) + } + s, err := store.OpenReadOnly(ctx, path) + if err != nil { + return nil, err + } + if err := validateOwnership(ctx, s.DB()); err != nil { + s.Close() + return nil, err + } + return s, nil +} + +type ownershipReader interface { + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func validateOwnership(ctx context.Context, db ownershipReader) error { + var owner, version string + err := db.QueryRowContext(ctx, "SELECT value FROM metric_meta WHERE key='owner'").Scan(&owner) + if err == nil { + err = db.QueryRowContext(ctx, "SELECT value FROM metric_meta WHERE key='version'").Scan(&version) + } + var foreign int + if err == nil { + err = db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT IN ('metric_meta','metric_observations','metric_events','metric_runs','sqlite_sequence') AND name NOT LIKE 'sqlite_%'`).Scan(&foreign) + } + if err != nil || owner != Owner || version != "1" || foreign != 0 { + return errors.New("refusing a database not exclusively owned by gitcrawl metrics schema version 1") + } + return nil +} + +// beforeWritableOpen is nil except in tests that replace a checked path. +var beforeWritableOpen func() + +func Open(ctx context.Context, path string) (_ *store.Store, err error) { + if !filepath.IsAbs(path) || strings.TrimSpace(path) != path { + return nil, errors.New("metrics database path must be absolute") + } + info, err := os.Lstat(path) + newFile := errors.Is(err, os.ErrNotExist) + if err == nil { + if !info.Mode().IsRegular() { + return nil, errors.New("refusing a non-regular metrics database") + } + read, err := ownedReadOnly(ctx, path) + if err != nil { + return nil, err + } + if err = read.Close(); err != nil { + return nil, err + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } else { + if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, err + } + // Never initialize an existing empty archive or follow a database symlink. + var f *os.File + f, err = os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return nil, err + } + created, statErr := f.Stat() + if statErr != nil { + _ = f.Close() + return nil, statErr + } + info = created + // Remove only our newly created file on failure, never an existing + // database or a replacement installed at the same path. + defer func() { + if err != nil { + if current, e := os.Lstat(path); e == nil && os.SameFile(created, current) { + err = errors.Join(err, os.Remove(path)) + } + } + }() + if err = f.Close(); err != nil { + return nil, err + } + } + if beforeWritableOpen != nil { + beforeWritableOpen() + } + // Connecting must not apply schema before the opened database is checked. + s, err := store.Open(ctx, store.Options{Path: path, MaxOpenConns: 1, MaxIdleConns: 1}) + if err != nil { + return nil, err + } + if err = initialize(ctx, s.DB(), path, info, newFile); err != nil { + return nil, errors.Join(err, s.Close()) + } + return s, nil +} + +func initialize(ctx context.Context, db *sql.DB, path string, expected os.FileInfo, newFile bool) (err error) { + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer func() { err = errors.Join(err, conn.Close()) }() + if _, err = conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + return err + } + committed := false + defer func() { + if !committed { + rollbackCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, rollbackErr := conn.ExecContext(rollbackCtx, "ROLLBACK") + err = errors.Join(err, rollbackErr) + } + }() + // Validate through the pinned connection under the write lock, not through + // another pathname lookup that could inspect a different database. + if newFile { + var objects int + if err = conn.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master").Scan(&objects); err != nil { + return err + } + if objects != 0 { + return errors.New("refusing to initialize a nonempty metrics database") + } + } else if err = validateOwnership(ctx, conn); err != nil { + return err + } + current, err := os.Lstat(path) + if err != nil { + return err + } + if !current.Mode().IsRegular() || !os.SameFile(expected, current) { + return errors.New("metrics database path changed before initialization") + } + if _, err = conn.ExecContext(ctx, Schema); err != nil { + return err + } + if _, err = conn.ExecContext(ctx, "INSERT OR IGNORE INTO metric_meta VALUES('owner',?),('version','1')", Owner); err != nil { + return err + } + _, err = conn.ExecContext(ctx, "COMMIT") + committed = err == nil + return err +} + +func Validate(r Row) error { + if r.Type != "metric" && r.Type != "event" { + return errors.New("invalid observation type") + } + if strings.TrimSpace(r.Entity) == "" || !validRepository(r.Target) || len(r.Entity) > 200 || len(r.Target) > 300 || strings.TrimSpace(r.Provenance) == "" { + return errors.New("invalid observation identity") + } + for _, v := range []string{r.TS, r.ObservedAt} { + if _, err := time.Parse(time.RFC3339Nano, v); err != nil { + return errors.New("invalid observation time") + } + } + if r.Type == "metric" { + if r.Metric == "" || (r.Kind != "counter" && r.Kind != "daily") || (r.Value != nil && Value(*r.Value) == nil) { + return errors.New("invalid metric") + } + if r.Kind == "daily" { + at, _ := time.Parse(time.RFC3339Nano, r.TS) + observed, _ := time.Parse(time.RFC3339Nano, r.ObservedAt) + if !at.UTC().Truncate(24 * time.Hour).Before(observed.UTC().Truncate(24 * time.Hour)) { + return errors.New("daily observation must describe a completed UTC day") + } + } + } else if r.Kind == "" || r.Label == "" { + return errors.New("event kind and label are required") + } + return nil +} + +func insert(ctx context.Context, tx *sql.Tx, r Row) (int, error) { + if err := Validate(r); err != nil { + return 0, err + } + // Imported IDs preserve their exact history. Only freshly collected daily + // values suppress unchanged re-reads; later corrections append a new sequence. + if r.ID == "" && r.Type == "metric" && r.Kind == "daily" { + previous, provenance, err := latestDaily(ctx, tx, r) + if err == nil && provenance == r.Provenance && ((r.Value == nil && !previous.Valid) || (r.Value != nil && previous.Valid && previous.Float64 == *r.Value)) { + return 0, nil + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + } + if r.ID == "" { + b, _ := json.Marshal(r) + h := sha256.Sum256(b) + r.ID = hex.EncodeToString(h[:]) + } + var result sql.Result + var err error + if r.Type == "metric" { + result, err = tx.ExecContext(ctx, "INSERT INTO metric_observations(id,entity,target,metric,kind,ts,value,observed_at,provenance) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING", r.ID, r.Entity, r.Target, r.Metric, r.Kind, r.TS, r.Value, r.ObservedAt, r.Provenance) + } else { + result, err = tx.ExecContext(ctx, "INSERT INTO metric_events(id,entity,target,kind,ts,label,url,observed_at,provenance) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING", r.ID, r.Entity, r.Target, r.Kind, r.TS, r.Label, r.URL, r.ObservedAt, r.Provenance) + } + if err != nil { + return 0, err + } + n, err := result.RowsAffected() + return int(n), err +} + +// A daily import can spell the same UTC day with another offset or time of day. +// Compare parsed days without rewriting imported timestamps or delivery IDs. +func latestDaily(ctx context.Context, tx *sql.Tx, r Row) (sql.NullFloat64, string, error) { + rows, err := tx.QueryContext(ctx, `SELECT ts,value,provenance FROM metric_observations WHERE entity=? AND target=? AND metric=? AND kind='daily' ORDER BY sequence DESC`, r.Entity, r.Target, r.Metric) + if err != nil { + return sql.NullFloat64{}, "", err + } + defer rows.Close() + at, _ := time.Parse(time.RFC3339Nano, r.TS) + day := at.UTC().Truncate(24 * time.Hour) + for rows.Next() { + var raw, provenance string + var value sql.NullFloat64 + if err := rows.Scan(&raw, &value, &provenance); err != nil { + return value, "", err + } + previous, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + return value, "", errors.New("invalid stored daily observation time") + } + if previous.UTC().Truncate(24 * time.Hour).Equal(day) { + return value, provenance, nil + } + } + if err := rows.Err(); err != nil { + return sql.NullFloat64{}, "", err + } + return sql.NullFloat64{}, "", sql.ErrNoRows +} + +func Write(ctx context.Context, s *store.Store, rows []Row) (int, error) { + written := 0 + err := s.WithTx(ctx, func(tx *sql.Tx) error { + for _, r := range rows { + n, err := insert(ctx, tx, r) + if err != nil { + return err + } + written += n + } + return nil + }) + if err != nil { + return 0, err + } + return written, nil +} + +func inScope(c Config, r Row) bool { + for _, t := range c.Targets { + if t.Target == r.Target && t.Entity == r.Entity { + return true + } + } + return false +} + +// Import validates scope and streams the whole input in one transaction. A bad +// row (including one past a batch boundary) never leaves a partial history. +func Import(ctx context.Context, s *store.Store, c Config, in io.Reader) (int, error) { + written := 0 + err := s.WithTx(ctx, func(tx *sql.Tx) error { + scanner := bufio.NewScanner(in) + scanner.Buffer(make([]byte, 65536), 4*1024*1024) + line := 0 + for scanner.Scan() { + line++ + var r Row + d := json.NewDecoder(strings.NewReader(scanner.Text())) + d.DisallowUnknownFields() + if d.Decode(&r) != nil || d.Decode(new(any)) != io.EOF || !inScope(c, r) { + return fmt.Errorf("invalid import scope or JSON at line %d", line) + } + if r.ID == "" { + return fmt.Errorf("import ID required at line %d", line) + } + n, err := insert(ctx, tx, r) + if err != nil { + return fmt.Errorf("import line %d: %w", line, err) + } + written += n + } + return scanner.Err() + }) + if err != nil { + return 0, err + } + return written, nil +} + +func Execute(ctx context.Context, command string, c Config, collect Collector, in io.Reader) (Result, error) { + result := Result{Source: Owner, Command: command} + if err := c.Validate(); err != nil { + return result, err + } + if command != "status" && command != "import" && command != "collect" { + return result, errors.New("unknown metrics command") + } + if command == "status" { + s, err := ownedReadOnly(ctx, c.Database) + if err != nil { + return result, err + } + defer s.Close() + rows, err := s.DB().QueryContext(ctx, "SELECT observed_at FROM metric_observations") + if err != nil { + return result, err + } + var latest time.Time + for rows.Next() { + var raw string + if err = rows.Scan(&raw); err != nil { + break + } + var at time.Time + at, err = time.Parse(time.RFC3339Nano, raw) + if err != nil { + err = errors.New("invalid stored observation time") + break + } + result.Observations++ + // Imported timestamps retain their original offsets and precision. + if result.LastObserved == nil || at.After(latest) { + latest = at + result.LastObserved = &raw + } + } + err = errors.Join(err, rows.Err(), rows.Close()) + if err == nil { + err = s.DB().QueryRowContext(ctx, "SELECT count(*) FROM metric_events").Scan(&result.Events) + } + result.OK = err == nil + return result, err + } + // Own the database from before initialization through provider reads, commit, + // and close. SQLite transactions alone do not serialize collection attempts. + lock, err := acquireWriter(c.Database) + if err != nil { + return result, err + } + defer lock.Close() + s, err := Open(ctx, c.Database) + if err != nil { + return result, err + } + defer s.Close() + if command == "import" { + result.RowsWritten, err = Import(ctx, s, c, in) + result.OK = err == nil + return result, err + } + ts := time.Now().UTC().Format(time.RFC3339Nano) + rows, collectionErr := collect(ctx, c, ts) + for _, r := range rows { + if !inScope(c, r) { + return result, errors.New("collector returned invalid target") + } + } + // Preserve completed reads even when collection is canceled. This bounded, + // independent write does not start another request or retry collection. + writeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + err = s.WithTx(writeCtx, func(tx *sql.Tx) error { + for _, r := range rows { + n, err := insert(writeCtx, tx, r) + if err != nil { + return err + } + result.RowsWritten += n + } + status := "ok" + if collectionErr != nil { + status = "partial" + } + _, err := tx.ExecContext(writeCtx, "INSERT INTO metric_runs(ts,status,rows_written) VALUES(?,?,?)", ts, status, result.RowsWritten) + return err + }) + if err != nil { + result.RowsWritten = 0 + return result, err + } + result.OK = collectionErr == nil + if collectionErr != nil { + return result, ErrPartialCollection + } + return result, nil +} diff --git a/internal/headlinemetrics/metrics_test.go b/internal/headlinemetrics/metrics_test.go new file mode 100644 index 00000000..c4748835 --- /dev/null +++ b/internal/headlinemetrics/metrics_test.go @@ -0,0 +1,646 @@ +package headlinemetrics + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/openclaw/crawlkit/store" +) + +func testConfig(t *testing.T) Config { + t.Helper() + return Config{Database: filepath.Join(t.TempDir(), "metrics.sqlite"), Targets: []Target{{Entity: "OpenClaw", Target: "openclaw/openclaw"}, {Entity: "Example", Target: "example/project"}}} +} +func testRow() Row { + return Counter(Target{"OpenClaw", "openclaw/openclaw"}, "stars", Value(12), "2026-09-15T01:00:00Z", "github_rest") +} +func openTestStore(t *testing.T, c Config) *store.Store { + t.Helper() + s, err := Open(context.Background(), c.Database) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} +func ndjson(t *testing.T, rows ...Row) string { + t.Helper() + var out strings.Builder + for _, r := range rows { + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + out.Write(b) + out.WriteByte('\n') + } + return out.String() +} + +func TestStorePreservesZerosNullsDecreasesAndIdempotentImports(t *testing.T) { + ctx := context.Background() + c := testConfig(t) + s := openTestStore(t, c) + rows := []Row{} + for i, v := range []*float64{Value(12), Value(8), Value(0), nil} { + r := testRow() + r.ID = fmt.Sprint(i) + r.Value = v + rows = append(rows, r) + } + event := Row{Type: "event", ID: "import-release", Entity: "Example", Target: "example/project", Kind: "release", TS: "2026-09-14T00:00:00Z", ObservedAt: "2026-09-15T00:00:00Z", Provenance: "claw-track", Label: "v1", URL: "https://github.com/example/project/releases/tag/v1"} + rows = append(rows, event) + for _, want := range []int{5, 0} { + n, err := Import(ctx, s, c, strings.NewReader(ndjson(t, rows...))) + if err != nil || n != want { + t.Fatalf("import = %d,%v want %d", n, err, want) + } + } + cursor, err := s.DB().Query("SELECT value FROM metric_observations ORDER BY sequence") + if err != nil { + t.Fatal(err) + } + defer cursor.Close() + for _, want := range []*float64{Value(12), Value(8), Value(0), nil} { + if !cursor.Next() { + t.Fatal("missing observation") + } + var got sql.NullFloat64 + if err := cursor.Scan(&got); err != nil { + t.Fatal(err) + } + if got.Valid != (want != nil) || (want != nil && got.Float64 != *want) { + t.Fatalf("value = %+v want %v", got, want) + } + } + if cursor.Next() { + t.Fatal("duplicate observations") + } + result, err := Execute(ctx, "status", c, nil, nil) + if err != nil || result.Observations != 4 || result.Events != 1 || result.LastObserved == nil { + t.Fatalf("status = %+v %v", result, err) + } + info, err := os.Stat(c.Database) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0077 != 0 { + t.Fatalf("database permissions: %v", info.Mode()) + } +} + +func TestImportRollbackAfterBatchBoundaryAndInvalidInput(t *testing.T) { + for _, bad := range []string{"not json", `{"type":"metric"}`, "", strings.Repeat("x", 4*1024*1024+1)} { + t.Run(fmt.Sprint(len(bad)), func(t *testing.T) { + c := testConfig(t) + s := openTestStore(t, c) + var input strings.Builder + for i := 0; i < 501; i++ { + r := testRow() + r.ID = fmt.Sprint(i) + input.WriteString(ndjson(t, r)) + } + input.WriteString(bad + "\n") + n, err := Import(context.Background(), s, c, strings.NewReader(input.String())) + if err == nil || n != 0 { + t.Fatalf("import=%d,%v", n, err) + } + var count int + if err := s.DB().QueryRow("SELECT count(*) FROM metric_observations").Scan(&count); err != nil || count != 0 { + t.Fatalf("partial import committed: %d,%v", count, err) + } + }) + } + c := testConfig(t) + s := openTestStore(t, c) + for _, mutate := range []func(*Row){func(r *Row) { r.Entity = "wrong" }, func(r *Row) { r.Target = "other/repo" }, func(r *Row) { r.ID = "" }, func(r *Row) { r.Value = Value(-1); r.TS = "invalid" }} { + r := testRow() + r.ID = "id" + mutate(&r) + if _, err := Import(context.Background(), s, c, strings.NewReader(ndjson(t, r))); err == nil { + t.Fatalf("accepted invalid row %+v", r) + } + } +} + +func TestDailyRevisionsAppendAndImportedIDsRemainIndependent(t *testing.T) { + c := testConfig(t) + s := openTestStore(t, c) + ctx := context.Background() + r := testRow() + r.Kind = "daily" + r.Metric = "clones" + r.TS = "2026-09-14T23:59:59.999Z" + r.Provenance = "github_traffic" + for i, v := range []float64{10, 10, 9, 10} { + r.Value = Value(v) + r.ObservedAt = fmt.Sprintf("2026-09-15T%02d:00:00Z", i) + n, err := Write(ctx, s, []Row{r}) + want := 1 + if i == 1 { + want = 0 + } + if err != nil || n != want { + t.Fatalf("daily revision %d: %d,%v", i, n, err) + } + } + r.ID = "historical-1" + r.Provenance = "claw-track" + n, err := Import(ctx, s, c, strings.NewReader(ndjson(t, r))) + if err != nil || n != 1 { + t.Fatalf("history: %d,%v", n, err) + } + r.ID = "historical-2" + n, err = Import(ctx, s, c, strings.NewReader(ndjson(t, r))) + if err != nil || n != 1 { + t.Fatalf("distinct history ID: %d,%v", n, err) + } + var count int + var latest float64 + if err = s.DB().QueryRow("SELECT count(*) FROM metric_observations").Scan(&count); err != nil || count != 5 { + t.Fatalf("count %d %v", count, err) + } + if err = s.DB().QueryRow("SELECT value FROM metric_observations ORDER BY sequence DESC LIMIT 1").Scan(&latest); err != nil || latest != 10 { + t.Fatalf("latest %f %v", latest, err) + } +} + +func TestRefuseArchiveWrongOwnerVersionAndLinksWithoutChangingBytes(t *testing.T) { + for _, schema := range []string{ + "CREATE TABLE threads(id INTEGER PRIMARY KEY)", + "CREATE TABLE metric_meta(key TEXT PRIMARY KEY,value TEXT);INSERT INTO metric_meta VALUES('owner','redcrawl'),('version','1')", + "CREATE TABLE metric_meta(key TEXT PRIMARY KEY,value TEXT);INSERT INTO metric_meta VALUES('owner','gitcrawl'),('version','2')", + "CREATE TABLE metric_meta(key TEXT PRIMARY KEY,value TEXT);INSERT INTO metric_meta VALUES('owner','gitcrawl'),('version','1');CREATE TABLE threads(id INTEGER)", + } { + t.Run(schema, func(t *testing.T) { + c := testConfig(t) + s, err := store.Open(context.Background(), store.Options{Path: c.Database, Schema: schema}) + if err != nil { + t.Fatal(err) + } + s.Close() + before, err := os.ReadFile(c.Database) + if err != nil { + t.Fatal(err) + } + if s, err := Open(context.Background(), c.Database); err == nil { + s.Close() + t.Fatal("archive accepted") + } + if _, err := Execute(context.Background(), "status", c, nil, nil); err == nil { + t.Fatal("wrong schema status accepted") + } + after, _ := os.ReadFile(c.Database) + if string(before) != string(after) { + t.Fatal("archive bytes changed") + } + }) + } + c := testConfig(t) + s := openTestStore(t, c) + s.Close() + link := filepath.Join(t.TempDir(), "linked.db") + if err := os.Symlink(c.Database, link); err != nil { + t.Skip(err) + } + if s, err := Open(context.Background(), link); err == nil { + s.Close() + t.Fatal("database symlink accepted") + } + empty := filepath.Join(t.TempDir(), "empty.db") + if err := os.WriteFile(empty, nil, 0600); err != nil { + t.Fatal(err) + } + if s, err := Open(context.Background(), empty); err == nil { + s.Close() + t.Fatal("empty existing database accepted") + } +} + +func TestValidationAndReadOnlyStatus(t *testing.T) { + c := testConfig(t) + if _, err := Execute(context.Background(), "status", c, nil, nil); err == nil { + t.Fatal("missing status accepted") + } + if _, err := os.Stat(c.Database); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("status created database: %v", err) + } + for _, mutate := range []func(*Config){func(c *Config) { c.Database = "relative.db" }, func(c *Config) { c.Targets = nil }, func(c *Config) { c.Targets = append(c.Targets, c.Targets[0]) }, func(c *Config) { c.Targets[0].Target = "../archive" }, func(c *Config) { c.TokenEnv = "bad-name" }, func(c *Config) { c.CookieJar = "/unused" }} { + cfg := testConfig(t) + mutate(&cfg) + if err := cfg.Validate(); err == nil { + t.Fatalf("config accepted: %+v", cfg) + } + } + for _, mutate := range []func(*Row){func(r *Row) { r.Type = "unknown" }, func(r *Row) { r.Entity = "" }, func(r *Row) { r.Provenance = "" }, func(r *Row) { r.TS = "bad" }, func(r *Row) { r.Kind = "gauge" }, func(r *Row) { r.Value = new(float64); *r.Value = -1 }, func(r *Row) { r.Value = new(float64); *r.Value = math.Inf(1) }, func(r *Row) { r.Kind = "daily" }, func(r *Row) { r.Type = "event"; r.Label = "" }} { + r := testRow() + mutate(&r) + if err := Validate(r); err == nil { + t.Fatalf("row accepted: %+v", r) + } + } + for _, value := range []float64{-1, math.Inf(1), math.NaN()} { + if Value(value) != nil { + t.Fatal("invalid value accepted") + } + } + for _, body := range []string{`{"database":"/tmp/test","targets":[]} invalid`, `{"database":"/tmp/test","targets":[],"unknown":true}`} { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + if _, err := ReadConfig(path); err == nil { + t.Fatal("invalid JSON config accepted") + } + } +} + +func TestDatabaseWhitespaceCannotBypassArchiveOwnership(t *testing.T) { + c := testConfig(t) + if err := os.WriteFile(c.Database, []byte("archive bytes"), 0600); err != nil { + t.Fatal(err) + } + for _, suffix := range []string{" ", "\t", "\n"} { + path := c.Database + suffix + if s, err := Open(context.Background(), path); err == nil { + s.Close() + t.Fatal("whitespace path accepted") + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("created alias path: %v", err) + } + cfg := c + cfg.Database = path + if err := cfg.Validate(); err == nil { + t.Fatal("config accepted whitespace alias") + } + } + if b, _ := os.ReadFile(c.Database); string(b) != "archive bytes" { + t.Fatal("archive changed") + } + c.Targets[0].Target = "openclaw/.github" + if err := c.Validate(); err != nil { + t.Fatal(err) + } + c.Targets[0].Target = "openclaw/.." + if err := c.Validate(); err == nil { + t.Fatal("traversal target accepted") + } +} + +func TestCollectRetainsPartialAndCancelledResultsAtomically(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(fmt.Sprint(cancelled), func(t *testing.T) { + c := testConfig(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + collect := func(_ context.Context, _ Config, ts string) ([]Row, error) { + if cancelled { + cancel() + } + r := testRow() + r.TS = ts + r.ObservedAt = ts + missing := r + missing.Metric = "watchers" + missing.Value = nil + return []Row{r, missing}, errors.New("do not expose source credentials") + } + result, err := Execute(ctx, "collect", c, collect, nil) + if err == nil || result.OK || result.RowsWritten != 2 || strings.Contains(err.Error(), "credentials") { + t.Fatalf("result=%+v error=%v", result, err) + } + s, err := ownedReadOnly(context.Background(), c.Database) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var status string + var n int + if err := s.DB().QueryRow("SELECT status,rows_written FROM metric_runs").Scan(&status, &n); err != nil || status != "partial" || n != 2 { + t.Fatalf("run=%s,%d %v", status, n, err) + } + }) + } + c := testConfig(t) + result, err := Execute(context.Background(), "collect", c, func(context.Context, Config, string) ([]Row, error) { + r := testRow() + r.Target = "outside/scope" + return []Row{r}, nil + }, nil) + if err == nil || result.RowsWritten != 0 { + t.Fatalf("scope leak: %+v %v", result, err) + } + s := openTestStore(t, c) + r := testRow() + bad := r + bad.Type = "invalid" + n, err := Write(context.Background(), s, []Row{r, bad}) + if err == nil || n != 0 { + t.Fatalf("write rollback=%d %v", n, err) + } +} + +func TestStatusOrdersExistingTimestampsChronologically(t *testing.T) { + for _, times := range [][]string{ + {"2026-09-15T01:00:00+02:00", "2026-09-15T00:00:00Z"}, + {"2026-09-15T00:00:00Z", "2026-09-15T00:00:00.000000001Z"}, + } { + c := testConfig(t) + s := openTestStore(t, c) + for i, at := range times { + r := testRow() + r.ID = fmt.Sprint(i) + r.ObservedAt = at + if _, err := Import(context.Background(), s, c, strings.NewReader(ndjson(t, r))); err != nil { + t.Fatal(err) + } + } + result, err := Execute(context.Background(), "status", c, nil, nil) + if err != nil || result.LastObserved == nil || *result.LastObserved != times[1] { + t.Fatalf("status = %+v, %v; latest instant = %s", result, err, times[1]) + } + } +} + +func TestFailedInitializationCanRetryWithoutAdoptingForeignFiles(t *testing.T) { + c := testConfig(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if s, err := Open(ctx, c.Database); err == nil { + s.Close() + t.Fatal("canceled initialization succeeded") + } + if _, err := os.Stat(c.Database); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed initialization stranded a file: %v", err) + } + s := openTestStore(t, c) + if _, err := Write(context.Background(), s, []Row{testRow()}); err != nil { + t.Fatal(err) + } + s.Close() + if s, err := Open(ctx, c.Database); err == nil { + s.Close() + t.Fatal("canceled reopen succeeded") + } + result, err := Execute(context.Background(), "status", c, nil, nil) + if err != nil || result.Observations != 1 { + t.Fatalf("existing history lost after failed reopen: %+v, %v", result, err) + } +} + +func TestRefuseMetricsDatabaseHardlinkAliases(t *testing.T) { + c := testConfig(t) + s := openTestStore(t, c) + s.Close() + alias := filepath.Join(t.TempDir(), "alias.sqlite") + if err := os.Link(c.Database, alias); err != nil { + t.Skip(err) + } + before, err := os.ReadFile(c.Database) + if err != nil { + t.Fatal(err) + } + for _, path := range []string{c.Database, alias} { + cfg := c + cfg.Database = path + if _, err := Execute(context.Background(), "import", cfg, nil, strings.NewReader("")); err == nil { + t.Fatal("hardlinked database bypasses writer serialization") + } + } + after, err := os.ReadFile(c.Database) + if err != nil || string(after) != string(before) { + t.Fatal("hardlinked database changed") + } +} + +func TestDailyCollectionUsesLatestImportedUTCDay(t *testing.T) { + c := testConfig(t) + s := openTestStore(t, c) + ctx := context.Background() + r := testRow() + r.ID, r.Kind, r.Metric = "imported-day", "daily", "clones" + r.TS, r.Provenance = "2026-09-15T01:00:00+02:00", "github_traffic" + if _, err := Import(ctx, s, c, strings.NewReader(ndjson(t, r))); err != nil { + t.Fatal(err) + } + r.ID, r.TS = "", "2026-09-14T23:59:59.999Z" + for i, value := range []*float64{Value(12), nil, nil, Value(0), Value(12)} { + r.Value = value + r.ObservedAt = fmt.Sprintf("2026-09-15T%02d:00:00Z", i+2) + n, err := Write(ctx, s, []Row{r}) + want := 1 + if i == 0 || i == 2 { + want = 0 + } + if err != nil || n != want { + t.Fatalf("daily revision %d = %d, %v; want %d", i, n, err, want) + } + } +} + +func TestConfigRejectsContentBeyondSizeLimit(t *testing.T) { + c := testConfig(t) + data, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "metrics.json") + data = append(data, []byte(strings.Repeat(" ", 1024*1024)+`{"ignored":true}`)...) + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + if _, err := ReadConfig(path); err == nil { + t.Fatal("oversized config with trailing JSON was accepted") + } +} + +func TestOpenRejectsReplacedDatabaseBeforeSchemaWrites(t *testing.T) { + ctx := context.Background() + for _, existing := range []bool{false, true} { + for _, parent := range []bool{false, true} { + for _, metrics := range []bool{false, true} { + t.Run(fmt.Sprintf("existing=%t/parent=%t/metrics=%t", existing, parent, metrics), func(t *testing.T) { + root := t.TempDir() + active := filepath.Join(root, "active") + if err := os.Mkdir(active, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(active, "metrics.sqlite") + if existing { + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + } + replacementDir := filepath.Join(root, "replacement") + replacement := filepath.Join(replacementDir, "metrics.sqlite") + schema := "CREATE TABLE threads(id INTEGER PRIMARY KEY,body TEXT); INSERT INTO threads VALUES(1,'archive retained')" + if metrics { + schema = Schema + "INSERT INTO metric_meta VALUES('owner','gitcrawl'),('version','1'); INSERT INTO metric_runs(ts,status,rows_written) VALUES('retained','ok',7)" + } + s, err := store.Open(ctx, store.Options{Path: replacement, Schema: schema}) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + replacementInfo, err := os.Stat(replacement) + if err != nil { + t.Fatal(err) + } + called := false + beforeWritableOpen = func() { + called = true + from, to := replacement, path + if parent { + from, to = replacementDir, active + } + if err := os.Rename(to, to+".original"); err != nil { + t.Fatal(err) + } + if err := os.Rename(from, to); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { beforeWritableOpen = nil }) + opened, err := Open(ctx, path) + if opened != nil { + opened.Close() + } + if !called || err == nil || opened != nil { + t.Fatalf("replaced database accepted: called=%t store=%v error=%v", called, opened, err) + } + current, err := os.Stat(path) + if err != nil || !os.SameFile(replacementInfo, current) { + t.Fatalf("replacement removed or changed: %v", err) + } + read, err := store.OpenReadOnly(ctx, path) + if err != nil { + t.Fatal(err) + } + defer read.Close() + if metrics { + var retained int + if err := read.DB().QueryRow("SELECT rows_written FROM metric_runs WHERE ts='retained'").Scan(&retained); err != nil || retained != 7 { + t.Fatalf("replacement metrics history changed: %d, %v", retained, err) + } + } else { + var objects int + if err := read.DB().QueryRow("SELECT count(*) FROM sqlite_master WHERE name GLOB 'metric_*'").Scan(&objects); err != nil || objects != 0 { + t.Fatalf("metrics schema landed in archive: %d, %v", objects, err) + } + var body string + if err := read.DB().QueryRow("SELECT body FROM threads WHERE id=1").Scan(&body); err != nil || body != "archive retained" { + t.Fatalf("archive contents changed: %q, %v", body, err) + } + } + }) + } + } + } +} + +func TestOpenRevalidatesDatabaseChangedInPlace(t *testing.T) { + ctx := context.Background() + for _, existing := range []bool{false, true} { + t.Run(fmt.Sprint(existing), func(t *testing.T) { + c := testConfig(t) + if existing { + s := openTestStore(t, c) + if err := s.Close(); err != nil { + t.Fatal(err) + } + } + beforeWritableOpen = func() { + schema := "CREATE VIEW unrelated AS SELECT 1" + if existing { + schema = "UPDATE metric_meta SET value='another-owner' WHERE key='owner'" + } + s, err := store.Open(ctx, store.Options{Path: c.Database, Schema: schema}) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { beforeWritableOpen = nil }) + s, err := Open(ctx, c.Database) + if s != nil { + s.Close() + } + if err == nil || s != nil { + t.Fatalf("changed database accepted: %v, %v", s, err) + } + if existing { + read, err := store.OpenReadOnly(ctx, c.Database) + if err != nil { + t.Fatal(err) + } + defer read.Close() + var owner string + if err := read.DB().QueryRow("SELECT value FROM metric_meta WHERE key='owner'").Scan(&owner); err != nil || owner != "another-owner" { + t.Fatalf("replaced ownership overwritten: %q, %v", owner, err) + } + } + }) + } +} + +func TestOpenRollsBackSchemaWhenOwnershipInsertFails(t *testing.T) { + ctx := context.Background() + c := testConfig(t) + s, err := store.Open(ctx, store.Options{Path: c.Database, Schema: ` +CREATE TABLE metric_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL); +INSERT INTO metric_meta VALUES('owner','gitcrawl'),('version','1'); +CREATE TRIGGER reject_ownership BEFORE INSERT ON metric_meta BEGIN SELECT RAISE(ABORT,'fixture rejection'); END; +`}) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if s, err := Open(ctx, c.Database); err == nil { + s.Close() + t.Fatal("ownership insert unexpectedly succeeded") + } + read, err := store.OpenReadOnly(ctx, c.Database) + if err != nil { + t.Fatal(err) + } + var tables int + err = read.DB().QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table' AND name IN ('metric_observations','metric_events','metric_runs')").Scan(&tables) + closeErr := read.Close() + if err != nil || closeErr != nil || tables != 0 { + t.Fatalf("schema was not rolled back: tables=%d error=%v close=%v", tables, err, closeErr) + } + // The failed open must release its transaction and connection for a retry. + s, err = store.Open(ctx, store.Options{Path: c.Database, Schema: "DROP TRIGGER reject_ownership"}) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(ctx, c.Database) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/store/analytics_actor_recovery_test.go b/internal/store/analytics_actor_recovery_test.go new file mode 100644 index 00000000..ffb024ef --- /dev/null +++ b/internal/store/analytics_actor_recovery_test.go @@ -0,0 +1,129 @@ +package store + +import ( + "context" + "encoding/json" + "path/filepath" + "reflect" + "slices" + "testing" + "time" +) + +func TestAnalyticsActorRecoverySeedsAndRefreshesNativeIdentitiesAtomically(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + // Historical rows predate the enqueue-on-capture path. + _, err = s.DB().ExecContext(ctx, `INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo','1','{}','2026-01-01'); + INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at) VALUES + (1,1,'1',1,'issue','open','fixture','','[]','[]','{"node_id":"content-unknown","user":null}','h','2026-01-01'), + (2,1,'2',2,'issue','open','fixture','','[]','[]','{"node_id":"content-known","user":{"node_id":"actor-known"}}','h','2026-01-01')`) + if err != nil { + t.Fatal(err) + } + for _, profiles := range []bool{false, true} { + if err = s.SeedAnalyticsNodes(ctx, profiles); err != nil { + t.Fatal(err) + } + } + assertNodes := func(profiles bool, want []string) { + t.Helper() + var got []string + var err error + if profiles { + got, err = s.AnalyticsProfileNodes(ctx, 100) + } else { + got, err = s.AnalyticsIdentityNodes(ctx, 100) + } + if err != nil { + t.Fatal(err) + } + slices.Sort(got) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Fatalf("profiles=%v nodes=%v want%v", profiles, got, want) + } + } + assertNodes(false, []string{"content-unknown"}) + assertNodes(true, []string{"actor-known"}) + now := time.Now().UTC() + at := now.Format(time.RFC3339Nano) + evidence := map[string]any{"id": "content-unknown", "author": map[string]any{"id": "actor-recovered", "login": "shared-login", "__typename": "User"}} + if err = s.SaveActorEvidence(ctx, []map[string]any{evidence}, at); err != nil { + t.Fatal(err) + } + assertNodes(false, nil) + assertNodes(true, []string{"actor-known", "actor-recovered"}) + profile := func(id, name string) map[string]any { + return map[string]any{"id": id, "login": "shared-login", "__typename": "User", "name": name, "bio": "fixture profile", "url": "https://github.com/shared-login", "createdAt": "2026-01-01T02:00:00+02:00"} + } + first, second := profile("actor-known", "Known"), profile("actor-recovered", "Recovered") + if err = s.SaveActorProfiles(ctx, []map[string]any{first, second}, at); err != nil { + t.Fatal(err) + } + assertNodes(true, nil) + var count int + if err = s.DB().QueryRowContext(ctx, "SELECT count(*) FROM actor_profiles WHERE login='shared-login'").Scan(&count); err != nil || count != 2 { + t.Fatalf("login reuse merged native actors: %d %v", count, err) + } + read := func(id string) (string, string, string) { + t.Helper() + var raw, created, observed string + if err := s.DB().QueryRowContext(ctx, "SELECT raw_json,created_at,observed_at FROM actor_profiles WHERE node_id=?", id).Scan(&raw, &created, &observed); err != nil { + t.Fatal(err) + } + return raw, created, observed + } + raw, created, observed := read("actor-known") + var retained map[string]any + if err = json.Unmarshal([]byte(raw), &retained); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(retained, first) || created != "2026-01-01T00:00:00Z" || observed != at { + t.Fatal("native profile evidence or provider date changed") + } + changed := profile("actor-known", "Changed") + invalid := profile("actor-recovered", "Must not apply") + invalid["unsupported"] = make(chan int) + if err = s.SaveActorProfiles(ctx, []map[string]any{changed, invalid}, at); err == nil { + t.Fatal("invalid batch accepted") + } + after, _, _ := read("actor-known") + if after != raw { + t.Fatal("failed later profile leaked an earlier update") + } + // Failed identity capture must roll back its newly enqueued profile as well. + invalidEvidence := map[string]any{"id": "other-content", "author": map[string]any{"id": "actor-leaked"}, "unsupported": make(chan int)} + if err = s.SaveActorEvidence(ctx, []map[string]any{invalidEvidence}, at); err == nil { + t.Fatal("invalid actor evidence accepted") + } + assertNodes(true, nil) + if err = s.DB().QueryRowContext(ctx, "SELECT count(*) FROM actor_identity_evidence").Scan(&count); err != nil || count != 1 { + t.Fatalf("failed identity batch changed evidence: %d %v", count, err) + } + // Fresh profiles are not polled repeatedly, but become eligible after 24 hours. + if err = s.SaveActorProfiles(ctx, []map[string]any{first}, now.Add(-25*time.Hour).Format(time.RFC3339Nano)); err != nil { + t.Fatal(err) + } + assertNodes(true, []string{"actor-known"}) + if err = s.SaveActorProfiles(ctx, []map[string]any{changed}, at); err != nil { + t.Fatal(err) + } + assertNodes(true, nil) + refreshed, _, _ := read("actor-known") + if err = json.Unmarshal([]byte(refreshed), &retained); err != nil || retained["name"] != "Changed" || retained["id"] != "actor-known" { + t.Fatalf("refresh lost native identity: %+v %v", retained, err) + } + if err = s.SeedAnalyticsNodes(ctx, true); err != nil { + t.Fatal(err) + } + if err = s.SeedAnalyticsNodes(ctx, false); err != nil { + t.Fatal(err) + } + assertNodes(true, nil) + assertNodes(false, nil) +} diff --git a/internal/store/analytics_integrity.go b/internal/store/analytics_integrity.go new file mode 100644 index 00000000..f4046b9e --- /dev/null +++ b/internal/store/analytics_integrity.go @@ -0,0 +1,428 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" +) + +// Diagnostic receipts and retry state are local operational evidence, not +// public conversation payloads. Resolved rows are retained, never reset/deleted. +func (s *Store) ensureAnalyticsIntegritySchema(ctx context.Context) error { + if err := s.ensureColumn(ctx, "pull_request_review_thread_syncs", "review_thread_ids_json", "text"); err != nil { + return err + } + _, err := s.q().ExecContext(ctx, ` +CREATE TABLE IF NOT EXISTS analytics_fetch_attempts( + id INTEGER PRIMARY KEY,repository TEXT NOT NULL,number INTEGER NOT NULL,operation TEXT NOT NULL, + started_at TEXT NOT NULL,finished_at TEXT NOT NULL,status TEXT NOT NULL, + error_class TEXT,error_text TEXT,evidence_json TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS analytics_attempt_item ON analytics_fetch_attempts(repository,number,id); +CREATE INDEX IF NOT EXISTS analytics_attempt_repository ON analytics_fetch_attempts(repository,id); +CREATE TABLE IF NOT EXISTS analytics_retries( + repository TEXT NOT NULL,number INTEGER NOT NULL,operation TEXT NOT NULL, + first_seen_at TEXT NOT NULL,last_seen_at TEXT NOT NULL,next_attempt_at TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0,last_attempt_id INTEGER,resolved_at TEXT, + PRIMARY KEY(repository,number,operation)); +CREATE INDEX IF NOT EXISTS analytics_retry_due ON analytics_retries(repository,resolved_at,next_attempt_at,number); +CREATE TABLE IF NOT EXISTS analytics_review_state_coverage( + repository TEXT PRIMARY KEY,cursor INTEGER NOT NULL,ceiling INTEGER NOT NULL, + scanned INTEGER NOT NULL,queued INTEGER NOT NULL,pending_items INTEGER NOT NULL, + scan_complete INTEGER NOT NULL,complete INTEGER NOT NULL,observed_at TEXT NOT NULL); +`) + return err +} + +type AnalyticsAttempt struct { + Repository string `json:"repository"` + Number int `json:"number"` + Operation string `json:"operation"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + Status string `json:"status"` + ErrorClass string `json:"error_class,omitempty"` + ErrorText string `json:"error_text,omitempty"` + Evidence json.RawMessage `json:"evidence"` +} + +func (s *Store) RecordAnalyticsAttempt(ctx context.Context, a AnalyticsAttempt) error { + if a.Repository == "" || a.Operation == "" || (a.Status != "success" && a.Status != "failed") || !json.Valid(a.Evidence) { + return fmt.Errorf("invalid analytics attempt") + } + finished, err := time.Parse(time.RFC3339Nano, a.FinishedAt) + if err != nil { + return err + } + return s.WithTx(ctx, func(tx *Store) error { + status, class, message := a.Status, a.ErrorClass, a.ErrorText + knownReview := true + if status == "success" && (a.Operation == "graphql_history" || a.Operation == "review_state") { + if err := tx.q().QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM threads t JOIN repositories r ON r.id=t.repo_id WHERE r.full_name=? COLLATE NOCASE AND t.number=? AND (t.kind<>'pull_request' OR EXISTS(SELECT 1 FROM pull_request_review_thread_syncs x WHERE x.thread_id=t.id AND x.review_thread_ids_json IS NOT NULL)))`, a.Repository, a.Number).Scan(&knownReview); err != nil { + return err + } + if a.Operation == "review_state" && !knownReview { + status = "failed" + class = "unapplied_review_state" + message = "Fetch did not establish a review-state membership observation" + } + } + r, err := tx.q().ExecContext(ctx, `INSERT INTO analytics_fetch_attempts(repository,number,operation,started_at,finished_at,status,error_class,error_text,evidence_json) VALUES(?,?,?,?,?,?,?,?,?)`, a.Repository, a.Number, a.Operation, a.StartedAt, a.FinishedAt, status, nullString(class), nullString(message), string(a.Evidence)) + if err != nil { + return err + } + id, err := r.LastInsertId() + if err != nil { + return err + } + if status == "success" { + _, err = tx.q().ExecContext(ctx, `UPDATE analytics_retries SET resolved_at=?,last_attempt_id=? WHERE repository=? AND number=? AND resolved_at IS NULL AND (operation=? OR (?='graphql_history' AND operation IN ('graphql_history','review_state'))) AND (operation<>'review_state' OR ?)`, a.FinishedAt, id, a.Repository, a.Number, a.Operation, a.Operation, knownReview) + return err + } + if a.Operation != "review_state" { + if _, err = tx.q().ExecContext(ctx, "UPDATE analytics_coverage SET complete=0 WHERE repository=?", a.Repository); err != nil { + return err + } + } + var attempts int + if err = tx.q().QueryRowContext(ctx, `SELECT coalesce((SELECT attempts FROM analytics_retries WHERE repository=? AND number=? AND operation=?),0)`, a.Repository, a.Number, a.Operation).Scan(&attempts); err != nil { + return err + } + delay := time.Duration(1<0 AND next_attempt_at<=? AND NOT EXISTS(SELECT 1 FROM analytics_retries core WHERE core.repository=analytics_retries.repository AND core.number=analytics_retries.number AND core.operation='graphql_history' AND core.resolved_at IS NULL)` + read := func(attempted bool, n int) ([]int, error) { + predicate := "attempts=0" + if attempted { + predicate = "attempts>0" + } + rows, err := s.q().QueryContext(ctx, "SELECT number FROM analytics_retries WHERE "+eligible+" AND "+predicate+" ORDER BY next_attempt_at,number LIMIT ?", repository, at, n) + if err != nil { + return nil, err + } + defer rows.Close() + var out []int + for rows.Next() { + var number int + if err = rows.Scan(&number); err != nil { + return nil, err + } + out = append(out, number) + } + return out, rows.Err() + } + retries, err := read(true, max(1, limit/4)) + if err != nil { + return nil, err + } + fresh, err := read(false, limit-len(retries)) + if err != nil { + return nil, err + } + if len(fresh)+len(retries) < limit { + retries, err = read(true, limit-len(fresh)) + if err != nil { + return nil, err + } + } + return append(retries, fresh...), nil +} + +func (s *Store) ReviewStateParent(ctx context.Context, repository string, number int, providerRepositoryID, providerNodeID string) (Thread, error) { + var t Thread + var repoID, rawRepo string + err := s.q().QueryRowContext(ctx, `SELECT t.id,t.repo_id,t.github_id,t.kind,t.raw_json,r.github_repo_id,r.raw_json FROM threads t JOIN repositories r ON r.id=t.repo_id WHERE r.full_name=? COLLATE NOCASE AND t.number=?`, repository, number).Scan(&t.ID, &t.RepoID, &t.GitHubID, &t.Kind, &t.RawJSON, &repoID, &rawRepo) + if err != nil { + return t, err + } + if t.Kind != "pull_request" || repoID != providerRepositoryID { + return t, fmt.Errorf("review-state archived repository/PR identity mismatch") + } + var repo map[string]any + if err = json.Unmarshal([]byte(rawRepo), &repo); err != nil { + return t, err + } + if node, ok := repo["node_id"].(string); ok && node != "" && node != providerNodeID { + return t, fmt.Errorf("review-state repository node mismatch") + } + return t, nil +} + +func (s *Store) DueAnalyticsRetries(ctx context.Context, repository, at string, limit int, operations ...string) ([]int, error) { + operation := "" + if len(operations) > 0 { + operation = operations[0] + } + rows, err := s.q().QueryContext(ctx, `SELECT r.number FROM analytics_retries r WHERE r.repository=? AND r.resolved_at IS NULL AND r.number>0 AND r.next_attempt_at<=? AND (?='' OR r.operation=?) + AND (r.operation<>'review_state' OR NOT EXISTS(SELECT 1 FROM analytics_retries core WHERE core.repository=r.repository AND core.number=r.number AND core.operation='graphql_history' AND core.resolved_at IS NULL)) + GROUP BY r.number ORDER BY min(r.next_attempt_at),r.number LIMIT ?`, repository, at, operation, operation, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []int + for rows.Next() { + var n int + if err = rows.Scan(&n); err != nil { + return nil, err + } + out = append(out, n) + } + return out, rows.Err() +} + +func (s *Store) AnalyticsOutstanding(ctx context.Context, repository string) (int, error) { + var n int + err := s.q().QueryRowContext(ctx, `SELECT count(*) FROM analytics_retries WHERE repository=? AND resolved_at IS NULL`, repository).Scan(&n) + return n, err +} + +func (s *Store) AnalyticsCoreOutstanding(ctx context.Context, repository string) (int, error) { + var n int + err := s.q().QueryRowContext(ctx, `SELECT count(*) FROM analytics_retries WHERE repository=? AND resolved_at IS NULL AND operation<>'review_state'`, repository).Scan(&n) + return n, err +} + +func (s *Store) AnalyticsItemQueued(ctx context.Context, repository string, number int, operations ...string) (bool, error) { + operation := "graphql_history" + if len(operations) > 0 { + operation = operations[0] + } + var n int + err := s.q().QueryRowContext(ctx, `SELECT count(*) FROM analytics_retries WHERE repository=? AND number=? AND operation=? AND resolved_at IS NULL`, repository, number, operation).Scan(&n) + return n > 0, err +} + +type ReviewStateRecovery struct { + Cursor int64 `json:"cursor"` + Ceiling int64 `json:"ceiling"` + Scanned int64 `json:"scanned"` + Queued int64 `json:"queued"` + Done bool `json:"done"` +} + +// SeedReviewStateRecovery walks at most limit primary-key rows per watch cycle. +// It queues only PRs with retained review threads (or unknown connection data), +// and never restarts historical discovery or invents missing resolution state. +func (s *Store) SeedReviewStateRecovery(ctx context.Context, repository string, limit int) (ReviewStateRecovery, error) { + key := "review_state_recovery:" + repository + value, err := s.AnalyticsState(ctx, key) + if err != nil { + return ReviewStateRecovery{}, err + } + var progress ReviewStateRecovery + if value != "" { + if err = json.Unmarshal([]byte(value), &progress); err != nil { + return progress, err + } + } else { + if err = s.q().QueryRowContext(ctx, "SELECT coalesce(max(id),0) FROM threads").Scan(&progress.Ceiling); err != nil { + return progress, err + } + } + if progress.Done { + return progress, nil + } + rows, err := s.q().QueryContext(ctx, `SELECT t.id,t.number,t.kind,r.full_name, + CASE WHEN json_valid(t.raw_json) THEN coalesce( + json_extract(t.raw_json,'$._gitcrawl_source')='graphql' AND + json_type(t.raw_json,'$._graphql.reviewThreads.totalCount')='integer' AND + json_extract(t.raw_json,'$._graphql.reviewThreads.totalCount')=0 AND + json_type(t.raw_json,'$._graphql.reviewThreads.nodes')='array' AND + json_array_length(t.raw_json,'$._graphql.reviewThreads.nodes')=0 AND + json_type(t.raw_json,'$._graphql.reviewThreads.pageInfo.hasNextPage')='false',0) ELSE 0 END, + t.observation_sequence,coalesce(t.last_pulled_at,''), + EXISTS(SELECT 1 FROM pull_request_review_thread_syncs x WHERE x.thread_id=t.id AND x.review_thread_ids_json IS NOT NULL) + FROM threads t JOIN repositories r ON r.id=t.repo_id WHERE t.id>? AND t.id<=? ORDER BY t.id LIMIT ?`, progress.Cursor, progress.Ceiling, limit) + if err != nil { + return progress, err + } + type item struct { + id int64 + number int + kind, repo string + empty bool + sequence int64 + pulled string + known bool + } + var items []item + for rows.Next() { + var v item + if err = rows.Scan(&v.id, &v.number, &v.kind, &v.repo, &v.empty, &v.sequence, &v.pulled, &v.known); err != nil { + rows.Close() + return progress, err + } + items = append(items, v) + } + err = rows.Err() + rows.Close() + if err != nil { + return progress, err + } + var committed ReviewStateRecovery + err = s.WithTx(ctx, func(tx *Store) error { + next := progress // A busy-transaction retry must not double-count progress. + at := time.Now().UTC().Format(time.RFC3339Nano) + for _, v := range items { + next.Cursor = v.id + next.Scanned++ + if !strings.EqualFold(v.repo, repository) || v.kind != "pull_request" || v.known { + continue + } + if _, e := time.Parse(time.RFC3339Nano, v.pulled); v.empty && e == nil { + // Materialize only an actual retained complete-empty observation. + // CAS checks prevent an older scan from overwriting a live refresh. + _, e = tx.q().ExecContext(ctx, `INSERT INTO pull_request_review_thread_syncs(thread_id,fetched_at,review_thread_ids_json) + SELECT id,last_pulled_at,'[]' FROM threads WHERE id=? AND observation_sequence=? AND last_pulled_at=? + ON CONFLICT(thread_id) DO UPDATE SET fetched_at=excluded.fetched_at,review_thread_ids_json=excluded.review_thread_ids_json + WHERE pull_request_review_thread_syncs.review_thread_ids_json IS NULL AND pull_request_review_thread_syncs.fetched_at<=excluded.fetched_at`, v.id, v.sequence, v.pulled) + if e != nil { + return e + } + var known bool + if e = tx.q().QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pull_request_review_thread_syncs WHERE thread_id=? AND review_thread_ids_json IS NOT NULL)`, v.id).Scan(&known); e != nil { + return e + } + if known { + continue + } + } + r, e := tx.q().ExecContext(ctx, `INSERT OR IGNORE INTO analytics_retries(repository,number,operation,first_seen_at,last_seen_at,next_attempt_at) VALUES(?,?,'review_state',?,?,?)`, repository, v.number, at, at, at) + if e != nil { + return e + } + n, e := r.RowsAffected() + if e != nil { + return e + } + next.Queued += n + } + next.Done = len(items) < limit || next.Cursor >= next.Ceiling + encoded, _ := json.Marshal(next) + if e := tx.SetAnalyticsState(ctx, key, string(encoded)); e != nil { + return e + } + committed = next + return nil + }) + if err == nil { + progress = committed + } + return progress, err +} + +// Core completeness concerns issue/PR/comment traversal. Review-state enrichment +// is deliberately independent so existing response/contributor KPIs remain usable. +func (s *Store) SetAnalyticsCoverageComplete(ctx context.Context, repository string, complete bool) error { + _, err := s.q().ExecContext(ctx, "UPDATE analytics_coverage SET complete=? WHERE repository=?", boolInt(complete), repository) + return err +} + +func (s *Store) SaveReviewStateCoverage(ctx context.Context, repository string, progress ReviewStateRecovery) error { + var pending int + if err := s.q().QueryRowContext(ctx, `SELECT count(*) FROM analytics_retries WHERE repository=? AND operation='review_state' AND resolved_at IS NULL`, repository).Scan(&pending); err != nil { + return err + } + _, err := s.q().ExecContext(ctx, `INSERT INTO analytics_review_state_coverage VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(repository) DO UPDATE SET cursor=excluded.cursor,ceiling=excluded.ceiling,scanned=excluded.scanned,queued=excluded.queued,pending_items=excluded.pending_items,scan_complete=excluded.scan_complete,complete=excluded.complete,observed_at=excluded.observed_at`, repository, progress.Cursor, progress.Ceiling, progress.Scanned, progress.Queued, pending, boolInt(progress.Done), boolInt(progress.Done && pending == 0), time.Now().UTC().Format(time.RFC3339Nano)) + return err +} + +func (s *Store) AnalyticsIntegrityStatus(ctx context.Context, repository string) (map[string]any, error) { + out := map[string]any{"repository": repository, "checked_at": time.Now().UTC().Format(time.RFC3339Nano)} + var through, observed string + var issues, prs, complete int + coverageErr := s.q().QueryRowContext(ctx, "SELECT through,issues,pull_requests,complete,observed_at FROM analytics_coverage WHERE repository=?", repository).Scan(&through, &issues, &prs, &complete, &observed) + if coverageErr == sql.ErrNoRows { + out["coverage"] = map[string]any{"state": "not_started", "through": nil, "issues": nil, "pull_requests": nil, "complete": false, "observed_at": nil} + } else if coverageErr != nil { + return nil, coverageErr + } else { + out["coverage"] = map[string]any{"state": "observed", "through": through, "issues": issues, "pull_requests": prs, "complete": complete == 1, "observed_at": observed} + } + n, err := s.AnalyticsOutstanding(ctx, repository) + if err != nil { + return nil, err + } + out["unresolved_retries"] = n + progress, err := s.AnalyticsState(ctx, "review_state_recovery:"+repository) + if err != nil { + return nil, err + } + if progress != "" { + out["review_state_recovery"] = json.RawMessage(progress) + var recovery ReviewStateRecovery + if err = json.Unmarshal([]byte(progress), &recovery); err != nil { + return nil, err + } + out["review_state_scan_complete"] = recovery.Done + } else { + out["review_state_recovery"] = nil + out["review_state_scan_complete"] = false + } + corePending, err := s.AnalyticsCoreOutstanding(ctx, repository) + if err != nil { + return nil, err + } + out["core_unresolved_retries"] = corePending + out["coverage"].(map[string]any)["complete"] = complete == 1 && corePending == 0 + var reviewPending, reviewComplete int + if err = s.q().QueryRowContext(ctx, `SELECT pending_items,complete FROM analytics_review_state_coverage WHERE repository=?`, repository).Scan(&reviewPending, &reviewComplete); err == nil { + out["review_state_coverage"] = map[string]any{"pending_items": reviewPending, "complete": reviewComplete == 1} + } else if err == sql.ErrNoRows { + out["review_state_coverage"] = nil + } else { + return nil, err + } + rows, err := s.q().QueryContext(ctx, `SELECT id,number,operation,started_at,finished_at,status,coalesce(error_class,'') FROM analytics_fetch_attempts WHERE repository=? ORDER BY id DESC LIMIT 10`, repository) + if err != nil { + return nil, err + } + defer rows.Close() + items := []map[string]any{} + for rows.Next() { + var id, number int64 + var operation, start, finish, status, class string + if err = rows.Scan(&id, &number, &operation, &start, &finish, &status, &class); err != nil { + return nil, err + } + items = append(items, map[string]any{"id": id, "number": number, "operation": operation, "started_at": start, "finished_at": finish, "status": status, "error_class": class}) + } + out["latest_attempts"] = items + return out, rows.Err() +} diff --git a/internal/store/analytics_integrity_test.go b/internal/store/analytics_integrity_test.go new file mode 100644 index 00000000..4b16c63d --- /dev/null +++ b/internal/store/analytics_integrity_test.go @@ -0,0 +1,404 @@ +package store + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" +) + +func TestAnalyticsFailureRecoveryRetainsReceiptsAndFairRetryTimes(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "archive.db") + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + attempt := AnalyticsAttempt{Repository: "fixture/repo", Number: 7, Operation: "graphql_history", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", ErrorClass: "validation", ErrorText: "incomplete connection", Evidence: json.RawMessage(`{"totalCount":2,"received":1}`)} + if err = s.RecordAnalyticsAttempt(ctx, attempt); err != nil { + t.Fatal(err) + } + if due, err := s.DueAnalyticsRetries(ctx, "fixture/repo", "2026-01-01T00:00:10Z", 5); err != nil || len(due) != 0 { + t.Fatalf("early retry %v %v", due, err) + } + s.Close() + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if due, err := s.DueAnalyticsRetries(ctx, "fixture/repo", "2026-01-01T00:00:40Z", 5); err != nil || len(due) != 1 || due[0] != 7 { + t.Fatalf("restart lost failure %v %v", due, err) + } + attempt.Number = 8 + attempt.FinishedAt = "2026-01-01T00:00:02Z" + if err = s.RecordAnalyticsAttempt(ctx, attempt); err != nil { + t.Fatal(err) + } + attempt.Number = 7 + attempt.FinishedAt = "2026-01-01T00:00:40Z" + if err = s.RecordAnalyticsAttempt(ctx, attempt); err != nil { + t.Fatal(err) + } + if due, err := s.DueAnalyticsRetries(ctx, "fixture/repo", "2026-01-01T00:00:41Z", 1); err != nil || len(due) != 1 || due[0] != 8 { + t.Fatalf("poison item starved peer %v %v", due, err) + } + attempt.Status = "success" + attempt.FinishedAt = "2026-01-01T00:02:00Z" + attempt.ErrorClass = "" + attempt.ErrorText = "" + if err = s.RecordAnalyticsAttempt(ctx, attempt); err != nil { + t.Fatal(err) + } + var receipts, resolved int + s.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts").Scan(&receipts) + s.DB().QueryRow("SELECT count(*) FROM analytics_retries WHERE number=7 AND resolved_at IS NOT NULL").Scan(&resolved) + if receipts != 4 || resolved != 1 { + t.Fatalf("history lost receipts=%d resolved=%d", receipts, resolved) + } +} + +func TestReviewStateRecoveryIsBoundedResumableAndPreservesHistory(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + _, err = s.DB().Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','Fixture/Repo',1,'{}','2026-01-01'); +INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at,last_pulled_at) VALUES +(1,1,'P1',1,'pull_request','open','','','[]','[]','{"_graphql":{"reviewThreads":{"totalCount":1}}}','h1','2026-01-01','2026-01-01T00:00:00Z'), +(2,1,'P2',2,'pull_request','open','','','[]','[]','{"_gitcrawl_source":"graphql","_graphql":{"reviewThreads":{"totalCount":0,"nodes":[],"pageInfo":{"hasNextPage":false}}}}','h2','2026-01-01','2026-01-01T00:00:00Z');`) + if err != nil { + t.Fatal(err) + } + p, err := s.SeedReviewStateRecovery(ctx, "fixture/repo", 1) + if err != nil || p.Done || p.Queued != 1 || p.Cursor != 1 { + t.Fatalf("first step %+v %v", p, err) + } + p, err = s.SeedReviewStateRecovery(ctx, "fixture/repo", 1) + if err != nil || !p.Done || p.Queued != 1 || p.Scanned != 2 { + t.Fatalf("resume %+v %v", p, err) + } + var emptyIDs, observed string + if err = s.DB().QueryRow("SELECT review_thread_ids_json,fetched_at FROM pull_request_review_thread_syncs WHERE thread_id=2").Scan(&emptyIDs, &observed); err != nil || emptyIDs != "[]" || observed != "2026-01-01T00:00:00Z" { + t.Fatalf("known empty evidence not materialized: %s %s %v", emptyIDs, observed, err) + } + threads := []PullRequestReviewThread{{ReviewThreadID: "T1", ThreadID: 1, IsResolved: true, IsOutdated: false, RawJSON: `{"id":"T1","isResolved":true}`, CommentsJSON: `[]`, FetchedAt: "2026-01-02T00:00:00Z"}} + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-02T00:00:00Z", threads); err != nil { + t.Fatal(err) + } + threads[0].IsResolved = false + threads[0].RawJSON = `{"id":"T1","isResolved":false}` + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-03T00:00:00Z", threads); err != nil { + t.Fatal(err) + } + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-04T00:00:00Z", nil); err != nil { + t.Fatal(err) + } + var revisions, retained int + var membership string + s.DB().QueryRow("SELECT count(*) FROM pull_request_review_thread_revisions WHERE thread_id=1").Scan(&revisions) + s.DB().QueryRow("SELECT count(*) FROM pull_request_review_threads WHERE thread_id=1 AND deleted_at IS NULL").Scan(&retained) + s.DB().QueryRow("SELECT review_thread_ids_json FROM pull_request_review_thread_syncs WHERE thread_id=1").Scan(&membership) + if revisions != 2 || retained != 1 || membership != "[]" { + t.Fatalf("absence became deletion or history lost: %d %d %s", revisions, retained, membership) + } + for _, invalid := range [][]PullRequestReviewThread{{{ReviewThreadID: ""}}, {{ReviewThreadID: "duplicate"}, {ReviewThreadID: "duplicate"}}} { + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-05T00:00:00Z", invalid); err == nil { + t.Fatal("invalid membership was certified") + } + } + s.DB().QueryRow("SELECT review_thread_ids_json FROM pull_request_review_thread_syncs WHERE thread_id=1").Scan(&membership) + if membership != "[]" { + t.Fatal("rejected input changed prior membership") + } + if err = s.UpsertPullRequestReviewThreads(ctx, 1, "2026-01-01T00:00:00Z", threads); err != nil { + t.Fatal(err) + } + s.DB().QueryRow("SELECT review_thread_ids_json,fetched_at FROM pull_request_review_thread_syncs WHERE thread_id=1").Scan(&membership, &observed) + if membership != "[]" || observed != "2026-01-04T00:00:00Z" { + t.Fatalf("older observation overwrote newer membership: %s %s", membership, observed) + } +} + +func TestAnalyticsV14MigrationPreservesReceiptsAndCoverageWatermark(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "archive.db") + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 2); err != nil { + t.Fatal(err) + } + if err = s.SetAnalyticsState(ctx, "updates:fixture/repo", `{"kind":1,"cursor":"provider-cursor"}`); err != nil { + t.Fatal(err) + } + if _, err = s.DB().Exec(`DROP TABLE analytics_fetch_attempts; DROP TABLE analytics_retries; ALTER TABLE pull_request_review_thread_syncs DROP COLUMN review_thread_ids_json; PRAGMA user_version=14`); err != nil { + t.Fatal(err) + } + if err = s.markObservationSchemaConverged(ctx); err != nil { + t.Fatal(err) + } + s.Close() + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var version, complete int + var through string + s.DB().QueryRow("PRAGMA user_version").Scan(&version) + s.DB().QueryRow("SELECT through,complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&through, &complete) + cp, err := s.AnalyticsState(ctx, "updates:fixture/repo") + if err != nil || version != 15 || through != "2026-01-01T00:00:00Z" || complete != 1 || cp != `{"kind":1,"cursor":"provider-cursor"}` { + t.Fatalf("migration changed evidence: %d %s %d %s %v", version, through, complete, cp, err) + } +} + +func TestReviewStateFailuresDoNotInvalidateVerifiedCoreCoverage(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err = s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 2); err != nil { + t.Fatal(err) + } + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 2, Operation: "review_state", StartedAt: "2026-01-02T00:00:00Z", FinishedAt: "2026-01-02T00:00:01Z", Status: "failed", ErrorClass: "validation", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + if err = s.SaveReviewStateCoverage(ctx, "fixture/repo", ReviewStateRecovery{Done: true, Scanned: 2, Ceiling: 2, Cursor: 2, Queued: 1}); err != nil { + t.Fatal(err) + } + var core, review int + s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&core) + s.DB().QueryRow("SELECT complete FROM analytics_review_state_coverage WHERE repository='fixture/repo'").Scan(&review) + if core != 1 || review != 0 { + t.Fatalf("enrichment invalidated core: core=%d review=%d", core, review) + } + if n, err := s.AnalyticsCoreOutstanding(ctx, "fixture/repo"); err != nil || n != 0 { + t.Fatalf("wrong core queue %d %v", n, err) + } + a.Operation = "graphql_history" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + s.DB().QueryRow("SELECT complete FROM analytics_coverage WHERE repository='fixture/repo'").Scan(&core) + if core != 0 { + t.Fatal("core failure left core coverage complete") + } +} + +func TestAnalyticsStatusSelectsRepositoryBeforeLimit(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.SaveAnalyticsCoverage(ctx, "fixture/repo", "2026-01-01T00:00:00Z", 1, 2) + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "graphql_history", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "success", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + a.Repository = "fixture/other" + for i := 0; i < 60; i++ { + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + } + status, err := s.AnalyticsIntegrityStatus(ctx, "fixture/repo") + if err != nil { + t.Fatal(err) + } + if len(status["latest_attempts"].([]map[string]any)) != 1 { + t.Fatal("another repository hid scoped history") + } +} + +func TestAnalyticsStatusBeforeCollectionIsUnknown(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + status, err := s.AnalyticsIntegrityStatus(ctx, "fixture/repo") + if err != nil { + t.Fatal(err) + } + coverage := status["coverage"].(map[string]any) + if coverage["state"] != "not_started" || coverage["complete"] != false || coverage["issues"] != nil { + t.Fatalf("invented coverage: %+v", coverage) + } + var n int + s.DB().QueryRow("SELECT count(*) FROM analytics_coverage").Scan(&n) + if n != 0 { + t.Fatal("read-only status wrote a baseline") + } +} + +func TestAnalyticsV14WithoutExtensionTablesCanUpgrade(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "archive.db") + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if _, err = s.DB().Exec(`DROP TABLE actor_profiles; DROP TABLE analytics_collection_state; PRAGMA user_version=14`); err != nil { + t.Fatal(err) + } + if err = s.markObservationSchemaConverged(ctx); err != nil { + t.Fatal(err) + } + s.Close() + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if _, err = s.AnalyticsState(ctx, "missing"); err != nil { + t.Fatal(err) + } + if _, err = s.AnalyticsProfileNodes(ctx, 1); err != nil { + t.Fatal(err) + } +} + +func TestAnalyticsCoreRetryOwnsBackoffWhenReviewRetryAlsoExists(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "graphql_history", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", Evidence: json.RawMessage(`{}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + a.Operation = "review_state" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + if due, err := s.DueAnalyticsRetries(ctx, a.Repository, "2026-01-01T00:00:40Z", 8, "review_state"); err != nil || len(due) != 0 { + t.Fatalf("review bypassed due core scheduler: %v %v", due, err) + } + a.Operation = "graphql_history" + a.FinishedAt = "2026-01-01T00:00:50Z" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + if due, err := s.DueAnalyticsRetries(ctx, a.Repository, "2026-01-01T00:01:00Z", 8, "review_state"); err != nil || len(due) != 0 { + t.Fatalf("review bypassed core backoff: %v %v", due, err) + } +} + +func TestAnalyticsRecoveryRequiresAnAcceptedMembershipObservation(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 1, Operation: "review_state", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "success", Evidence: json.RawMessage(`{"threads_skipped_stale":1}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + var status, class string + s.DB().QueryRow("SELECT status,error_class FROM analytics_fetch_attempts ORDER BY id DESC LIMIT 1").Scan(&status, &class) + if status != "failed" || class != "unapplied_review_state" { + t.Fatal("fetch success certified missing membership") + } + repo, err := s.UpsertRepository(ctx, Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", RawJSON: "{}", UpdatedAt: "2026-01-02T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + tid, err := s.UpsertThread(ctx, Thread{RepoID: repo, GitHubID: "P1", Number: 1, Kind: "pull_request", State: "open", Title: "fixture", HTMLURL: "https://github.com/fixture/repo/pull/1", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h", UpdatedAt: "2026-01-02T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + if err = s.UpsertPullRequestReviewThreads(ctx, tid, "2026-01-02T00:00:00Z", nil); err != nil { + t.Fatal(err) + } + a.FinishedAt = "2026-01-02T00:00:01Z" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + if n, err := s.AnalyticsOutstanding(ctx, a.Repository); err != nil || n != 0 { + t.Fatalf("proven empty membership did not reconcile: %d %v", n, err) + } +} + +func TestReviewRetryFairShareAndExplicitUnavailableBackoff(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for n := 1; n <= 100; n++ { + if _, err = s.DB().Exec("insert into analytics_retries(repository,number,operation,first_seen_at,last_seen_at,next_attempt_at) values('fixture/repo',?,'review_state','2000-01-01T00:00:00Z','2000-01-01T00:00:00Z','2000-01-01T00:00:00Z')", n); err != nil { + t.Fatal(err) + } + } + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 999, Operation: "review_state", StartedAt: "2026-01-01T00:00:00Z", FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", ErrorClass: "partial_response", Evidence: json.RawMessage(`{"graphql_errors":{"items":[{"type":"NOT_FOUND"}]}}`)} + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + due, err := s.DueReviewStateWork(ctx, "fixture/repo", "2026-01-01T00:02:00Z", 16) + if err != nil { + t.Fatal(err) + } + for _, n := range due { + if n == 999 { + t.Fatal("unavailable backoff ignored") + } + } + due, err = s.DueReviewStateWork(ctx, "fixture/repo", "2026-01-01T00:16:00Z", 16) + if err != nil || len(due) != 16 || due[0] != 999 { + t.Fatalf("failed retry starved by bulk: %v %v", due, err) + } + a.Operation = "graphql_history" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + due, err = s.DueReviewStateWork(ctx, "fixture/repo", "2026-01-01T00:16:00Z", 16) + if err != nil { + t.Fatal(err) + } + for _, n := range due { + if n == 999 { + t.Fatal("review bypassed core obligation") + } + } + repoID, err := s.UpsertRepository(ctx, Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", RawJSON: "{}", UpdatedAt: a.FinishedAt}) + if err != nil { + t.Fatal(err) + } + tid, err := s.UpsertThread(ctx, Thread{RepoID: repoID, GitHubID: "P999", Number: 999, Kind: "pull_request", State: "open", Title: "fixture", HTMLURL: "https://github.com/fixture/repo/pull/999", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h", UpdatedAt: a.FinishedAt}) + if err != nil { + t.Fatal(err) + } + if err = s.UpsertPullRequestReviewThreads(ctx, tid, a.FinishedAt, nil); err != nil { + t.Fatal(err) + } + a.Operation = "review_state" + a.Status = "success" + if err = s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + var status string + s.DB().QueryRow("select status from analytics_fetch_attempts order by id desc limit 1").Scan(&status) + if status != "success" { + t.Fatal("review success not proven", status) + } + var unresolved int + s.DB().QueryRow("select count(*) from analytics_retries where operation='graphql_history' and resolved_at is null").Scan(&unresolved) + if unresolved != 1 { + t.Fatal("review-only success cleared core failure") + } +} diff --git a/internal/store/analytics_recovery_contract_test.go b/internal/store/analytics_recovery_contract_test.go new file mode 100644 index 00000000..a564e9e6 --- /dev/null +++ b/internal/store/analytics_recovery_contract_test.go @@ -0,0 +1,145 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "path/filepath" + "testing" +) + +func recoveryContractStore(t *testing.T) (*Store, int64) { + t.Helper() + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + repoID, err := s.UpsertRepository(ctx, Repository{Owner: "fixture", Name: "repo", FullName: "fixture/repo", GitHubRepoID: "101", RawJSON: `{"node_id":"R1"}`, UpdatedAt: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + id, err := s.UpsertThread(ctx, Thread{RepoID: repoID, GitHubID: "201", Number: 7, Kind: "pull_request", State: "open", Title: "retained", HTMLURL: "https://github.com/fixture/repo/pull/7", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: `{"node_id":"P7"}`, ContentHash: "retained-hash", UpdatedAt: "2026-01-01T00:00:00Z"}) + if err != nil { + t.Fatal(err) + } + return s, id +} + +func TestReviewStateParentBindsIdentityInOwningTransaction(t *testing.T) { + ctx := context.Background() + s, id := recoveryContractStore(t) + parent, err := s.ReviewStateParent(ctx, "FIXTURE/REPO", 7, "101", "R1") + if err != nil || parent.ID != id || parent.GitHubID != "201" || parent.RawJSON != `{"node_id":"P7"}` { + t.Fatalf("parent=%+v err=%v", parent, err) + } + for _, tc := range []struct { + repo string + number int + database, node string + }{ + {"fixture/other", 7, "101", "R1"}, {"fixture/repo", 8, "101", "R1"}, {"fixture/repo", 7, "102", "R1"}, {"fixture/repo", 7, "101", "R2"}, + } { + if _, err := s.ReviewStateParent(ctx, tc.repo, tc.number, tc.database, tc.node); err == nil { + t.Fatalf("mismatched identity accepted: %+v", tc) + } + } + rolledBack := errors.New("fixture rollback") + err = s.WithTx(ctx, func(tx *Store) error { + if _, err := tx.q().ExecContext(ctx, `UPDATE repositories SET raw_json='{"node_id":"R-new"}' WHERE id=?`, parent.RepoID); err != nil { + return err + } + if _, err := tx.ReviewStateParent(ctx, "fixture/repo", 7, "101", "R1"); err == nil { + t.Fatal("read stale identity outside owning transaction") + } + got, err := tx.ReviewStateParent(ctx, "fixture/repo", 7, "101", "R-new") + if err != nil || got.ID != id { + t.Fatalf("transaction-local binding failed: %+v %v", got, err) + } + return rolledBack + }) + if !errors.Is(err, rolledBack) { + t.Fatal(err) + } + if _, err = s.ReviewStateParent(ctx, "fixture/repo", 7, "101", "R1"); err != nil { + t.Fatal("identity mutation escaped rollback", err) + } + if _, err = s.DB().ExecContext(ctx, "UPDATE threads SET kind='issue' WHERE id=?", id); err != nil { + t.Fatal(err) + } + if _, err = s.ReviewStateParent(ctx, "fixture/repo", 7, "101", "R1"); err == nil { + t.Fatal("issue accepted as review-state parent") + } +} + +func TestReviewRecoveryCannotDischargeCoreRetryOrCertifyCoreCoverage(t *testing.T) { + ctx := context.Background() + s, id := recoveryContractStore(t) + through := "2026-01-01T00:00:00Z" + if err := s.SaveAnalyticsCoverage(ctx, "fixture/repo", through, 2, 1); err != nil { + t.Fatal(err) + } + a := AnalyticsAttempt{Repository: "fixture/repo", Number: 7, Operation: "review_state", StartedAt: through, FinishedAt: "2026-01-01T00:00:01Z", Status: "failed", ErrorClass: "validation", Evidence: json.RawMessage(`{}`)} + record := func() { + t.Helper() + if err := s.RecordAnalyticsAttempt(ctx, a); err != nil { + t.Fatal(err) + } + } + queued := func(want bool, operations ...string) { + t.Helper() + got, err := s.AnalyticsItemQueued(ctx, a.Repository, a.Number, operations...) + if err != nil || got != want { + t.Fatalf("queued(%v)=%v want%v err%v", operations, got, want, err) + } + } + coverage := func(want int) { + t.Helper() + var complete int + var watermark string + if err := s.DB().QueryRowContext(ctx, "SELECT complete,through FROM analytics_coverage WHERE repository=?", a.Repository).Scan(&complete, &watermark); err != nil || complete != want || watermark != through { + t.Fatalf("coverage=%d through=%s err=%v", complete, watermark, err) + } + } + record() + queued(false) + queued(true, "review_state") + coverage(1) + a.Operation = "graphql_history" + record() + queued(true) + coverage(0) + if err := s.UpsertPullRequestReviewThreads(ctx, id, a.FinishedAt, nil); err != nil { + t.Fatal(err) + } + a.Operation = "review_state" + a.Status = "success" + record() + queued(false, "review_state") + queued(true) + coverage(0) + a.Operation = "graphql_history" + record() + queued(false) + coverage(0) + // A successful item is not itself a completed traversal watermark. + if err := s.SetAnalyticsCoverageComplete(ctx, a.Repository, true); err != nil { + t.Fatal(err) + } + coverage(1) + a.Status = "failed" + a.FinishedAt = "2026-01-01T00:01:00Z" + record() + queued(true) + coverage(0) + var receipts int + if err := s.DB().QueryRowContext(ctx, "SELECT count(*) FROM analytics_fetch_attempts WHERE repository=?", a.Repository).Scan(&receipts); err != nil || receipts != 5 { + t.Fatalf("lost recovery history: %d %v", receipts, err) + } + var resolved sql.NullString + if err := s.DB().QueryRowContext(ctx, "SELECT resolved_at FROM analytics_retries WHERE repository=? AND operation='review_state'", a.Repository).Scan(&resolved); err != nil || !resolved.Valid { + t.Fatalf("independent review resolution lost: %v %v", resolved, err) + } +} diff --git a/internal/store/analytics_source.go b/internal/store/analytics_source.go new file mode 100644 index 00000000..bb882569 --- /dev/null +++ b/internal/store/analytics_source.go @@ -0,0 +1,345 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" +) + +// Publication describes provider-authored timestamps, not capture/repair time. +type Publication struct { + Submitted string `json:"submitted_at_gh,omitempty"` + Published string `json:"publication_at_gh,omitempty"` +} + +func ProviderTime(value any) string { + s, ok := value.(string) + if !ok || strings.TrimSpace(s) != s { + return "" + } + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return "" + } + return t.UTC().Format(time.RFC3339Nano) +} +func CommentPublication(kind, raw string) Publication { + var p map[string]any + if json.Unmarshal([]byte(raw), &p) != nil { + return Publication{} + } + g, _ := p["_graphql"].(map[string]any) + pick := func(values ...any) string { + for _, v := range values { + if s := ProviderTime(v); s != "" { + return s + } + } + return "" + } + if kind == "pull_review" { + if strings.EqualFold(fmt.Sprint(p["state"]), "pending") { + return Publication{} + } + at := pick(p["submitted_at"], g["submittedAt"]) + return Publication{Submitted: at, Published: at} + } + // A retained null publication event is evidence of an unknown/draft event. + if value, present := g["publishedAt"]; present { + return Publication{Published: ProviderTime(value)} + } + if value, present := p["published_at"]; present { + return Publication{Published: ProviderTime(value)} + } + if kind == "pull_review_comment" { + return Publication{} + } + return Publication{Published: pick(p["created_at"], g["createdAt"])} +} +func (s *Store) ensureAnalyticsSourceSchema(ctx context.Context) error { + for _, table := range []string{"comments", "comment_revisions"} { + for _, col := range []string{"submitted_at_gh", "publication_at_gh"} { + if err := s.ensureColumn(ctx, table, col, "text"); err != nil { + return err + } + } + } + _, err := s.q().ExecContext(ctx, `CREATE TABLE IF NOT EXISTS actor_identity_evidence( + node_id TEXT PRIMARY KEY,actor_node_id TEXT,login TEXT,actor_type TEXT,observed_at TEXT NOT NULL,raw_json TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS actor_profiles(node_id TEXT PRIMARY KEY,login TEXT NOT NULL,actor_type TEXT NOT NULL,name TEXT,bio TEXT,url TEXT,created_at TEXT,observed_at TEXT NOT NULL,raw_json TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS analytics_coverage(repository TEXT PRIMARY KEY,through TEXT NOT NULL,issues INTEGER,pull_requests INTEGER,complete INTEGER NOT NULL,observed_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS analytics_repair_receipts(name TEXT PRIMARY KEY,cursor INTEGER NOT NULL DEFAULT 0,updated_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS analytics_pending_nodes(node_id TEXT NOT NULL,kind TEXT NOT NULL,PRIMARY KEY(kind,node_id)); + CREATE TABLE IF NOT EXISTS analytics_collection_state(name TEXT PRIMARY KEY,value TEXT NOT NULL,updated_at TEXT NOT NULL);`) + return err +} +func (s *Store) queueAnalyticsActor(ctx context.Context, raw string) error { + var payload struct { + NodeID string `json:"node_id"` + User struct { + NodeID string `json:"node_id"` + } `json:"user"` + } + if json.Unmarshal([]byte(raw), &payload) != nil { + return nil + } + id, kind := payload.User.NodeID, "profile" + if id == "" { + id, kind = payload.NodeID, "identity" + } + if id == "" { + return nil + } + _, err := s.q().ExecContext(ctx, "INSERT OR IGNORE INTO analytics_pending_nodes(node_id,kind) VALUES(?,?)", id, kind) + return err +} +func (s *Store) upsertCommentPublication(ctx context.Context, id int64, kind, raw string) error { + p := CommentPublication(kind, raw) + if err := s.queueAnalyticsActor(ctx, raw); err != nil { + return err + } + _, err := s.q().ExecContext(ctx, "UPDATE comments SET submitted_at_gh=?,publication_at_gh=? WHERE id=?", nullString(p.Submitted), nullString(p.Published), id) + return err +} + +type PublicationRepair struct { + Scanned int `json:"scanned"` + Changed int `json:"changed"` + WouldChange int `json:"would_change"` + Unknown int `json:"unknown"` +} + +// RepairPublication uses retained native payloads. Raw JSON, IDs and original +// recorded_at values are never rewritten, and no synthetic revision is appended. +func (s *Store) RepairPublication(ctx context.Context, apply bool) (PublicationRepair, error) { + result := PublicationRepair{} + pending := false + if apply { + value, e := s.AnalyticsState(ctx, "publication_repair_in_progress") + if e != nil { + return result, e + } + pending = value == "1" + if e = s.SetAnalyticsState(ctx, "publication_repair_in_progress", "1"); e != nil { + return result, e + } + } + for _, table := range []string{"comments", "comment_revisions"} { + var cursor int64 + var hasFields int + if err := s.q().QueryRowContext(ctx, "SELECT count(*) FROM pragma_table_info(?) WHERE name IN('submitted_at_gh','publication_at_gh')", table).Scan(&hasFields); err != nil { + return result, err + } + for { + fields := ",NULL,NULL" + if hasFields == 2 { + fields = ",c.submitted_at_gh,c.publication_at_gh" + } + query := "SELECT c.id,c.comment_type,c.raw_json" + fields + " FROM comments c WHERE c.id>? ORDER BY c.id LIMIT 1000" + if table == "comment_revisions" { + fields = ",NULL,NULL" + if hasFields == 2 { + fields = ",r.submitted_at_gh,r.publication_at_gh" + } + query = "SELECT r.id,c.comment_type,r.raw_json" + fields + " FROM comment_revisions r JOIN comments c ON c.id=r.comment_id WHERE r.id>? ORDER BY r.id LIMIT 1000" + } + rows, err := s.q().QueryContext(ctx, query, cursor) + if err != nil { + return result, err + } + type item struct { + id int64 + kind, raw string + submitted, published sql.NullString + } + var batch []item + for rows.Next() { + var r item + if err := rows.Scan(&r.id, &r.kind, &r.raw, &r.submitted, &r.published); err != nil { + rows.Close() + return result, err + } + batch = append(batch, r) + } + err = rows.Err() + rows.Close() + if err != nil { + return result, err + } + if len(batch) == 0 { + break + } + if apply { + err = s.WithTx(ctx, func(tx *Store) error { + for _, r := range batch { + p := CommentPublication(r.kind, r.raw) + if p.Submitted != r.submitted.String || p.Published != r.published.String { + result.WouldChange++ + } + result.Scanned++ + if r.kind == "pull_review" && p.Submitted == "" { + result.Unknown++ + } + res, e := tx.q().ExecContext(ctx, "UPDATE "+table+" SET submitted_at_gh=?,publication_at_gh=? WHERE id=? AND raw_json=? AND (submitted_at_gh IS NOT ? OR publication_at_gh IS NOT ?)", nullString(p.Submitted), nullString(p.Published), r.id, r.raw, nullString(p.Submitted), nullString(p.Published)) + if e != nil { + return e + } + n, e := res.RowsAffected() + if e != nil { + return e + } + result.Changed += int(n) + } + _, e := tx.q().ExecContext(ctx, "INSERT INTO analytics_repair_receipts(name,cursor,updated_at) VALUES(?,?,?) ON CONFLICT(name) DO UPDATE SET cursor=excluded.cursor,updated_at=excluded.updated_at", table, batch[len(batch)-1].id, time.Now().UTC().Format(time.RFC3339Nano)) + return e + }) + if err != nil { + return result, err + } + } else { + for _, r := range batch { + result.Scanned++ + p := CommentPublication(r.kind, r.raw) + if p.Submitted != r.submitted.String || p.Published != r.published.String { + result.WouldChange++ + } + if r.kind == "pull_review" && p.Submitted == "" { + result.Unknown++ + } + } + } + cursor = batch[len(batch)-1].id + } + } + if apply { + if result.Changed > 0 || pending { + if e := s.SetAnalyticsState(ctx, "publication_repair_generation", time.Now().UTC().Format(time.RFC3339Nano)); e != nil { + return result, e + } + } + if e := s.SetAnalyticsState(ctx, "publication_repair_in_progress", "0"); e != nil { + return result, e + } + } + return result, nil +} + +// Use the native node itself as the identity evidence key: login reuse is not an +// identity join. Deleted/unavailable actors are retained as explicit unknowns. +func (s *Store) SaveActorEvidence(ctx context.Context, nodes []map[string]any, at string) error { + return s.WithTx(ctx, func(tx *Store) error { + for _, n := range nodes { + id, _ := n["id"].(string) + if id == "" { + continue + } + a, _ := n["author"].(map[string]any) + actor, _ := a["id"].(string) + if actor != "" { + if _, e := tx.q().ExecContext(ctx, "INSERT OR IGNORE INTO analytics_pending_nodes(node_id,kind) VALUES(?,'profile')", actor); e != nil { + return e + } + } + login, _ := a["login"].(string) + typ, _ := a["__typename"].(string) + raw, e := json.Marshal(n) + if e != nil { + return e + } + if _, e = tx.q().ExecContext(ctx, `INSERT INTO actor_identity_evidence VALUES(?,?,?,?,?,?) ON CONFLICT(node_id) DO UPDATE SET actor_node_id=excluded.actor_node_id,login=excluded.login,actor_type=excluded.actor_type,observed_at=excluded.observed_at,raw_json=excluded.raw_json`, id, nullString(actor), nullString(login), nullString(typ), at, string(raw)); e != nil { + return e + } + } + return nil + }) +} +func (s *Store) SaveActorProfiles(ctx context.Context, nodes []map[string]any, at string) error { + return s.WithTx(ctx, func(tx *Store) error { + for _, n := range nodes { + id, _ := n["id"].(string) + login, _ := n["login"].(string) + typ, _ := n["__typename"].(string) + if id == "" { + continue + } + raw, e := json.Marshal(n) + if e != nil { + return e + } + name, _ := n["name"].(string) + bio, _ := n["bio"].(string) + url, _ := n["url"].(string) + _, e = tx.q().ExecContext(ctx, `INSERT INTO actor_profiles VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(node_id) DO UPDATE SET login=excluded.login,actor_type=excluded.actor_type,name=excluded.name,bio=excluded.bio,url=excluded.url,created_at=excluded.created_at,observed_at=excluded.observed_at,raw_json=excluded.raw_json`, id, login, typ, nullString(name), nullString(bio), nullString(url), nullString(ProviderTime(n["createdAt"])), at, string(raw)) + if e != nil { + return e + } + } + return nil + }) +} +func (s *Store) SeedAnalyticsNodes(ctx context.Context, profiles bool) error { + query := `INSERT OR IGNORE INTO analytics_pending_nodes(node_id,kind) SELECT json_extract(raw_json,'$.node_id'),'identity' FROM threads WHERE json_valid(raw_json) AND json_extract(raw_json,'$.user.node_id') IS NULL AND json_extract(raw_json,'$.node_id') IS NOT NULL + UNION SELECT json_extract(raw_json,'$.node_id'),'identity' FROM comments WHERE json_valid(raw_json) AND json_extract(raw_json,'$.user.node_id') IS NULL AND json_extract(raw_json,'$.node_id') IS NOT NULL` + if profiles { + query = `INSERT OR IGNORE INTO analytics_pending_nodes(node_id,kind) SELECT json_extract(raw_json,'$.user.node_id'),'profile' FROM threads WHERE json_valid(raw_json) AND json_extract(raw_json,'$.user.node_id') IS NOT NULL + UNION SELECT json_extract(raw_json,'$.user.node_id'),'profile' FROM comments WHERE json_valid(raw_json) AND json_extract(raw_json,'$.user.node_id') IS NOT NULL + UNION SELECT actor_node_id,'profile' FROM actor_identity_evidence WHERE actor_node_id IS NOT NULL` + } + _, err := s.q().ExecContext(ctx, query) + return err +} +func (s *Store) AnalyticsIdentityNodes(ctx context.Context, limit int, after ...string) ([]string, error) { + return s.analyticsNodes(ctx, limit, false, after...) +} +func (s *Store) AnalyticsProfileNodes(ctx context.Context, limit int, after ...string) ([]string, error) { + return s.analyticsNodes(ctx, limit, true, after...) +} +func (s *Store) analyticsNodes(ctx context.Context, limit int, profiles bool, after ...string) ([]string, error) { + query := `SELECT q.node_id FROM analytics_pending_nodes q WHERE kind='identity' AND q.node_id>? AND NOT EXISTS(SELECT 1 FROM actor_identity_evidence e WHERE e.node_id=q.node_id) ORDER BY q.node_id LIMIT ?` + if profiles { + query = `SELECT q.node_id FROM analytics_pending_nodes q WHERE kind='profile' AND q.node_id>? AND NOT EXISTS(SELECT 1 FROM actor_profiles p WHERE p.node_id=q.node_id AND p.observed_at>=?) ORDER BY q.node_id LIMIT ?` + } + cursor := "" + if len(after) > 0 { + cursor = after[0] + } + var rows *sql.Rows + var e error + if profiles { + rows, e = s.q().QueryContext(ctx, query, cursor, time.Now().UTC().Add(-24*time.Hour).Format(time.RFC3339Nano), limit) + } else { + rows, e = s.q().QueryContext(ctx, query, cursor, limit) + } + if e != nil { + return nil, e + } + defer rows.Close() + var out []string + for rows.Next() { + var id string + if e = rows.Scan(&id); e != nil { + return nil, e + } + out = append(out, id) + } + return out, rows.Err() +} +func (s *Store) AnalyticsState(ctx context.Context, key string) (string, error) { + var value string + err := s.q().QueryRowContext(ctx, "SELECT value FROM analytics_collection_state WHERE name=?", key).Scan(&value) + if err == sql.ErrNoRows { + return "", nil + } + return value, err +} +func (s *Store) SetAnalyticsState(ctx context.Context, key, value string) error { + _, err := s.q().ExecContext(ctx, "INSERT INTO analytics_collection_state VALUES(?,?,?) ON CONFLICT(name) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", key, value, time.Now().UTC().Format(time.RFC3339Nano)) + return err +} +func (s *Store) SaveAnalyticsCoverage(ctx context.Context, repo, through string, issues, prs int) error { + _, e := s.q().ExecContext(ctx, `INSERT INTO analytics_coverage VALUES(?,?,?,?,1,?) ON CONFLICT(repository) DO UPDATE SET through=excluded.through,issues=excluded.issues,pull_requests=excluded.pull_requests,complete=1,observed_at=excluded.observed_at`, repo, through, issues, prs, time.Now().UTC().Format(time.RFC3339Nano)) + return e +} diff --git a/internal/store/analytics_source_test.go b/internal/store/analytics_source_test.go new file mode 100644 index 00000000..9d1cc1b1 --- /dev/null +++ b/internal/store/analytics_source_test.go @@ -0,0 +1,105 @@ +package store + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" +) + +func TestPublicationRepairPreservesSourceEvidence(t *testing.T) { + ctx := context.Background() + s, e := Open(ctx, filepath.Join(t.TempDir(), "source.db")) + if e != nil { + t.Fatal(e) + } + defer s.Close() + _, e = s.db.Exec(`INSERT INTO repositories(id,owner,name,full_name,github_repo_id,raw_json,updated_at) VALUES(1,'fixture','repo','fixture/repo',1,'{}','2026-01-01'); + INSERT INTO threads(id,repo_id,github_id,number,kind,state,title,body,html_url,labels_json,assignees_json,raw_json,content_hash,updated_at) VALUES(1,1,'pr',1,'pull_request','open','','','','[]','[]','{}','h','2026-01-01')`) + if e != nil { + t.Fatal(e) + } + raw := `{"state":"APPROVED","created_at":"2026-01-01T00:00:00Z","submitted_at":"2026-01-03T12:34:56Z","user":{"login":"a","type":"User"}}` + id, e := s.UpsertComment(ctx, Comment{ThreadID: 1, GitHubID: "review", CommentType: "pull_review", RawJSON: raw, CreatedAtGitHub: "2026-01-01T00:00:00Z"}) + if e != nil { + t.Fatal(e) + } + _, e = s.db.Exec("UPDATE comments SET submitted_at_gh=NULL,publication_at_gh=NULL; UPDATE comment_revisions SET submitted_at_gh=NULL,publication_at_gh=NULL") + if e != nil { + t.Fatal(e) + } + var originalRecorded string + s.db.QueryRow("SELECT recorded_at FROM comment_revisions").Scan(&originalRecorded) + preview, e := s.RepairPublication(ctx, false) + if e != nil || preview.WouldChange != 2 || preview.Changed != 0 { + t.Fatalf("dry-run %+v %v", preview, e) + } + r, e := s.RepairPublication(ctx, true) + if e != nil || r.Changed != 2 { + t.Fatalf("%+v %v", r, e) + } + r, e = s.RepairPublication(ctx, true) + if e != nil || r.Changed != 0 { + t.Fatalf("replay %+v %v", r, e) + } + var published, submitted, created, stored string + s.db.QueryRow("SELECT publication_at_gh,submitted_at_gh,created_at_gh,raw_json FROM comments WHERE id=?", id).Scan(&published, &submitted, &created, &stored) + if published != "2026-01-03T12:34:56Z" || submitted != published || created != "2026-01-01T00:00:00Z" || stored != raw { + t.Fatalf("source timestamps/evidence changed incorrectly") + } + var n int + var recorded string + s.db.QueryRow("SELECT count(*),min(recorded_at) FROM comment_revisions").Scan(&n, &recorded) + if n != 1 || recorded != originalRecorded { + t.Fatal("repair manufactured a source revision") + } + for _, c := range []struct{ kind, raw, want string }{ + {"pull_review", `{"state":"PENDING","submitted_at":"2026-01-01T00:00:00Z"}`, ""}, + {"pull_review", `{"submitted_at":"not a date"}`, ""}, + {"pull_review", `{"_graphql":{"submittedAt":"2026-01-01T00:00:00+02:00"}}`, "2025-12-31T22:00:00Z"}, + {"pull_review_comment", `{"created_at":"2026-01-01T00:00:00Z","_graphql":{"publishedAt":"2026-01-02T00:00:00Z"}}`, "2026-01-02T00:00:00Z"}, + } { + if got := CommentPublication(c.kind, c.raw).Published; got != c.want { + t.Fatalf("publication=%q want %q", got, c.want) + } + } + nodes := []map[string]any{{"id": "comment-node", "author": map[string]any{"id": "user-one", "login": "same", "__typename": "User"}}, {"id": "other-node", "author": map[string]any{"id": "user-two", "login": "same", "__typename": "User"}}} + if e = s.SaveActorEvidence(ctx, nodes, "2026-01-01T00:00:00Z"); e != nil { + t.Fatal(e) + } + s.db.QueryRow("SELECT count(distinct actor_node_id) FROM actor_identity_evidence").Scan(&n) + if n != 2 { + t.Fatal("login reuse merged identities") + } + _, _ = json.Marshal(nodes) +} + +func TestAnalyticsQueuesNewUnresolvedSourceIdentities(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "source.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for _, raw := range []string{`{"node_id":"content","user":{"node_id":"actor"}}`, `{"node_id":"unresolved","user":null}`} { + if err := s.queueAnalyticsActor(ctx, raw); err != nil { + t.Fatal(err) + } + } + var count int + if err := s.DB().QueryRowContext(ctx, `SELECT count(*) FROM analytics_pending_nodes WHERE (node_id='actor' AND kind='profile') OR (node_id='unresolved' AND kind='identity')`).Scan(&count); err != nil || count != 2 { + t.Fatalf("count=%d err=%v", count, err) + } +} + +func TestCommentPublicationPreservesExplicitUnknown(t *testing.T) { + for _, raw := range []string{ + `{"created_at":"2026-01-01T00:00:00Z","pull_request_review_id":123}`, + `{"created_at":"2026-01-01T00:00:00Z","_graphql":{"publishedAt":null,"createdAt":"2026-01-01T00:00:00Z"}}`, + `{"created_at":"2026-01-01T00:00:00Z","published_at":null}`, + } { + if got := CommentPublication("pull_review_comment", raw).Published; got != "" { + t.Errorf("draft published at %q", got) + } + } +} diff --git a/internal/store/comments.go b/internal/store/comments.go index f2c599ed..d523faf6 100644 --- a/internal/store/comments.go +++ b/internal/store/comments.go @@ -12,15 +12,16 @@ import ( ) type Comment struct { - ID int64 `json:"id"` - ThreadID int64 `json:"thread_id"` - GitHubID string `json:"github_id"` - CommentType string `json:"comment_type"` - AuthorLogin string `json:"author_login,omitempty"` - AuthorType string `json:"author_type,omitempty"` - Body string `json:"body"` - IsBot bool `json:"is_bot"` - ReviewState string `json:"review_state,omitempty"` + ID int64 `json:"id"` + ThreadID int64 `json:"thread_id"` + GitHubID string `json:"github_id"` + CommentType string `json:"comment_type"` + AuthorLogin string `json:"author_login,omitempty"` + AuthorType string `json:"author_type,omitempty"` + Body string `json:"body"` + IsBot bool `json:"is_bot"` + ReviewState string `json:"review_state,omitempty"` + Publication RawJSON string `json:"-"` CreatedAtGitHub string `json:"created_at_gh,omitempty"` UpdatedAtGitHub string `json:"updated_at_gh,omitempty"` @@ -71,6 +72,9 @@ func (s *Store) upsertComment(ctx context.Context, comment Comment) (int64, erro if err != nil { return 0, fmt.Errorf("upsert comment: %w", err) } + if err := s.upsertCommentPublication(ctx, id, comment.CommentType, comment.RawJSON); err != nil { + return 0, err + } if s.portableCommentBodyMetadata { if _, err := s.q().ExecContext(ctx, ` update comments @@ -131,10 +135,10 @@ func (s *Store) recordCommentRevision(ctx context.Context, commentID int64, reco if _, err := s.q().ExecContext(ctx, ` insert into comment_revisions( comment_id, author_login, author_type, body, is_bot, raw_json, - created_at_gh, updated_at_gh, deleted_at, deletion_reason, recorded_at + created_at_gh, updated_at_gh, deleted_at, deletion_reason, recorded_at,submitted_at_gh,publication_at_gh ) select c.id, c.author_login, c.author_type, c.body, c.is_bot, c.raw_json, - c.created_at_gh, c.updated_at_gh, c.deleted_at, c.deletion_reason, ? + c.created_at_gh, c.updated_at_gh, c.deleted_at, c.deletion_reason, ?,c.submitted_at_gh,c.publication_at_gh from comments c where c.id = ? and not exists ( @@ -180,6 +184,7 @@ func (s *Store) ListComments(ctx context.Context, threadID int64) ([]Comment, er Body: row.Body, IsBot: int64Bool(row.IsBot), ReviewState: reviewStateFromRawJSON(row.RawJson), + Publication: CommentPublication(row.CommentType, row.RawJson), RawJSON: row.RawJson, CreatedAtGitHub: stringValue(row.CreatedAtGh), UpdatedAtGitHub: stringValue(row.UpdatedAtGh), diff --git a/internal/store/portable.go b/internal/store/portable.go index 062e9801..9fcc4cc9 100644 --- a/internal/store/portable.go +++ b/internal/store/portable.go @@ -283,11 +283,17 @@ func (s *Store) vacuumPortableDatabase(ctx context.Context) error { func (s *Store) scrubPortableSyncFailures(ctx context.Context, include bool, stats *PortablePruneStats) (bool, error) { ledgerExists := s.tableExists(ctx, "sync_attempt_failures") + var analyticsTables []string + for _, table := range portableAnalyticsTables() { + if s.tableExists(ctx, table) { + analyticsTables = append(analyticsTables, table) + } + } pending, err := s.portableSyncFailureScrubPending(ctx) if err != nil { return false, err } - if !ledgerExists && !pending { + if !ledgerExists && len(analyticsTables) == 0 && !pending { return false, nil } if !pending { @@ -302,14 +308,24 @@ func (s *Store) scrubPortableSyncFailures(ctx context.Context, include bool, sta return false, fmt.Errorf("mark portable sync failure scrub pending: %w", err) } } - if !ledgerExists { - return true, nil - } conn, err := s.db.Conn(ctx) if err != nil { return false, fmt.Errorf("open portable sync failure scrub connection: %w", err) } defer conn.Close() + if len(analyticsTables) > 0 { + if _, err := conn.ExecContext(ctx, `pragma secure_delete = on`); err != nil { + return false, err + } + for _, table := range analyticsTables { + if _, err := conn.ExecContext(ctx, `delete from `+sqliteIdentifier(table)); err != nil { + return false, fmt.Errorf("scrub portable analytics table %s: %w", table, err) + } + } + } + if !ledgerExists { + return true, nil + } var hasRows bool if err := conn.QueryRowContext(ctx, `select exists(select 1 from sync_attempt_failures limit 1)`).Scan(&hasRows); err != nil { return false, fmt.Errorf("inspect portable sync failures: %w", err) diff --git a/internal/store/portable_schema.go b/internal/store/portable_schema.go index d1f110b9..46a06eff 100644 --- a/internal/store/portable_schema.go +++ b/internal/store/portable_schema.go @@ -79,7 +79,7 @@ func (s *Store) canonicalizePortableSchema(ctx context.Context, options Portable } capabilities := "body_excerpts,comment_excerpts,author_association,thread_revisions,thread_fingerprints,thread_key_summaries,pr_details,pr_files,pr_commits,pr_checks,pr_review_threads,workflow_runs,family_tombstones,comment_revisions,pr_review_thread_revisions,raw_json_stripped" includes := "repositories,threads,comments,comment_revisions,thread_revisions,thread_fingerprints,thread_key_summaries,pull_request_details,pull_request_files,pull_request_commits,pull_request_checks,pull_request_review_threads,pull_request_review_thread_revisions,pull_request_review_thread_syncs,github_workflow_runs" - excluded := "raw_json,pull_request_file_patches,documents,fts,vectors,code_snapshots,code_documents,cluster_events,run_history,similarity_edges,blobs,sync_attempt_failures" + excluded := "analytics,actor_evidence,raw_json,pull_request_file_patches,documents,fts,vectors,code_snapshots,code_documents,cluster_events,run_history,similarity_edges,blobs,sync_attempt_failures" if stats.SyncFailuresIncluded { capabilities += ",sync_failure_ledger_redacted" includes += ",sync_attempt_failures" @@ -422,7 +422,7 @@ func (s *Store) sanitizePortableRepositoryCompatibilityColumn(ctx context.Contex } func canonicalPortableDroppedTables() []string { - return []string{ + return append([]string{ "code_documents_fts", "code_documents_fts_config", "code_documents_fts_data", @@ -452,11 +452,11 @@ func canonicalPortableDroppedTables() []string { "similarity_edges", "blobs", "sync_attempt_failures", - } + }, portableAnalyticsTables()...) } func canonicalPortableBulkDropOrder() []string { - return []string{ + return append([]string{ "code_documents_fts", "code_documents_fts_config", "code_documents_fts_data", @@ -486,5 +486,15 @@ func canonicalPortableBulkDropOrder() []string { "embedding_runs", "cluster_runs", "blobs", + }, portableAnalyticsTables()...) +} + +// Analytics source datasets have no repository-scoped portable contract. +func portableAnalyticsTables() []string { + return []string{ + "analytics_fetch_attempts", "analytics_retries", "analytics_pending_nodes", + "analytics_collection_state", "analytics_repair_receipts", + "analytics_coverage", "analytics_review_state_coverage", + "actor_identity_evidence", "actor_profiles", } } diff --git a/internal/store/review_threads.go b/internal/store/review_threads.go index baf3d1c6..8bc50795 100644 --- a/internal/store/review_threads.go +++ b/internal/store/review_threads.go @@ -3,7 +3,9 @@ package store import ( "context" "database/sql" + "encoding/json" "fmt" + "strings" "time" "github.com/openclaw/gitcrawl/internal/store/storedb" @@ -34,7 +36,12 @@ type PullRequestReviewThread struct { } func (s *Store) UpsertPullRequestReviewThreads(ctx context.Context, threadID int64, fetchedAt string, threads []PullRequestReviewThread) error { + seen := map[string]bool{} for _, thread := range threads { + if thread.ReviewThreadID == "" || strings.TrimSpace(thread.ReviewThreadID) != thread.ReviewThreadID || seen[thread.ReviewThreadID] { + return fmt.Errorf("review-thread membership requires nonempty unique native IDs") + } + seen[thread.ReviewThreadID] = true if err := validateTombstone(thread.DeletedAt, thread.DeletionReason); err != nil { return fmt.Errorf("upsert pull request review thread %q: %w", thread.ReviewThreadID, err) } @@ -48,12 +55,43 @@ func (s *Store) UpsertPullRequestReviewThreads(ctx context.Context, threadID int } func (s *Store) upsertPullRequestReviewThreads(ctx context.Context, threadID int64, fetchedAt string, threads []PullRequestReviewThread) error { + observed, err := time.Parse(time.RFC3339Nano, fetchedAt) + if err != nil { + return fmt.Errorf("invalid review-thread observation time: %w", err) + } + var previous string + err = s.q().QueryRowContext(ctx, "SELECT fetched_at FROM pull_request_review_thread_syncs WHERE thread_id=?", threadID).Scan(&previous) + if err != nil && err != sql.ErrNoRows { + return err + } + if err == nil { + prior, parseErr := time.Parse(time.RFC3339Nano, previous) + if parseErr != nil { + return fmt.Errorf("invalid prior review-thread observation time: %w", parseErr) + } + // This runs inside the same native transaction as state/history writes. + // An older completion must not replace either membership or row state. + if observed.Before(prior) { + return nil + } + } if err := s.qsql().UpsertPullRequestReviewThreadSync(ctx, storedb.UpsertPullRequestReviewThreadSyncParams{ ThreadID: threadID, FetchedAt: fetchedAt, }); err != nil { return fmt.Errorf("mark pull request review threads fetched: %w", err) } + ids := make([]string, 0, len(threads)) + for _, thread := range threads { + ids = append(ids, thread.ReviewThreadID) + } + encoded, err := json.Marshal(ids) + if err != nil { + return err + } + if _, err = s.q().ExecContext(ctx, "UPDATE pull_request_review_thread_syncs SET review_thread_ids_json=? WHERE thread_id=? AND fetched_at=?", string(encoded), threadID, fetchedAt); err != nil { + return err + } for _, thread := range threads { if thread.ReviewThreadID == "" { continue diff --git a/internal/store/store.go b/internal/store/store.go index 501a1e32..ca464e80 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,7 +17,7 @@ import ( ) const ( - schemaVersion = 13 + schemaVersion = 15 timeLayout = time.RFC3339Nano ) @@ -285,6 +285,30 @@ func (s *Store) migrate(ctx context.Context) error { if _, err := s.db.ExecContext(ctx, schemaSQL); err != nil { return fmt.Errorf("apply schema: %w", err) } + // Version 15 only adds operational receipts and an explicit review-thread + // membership column. A converged v14 archive needs no row/history rebuild. + if current == 14 { + structural, e := inspectStructuralCompatibilityMigrations(ctx, s, current, inspectPRDetailSchema(ctx, s)) + if e != nil { + return e + } + converged, e := s.observationSchemaConvergenceIsCurrent(ctx) + if e != nil { + return e + } + if len(structural) == 1 && structural[0] == "schema_version_14_to_15" && converged { + // Some v14 archives predate the optional analytics extension. These + // additive tables/columns are cheap to ensure and require no row scan. + if e = s.ensureAnalyticsSourceSchema(ctx); e != nil { + return e + } + if e = s.ensureAnalyticsIntegritySchema(ctx); e != nil { + return e + } + _, e = s.db.ExecContext(ctx, fmt.Sprintf("PRAGMA user_version=%d", schemaVersion)) + return e + } + } if current == schemaVersion { prDetails := inspectPRDetailSchema(ctx, s) structural, err := inspectStructuralCompatibilityMigrations( @@ -319,6 +343,12 @@ func (s *Store) migrate(ctx context.Context) error { if err := s.ensureFamilyTombstoneSchema(ctx); err != nil { return err } + if err := s.ensureAnalyticsSourceSchema(ctx); err != nil { + return err + } + if err := s.ensureAnalyticsIntegritySchema(ctx); err != nil { + return err + } if err := s.ensureCanonicalObservationTables(ctx); err != nil { return err } diff --git a/internal/store/threads.go b/internal/store/threads.go index 0a3dd744..7b1b2a32 100644 --- a/internal/store/threads.go +++ b/internal/store/threads.go @@ -254,6 +254,9 @@ func (s *Store) upsertThreadObservation(ctx context.Context, thread Thread, opti if err != nil { return UpsertThreadResult{}, fmt.Errorf("upsert thread: %w", err) } + if err := s.queueAnalyticsActor(ctx, thread.RawJSON); err != nil { + return UpsertThreadResult{}, err + } if s.portableThreadBodyMetadata { if _, err := s.q().ExecContext(ctx, ` update threads diff --git a/internal/syncer/graphql_history_test.go b/internal/syncer/graphql_history_test.go index d1ce8928..e8faa959 100644 --- a/internal/syncer/graphql_history_test.go +++ b/internal/syncer/graphql_history_test.go @@ -59,6 +59,7 @@ func TestGraphQLHistoryUsesNativeTransactionsAndPreservesLegacyIdentity(t *testi defer server.Close() client := historyFixtureClient{Client: gh.New(gh.Options{BaseURL: server.URL}), batch: gh.HistoryBatch{Repository: rawRepo, Items: []gh.HistoryItem{{Thread: row, Pull: pull, Comments: comments, Reviews: reviews, ReviewComments: inline}}}} options.GraphQLHistory = true + client.batch.Items[0].ReviewThreads = []map[string]any{{"id": "RT_fixture", "isResolved": true, "isOutdated": false, "comments": map[string]any{"nodes": []any{map[string]any{"id": "inline-node", "body": "reply", "replyTo": map[string]any{"id": "parent-node"}}}}}} stats, err := New(client, st).Sync(ctx, options) if err != nil { t.Fatal(err) @@ -73,6 +74,16 @@ func TestGraphQLHistoryUsesNativeTransactionsAndPreservesLegacyIdentity(t *testi if stats.ThreadsSynced != 1 || stats.PRDetailsSynced != 1 || stats.CommentsSynced == 0 { t.Fatalf("stats %+v", stats) } + if stats.ReviewThreadsSynced != 1 { + t.Fatalf("review states missing: %+v", stats) + } + var resolved int + var members string + st.DB().QueryRow("SELECT is_resolved FROM pull_request_review_threads WHERE review_thread_id='RT_fixture'").Scan(&resolved) + st.DB().QueryRow("SELECT review_thread_ids_json FROM pull_request_review_thread_syncs WHERE thread_id=?", after[0].ID).Scan(&members) + if resolved != 1 || members != `["RT_fixture"]` { + t.Fatalf("review projection %d %s", resolved, members) + } if _, err := New(client, st).Sync(ctx, options); err != nil { t.Fatal(err) } @@ -82,8 +93,26 @@ func TestGraphQLHistoryUsesNativeTransactionsAndPreservesLegacyIdentity(t *testi t.Fatal("failed batch persisted") } assertTableRowCount(t, st, "threads", 1) + var failed int + if err := st.DB().QueryRow("SELECT count(*) FROM analytics_fetch_attempts WHERE status='failed'").Scan(&failed); err != nil || failed != 1 { + t.Fatalf("missing durable fetch failure: %d %v", failed, err) + } options.IncludePRDetails = true if _, err := New(client, st).Sync(ctx, options); err == nil { t.Fatal("unsupported hydration accepted") } } + +func TestGraphQLCancellationDoesNotHideReceiptPersistenceFailure(t *testing.T) { + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + st.Close() + client := historyFixtureClient{err: context.DeadlineExceeded} + _, err = New(client, st).Sync(ctx, Options{Owner: "fixture", Repo: "repo", Numbers: []int{1}, State: "all", GraphQLHistory: true, IncludeComments: true, IncludePRMetadata: true, ReceiptOperation: "review_state"}) + if !errors.Is(err, context.DeadlineExceeded) || !errors.Is(err, errAnalyticsReceipt) { + t.Fatalf("missing distinguishable receipt failure: %v", err) + } +} diff --git a/internal/syncer/review_state.go b/internal/syncer/review_state.go new file mode 100644 index 00000000..4a118c96 --- /dev/null +++ b/internal/syncer/review_state.go @@ -0,0 +1,75 @@ +package syncer + +import ( + "context" + "encoding/json" + "fmt" + "time" + + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" +) + +func (s *Syncer) syncReviewState(ctx context.Context, options Options, started string) (Stats, error) { + stats := Stats{Repository: options.Owner + "/" + options.Repo, Numbers: uniquePositiveNumbers(options.Numbers), StartedAt: started, ReviewStateOnly: true} + client, ok := s.client.(interface { + FetchGraphQLReviewState(context.Context, string, string, []int, gh.Reporter) ([]gh.ReviewStateItem, error) + }) + if !ok { + return stats, fmt.Errorf("client does not support targeted review state") + } + // Reserve an observation sequence before fetching, as in normal capture. + sequence, err := s.store.NextThreadObservationSequence(ctx, started) + if err != nil { + return stats, err + } + fetched := time.Now() + items, err := client.FetchGraphQLReviewState(ctx, options.Owner, options.Repo, stats.Numbers, options.Reporter) + stats.FetchMillis = time.Since(fetched).Milliseconds() + if err != nil { + return stats, err + } + if len(items) != len(stats.Numbers) { + return stats, fmt.Errorf("incomplete review-state batch") + } + persist := time.Now() + err = s.store.WithTx(ctx, func(tx *store.Store) error { + applied, threads := 0, 0 + for i, item := range items { + if item.Number != stats.Numbers[i] { + return fmt.Errorf("review-state selection mismatch") + } + t, err := tx.ReviewStateParent(ctx, stats.Repository, item.Number, item.RepositoryID, item.RepositoryNodeID) + if err != nil { + return err + } + var raw map[string]any + if err := json.Unmarshal([]byte(t.RawJSON), &raw); err != nil { + return err + } + if known := stringValue(raw["node_id"]); known != "" && known != item.NodeID { + return fmt.Errorf("review-state archived node identity mismatch") + } + reserved, err := tx.ReserveThreadChildObservation(ctx, t.ID, store.ThreadChildReviewThreads, item.UpdatedAt, sequence) + if err != nil { + return err + } + if !reserved { + continue + } + count, err := s.persistPullReviewThreads(ctx, tx, t, item.Threads, started) + if err != nil { + return err + } + applied++ + threads += count + } + stats.ThreadsSynced = applied + stats.PullRequestsSynced = applied + stats.ReviewThreadsSynced = threads + return nil + }) + stats.PersistMillis = time.Since(persist).Milliseconds() + stats.FinishedAt = s.now().Format(time.RFC3339Nano) + return stats, err +} diff --git a/internal/syncer/review_state_test.go b/internal/syncer/review_state_test.go new file mode 100644 index 00000000..8c207fce --- /dev/null +++ b/internal/syncer/review_state_test.go @@ -0,0 +1,154 @@ +package syncer + +import ( + "context" + "encoding/json" + "errors" + gh "github.com/openclaw/gitcrawl/internal/github" + "github.com/openclaw/gitcrawl/internal/store" + "path/filepath" + "testing" + "time" +) + +type reviewOnlyFixture struct { + *gh.Client + items []gh.ReviewStateItem + err error +} + +func (f reviewOnlyFixture) FetchGraphQLReviewState(context.Context, string, string, []int, gh.Reporter) ([]gh.ReviewStateItem, error) { + return f.items, f.err +} +func TestReviewOnlyNeverOverwritesCanonicalContentHistoryOrVectors(t *testing.T) { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + legacy := &metadataGitHub{} + opts := Options{Owner: "openclaw", Repo: "gitcrawl", Numbers: []int{8}, State: "all", IncludeComments: true, IncludePRMetadata: true} + if _, err = New(legacy, s).Sync(ctx, opts); err != nil { + t.Fatal(err) + } + repo, err := s.RepositoryByFullName(ctx, "openclaw/gitcrawl") + if err != nil { + t.Fatal(err) + } + heads, err := s.ListThreads(ctx, repo.ID, true) + if err != nil { + t.Fatal(err) + } + head := heads[0] + if _, err = s.DB().Exec("update threads set raw_json=json_set(raw_json,'$.node_id','PR8') where id=?", head.ID); err != nil { + t.Fatal(err) + } + if _, err = s.DB().Exec("update repositories set raw_json=json_set(raw_json,'$.node_id','R1') where id=?", repo.ID); err != nil { + t.Fatal(err) + } + s.DB().QueryRow("select raw_json from threads where id=?", head.ID).Scan(&head.RawJSON) + s.DB().QueryRow("select raw_json from repositories where id=?", repo.ID).Scan(&repo.RawJSON) + if err = s.UpsertThreadVector(ctx, store.ThreadVector{ThreadID: head.ID, Basis: "title_original", Model: "fixture", Dimensions: 2, ContentHash: head.ContentHash, Vector: []float64{1, 2}, CreatedAt: "2026-01-01T00:00:00Z", UpdatedAt: "2026-01-01T00:00:00Z"}); err != nil { + t.Fatal(err) + } + snapshot := func() string { + tables := []string{"threads", "comments", "comment_revisions", "thread_revisions", "thread_fingerprints", "thread_vectors"} + out := map[string][][]any{} + for _, table := range tables { + rows, e := s.DB().Query("select * from " + table) + if e != nil { + t.Fatal(e) + } + cols, _ := rows.Columns() + for rows.Next() { + v := make([]any, len(cols)) + p := make([]any, len(v)) + for i := range v { + p[i] = &v[i] + } + if e = rows.Scan(p...); e != nil { + t.Fatal(e) + } + out[table] = append(out[table], v) + } + rows.Close() + } + b, _ := json.Marshal(out) + return string(b) + } + before := snapshot() + var raw map[string]any + json.Unmarshal([]byte(head.RawJSON), &raw) + nodeID := stringValue(raw["node_id"]) + if nodeID == "" { + nodeID = "PR8" + } + // Fixture repo supplies its retained database identity to the targeted path. + var repoRaw map[string]any + json.Unmarshal([]byte(repo.RawJSON), &repoRaw) + item := gh.ReviewStateItem{Number: 8, NodeID: nodeID, RepositoryID: repo.GitHubRepoID, RepositoryNodeID: stringValue(repoRaw["node_id"]), UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano), Threads: []map[string]any{{"id": "RT1", "isResolved": true, "isOutdated": true, "path": "file.go", "line": 3, "comments": map[string]any{"nodes": []any{map[string]any{"id": "RC1", "body": "inline body", "url": "https://github.com/openclaw/gitcrawl/pull/8#r1", "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-02T00:00:00Z", "author": map[string]any{"login": "fixture", "__typename": "User"}, "replyTo": map[string]any{"id": "RC0"}}}}}}} + f := reviewOnlyFixture{items: []gh.ReviewStateItem{item}} + opts.GraphQLHistory = true + opts.ReviewStateOnly = true + opts.ReceiptOperation = "review_state" + stats, err := New(f, s).Sync(ctx, opts) + if err != nil { + t.Fatal(err) + } + if !stats.ReviewStateOnly || stats.CommentsSynced != 0 || stats.ReviewThreadsSynced != 1 { + t.Fatalf("unexpected stats %+v", stats) + } + if before != snapshot() { + t.Fatal("canonical content/history/vector changed") + } + var membership, body, login, comments, url, created, updated, authorType string + var resolved, outdated int + s.DB().QueryRow("select review_thread_ids_json from pull_request_review_thread_syncs where thread_id=?", head.ID).Scan(&membership) + s.DB().QueryRow("select first_comment_body,first_author_login,comments_json,is_resolved,is_outdated,first_comment_url,first_comment_created_at,first_comment_updated_at,first_author_type from pull_request_review_threads where thread_id=?", head.ID).Scan(&body, &login, &comments, &resolved, &outdated, &url, &created, &updated, &authorType) + if membership != `["RT1"]` || body != "inline body" || login != "fixture" || resolved != 1 || outdated != 1 { + t.Fatal("review contract lost", membership, body, login) + } + if url != "https://github.com/openclaw/gitcrawl/pull/8#r1" || created != "2026-01-01T00:00:00Z" || updated != "2026-01-02T00:00:00Z" || authorType != "User" { + t.Fatal("first-comment metadata lost") + } + saved := membership + comments + for _, failure := range []error{errors.New("partial response"), context.Canceled} { + f.err = failure + if _, err = New(f, s).Sync(ctx, opts); err == nil { + t.Fatal("failure accepted") + } + s.DB().QueryRow("select review_thread_ids_json from pull_request_review_thread_syncs where thread_id=?", head.ID).Scan(&membership) + s.DB().QueryRow("select comments_json from pull_request_review_threads where thread_id=?", head.ID).Scan(&comments) + if saved != membership+comments || before != snapshot() { + t.Fatal("failed observation overwrote retained data") + } + } + f.err = nil + f.items[0].RepositoryID = "wrong" + if _, err = New(f, s).Sync(ctx, opts); err == nil { + t.Fatal("wrong repository accepted") + } + f.items[0] = item + f.items[0].NodeID = "wrong" + if stringValue(raw["node_id"]) != "" { + if _, err = New(f, s).Sync(ctx, opts); err == nil { + t.Fatal("wrong node accepted") + } + } + // A later identity failure rolls back earlier state/history in the batch. + f.items = []gh.ReviewStateItem{item, item} + f.items[1].Number = 9 + f.items[0].Threads[0]["isResolved"] = false + opts.Numbers = []int{8, 9} + var revisionsBefore, revisionsAfter int + s.DB().QueryRow("select count(*) from pull_request_review_thread_revisions").Scan(&revisionsBefore) + if _, err = New(f, s).Sync(ctx, opts); err == nil { + t.Fatal("missing parent accepted") + } + s.DB().QueryRow("select count(*) from pull_request_review_thread_revisions").Scan(&revisionsAfter) + s.DB().QueryRow("select is_resolved from pull_request_review_threads where thread_id=?", head.ID).Scan(&resolved) + if resolved != 1 || revisionsAfter != revisionsBefore || before != snapshot() { + t.Fatal("partial batch publication or canonical mutation") + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 6ab6c239..a538fd7c 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -43,7 +43,11 @@ type Syncer struct { } type Options struct { - GraphQLHistory bool + GraphQLHistory bool + ReviewStateOnly bool + // ReceiptOperation separates targeted review-state recovery from verified + // core traversal; both still fetch and validate complete conversations. + ReceiptOperation string Owner string Repo string State string @@ -60,6 +64,9 @@ type Options struct { } type Stats struct { + ReviewStateOnly bool `json:"review_state_only,omitempty"` + FetchMillis int64 `json:"fetch_ms,omitempty"` + PersistMillis int64 `json:"persist_ms,omitempty"` Repository string `json:"repository"` ThreadsSynced int `json:"threads_synced"` IssuesSynced int `json:"issues_synced"` @@ -121,7 +128,12 @@ func New(client GitHubClient, st *store.Store) *Syncer { } } -func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { +var errAnalyticsReceipt = errors.New("persist GraphQL attempt") + +func (s *Syncer) Sync(ctx context.Context, options Options) (result Stats, resultErr error) { + if options.ReviewStateOnly && !options.GraphQLHistory { + return Stats{}, fmt.Errorf("review-state-only requires GraphQL history transport") + } startedAt := s.now() started := startedAt.Format(time.RFC3339Nano) if err := reportSyncProgress(options.Progress, SyncProgress{ @@ -143,6 +155,40 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { if len(options.Numbers) == 0 || !options.IncludeComments || !options.IncludePRMetadata || options.IncludePRDetails || since != "" || options.Limit != 0 || state != "all" { return Stats{}, fmt.Errorf("--graphql-history requires --numbers, --state all, --include-comments and --with pr-metadata; since/limit/pr-details are unsupported") } + operation := options.ReceiptOperation + if operation == "" { + operation = "graphql_history" + } + if operation != "graphql_history" && operation != "review_state" { + return Stats{}, fmt.Errorf("unsupported GraphQL receipt operation") + } + if options.ReviewStateOnly && operation != "review_state" { + return Stats{}, fmt.Errorf("review-state-only fetch requires review recovery operation") + } + // Fetch/validation failures happen before conversation transactions and + // were previously invisible to durable run tables. Keep a receipt even + // when the request is cancelled; accepted content remains untouched. + defer func() { + receiptCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + finished := s.now().Format(time.RFC3339Nano) + status, class, message := "success", "", "" + evidence, _ := json.Marshal(result) + if resultErr != nil { + status = "failed" + class, message, evidence = gh.HistoryFailureDetails(resultErr) + } + for _, number := range uniquePositiveNumbers(options.Numbers) { + err := s.store.RecordAnalyticsAttempt(receiptCtx, store.AnalyticsAttempt{Repository: options.Owner + "/" + options.Repo, Number: number, Operation: operation, StartedAt: started, FinishedAt: finished, Status: status, ErrorClass: class, ErrorText: message, Evidence: evidence}) + if err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("%w: %w", errAnalyticsReceipt, err)) + return + } + } + }() + if options.ReviewStateOnly { + return s.syncReviewState(ctx, options, started) + } client, ok := s.client.(interface { FetchGraphQLHistory(context.Context, string, string, []int, gh.Reporter) (gh.HistoryBatch, error) }) @@ -290,6 +336,8 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { if item.Pull != nil { payload.hasPullDetails = true payload.pullDetails = pullRequestDetailRows{pull: item.Pull, fetchedAt: s.now().Format(time.RFC3339Nano)} + payload.reviewThreads = item.ReviewThreads + payload.reviewThreadsFetchedAt = payload.pullDetails.fetchedAt } } received.CommentsReceived += len(payload.commentRows) @@ -516,6 +564,11 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { if err := reserveChild(store.ThreadChildPullRequestDetails); err != nil { return err } + if history != nil { + if err := reserveChild(store.ThreadChildReviewThreads); err != nil { + return err + } + } } if options.IncludePRDetails && thread.Kind == "pull_request" { for _, family := range []store.ThreadChildObservationFamily{ diff --git a/internal/vector/exact.go b/internal/vector/exact.go index 60f58639..3b236605 100644 --- a/internal/vector/exact.go +++ b/internal/vector/exact.go @@ -56,6 +56,8 @@ func queryExact(ctx context.Context, items []Item, query []float64, limit int, e if err := validateExactQuery(query); err != nil { return nil, err } + preparedQuery := Prepare(query) + scratch := make([]float64, len(query)) scored := make([]crawlvector.Scored[Neighbor], 0, len(items)) for _, item := range items { if err := ctx.Err(); err != nil { @@ -64,7 +66,10 @@ func queryExact(ctx context.Context, items []Item, query []float64, limit int, e if item.ThreadID == excludeThreadID { continue } - score := Cosine(query, item.Vector) + if len(item.Vector) != len(query) { + continue + } + score := preparedQuery.Cosine(prepareInto(scratch, item.Vector)) if math.IsNaN(score) || math.IsInf(score, 0) || score <= 0 { continue } diff --git a/internal/vector/prepared.go b/internal/vector/prepared.go new file mode 100644 index 00000000..654eb075 --- /dev/null +++ b/internal/vector/prepared.go @@ -0,0 +1,52 @@ +package vector + +import "math" + +// Prepared holds an immutable, validated copy of a vector for repeated scoring. +// Its zero value scores zero against every vector. +type Prepared struct { + values []float64 + magnitude float64 +} + +// Prepare scales by max-abs before computing the magnitude, so even very large +// or subnormal finite inputs are safe. Empty, zero and non-finite inputs are invalid. +func Prepare(values []float64) Prepared { + return prepareInto(make([]float64, len(values)), values) +} + +func prepareScalar(dst, values []float64) Prepared { + var maxAbs float64 + for _, value := range values { + if math.IsNaN(value) || math.IsInf(value, 0) { + return Prepared{} + } + maxAbs = max(maxAbs, math.Abs(value)) + } + if maxAbs == 0 { + return Prepared{} + } + var magnitude float64 + for i, value := range values { + scaled := value / maxAbs + dst[i] = scaled + magnitude += scaled * scaled + } + return Prepared{values: dst[:len(values)], magnitude: math.Sqrt(magnitude)} +} + +// Cosine returns zero for invalid or mismatched vectors and clamps to [-1, 1]. +func (left Prepared) Cosine(right Prepared) float64 { + if len(left.values) == 0 || len(left.values) != len(right.values) { + return 0 + } + return max(-1, min(1, preparedDot(left.values, right.values)/(left.magnitude*right.magnitude))) +} + +func dotScalar(left, right []float64) float64 { + var dot float64 + for i, value := range left { + dot += value * right[i] + } + return dot +} diff --git a/internal/vector/prepared_nosimd.go b/internal/vector/prepared_nosimd.go new file mode 100644 index 00000000..6836af00 --- /dev/null +++ b/internal/vector/prepared_nosimd.go @@ -0,0 +1,11 @@ +//go:build !goexperiment.simd + +package vector + +func preparedDot(left, right []float64) float64 { + return dotScalar(left, right) +} + +func prepareInto(dst, values []float64) Prepared { + return prepareScalar(dst, values) +} diff --git a/internal/vector/prepared_simd.go b/internal/vector/prepared_simd.go new file mode 100644 index 00000000..8816dab5 --- /dev/null +++ b/internal/vector/prepared_simd.go @@ -0,0 +1,115 @@ +//go:build goexperiment.simd + +package vector + +import ( + "math" + "simd" +) + +// Emulation is slower than the scalar kernels. +var emulatedSIMD = simd.Emulated() + +// Lane buffers cover 2048-bit vectors (arm64 SVE is planned for Go 1.28), since +// release builds enable this experiment and Store panics on a short slice. +const maxFloat64Lanes = 32 + +func preparedDot(left, right []float64) float64 { + if emulatedSIMD { + return dotScalar(left, right) + } + var a, b, c, d simd.Float64s + lanes := a.Len() + i := 0 + for ; i+4*lanes <= len(left); i += 4 * lanes { + a = simd.LoadFloat64s(left[i:]).MulAdd(simd.LoadFloat64s(right[i:]), a) + b = simd.LoadFloat64s(left[i+lanes:]).MulAdd(simd.LoadFloat64s(right[i+lanes:]), b) + c = simd.LoadFloat64s(left[i+2*lanes:]).MulAdd(simd.LoadFloat64s(right[i+2*lanes:]), c) + d = simd.LoadFloat64s(left[i+3*lanes:]).MulAdd(simd.LoadFloat64s(right[i+3*lanes:]), d) + } + for ; i+lanes <= len(left); i += lanes { + a = simd.LoadFloat64s(left[i:]).MulAdd(simd.LoadFloat64s(right[i:]), a) + } + // Reduce once, outside the loop. + var sums [maxFloat64Lanes]float64 + a.Add(b).Add(c.Add(d)).Store(sums[:]) + var dot float64 + for _, value := range sums[:lanes] { + dot += value + } + for ; i < len(left); i++ { + dot += left[i] * right[i] + } + return dot +} + +func prepareInto(dst, values []float64) Prepared { + if emulatedSIMD { + return prepareScalar(dst, values) + } + var maxima simd.Float64s + var invalid simd.Mask64s + finiteLimit := simd.BroadcastFloat64s(math.MaxFloat64) + lanes := maxima.Len() + i := 0 + for ; i+lanes <= len(values); i += lanes { + v := simd.LoadFloat64s(values[i:]) + abs := v.Abs() + invalid = invalid.Or(v.NotEqual(v)).Or(abs.Greater(finiteLimit)) + maxima = maxima.Max(abs) + } + var maxValues [maxFloat64Lanes]float64 + var bad [maxFloat64Lanes]int64 + maxima.Store(maxValues[:]) + invalid.ToInt64s().Store(bad[:]) + var maxAbs float64 + for lane := 0; lane < lanes; lane++ { + if bad[lane] != 0 { + return Prepared{} + } + maxAbs = max(maxAbs, maxValues[lane]) + } + for _, value := range values[i:] { + if math.IsNaN(value) || math.IsInf(value, 0) { + return Prepared{} + } + maxAbs = max(maxAbs, math.Abs(value)) + } + if maxAbs == 0 { + return Prepared{} + } + divisor := simd.BroadcastFloat64s(maxAbs) + var a, b, c, d simd.Float64s + i = 0 + for ; i+4*lanes <= len(values); i += 4 * lanes { + v0 := simd.LoadFloat64s(values[i:]).Div(divisor) + v1 := simd.LoadFloat64s(values[i+lanes:]).Div(divisor) + v2 := simd.LoadFloat64s(values[i+2*lanes:]).Div(divisor) + v3 := simd.LoadFloat64s(values[i+3*lanes:]).Div(divisor) + v0.Store(dst[i:]) + v1.Store(dst[i+lanes:]) + v2.Store(dst[i+2*lanes:]) + v3.Store(dst[i+3*lanes:]) + a = v0.MulAdd(v0, a) + b = v1.MulAdd(v1, b) + c = v2.MulAdd(v2, c) + d = v3.MulAdd(v3, d) + } + for ; i+lanes <= len(values); i += lanes { + v := simd.LoadFloat64s(values[i:]).Div(divisor) + v.Store(dst[i:]) + a = v.MulAdd(v, a) + } + var sums [maxFloat64Lanes]float64 + a.Add(b).Add(c.Add(d)).Store(sums[:]) + var magnitude float64 + for _, value := range sums[:lanes] { + magnitude += value + } + for ; i < len(values); i++ { + scaled := values[i] / maxAbs + dst[i] = scaled + magnitude += scaled * scaled + } + return Prepared{values: dst[:len(values)], magnitude: math.Sqrt(magnitude)} +} diff --git a/internal/vector/prepared_simd_test.go b/internal/vector/prepared_simd_test.go new file mode 100644 index 00000000..1cce7355 --- /dev/null +++ b/internal/vector/prepared_simd_test.go @@ -0,0 +1,22 @@ +//go:build goexperiment.simd + +package vector + +import ( + "simd" + "testing" +) + +func TestSIMDMode(t *testing.T) { + var lanes simd.Float64s + t.Logf("float64 lanes=%d emulated=%t", lanes.Len(), emulatedSIMD) + if !emulatedSIMD { + return + } + items := benchmarkItems(2, 1024) + left := Prepare(items[0].Vector) + right := Prepare(items[1].Vector) + if got, want := left.Cosine(right), Cosine(items[0].Vector, items[1].Vector); got != want { + t.Fatalf("emulated fallback: got %.17g want scalar %.17g", got, want) + } +} diff --git a/internal/vector/prepared_test.go b/internal/vector/prepared_test.go new file mode 100644 index 00000000..3d90bcdf --- /dev/null +++ b/internal/vector/prepared_test.go @@ -0,0 +1,168 @@ +package vector + +import ( + "context" + "math" + "math/rand/v2" + "sort" + "testing" + + "github.com/openclaw/gitcrawl/internal/config" +) + +// Lane reduction and FMA change rounding; normalized scores must agree within 1e-12. +func TestPreparedCosine(t *testing.T) { + rng := rand.New(rand.NewPCG(17, 29)) + lengths := []int{1024, 1536, 3072} + // Cover three times the maximum float32 lane count, including odd tails. + for n := 0; n <= 3*16+7; n++ { + lengths = append(lengths, n) + } + var maxDeviation, maxDotDeviation float64 + for _, n := range lengths { + left, right := make([]float64, n), make([]float64, n) + for trial := 0; trial < 24; trial++ { + for i := range left { + left[i], right[i] = rng.NormFloat64(), rng.NormFloat64() + if trial%3 == 0 { + right[i] = left[i] + right[i]*0.01 + } + if trial%3 == 1 { + right[i] = -left[i] + } + if trial%4 == 0 { + left[i] *= 1e300 + right[i] *= 1e-300 + } + } + l, r := Prepare(left), Prepare(right) + want, got := Cosine(left, right), l.Cosine(r) + deviation := math.Abs(want - got) + maxDeviation = max(maxDeviation, deviation) + if math.IsNaN(got) || deviation > 1e-12 { + t.Fatalf("n=%d trial=%d: got %.17g want %.17g", n, trial, got, want) + } + if n > 0 { + scalar := dotScalar(l.values, r.values) / (l.magnitude * r.magnitude) + actual := preparedDot(l.values, r.values) / (l.magnitude * r.magnitude) + maxDotDeviation = max(maxDotDeviation, math.Abs(scalar-actual)) + if math.Abs(scalar-actual) > 1e-12 { + t.Fatalf("dot n=%d: got %.17g want %.17g", n, actual, scalar) + } + } + } + } + t.Logf("max cosine deviation from original: %.17g; dispatch-vs-scalar normalized dot: %.17g", maxDeviation, maxDotDeviation) +} + +func TestPreparedSpecialValues(t *testing.T) { + cases := [][]float64{nil, {}, {0}, {0, 0}, {1}, {-1}, {1, -1}, {math.MaxFloat64, -math.MaxFloat64}, {math.SmallestNonzeroFloat64, -math.SmallestNonzeroFloat64}, {math.NaN(), 1}, {1, math.Inf(1)}, {math.Inf(-1), 1}} + for _, left := range cases { + for _, right := range cases { + want, got := Cosine(left, right), Prepare(left).Cosine(Prepare(right)) + if math.IsNaN(got) || math.Abs(want-got) > 1e-12 { + t.Fatalf("%v / %v: got %.17g want %.17g", left, right, got, want) + } + } + } + for n := 1; n <= 3*16+7; n++ { + for i := 0; i < n; i++ { + for _, bad := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + values := make([]float64, n) + values[0], values[i] = 1, bad + if got := Prepare(values).Cosine(Prepare(values)); got != 0 { + t.Fatalf("n=%d index=%d: %g", n, i, got) + } + } + } + } + values := []float64{1, 2, 3} + prepared := Prepare(values) + values[0] = math.NaN() + if got := prepared.Cosine(Prepare([]float64{1, 2, 3})); math.Abs(got-1) > 1e-12 { + t.Fatalf("Prepare aliases input: %g", got) + } + if got := preparedDot(nil, nil); got != 0 { + t.Fatalf("empty dot = %g", got) + } + if got := preparedDot(make([]float64, 55), make([]float64, 55)); got != 0 { + t.Fatalf("zero dot = %g", got) + } +} + +func TestPreparedQueryMatchesOriginal(t *testing.T) { + items := benchmarkItems(256, 1024) + query := items[0].Vector + var want []Neighbor + for _, item := range items[1:] { + score := Cosine(query, item.Vector) + if score > 0 { + want = append(want, Neighbor{ThreadID: item.ThreadID, Score: score}) + } + } + sort.Slice(want, func(i, j int) bool { + if want[i].Score == want[j].Score { + return want[i].ThreadID < want[j].ThreadID + } + return want[i].Score > want[j].Score + }) + want = want[:20] + got := Query(items, query, 20, items[0].ThreadID) + if len(got) != len(want) { + t.Fatalf("neighbors: %d want %d", len(got), len(want)) + } + for i := range want { + if got[i].ThreadID != want[i].ThreadID || math.Abs(got[i].Score-want[i].Score) > 1e-12 { + t.Fatalf("neighbor %d: %+v want %+v", i, got[i], want[i]) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := queryExact(ctx, items, query, 20, 0); err != context.Canceled { + t.Fatalf("cancellation: %v", err) + } +} + +func benchmarkItems(count, dims int) []Item { + rng := rand.New(rand.NewPCG(7, 11)) + items := make([]Item, count) + for i := range items { + values := make([]float64, dims) + for j := range values { + values[j] = rng.NormFloat64() + } + items[i] = Item{ThreadID: int64(i + 1), Vector: values} + } + return items +} + +var benchmarkScore float64 +var benchmarkNeighbors []Neighbor + +func BenchmarkCosine(b *testing.B) { + items := benchmarkItems(2, config.Default().OpenAI.EmbedDimensions) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + benchmarkScore = Cosine(items[0].Vector, items[1].Vector) + } +} + +func BenchmarkPreparedCosine(b *testing.B) { + items := benchmarkItems(2, config.Default().OpenAI.EmbedDimensions) + left, right := Prepare(items[0].Vector), Prepare(items[1].Vector) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + benchmarkScore = left.Cosine(right) + } +} + +func BenchmarkQueryExact20000(b *testing.B) { + items := benchmarkItems(20000, config.Default().OpenAI.EmbedDimensions) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + benchmarkNeighbors = Query(items, items[0].Vector, 20, items[0].ThreadID) + } +} diff --git a/scripts/build-docs-site.mjs b/scripts/build-docs-site.mjs index bf09cb3f..12086d8a 100644 --- a/scripts/build-docs-site.mjs +++ b/scripts/build-docs-site.mjs @@ -16,7 +16,7 @@ const sections = [ ["Start", ["index.md", "installation.md", "quickstart.md", "concepts.md"]], ["Configure", ["configuration.md", "sync.md", "refresh-and-embed.md"]], ["Use", ["search.md", "clustering.md", "governance.md", "tui.md", "gh-shim.md"]], - ["Operate", ["portable-stores.md", "maintainer-archive.md", "automation.md", "releasing.md"]], + ["Operate", ["portable-stores.md", "maintainer-archive.md", "metrics.md", "automation.md", "releasing.md"]], ["Reference", ["commands.md", "reference.md"]], ];