feat: collect repository metrics in an independent store

README.md

@@ -64,6 +64,7 @@ Octopool owns pooled live `gh` reads. Gitcrawl keeps local mirror, search, clust
 | Check archive health | `gitcrawl status` / `gitcrawl doctor` | [Configuration](docs/configuration.md) |
 | Refresh a portable subscriber | `gitcrawl portable refresh --expected-remote URL` | [Portable stores](docs/portable-stores.md#routine-subscriber-refresh) |
 | Mirror GitHub threads | `gitcrawl sync owner/repo` | [Sync](docs/sync.md) |
+| Collect repository counters and releases | `gitcrawl metrics collect\|import\|status --config metrics.json` | [Repository metrics](docs/metrics.md) |
 | Search threads or indexed code | `gitcrawl search ...` | [Search](docs/search.md) |
 | Build and inspect clusters | `gitcrawl refresh`, `clusters`, `tui` | [Clustering](docs/clustering.md) |
 | Export a code-free conversation snapshot | `gitcrawl capture owner/repo` | [Capture](docs/capture.md) |

docs/commands.md

@@ -38,6 +38,17 @@ These work on every command.
 | `gitcrawl configure [--summary-model --embed-model --embedding-basis --json]` | Update model fields in `config.toml` | [Configuration](/configuration/#gitcrawl-configure) |
 | `gitcrawl version` | Print version | — |
 
+## Repository metrics
+
+| Command | Purpose | Docs |
+| --- | --- | --- |
+| `gitcrawl metrics collect --config metrics.json [--json]` | Observe stars, forks, actual watchers, open PRs/issues, optional daily clones, and stable releases in a separate database | [Repository metrics](/metrics/) |
+| `gitcrawl metrics import --config metrics.json [--json]` | Atomically import scoped NDJSON history from stdin, preserving NULLs and IDs | [Repository metrics](/metrics/#storage-imports-and-failures) |
+| `gitcrawl metrics status --config metrics.json [--json]` | Inspect the metrics database without writes or network calls | [Repository metrics](/metrics/) |
+
+For `metrics`, `--config` selects an independent JSON config; it never selects or
+initializes the normal thread archive. No embedding or model calls are made.
+
 ## Sync
 
 | Command | Purpose | Docs |

docs/installation.md

@@ -76,6 +76,11 @@ go build \
 
 Symlink or copy `bin/gitcrawl` somewhere on your `PATH` (`~/bin`, `/usr/local/bin`, `~/.local/bin`).
 
+For a metrics-only source deployment alongside an existing portable subscriber,
+use an [isolated versioned metrics runtime](/metrics/#isolated-source-built-installation)
+and verify it with `metrics status` and its separate config. Keep the subscriber's
+binary selection and refresh job unchanged.
+
 Source builds use scalar vector scoring by default. Set `GOEXPERIMENT=simd` on
 `go build` to enable Go 1.27's experimental portable SIMD kernels for clustering
 and exact neighbors. The API may change in Go 1.28; unsupported hardware and

internal/cli/app.go

@@ -30,6 +30,7 @@ type App struct {
 	observedGitHubToken string
 	Stdout              io.Writer
 	Stderr              io.Writer
+	Stdin               io.Reader
 
 	configPath            string
 	format                OutputFormat
@@ -53,6 +54,7 @@ func New() *App {
 	return &App{
 		Stdout:              os.Stdout,
 		Stderr:              os.Stderr,
+		Stdin:               os.Stdin,
 		format:              FormatText,
 		getWorkingDirectory: os.Getwd,
 	}
@@ -126,6 +128,8 @@ func (a *App) Run(ctx context.Context, args []string) error {
 		return a.runDoctor(ctx, rest[1:])
 	case "status":
 		return a.runStatus(ctx, rest[1:])
+	case "metrics":
+		return a.runMetrics(ctx, rest[1:])
 	case "sync":
 		return a.runSync(ctx, rest[1:])
 	case "fill-pr-details":

internal/cli/control.go

@@ -37,9 +37,12 @@ func (a *App) runMetadata(args []string) error {
 		DefaultCache:    cfg.CacheDir,
 		DefaultLogs:     cfg.LogDir,
 	}
-	manifest.Capabilities = []string{"metadata", "status", "doctor", "sync", "capture", "coverage", "search", "code-index", "tui", "portable", "remote", "cloud-publish", "clusters", "summaries", "embeddings"}
+	manifest.Capabilities = []string{"metadata", "status", "metrics", "doctor", "sync", "capture", "coverage", "search", "code-index", "tui", "portable", "remote", "cloud-publish", "clusters", "summaries", "embeddings"}
 	manifest.Privacy = control.Privacy{ContainsPrivateMessages: true, ExportsSecrets: false, LocalOnlyScopes: []string{"github", "git", "sqlite", "portable"}}
 	manifest.Commands = map[string]control.Command{
+		"metrics-collect":  {Title: "Collect repository metrics", Argv: []string{"gitcrawl", "metrics", "collect", "--config", "METRICS_CONFIG", "--json"}, JSON: true, Mutates: true},
+		"metrics-import":   {Title: "Import metric history from stdin", Argv: []string{"gitcrawl", "metrics", "import", "--config", "METRICS_CONFIG", "--json"}, JSON: true, Mutates: true},
+		"metrics-status":   {Title: "Metrics database status", Argv: []string{"gitcrawl", "metrics", "status", "--config", "METRICS_CONFIG", "--json"}, JSON: true},
 		"status":           {Title: "Status", Argv: []string{"gitcrawl", "status", "--json"}, JSON: true},
 		"remote-status":    {Title: "Remote archive status", Argv: []string{"gitcrawl", "remote", "status", "--json"}, JSON: true},
 		"remote-archives":  {Title: "Remote archive list", Argv: []string{"gitcrawl", "remote", "archives", "--json"}, JSON: true},

internal/cli/help.go

@@ -45,6 +45,7 @@ Core commands:
   metadata             print crawlkit control metadata
   check-update         check for a newer gitcrawl release
   status               print fast read-only archive status
+  metrics              collect, import, or inspect an independent metrics database
   remote status        print remote archive status
   remote archives      list remote archives visible to the current identity
   remote login         authenticate with GitHub org access for a remote archive
@@ -90,6 +91,19 @@ No API server is provided. There is intentionally no serve command.
 `
 
 var commandUsageTexts = map[string]string{
+	"metrics": `gitcrawl metrics collects repository headline metrics in a separate SQLite database.
+
+Usage:
+  gitcrawl metrics collect --config /absolute/metrics.json [--json]
+  gitcrawl metrics import --config /absolute/metrics.json [--json] < history.ndjson
+  gitcrawl metrics status --config /absolute/metrics.json [--json]
+
+The JSON config requires database (absolute path) and targets [{entity,target}].
+Optional tokenEnv overrides GITHUB_TOKEN; native gh auth and --github-token-command
+are supported. No archive, portable-store, embedding, or model operations occur.
+Import validates target scope and IDs atomically. Unknown values remain SQL NULL.
+Status is read-only. Clone traffic 403/404 is optional unavailability.
+`,
 	"metadata": `gitcrawl metadata prints crawlkit control metadata.
 
 Usage:

internal/cli/metrics.go

@@ -0,0 +1,84 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/openclaw/gitcrawl/internal/config"
+	"github.com/openclaw/gitcrawl/internal/github"
+	"github.com/openclaw/gitcrawl/internal/headlinemetrics"
+)
+
+func (a *App) runMetrics(ctx context.Context, args []string) error {
+	if len(args) == 0 || args[0] == "help" || args[0] == "--help" || args[0] == "-h" {
+		return a.printCommandUsage("metrics")
+	}
+	command := args[0]
+	if command != "collect" && command != "import" && command != "status" {
+		return usageErr(fmt.Errorf("unknown metrics command %q", command))
+	}
+	fs := flag.NewFlagSet("metrics "+command, flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	path := fs.String("config", a.configPath, "metrics JSON config path")
+	jsonOut := fs.Bool("json", false, "write JSON output")
+	if err := fs.Parse(args[1:]); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return a.printCommandUsage("metrics")
+		}
+		return usageErr(err)
+	}
+	if *path == "" || fs.NArg() != 0 {
+		return usageErr(errors.New("metrics requires --config and no positional arguments"))
+	}
+	c, err := headlinemetrics.ReadConfig(*path)
+	if err != nil {
+		return usageErr(err)
+	}
+	a.applyCommandJSON(*jsonOut)
+	var collect headlinemetrics.Collector
+	if command == "collect" {
+		cfg := config.Default()
+		if c.TokenEnv != "" {
+			cfg.GitHub.TokenEnv = c.TokenEnv
+		}
+		token := a.resolveGitHubToken(ctx, cfg)
+		var provider func(context.Context) (string, error)
+		if a.githubTokenCommand != nil {
+			provider, err = githubTokenProvider(*a.githubTokenCommand)
+			if err != nil {
+				return usageErr(err)
+			}
+		}
+		client := github.New(github.Options{Token: token.Value, TokenProvider: provider, BaseURL: githubBaseURL(), HTTPClient: &http.Client{
+			Timeout:       30 * time.Second,
+			CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+		}})
+		collect = headlinemetrics.GitHubCollector(client, token.Value != "" || provider != nil)
+	}
+	in := a.Stdin
+	if in == nil {
+		in = http.NoBody
+	}
+	result, err := headlinemetrics.Execute(ctx, command, c, collect, in)
+	// Partial collection is still a useful structured result; diagnostics stay
+	// on stderr and a nonzero exit communicates unavailable required metrics.
+	if err == nil || errors.Is(err, headlinemetrics.ErrPartialCollection) {
+		var output any = result
+		if command == "status" {
+			output = struct {
+				headlinemetrics.Result
+				Observations int `json:"observations"`
+				Events       int `json:"events"`
+			}{result, result.Observations, result.Events}
+		}
+		if writeErr := a.writeOutput("metrics "+command, output, false); writeErr != nil {
+			return writeErr
+		}
+	}
+	return err
+}