ci: benchmark inup against similar tools

.github/workflows/benchmark.yml

@@ -0,0 +1,176 @@
+name: Benchmark
+
+# Times inup next to npm-check-updates, taze, npm-check and npm/pnpm outdated on
+# public repositories pinned to a commit (scripts/bench-compare.mjs), then opens a
+# pull request that updates the README's speed section, docs/guide/benchmarks.md
+# and the chart (scripts/bench-report.mjs). Pull requests that change the
+# benchmark get a short run, with the tables in the job summary only.
+
+on:
+  workflow_dispatch:
+    inputs:
+      runs:
+        description: Timed runs per tool, for first runs and repeat runs each
+        type: number
+        default: 10
+  workflow_run:
+    workflows: [Publish to npm]
+    types: [completed]
+  pull_request:
+    paths:
+      - scripts/bench-compare.mjs
+      - scripts/bench-report.mjs
+      - .github/workflows/benchmark.yml
+
+concurrency:
+  group: benchmark-${{ github.event.pull_request.number || 'main' }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  bench:
+    name: Benchmark (${{ matrix.fixture }})
+    # After a release, only once it is published.
+    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    # A fixed image, so successive results come from the same software.
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        fixture: [express, vue-core, tanstack-query]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          # After a release: the published commit.
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+          cache: 'pnpm'
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build
+        run: pnpm build
+
+      - name: Install hyperfine
+        env:
+          HYPERFINE_VERSION: '1.20.0'
+          HYPERFINE_SHA256: 3285ec7959285288137043dd81dce0dde056227018a8277532d9a364b4f03c2b
+        run: |
+          set -euo pipefail
+          NAME="hyperfine-v${HYPERFINE_VERSION}-x86_64-unknown-linux-musl"
+          curl -fsSL -o "$RUNNER_TEMP/hyperfine.tar.gz" \
+            "https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${NAME}.tar.gz"
+          echo "${HYPERFINE_SHA256}  $RUNNER_TEMP/hyperfine.tar.gz" | sha256sum --check --strict
+          tar -xzf "$RUNNER_TEMP/hyperfine.tar.gz" -C "$RUNNER_TEMP"
+          sudo install -m 0755 "$RUNNER_TEMP/${NAME}/hyperfine" /usr/local/bin/hyperfine
+          hyperfine --version
+
+      - name: Benchmark
+        env:
+          FIXTURE: ${{ matrix.fixture }}
+          # Pull requests check that the benchmark works; their numbers are not published.
+          RUNS: ${{ github.event_name == 'pull_request' && 3 || inputs.runs || 10 }}
+        run: |
+          node scripts/bench-compare.mjs --fixture "$FIXTURE" --runs "$RUNS" \
+            --workdir "$RUNNER_TEMP/bench" --out "bench-results/$FIXTURE.json"
+
+      - name: Upload results
+        uses: actions/upload-artifact@v7
+        with:
+          name: bench-${{ matrix.fixture }}
+          path: bench-results/${{ matrix.fixture }}.json
+          retention-days: 90
+
+  report:
+    name: Report
+    needs: bench
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          # Results go to main whichever commit was measured; a pull request renders its own scripts.
+          ref: ${{ github.event_name == 'pull_request' && github.sha || 'main' }}
+          token: ${{ github.event_name == 'pull_request' && github.token || secrets.GH_PAT }}
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Download results
+        uses: actions/download-artifact@v8
+        with:
+          pattern: bench-*
+          path: bench-results
+          merge-multiple: true
+
+      - name: Render README section, docs page and chart
+        run: |
+          node scripts/bench-report.mjs bench-results/*.json
+          cat bench-results/summary.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Open pull request with results
+        if: github.event_name != 'pull_request'
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT }}
+          GIT_EMAIL: ${{ secrets.GH_EMAIL }}
+          GIT_NAME: ${{ secrets.GH_USERNAME }}
+          PR_BRANCH: automated-benchmark
+          MEASURED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPOSITORY: ${{ github.repository }}
+        run: |
+          set -euo pipefail
+          FILES=(README.md docs/guide/benchmarks.md docs/benchmark/speed-light.svg docs/benchmark/speed-dark.svg)
+
+          git config user.email "$GIT_EMAIL"
+          git config user.name "$GIT_NAME"
+          git checkout -B "$PR_BRANCH"
+          git add "${FILES[@]}"
+
+          if git diff --cached --quiet; then
+            echo "Results are identical to the ones on main; nothing to publish."
+            exit 0
+          fi
+
+          git commit -m "docs: update benchmark results"
+          git push --force origin "$PR_BRANCH"
+
+          BODY_FILE="$RUNNER_TEMP/pr-body.md"
+          {
+            printf '%s\n\n' "Benchmark of inup at \`${MEASURED_SHA:0:7}\` ([run](${RUN_URL}))."
+            printf '%s\n\n' "![Benchmark chart](https://raw.githubusercontent.com/${REPOSITORY}/${PR_BRANCH}/docs/benchmark/speed-light.svg)"
+            cat bench-results/summary.md
+            printf '\n%s\n' "_Generated by the Benchmark workflow._"
+          } > "$BODY_FILE"
+
+          if gh pr view "$PR_BRANCH" --json state --jq '.state' 2>/dev/null | grep -qx OPEN; then
+            PR_URL="$(gh pr view "$PR_BRANCH" --json url --jq '.url')"
+            gh pr edit "$PR_BRANCH" --body-file "$BODY_FILE"
+            echo "Updated existing PR: $PR_URL"
+          else
+            PR_URL="$(gh pr create \
+              --base main \
+              --head "$PR_BRANCH" \
+              --title "docs: update benchmark results" \
+              --body-file "$BODY_FILE")"
+            echo "Opened PR: $PR_URL"
+          fi
+
+          echo "### Benchmark PR ready" >> "$GITHUB_STEP_SUMMARY"
+          echo "$PR_URL" >> "$GITHUB_STEP_SUMMARY"

.gitignore

@@ -68,6 +68,9 @@ dist
 tmp/
 temp/
 
+# pnpm bench:compare output
+bench-results/
+
 # Demo backups
 docs/demo-project/package.json.backup
 docs/demo-project/node_modules/

README.md

@@ -78,6 +78,9 @@ inup detects your package manager and finds dependencies across workspaces. Priv
 
 Need to leave a package alone? Run `npx inup --ignore "react,react-dom"`, or save your rules in [`.inuprc`](docs/guide/configuration.md).
 
+<!-- BENCHMARK:START -->
+<!-- BENCHMARK:END -->
+
 ## Let a release age before you take it
 
 A version published an hour ago is the one most likely to be a compromised release nobody has caught yet. Give releases a cooldown and inup stops offering anything younger — in the picker, in reports, and in `--apply`:
@@ -164,7 +167,7 @@ No telemetry or tracking. inup contacts your package registry for metadata and s
 <summary>Tests and coverage</summary>
 
 <!-- TEST-BADGES:START -->
-[![Tests](https://img.shields.io/badge/tests-1945_passing-brightgreen?style=for-the-badge&logo=vitest&logoColor=white)](https://github.com/donfear/inup/actions/workflows/ci.yml)
+[![Tests](https://img.shields.io/badge/tests-1986_passing-brightgreen?style=for-the-badge&logo=vitest&logoColor=white)](https://github.com/donfear/inup/actions/workflows/ci.yml)
 [![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen?style=for-the-badge)](https://github.com/donfear/inup/actions/workflows/ci.yml)
 <!-- TEST-BADGES:END -->
 

docs/guide/README.md

@@ -5,4 +5,5 @@
 - [CI & scripting](ci.md) — gate builds, JSON reports, auto-apply
 - [JSON report schema](json-schema.md) — field-by-field reference for `--json`
 - [GitHub Action](github-action.md) — one rolling upgrade PR on a schedule
+- [Benchmarks](benchmarks.md) — how long inup and similar tools take to find outdated dependencies
 - [FAQ & troubleshooting](faq.md)

docs/guide/benchmarks.md

@@ -0,0 +1,53 @@
+# Benchmarks
+
+How long inup takes to find outdated dependencies, next to the tools people use for the same job: [npm-check-updates](https://github.com/raineorshine/npm-check-updates), [taze](https://github.com/antfu-collective/taze), [npm-check](https://github.com/dylang/npm-check), and the package manager's own `npm outdated` or `pnpm outdated`. The [Benchmark workflow](../../.github/workflows/benchmark.yml) produces the results below and the chart in the README.
+
+## What is measured
+
+Every tool answers the same question: which dependencies, in every `package.json` of the repository, have a newer version than the one declared? Each one runs headless with the flags that make it check every dependency against its latest version and change nothing (the exact commands are listed with each result). The time runs from launch to exit, with the output discarded.
+
+- **First run**: every cache the tool keeps is emptied before each run. Each run gets a fresh home folder, fresh XDG cache, config and data folders, and a fresh temp folder, so nothing is left from the run before. It matches a first run on a new machine, or a CI job without a cache, with the tool itself already installed.
+- **Repeat run**: the tool keeps its caches, filled by an untimed warmup run. This is the second run on your own machine.
+
+The interactive pickers and installing the upgrades are not measured. inup's picker loads versions with the same lookups as the headless run.
+
+## The repositories
+
+Public repositories, each pinned to a commit so every run checks the same manifests:
+
+- [expressjs/express](https://github.com/expressjs/express): one package, npm
+- [vuejs/core](https://github.com/vuejs/core): a pnpm workspace with about 20 packages
+- [TanStack/query](https://github.com/TanStack/query): a pnpm workspace with about 100 packages, examples included
+
+Each repository is installed (`npm install` or `pnpm install --frozen-lockfile`, scripts off) before any timing, because `npm outdated`, `pnpm outdated` and npm-check read `node_modules`. pnpm runs at the version in the repository's `packageManager` field.
+
+## How it runs
+
+- [hyperfine](https://github.com/sharkdp/hyperfine) times each tool: one warmup run, then the timed runs. The tables show the median, and the details below them show the standard deviation and the fastest and slowest runs.
+- Each repository runs as its own job on a GitHub-hosted Ubuntu runner, and the tools run one after another on that runner. Compare tools within a column. Two columns, or two workflow runs, may come from different machines and network conditions.
+- inup is built from the commit being tested. The other tools are the latest versions on npm when the workflow runs, and the results list each version.
+- Before any timing, each tool runs once and must exit successfully and find updates. The **Updates found** column shows what each tool reported, as a check that they all did the full job. Counts differ because tools group results differently: `pnpm outdated` lists each package once for the whole workspace, while the others list it once for each `package.json` that declares it. npm-check prints only a table, so it has no count.
+
+## Where tools differ
+
+- inup also checks every dependency for known vulnerabilities during the run, and its times include that check.
+- taze looks versions up through its own metadata service (`npm.antfu.dev`) by default. On a repeat run it reuses what it fetched in the last 30 minutes without contacting the registry. inup revalidates each package with the registry on every run, so a version published a minute ago is already in the report.
+- npm-check-updates keeps no cache by default, so its repeat runs take as long as its first runs.
+- npm-check doesn't support workspaces, so it only runs on the single-package repository.
+
+## Run it yourself
+
+You need [hyperfine](https://github.com/sharkdp/hyperfine#installation) and network access:
+
+```bash
+pnpm bench:compare --fixture vue-core --runs 10   # writes bench-results/vue-core.json
+node scripts/bench-report.mjs bench-results/*.json  # rewrites the README section, this page and the chart
+```
+
+Fixtures: `express`, `vue-core`, `tanstack-query`. Times from your machine and network will differ from the runner's.
+
+## Results
+
+<!-- BENCHMARK:START -->
+_No results yet. They appear here after the Benchmark workflow runs._
+<!-- BENCHMARK:END -->

package.json

@@ -22,6 +22,7 @@
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage --reporter=default --reporter=json --outputFile.json=coverage/test-results.json",
     "bench": "pnpm build && node --expose-gc test/benchmarks/performance.cjs",
+    "bench:compare": "pnpm build && node scripts/bench-compare.mjs",
     "native:build": "napi build --platform --release --no-js --manifest-path native/napi/Cargo.toml --output-dir native/out",
     "native:check": "cargo fmt --manifest-path native/Cargo.toml --all --check && cargo clippy --manifest-path native/Cargo.toml --workspace --release --all-targets -- -D warnings && cargo test --manifest-path native/Cargo.toml --workspace --release",
     "docs:keys": "UPDATE_README=1 vitest run keymap-readme",