diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..5c888216 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,176 @@ +name: Benchmark + +# Times inup next to npm-check-updates, taze, npm-check and npm/pnpm outdated on +# public repositories pinned to a commit (scripts/bench-compare.mjs), then opens a +# pull request that updates the README's speed section, docs/guide/benchmarks.md +# and the chart (scripts/bench-report.mjs). Pull requests that change the +# benchmark get a short run, with the tables in the job summary only. + +on: + workflow_dispatch: + inputs: + runs: + description: Timed runs per tool, for first runs and repeat runs each + type: number + default: 10 + workflow_run: + workflows: [Publish to npm] + types: [completed] + pull_request: + paths: + - scripts/bench-compare.mjs + - scripts/bench-report.mjs + - .github/workflows/benchmark.yml + +concurrency: + group: benchmark-${{ github.event.pull_request.number || 'main' }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + bench: + name: Benchmark (${{ matrix.fixture }}) + # After a release, only once it is published. + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + # A fixed image, so successive results come from the same software. + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + matrix: + fixture: [express, vue-core, tanstack-query] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + # After a release: the published commit. + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + + - name: Enable Corepack + run: corepack enable + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build + run: pnpm build + + - name: Install hyperfine + env: + HYPERFINE_VERSION: '1.20.0' + HYPERFINE_SHA256: 3285ec7959285288137043dd81dce0dde056227018a8277532d9a364b4f03c2b + run: | + set -euo pipefail + NAME="hyperfine-v${HYPERFINE_VERSION}-x86_64-unknown-linux-musl" + curl -fsSL -o "$RUNNER_TEMP/hyperfine.tar.gz" \ + "https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${NAME}.tar.gz" + echo "${HYPERFINE_SHA256} $RUNNER_TEMP/hyperfine.tar.gz" | sha256sum --check --strict + tar -xzf "$RUNNER_TEMP/hyperfine.tar.gz" -C "$RUNNER_TEMP" + sudo install -m 0755 "$RUNNER_TEMP/${NAME}/hyperfine" /usr/local/bin/hyperfine + hyperfine --version + + - name: Benchmark + env: + FIXTURE: ${{ matrix.fixture }} + # Pull requests check that the benchmark works; their numbers are not published. + RUNS: ${{ github.event_name == 'pull_request' && 3 || inputs.runs || 10 }} + run: | + node scripts/bench-compare.mjs --fixture "$FIXTURE" --runs "$RUNS" \ + --workdir "$RUNNER_TEMP/bench" --out "bench-results/$FIXTURE.json" + + - name: Upload results + uses: actions/upload-artifact@v7 + with: + name: bench-${{ matrix.fixture }} + path: bench-results/${{ matrix.fixture }}.json + retention-days: 90 + + report: + name: Report + needs: bench + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + # Results go to main whichever commit was measured; a pull request renders its own scripts. + ref: ${{ github.event_name == 'pull_request' && github.sha || 'main' }} + token: ${{ github.event_name == 'pull_request' && github.token || secrets.GH_PAT }} + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Download results + uses: actions/download-artifact@v8 + with: + pattern: bench-* + path: bench-results + merge-multiple: true + + - name: Render README section, docs page and chart + run: | + node scripts/bench-report.mjs bench-results/*.json + cat bench-results/summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Open pull request with results + if: github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ secrets.GH_PAT }} + GIT_EMAIL: ${{ secrets.GH_EMAIL }} + GIT_NAME: ${{ secrets.GH_USERNAME }} + PR_BRANCH: automated-benchmark + MEASURED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + FILES=(README.md docs/guide/benchmarks.md docs/benchmark/speed-light.svg docs/benchmark/speed-dark.svg) + + git config user.email "$GIT_EMAIL" + git config user.name "$GIT_NAME" + git checkout -B "$PR_BRANCH" + git add "${FILES[@]}" + + if git diff --cached --quiet; then + echo "Results are identical to the ones on main; nothing to publish." + exit 0 + fi + + git commit -m "docs: update benchmark results" + git push --force origin "$PR_BRANCH" + + BODY_FILE="$RUNNER_TEMP/pr-body.md" + { + printf '%s\n\n' "Benchmark of inup at \`${MEASURED_SHA:0:7}\` ([run](${RUN_URL}))." + printf '%s\n\n' "![Benchmark chart](https://raw.githubusercontent.com/${REPOSITORY}/${PR_BRANCH}/docs/benchmark/speed-light.svg)" + cat bench-results/summary.md + printf '\n%s\n' "_Generated by the Benchmark workflow._" + } > "$BODY_FILE" + + if gh pr view "$PR_BRANCH" --json state --jq '.state' 2>/dev/null | grep -qx OPEN; then + PR_URL="$(gh pr view "$PR_BRANCH" --json url --jq '.url')" + gh pr edit "$PR_BRANCH" --body-file "$BODY_FILE" + echo "Updated existing PR: $PR_URL" + else + PR_URL="$(gh pr create \ + --base main \ + --head "$PR_BRANCH" \ + --title "docs: update benchmark results" \ + --body-file "$BODY_FILE")" + echo "Opened PR: $PR_URL" + fi + + echo "### Benchmark PR ready" >> "$GITHUB_STEP_SUMMARY" + echo "$PR_URL" >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index b034a4b9..107541ba 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,9 @@ dist tmp/ temp/ +# pnpm bench:compare output +bench-results/ + # Demo backups docs/demo-project/package.json.backup docs/demo-project/node_modules/ diff --git a/README.md b/README.md index aafa3b20..45cc7977 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,9 @@ inup detects your package manager and finds dependencies across workspaces. Priv Need to leave a package alone? Run `npx inup --ignore "react,react-dom"`, or save your rules in [`.inuprc`](docs/guide/configuration.md). + + + ## Let a release age before you take it A version published an hour ago is the one most likely to be a compromised release nobody has caught yet. Give releases a cooldown and inup stops offering anything younger — in the picker, in reports, and in `--apply`: @@ -164,7 +167,7 @@ No telemetry or tracking. inup contacts your package registry for metadata and s Tests and coverage -[![Tests](https://img.shields.io/badge/tests-1945_passing-brightgreen?style=for-the-badge&logo=vitest&logoColor=white)](https://github.com/donfear/inup/actions/workflows/ci.yml) +[![Tests](https://img.shields.io/badge/tests-1986_passing-brightgreen?style=for-the-badge&logo=vitest&logoColor=white)](https://github.com/donfear/inup/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen?style=for-the-badge)](https://github.com/donfear/inup/actions/workflows/ci.yml) diff --git a/docs/guide/README.md b/docs/guide/README.md index 945d7db8..cd429721 100644 --- a/docs/guide/README.md +++ b/docs/guide/README.md @@ -5,4 +5,5 @@ - [CI & scripting](ci.md) — gate builds, JSON reports, auto-apply - [JSON report schema](json-schema.md) — field-by-field reference for `--json` - [GitHub Action](github-action.md) — one rolling upgrade PR on a schedule +- [Benchmarks](benchmarks.md) — how long inup and similar tools take to find outdated dependencies - [FAQ & troubleshooting](faq.md) diff --git a/docs/guide/benchmarks.md b/docs/guide/benchmarks.md new file mode 100644 index 00000000..7f5923ef --- /dev/null +++ b/docs/guide/benchmarks.md @@ -0,0 +1,53 @@ +# Benchmarks + +How long inup takes to find outdated dependencies, next to the tools people use for the same job: [npm-check-updates](https://github.com/raineorshine/npm-check-updates), [taze](https://github.com/antfu-collective/taze), [npm-check](https://github.com/dylang/npm-check), and the package manager's own `npm outdated` or `pnpm outdated`. The [Benchmark workflow](../../.github/workflows/benchmark.yml) produces the results below and the chart in the README. + +## What is measured + +Every tool answers the same question: which dependencies, in every `package.json` of the repository, have a newer version than the one declared? Each one runs headless with the flags that make it check every dependency against its latest version and change nothing (the exact commands are listed with each result). The time runs from launch to exit, with the output discarded. + +- **First run**: every cache the tool keeps is emptied before each run. Each run gets a fresh home folder, fresh XDG cache, config and data folders, and a fresh temp folder, so nothing is left from the run before. It matches a first run on a new machine, or a CI job without a cache, with the tool itself already installed. +- **Repeat run**: the tool keeps its caches, filled by an untimed warmup run. This is the second run on your own machine. + +The interactive pickers and installing the upgrades are not measured. inup's picker loads versions with the same lookups as the headless run. + +## The repositories + +Public repositories, each pinned to a commit so every run checks the same manifests: + +- [expressjs/express](https://github.com/expressjs/express): one package, npm +- [vuejs/core](https://github.com/vuejs/core): a pnpm workspace with about 20 packages +- [TanStack/query](https://github.com/TanStack/query): a pnpm workspace with about 100 packages, examples included + +Each repository is installed (`npm install` or `pnpm install --frozen-lockfile`, scripts off) before any timing, because `npm outdated`, `pnpm outdated` and npm-check read `node_modules`. pnpm runs at the version in the repository's `packageManager` field. + +## How it runs + +- [hyperfine](https://github.com/sharkdp/hyperfine) times each tool: one warmup run, then the timed runs. The tables show the median, and the details below them show the standard deviation and the fastest and slowest runs. +- Each repository runs as its own job on a GitHub-hosted Ubuntu runner, and the tools run one after another on that runner. Compare tools within a column. Two columns, or two workflow runs, may come from different machines and network conditions. +- inup is built from the commit being tested. The other tools are the latest versions on npm when the workflow runs, and the results list each version. +- Before any timing, each tool runs once and must exit successfully and find updates. The **Updates found** column shows what each tool reported, as a check that they all did the full job. Counts differ because tools group results differently: `pnpm outdated` lists each package once for the whole workspace, while the others list it once for each `package.json` that declares it. npm-check prints only a table, so it has no count. + +## Where tools differ + +- inup also checks every dependency for known vulnerabilities during the run, and its times include that check. +- taze looks versions up through its own metadata service (`npm.antfu.dev`) by default. On a repeat run it reuses what it fetched in the last 30 minutes without contacting the registry. inup revalidates each package with the registry on every run, so a version published a minute ago is already in the report. +- npm-check-updates keeps no cache by default, so its repeat runs take as long as its first runs. +- npm-check doesn't support workspaces, so it only runs on the single-package repository. + +## Run it yourself + +You need [hyperfine](https://github.com/sharkdp/hyperfine#installation) and network access: + +```bash +pnpm bench:compare --fixture vue-core --runs 10 # writes bench-results/vue-core.json +node scripts/bench-report.mjs bench-results/*.json # rewrites the README section, this page and the chart +``` + +Fixtures: `express`, `vue-core`, `tanstack-query`. Times from your machine and network will differ from the runner's. + +## Results + + +_No results yet. They appear here after the Benchmark workflow runs._ + diff --git a/package.json b/package.json index df67c6f1..d8c50cb0 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage --reporter=default --reporter=json --outputFile.json=coverage/test-results.json", "bench": "pnpm build && node --expose-gc test/benchmarks/performance.cjs", + "bench:compare": "pnpm build && node scripts/bench-compare.mjs", "native:build": "napi build --platform --release --no-js --manifest-path native/napi/Cargo.toml --output-dir native/out", "native:check": "cargo fmt --manifest-path native/Cargo.toml --all --check && cargo clippy --manifest-path native/Cargo.toml --workspace --release --all-targets -- -D warnings && cargo test --manifest-path native/Cargo.toml --workspace --release", "docs:keys": "UPDATE_README=1 vitest run keymap-readme", diff --git a/scripts/bench-compare.mjs b/scripts/bench-compare.mjs new file mode 100644 index 00000000..2d9d5d33 --- /dev/null +++ b/scripts/bench-compare.mjs @@ -0,0 +1,489 @@ +#!/usr/bin/env node +// Time inup against the tools people use for the same job, on public +// repositories pinned to a commit. hyperfine does the timing. +// +// node scripts/bench-compare.mjs --fixture vue-core [--runs 10] [--out ] [--workdir ] +// pnpm bench:compare --fixture express (builds inup first) +// +// Every tool is asked the same question - which dependencies in every +// package.json are behind their latest version - and runs headless from launch +// to exit, its output discarded. Two modes: +// +// cold each run starts with the tool's caches empty: hyperfine's --prepare +// gives it a fresh HOME, XDG directories and TMPDIR +// warm the tool keeps its caches; an untimed warmup run fills them +// +// Cloning and installing the fixture, and installing the tools, happen before +// any timing. Before timing, each tool runs once to prove it works on the +// fixture and found updates. The output is one JSON file per fixture, which +// scripts/bench-report.mjs turns into the README section, docs page and chart. +import { execFileSync, spawnSync } from 'node:child_process' +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { cpus, release, tmpdir, totalmem, type } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..') + +/** Public repositories, pinned so every run measures the same manifests. */ +export const FIXTURES = [ + { id: 'express', repo: 'expressjs/express', sha: '0dee2c52c7c873498f8ed7d8ee4ddd0a04d3ada8' }, + { id: 'vue-core', repo: 'vuejs/core', sha: '65eae178383289b6a9f93505ab85017460f72395' }, + { + id: 'tanstack-query', + repo: 'TanStack/query', + sha: 'ce77be5e15c695df9d4d9122caf9256d334433ba', + }, +] + +/** Entries in a JSON report keyed by package (or by package.json, holding arrays). */ +const countEntries = (json) => + Object.values(json).reduce((n, value) => n + (Array.isArray(value) ? value.length : 1), 0) + +/** + * The tools, each set up to report every dependency against its latest + * version without writing anything. `fixture` is a described fixture: + * `{ pm, workspaces }`. `count` reads the number of updates from stdout; + * `failsWhenOutdated` marks the tools that exit 1 when they find updates. + */ +export const TOOLS = [ + { + id: 'inup', + name: () => 'inup', + args: () => ['--json'], + count: (stdout) => JSON.parse(stdout).summary.outdated, + }, + { + id: 'ncu', + name: () => 'npm-check-updates', + pkg: 'npm-check-updates', + bin: 'ncu', + args: (fixture) => ['--jsonUpgraded', ...(fixture.workspaces ? ['--workspaces'] : [])], + // One object of upgrades, or one per package.json with --workspaces. + count: (stdout) => + Object.values(JSON.parse(stdout)).reduce( + (n, value) => n + (typeof value === 'object' ? Object.keys(value).length : 1), + 0 + ), + }, + { + id: 'taze', + name: () => 'taze', + pkg: 'taze', + bin: 'taze', + // `major` reports every update, majors included, as the others do; + // --include-locked adds pinned versions. GitHub Actions and Node version + // files are not npm dependencies, and checking them calls GitHub's API. + args: (fixture) => [ + 'major', + ...(fixture.workspaces ? ['--recursive'] : []), + '--include-locked', + '--no-github-actions', + '--no-node-version', + '--json', + ], + count: (stdout) => + JSON.parse(stdout).packages.reduce( + (n, pkg) => n + pkg.resolved.filter((dep) => dep.update).length, + 0 + ), + }, + { + id: 'npm-check', + name: () => 'npm-check', + pkg: 'npm-check', + bin: 'npm-check', + // Finding unused dependencies is a different job. + args: () => ['--skip-unused'], + failsWhenOutdated: true, + unsupported: (fixture) => (fixture.workspaces ? 'no workspace support' : null), + count: () => null, // prints a table only + }, + { + id: 'pm', + name: (fixture) => `${fixture.pm} outdated`, + args: (fixture) => + fixture.pm === 'pnpm' + ? ['outdated', ...(fixture.workspaces ? ['--recursive'] : []), '--format', 'json'] + : [ + 'outdated', + ...(fixture.workspaces ? ['--workspaces', '--include-workspace-root'] : []), + '--json', + ], + failsWhenOutdated: true, + count: (stdout) => countEntries(JSON.parse(stdout)), + }, +] + +/** POSIX shell quoting. */ +export const quote = (value) => `'${String(value).replaceAll("'", "'\\''")}'` + +/** The environment that points every cache a tool might use into `box`. */ +export function sandboxEnv(box) { + return { + HOME: box, + XDG_CONFIG_HOME: join(box, '.config'), + XDG_CACHE_HOME: join(box, '.cache'), + XDG_DATA_HOME: join(box, '.local', 'share'), + XDG_STATE_HOME: join(box, '.local', 'state'), + TMPDIR: join(box, 'tmp'), + } +} + +/** The shell command hyperfine times: `argv` run inside `box`. */ +export function shellCommand(argv, box, { failsWhenOutdated = false } = {}) { + const env = Object.entries(sandboxEnv(box)).map(([key, value]) => `${key}=${quote(value)}`) + const command = [...env, ...argv.map(quote)].join(' ') + // hyperfine stops at a non-zero exit; for these tools, 1 means "found updates". + return failsWhenOutdated ? `${command} || [ $? -eq 1 ]` : command +} + +/** Empty `box` and recreate it, ready for a run. */ +export const resetBoxCommand = (box) => + `rm -rf ${quote(box)} && mkdir -p ${quote(join(box, 'tmp'))}` + +const DEPENDENCY_FIELDS = [ + 'dependencies', + 'devDependencies', + 'optionalDependencies', + 'peerDependencies', +] +const LOCAL_SPEC = /^(workspace|file|link|portal):/ + +/** package.json files under `dir` and the registry dependencies they name. */ +export function countManifests(dir) { + let manifests = 0 + const dependencies = new Set() + const walk = (path) => { + for (const entry of readdirSync(path, { withFileTypes: true })) { + if (entry.isDirectory()) { + if (entry.name !== 'node_modules' && !entry.name.startsWith('.')) + walk(join(path, entry.name)) + } else if (entry.name === 'package.json') { + manifests++ + const pkg = JSON.parse(readFileSync(join(path, entry.name), 'utf8')) + for (const field of DEPENDENCY_FIELDS) { + for (const [name, spec] of Object.entries(pkg[field] ?? {})) { + if (!LOCAL_SPEC.test(spec)) dependencies.add(name) + } + } + } + } + } + walk(dir) + return { manifests, dependencies: dependencies.size } +} + +const run = (command, args, options = {}) => + ( + execFileSync(command, args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'inherit'], + ...options, + }) ?? '' + ).trim() + +const log = (message) => console.error(`\n> ${message}`) + +function checkout(fixture, dir) { + if ( + existsSync(join(dir, '.git')) && + run('git', ['rev-parse', 'HEAD'], { cwd: dir }) === fixture.sha + ) + return + rmSync(dir, { recursive: true, force: true }) + mkdirSync(dir, { recursive: true }) + run('git', ['init', '-q'], { cwd: dir }) + run( + 'git', + ['fetch', '-q', '--depth', '1', `https://github.com/${fixture.repo}.git`, fixture.sha], + { + cwd: dir, + } + ) + run('git', ['checkout', '-q', 'FETCH_HEAD'], { cwd: dir }) +} + +/** The fixture's package manager (from `packageManager`) and whether it is a monorepo. */ +function inspectFixture(dir) { + const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) + const spec = pkg.packageManager?.split('+')[0] + const [pm, version] = spec ? spec.split('@') : ['npm'] + if (pm !== 'npm' && pm !== 'pnpm') throw new Error(`unsupported package manager: ${pm}`) + const workspaces = existsSync(join(dir, 'pnpm-workspace.yaml')) || pkg.workspaces !== undefined + return { pm, pmVersion: version ?? null, workspaces } +} + +/** Install npm packages into `dir`; returns the directory holding node_modules. */ +function npmInstall(dir, specs) { + mkdirSync(dir, { recursive: true }) + run( + 'npm', + ['install', '--prefix', dir, '--no-audit', '--no-fund', '--ignore-scripts', ...specs], + { + stdio: ['ignore', 'ignore', 'inherit'], + } + ) + return dir +} + +const readPackage = (prefix, name) => + JSON.parse(readFileSync(join(prefix, 'node_modules', name, 'package.json'), 'utf8')) + +/** `[node, ]` for an npm package installed under `prefix`. */ +function nodeBin(prefix, name, bin) { + const pkg = readPackage(prefix, name) + const path = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin[bin] + return { + argv: [process.execPath, join(prefix, 'node_modules', name, path)], + version: pkg.version, + } +} + +/** + * pnpm as people run it. Since pnpm 11 the `pnpm` bin is a native executable + * shipped in `@pnpm/exe.`; corepack starts it too. Older versions are + * a Node.js script. + */ +function pnpmBin(prefix, version) { + const scope = join(prefix, 'node_modules', '@pnpm') + const exe = existsSync(scope) + ? readdirSync(scope) + .filter((name) => name.startsWith('exe.')) + .map((name) => join(scope, name, 'pnpm')) + .find((path) => existsSync(path)) + : undefined + if (exe) return { argv: [exe], version } + return nodeBin(prefix, 'pnpm', 'pnpm') +} + +/** Where each tool's command comes from, and its version. */ +function resolveTools(fixture, workdir) { + const prefix = npmInstall( + join(workdir, 'tools'), + TOOLS.filter((tool) => tool.pkg).map((tool) => `${tool.pkg}@latest`) + ) + const inupVersion = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')).version + const commit = run('git', ['rev-parse', '--short', 'HEAD'], { cwd: ROOT }) + const bins = { + inup: { + argv: [process.execPath, join(ROOT, 'dist', 'cli.js')], + version: `${inupVersion} (${commit})`, + }, + pm: + fixture.pm === 'pnpm' + ? pnpmBin( + npmInstall(join(workdir, `pnpm-${fixture.pmVersion}`), [`pnpm@${fixture.pmVersion}`]), + fixture.pmVersion + ) + : { argv: ['npm'], version: run('npm', ['--version']) }, + } + for (const tool of TOOLS) { + if (tool.pkg) bins[tool.id] = nodeBin(prefix, tool.pkg, tool.bin) + } + return bins +} + +/** Run each tool once, untimed: it must succeed and report updates. */ +function smokeTest(tool, argv, cwd, box) { + rmSync(box, { recursive: true, force: true }) + mkdirSync(join(box, 'tmp'), { recursive: true }) + // Into a file, as hyperfine's /dev/null is: some tools exit before a pipe + // drains, cutting their output off at 64 KB. + const outFile = join(box, 'stdout') + const out = openSync(outFile, 'w') + const result = spawnSync(argv[0], argv.slice(1), { + cwd, + encoding: 'utf8', + env: { ...process.env, ...sandboxEnv(box) }, + stdio: ['ignore', out, 'pipe'], + }) + closeSync(out) + const stdout = readFileSync(outFile, 'utf8') + const ok = result.status === 0 || (tool.failsWhenOutdated && result.status === 1) + if (!ok || !stdout.trim()) { + throw new Error( + `${argv.join(' ')} exited with ${result.status ?? result.signal}\n${result.stderr.slice(-2000)}` + ) + } + const updates = tool.count(stdout) + if (updates === 0) throw new Error(`${argv.join(' ')} reported no updates; it checked nothing`) + return updates +} + +/** Time every tool in one hyperfine invocation; returns hyperfine's results in order. */ +function hyperfine({ tools, cwd, runs, mode, out }) { + const args = ['--runs', String(runs), '--warmup', '1', '--style', 'basic', '--export-json', out] + for (const tool of tools) { + if (mode === 'cold') args.push('--prepare', resetBoxCommand(tool.box)) + args.push('--command-name', tool.name) + } + args.push(...tools.map((tool) => tool.command)) + execFileSync('hyperfine', args, { cwd, stdio: ['ignore', 'inherit', 'inherit'] }) + return JSON.parse(readFileSync(out, 'utf8')).results +} + +const stats = ({ median, mean, stddev, min, max, times }) => ({ + median, + mean, + stddev, + min, + max, + times, +}) + +function environment() { + const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID, ImageOS, ImageVersion } = process.env + return { + os: `${type()} ${release()}`, + image: ImageOS ? `${ImageOS} ${ImageVersion}` : null, + cpu: cpus()[0]?.model.trim() ?? null, + cores: cpus().length, + memoryGb: Math.round(totalmem() / 2 ** 30), + node: process.version, + hyperfine: run('hyperfine', ['--version']).replace(/^hyperfine /, ''), + run: GITHUB_RUN_ID + ? `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}` + : null, + } +} + +export async function benchmark({ fixtureId, runs, out, workdir }) { + const fixture = FIXTURES.find((candidate) => candidate.id === fixtureId) + if (!fixture) { + const ids = FIXTURES.map((candidate) => candidate.id).join(', ') + throw new Error(`unknown fixture "${fixtureId}"; one of: ${ids}`) + } + if (spawnSync('hyperfine', ['--version']).error) { + throw new Error('hyperfine not found: https://github.com/sharkdp/hyperfine#installation') + } + if (!existsSync(join(ROOT, 'dist', 'cli.js'))) { + throw new Error('dist/cli.js not found; run pnpm build') + } + + const dir = join(workdir, 'fixtures', fixture.id) + log(`Checking out ${fixture.repo}@${fixture.sha.slice(0, 7)}`) + checkout(fixture, dir) + const described = { ...fixture, ...inspectFixture(dir) } + + log('Installing the tools') + const bins = resolveTools(described, workdir) + + log(`Installing ${fixture.repo} with ${described.pm}`) + const pm = bins.pm.argv + if (described.pm === 'pnpm') { + run(pm[0], [...pm.slice(1), 'install', '--frozen-lockfile', '--ignore-scripts'], { + cwd: dir, + stdio: ['ignore', 'ignore', 'inherit'], + }) + } else { + const install = existsSync(join(dir, 'package-lock.json')) ? 'ci' : 'install' + run('npm', [install, '--ignore-scripts', '--no-audit', '--no-fund'], { + cwd: dir, + stdio: ['ignore', 'ignore', 'inherit'], + }) + } + + const tools = [] + const skipped = [] + for (const tool of TOOLS) { + const name = tool.name(described) + const reason = tool.unsupported?.(described) + if (reason) { + skipped.push({ id: tool.id, name, reason }) + continue + } + const args = tool.args(described) + const argv = [...bins[tool.id].argv, ...args] + log(`Checking ${name} works on ${fixture.repo}`) + const updates = smokeTest(tool, argv, dir, join(workdir, 'boxes', 'smoke', tool.id)) + tools.push({ + id: tool.id, + name, + version: bins[tool.id].version, + display: [tool.bin ?? name.split(' ')[0], ...args].join(' '), + updates, + argv, + failsWhenOutdated: tool.failsWhenOutdated, + }) + } + + const results = {} + for (const mode of ['cold', 'warm']) { + log(`Timing ${mode} runs (${runs} per tool)`) + const timed = tools.map((tool) => { + const box = join(workdir, 'boxes', mode, tool.id) + rmSync(box, { recursive: true, force: true }) + mkdirSync(join(box, 'tmp'), { recursive: true }) + return { ...tool, box, command: shellCommand(tool.argv, box, tool) } + }) + const raw = join(workdir, `hyperfine-${fixture.id}-${mode}.json`) + const measured = hyperfine({ tools: timed, cwd: dir, runs, mode, out: raw }) + results[mode] = Object.fromEntries(timed.map((tool, i) => [tool.id, stats(measured[i])])) + } + + const report = { + schema: 1, + date: new Date().toISOString(), + runs, + fixture: { + id: fixture.id, + repo: fixture.repo, + sha: fixture.sha, + packageManager: described.pmVersion ? `${described.pm}@${described.pmVersion}` : described.pm, + workspaces: described.workspaces, + ...countManifests(dir), + }, + environment: environment(), + tools: tools.map(({ id, name, version, display, updates }) => ({ + id, + name, + version, + display, + updates, + })), + skipped, + results, + } + mkdirSync(dirname(out), { recursive: true }) + writeFileSync(out, `${JSON.stringify(report, null, 2)}\n`) + log(`Wrote ${out}`) + return report +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const { values } = parseArgs({ + options: { + fixture: { type: 'string' }, + runs: { type: 'string', default: '10' }, + out: { type: 'string' }, + workdir: { type: 'string', default: join(tmpdir(), 'inup-bench') }, + }, + }) + const runs = Number(values.runs) + if (!values.fixture || !Number.isInteger(runs) || runs < 2) { + console.error( + `Usage: node scripts/bench-compare.mjs --fixture <${FIXTURES.map((f) => f.id).join('|')}> [--runs ] [--out ] [--workdir ]` + ) + process.exit(1) + } + benchmark({ + fixtureId: values.fixture, + runs, + out: resolve(values.out ?? join('bench-results', `${values.fixture}.json`)), + workdir: resolve(values.workdir), + }).catch((error) => { + console.error(error.message) + process.exit(1) + }) +} diff --git a/scripts/bench-report.mjs b/scripts/bench-report.mjs new file mode 100644 index 00000000..3f234325 --- /dev/null +++ b/scripts/bench-report.mjs @@ -0,0 +1,295 @@ +#!/usr/bin/env node +// Turn scripts/bench-compare.mjs results into the README's speed section, the +// results on docs/guide/benchmarks.md and the chart beside them. +// +// node scripts/bench-report.mjs bench-results/*.json +// +// Writes README.md and docs/guide/benchmarks.md between their BENCHMARK +// markers, docs/benchmark/speed-{light,dark}.svg, and bench-results/summary.md +// (the tables, for the workflow's job summary and pull request). +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { FIXTURES } from './bench-compare.mjs' + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..') +export const CHART_PATHS = { + light: 'docs/benchmark/speed-light.svg', + dark: 'docs/benchmark/speed-dark.svg', +} + +/** Seconds, to two decimals under ten seconds and one above. */ +export const formatSeconds = (seconds) => `${seconds.toFixed(seconds < 10 ? 2 : 1)} s` + +const formatDate = (iso) => + new Date(iso).toLocaleDateString('en-GB', { + day: 'numeric', + month: 'short', + year: 'numeric', + timeZone: 'UTC', + }) + +const plural = (n, one, many) => `${n} ${n === 1 ? one : many}` + +/** "22 package.json files · 69 dependencies" */ +const sizes = ({ manifests, dependencies }) => + `${plural(manifests, 'package.json', 'package.json files')} · ${plural(dependencies, 'dependency', 'dependencies')}` + +const repoUrl = ({ repo, sha }) => `https://github.com/${repo}/tree/${sha}` + +/** Fixtures in their declared order, smallest first. */ +export function sortResults(results) { + const order = (result) => { + const index = FIXTURES.findIndex((fixture) => fixture.id === result.fixture.id) + return index === -1 ? FIXTURES.length : index + } + return [...results].sort((a, b) => order(a) - order(b)) +} + +/** One row per tool across all fixtures, in the order the tools ran. */ +function toolRows(results) { + const rows = new Map() + for (const result of results) { + for (const tool of [...result.tools, ...result.skipped]) { + const row = rows.get(tool.id) ?? { id: tool.id, names: new Set() } + row.names.add(tool.name) + rows.set(tool.id, row) + } + } + return [...rows.values()].map((row) => ({ id: row.id, name: [...row.names].join(' / ') })) +} + +const fastest = (timings) => Math.min(...Object.values(timings).map((timing) => timing.median)) + +function table(results, mode, { title, withSizes }) { + const header = results.map(({ fixture }) => { + const link = `[${fixture.repo}](${repoUrl(fixture)})` + return withSizes ? `${link}
${sizes(fixture)}` : link + }) + const lines = [ + `| ${title} | ${header.join(' | ')} |`, + `| :-- | ${results.map(() => '--:').join(' | ')} |`, + ] + for (const row of toolRows(results)) { + const cells = results.map((result) => { + const timing = result.results[mode][row.id] + if (!timing) return '—' + const text = formatSeconds(timing.median) + return timing.median === fastest(result.results[mode]) ? `**${text}**` : text + }) + lines.push(`| ${row.name} | ${cells.join(' | ')} |`) + } + return lines.join('\n') +} + +function where(environment) { + const machine = environment.image ? `GitHub Actions ${environment.image}` : environment.os + return `${machine}, ${plural(environment.cores, 'CPU', 'CPUs')}, Node ${environment.node}` +} + +/** The two tables (first run, repeat run) and the line under them. */ +export function renderTables(results) { + const [first] = results + const note = [ + 'First run: every cache the tool keeps is emptied before each run. Repeat run: caches kept.', + `Median of ${first.runs} runs per tool with [hyperfine](https://github.com/sharkdp/hyperfine) ${first.environment.hyperfine} on ${where(first.environment)}, ${formatDate(first.date)}.`, + "inup's times include its vulnerability check, which the other tools don't run.", + 'Fastest in **bold**.', + ].join(' ') + return [ + table(results, 'cold', { title: 'First run', withSizes: true }), + table(results, 'warm', { title: 'Repeat run', withSizes: false }), + `${note}`, + ].join('\n\n') +} + +/** Plain-text summary of the chart, for its alt text and SVG description. */ +export function describeChart(results) { + return results + .map(({ fixture, tools, results: timings }) => { + const parts = tools.map( + (tool) => `${tool.name} ${formatSeconds(timings.cold[tool.id].median)}` + ) + return `${fixture.repo}: ${parts.join(', ')}` + }) + .join('; ') +} + +/** The README section: heading, chart, tables. */ +export function renderReadme(results) { + return [ + '## Speed', + 'How long each tool takes to list every outdated dependency, from launch to exit, on public repositories pinned to a commit. Lower is better.', + [ + '', + ` `, + ` ${escapeXml(`First-run times in seconds. ${describeChart(results)}`)}`, + '', + ].join('\n'), + renderTables(results), + "[How it's measured, tool versions and exact commands](docs/guide/benchmarks.md)", + ].join('\n\n') +} + +function timingCell(timing, best) { + if (!timing) return '—' + const median = formatSeconds(timing.median) + const spread = `± ${timing.stddev.toFixed(2)}
${timing.min.toFixed(2)}–${timing.max.toFixed(2)}` + return `${timing.median === best ? `**${median}**` : median} ${spread}` +} + +/** The per-repository detail on docs/guide/benchmarks.md. */ +export function renderDetails(results) { + const sections = results.map((result) => { + const { fixture, environment, tools, skipped } = result + const bestCold = fastest(result.results.cold) + const bestWarm = fastest(result.results.warm) + const facts = [ + `[\`${fixture.repo}@${fixture.sha.slice(0, 7)}\`](${repoUrl(fixture)})`, + fixture.packageManager, + sizes(fixture), + ].join(' · ') + const rows = tools.map( + (tool) => + `| ${tool.name} | ${tool.version} | ${timingCell(result.results.cold[tool.id], bestCold)} | ${timingCell(result.results.warm[tool.id], bestWarm)} | ${tool.updates ?? '—'} |` + ) + for (const tool of skipped) rows.push(`| ${tool.name} | — | not run: ${tool.reason} | | |`) + const machine = [ + environment.image ? `GitHub Actions ${environment.image}` : null, + environment.os, + environment.cpu, + plural(environment.cores, 'CPU', 'CPUs'), + `${environment.memoryGb} GB`, + `Node ${environment.node}`, + `hyperfine ${environment.hyperfine}`, + ] + .filter(Boolean) + .join(' · ') + const run = environment.run ? ` · [workflow run](${environment.run})` : '' + return [ + `### ${fixture.repo}`, + facts, + [ + '| Tool | Version | First run | Repeat run | Updates found |', + '| :-- | :-- | --: | --: | --: |', + ...rows, + ].join('\n'), + `Median, ± standard deviation, min–max, over ${result.runs} runs. Measured ${formatDate(result.date)} on ${machine}${run}.`, + 'Commands, run in the repository root:', + ['```sh', ...tools.map((tool) => tool.display), '```'].join('\n'), + ].join('\n\n') + }) + return sections.join('\n\n') +} + +const THEMES = { + light: { ink: '#0b0b0b', muted: '#52514e', accent: '#2a78d6', other: '#898781' }, + dark: { ink: '#f0efec', muted: '#c3c2b7', accent: '#3987e5', other: '#898781' }, +} + +export function escapeXml(text) { + return String(text) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') +} + +/** A bar from the baseline at `x`, square there, with a 4px rounded end. */ +function barPath(x, y, width, height) { + const r = Math.min(4, width / 2, height / 2) + return `M${x} ${y}h${width - r}a${r} ${r} 0 0 1 ${r} ${r}v${height - 2 * r}a${r} ${r} 0 0 1 ${-r} ${r}h${r - width}z` +} + +/** + * First-run times as horizontal bars, one panel per repository, fastest at the + * top. inup carries the accent color; every other tool shares one gray. + * Each panel has its own scale: the repositories differ several-fold in size. + */ +export function renderChart(results, theme) { + const color = THEMES[theme] + const width = 720 + const labelWidth = 150 + const valueWidth = 64 + const barArea = width - labelWidth - valueWidth + const rowHeight = 26 + const barHeight = 16 + const parts = [] + const text = (x, y, content, attrs = '') => + parts.push(`${escapeXml(content)}`) + + text(0, 14, 'First run, empty caches · seconds, lower is better', ` fill="${color.muted}"`) + let y = 40 + for (const { fixture, tools, results: timings } of results) { + text(0, y + 14, fixture.repo, ` fill="${color.ink}" font-weight="600"`) + text(0, y + 32, sizes(fixture), ` fill="${color.muted}"`) + y += 44 + const rows = tools + .map((tool) => ({ ...tool, median: timings.cold[tool.id].median })) + .sort((a, b) => a.median - b.median) + const max = rows.at(-1).median + for (const row of rows) { + const ours = row.id === 'inup' + const weight = ours ? ' font-weight="600"' : '' + const barWidth = Math.max(4, (row.median / max) * barArea) + const middle = y + rowHeight / 2 + text(0, middle, row.name, ` fill="${color.ink}" dominant-baseline="central"${weight}`) + parts.push( + `` + ) + text( + labelWidth + barWidth + 6, + middle, + formatSeconds(row.median), + ` fill="${color.ink}" dominant-baseline="central"${weight}` + ) + y += rowHeight + } + y += 20 + } + const height = y - 20 + return [ + ``, + 'How long each tool takes to list outdated dependencies', + `${escapeXml(`First-run times in seconds. ${describeChart(results)}`)}`, + '', + ...parts, + '', + '', + ].join('\n') +} + +/** Replace what sits between `` and ``. */ +export function replaceBlock(text, name, content) { + const start = `` + const end = `` + const from = text.indexOf(start) + const to = text.indexOf(end) + if (from === -1 || to < from) throw new Error(`markers ${start} … ${end} not found`) + return `${text.slice(0, from + start.length)}\n${content}\n${text.slice(to)}` +} + +function main(files) { + if (files.length === 0) { + console.error('Usage: node scripts/bench-report.mjs ') + process.exit(1) + } + const results = sortResults(files.map((file) => JSON.parse(readFileSync(file, 'utf8')))) + const update = (path, name, content) => { + const file = join(ROOT, path) + writeFileSync(file, replaceBlock(readFileSync(file, 'utf8'), name, content)) + } + update('README.md', 'BENCHMARK', renderReadme(results)) + update('docs/guide/benchmarks.md', 'BENCHMARK', renderDetails(results)) + for (const [theme, path] of Object.entries(CHART_PATHS)) { + mkdirSync(dirname(join(ROOT, path)), { recursive: true }) + writeFileSync(join(ROOT, path), renderChart(results, theme)) + } + const summary = join(dirname(resolve(files[0])), 'summary.md') + writeFileSync(summary, `${renderTables(results)}\n`) + console.log(`Updated README.md, docs/guide/benchmarks.md and the charts; tables in ${summary}`) +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)) +} diff --git a/test/unit/scripts/bench-compare.test.ts b/test/unit/scripts/bench-compare.test.ts new file mode 100644 index 00000000..4106ae8e --- /dev/null +++ b/test/unit/scripts/bench-compare.test.ts @@ -0,0 +1,208 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + countManifests, + FIXTURES, + quote, + resetBoxCommand, + sandboxEnv, + shellCommand, + TOOLS, +} from '../../../scripts/bench-compare.mjs' + +const tool = (id: string) => { + const found = TOOLS.find((candidate: { id: string }) => candidate.id === id) + if (!found) throw new Error(`no tool ${id}`) + return found +} + +const single = { pm: 'npm', workspaces: false } +const npmWorkspace = { pm: 'npm', workspaces: true } +const pnpmWorkspace = { pm: 'pnpm', workspaces: true } + +describe('FIXTURES', () => { + it('pins every repository to a full commit', () => { + for (const fixture of FIXTURES) { + expect(fixture.sha).toMatch(/^[0-9a-f]{40}$/) + expect(fixture.repo).toMatch(/^[\w.-]+\/[\w.-]+$/) + } + expect(new Set(FIXTURES.map((fixture: { id: string }) => fixture.id)).size).toBe( + FIXTURES.length + ) + }) +}) + +describe('TOOLS', () => { + it('asks every tool for every update, workspaces included, without writing', () => { + expect(tool('inup').args(pnpmWorkspace)).toEqual(['--json']) + expect(tool('ncu').args(single)).toEqual(['--jsonUpgraded']) + expect(tool('ncu').args(pnpmWorkspace)).toEqual(['--jsonUpgraded', '--workspaces']) + expect(tool('taze').args(single)).toEqual([ + 'major', + '--include-locked', + '--no-github-actions', + '--no-node-version', + '--json', + ]) + expect(tool('taze').args(pnpmWorkspace)).toContain('--recursive') + expect(tool('npm-check').args(single)).toEqual(['--skip-unused']) + for (const { args } of TOOLS) { + for (const flag of ['-u', '--upgrade', '-w', '--write', '--install', '--update']) { + expect(args(pnpmWorkspace)).not.toContain(flag) + } + } + }) + + it("runs the fixture's own package manager", () => { + const pm = tool('pm') + expect(pm.name(single)).toBe('npm outdated') + expect(pm.args(single)).toEqual(['outdated', '--json']) + expect(pm.args(npmWorkspace)).toEqual([ + 'outdated', + '--workspaces', + '--include-workspace-root', + '--json', + ]) + expect(pm.name(pnpmWorkspace)).toBe('pnpm outdated') + expect(pm.args(pnpmWorkspace)).toEqual(['outdated', '--recursive', '--format', 'json']) + }) + + it('skips npm-check on workspaces only', () => { + expect(tool('npm-check').unsupported(single)).toBeNull() + expect(tool('npm-check').unsupported(pnpmWorkspace)).toBe('no workspace support') + }) + + it('accepts exit code 1 only from the tools that use it for "found updates"', () => { + const failing = TOOLS.filter( + (candidate: { failsWhenOutdated?: boolean }) => candidate.failsWhenOutdated + ) + expect(failing.map((candidate: { id: string }) => candidate.id)).toEqual(['npm-check', 'pm']) + }) + + it('counts the updates in each JSON report', () => { + expect(tool('inup').count(JSON.stringify({ summary: { outdated: 22 } }))).toBe(22) + expect(tool('ncu').count(JSON.stringify({ a: '^2.0.0', b: '^3.0.0' }))).toBe(2) + expect( + tool('ncu').count( + JSON.stringify({ + 'package.json': { a: '^2.0.0' }, + 'packages/x/package.json': { a: '^2.0.0', b: '1.0.0' }, + }) + ) + ).toBe(3) + expect( + tool('taze').count( + JSON.stringify({ + packages: [ + { resolved: [{ update: true }, { update: false }] }, + { resolved: [{ update: true }] }, + ], + }) + ) + ).toBe(2) + expect(tool('npm-check').count('table')).toBeNull() + // npm lists a package once, or once per workspace as an array; pnpm keys by package. + expect(tool('pm').count(JSON.stringify({ a: {}, b: [{}, {}] }))).toBe(3) + }) + + it('fails loudly on output it cannot read', () => { + expect(() => tool('inup').count('Error: something broke')).toThrow() + expect(() => tool('taze').count('{}')).toThrow() + }) +}) + +describe('sandbox', () => { + it('points home, the XDG folders and the temp folder into the box', () => { + const box = join('/bench', 'box') + expect(sandboxEnv(box)).toEqual({ + HOME: box, + XDG_CONFIG_HOME: join(box, '.config'), + XDG_CACHE_HOME: join(box, '.cache'), + XDG_DATA_HOME: join(box, '.local', 'share'), + XDG_STATE_HOME: join(box, '.local', 'state'), + TMPDIR: join(box, 'tmp'), + }) + }) + + it('quotes for a POSIX shell', () => { + expect(quote('/a b/cli.js')).toBe("'/a b/cli.js'") + expect(quote("it's")).toBe("'it'\\''s'") + }) + + it('prefixes the command with the sandbox and quotes every argument', () => { + const command = shellCommand(['node', '/a b/cli.js', '--json'], '/box') + expect(command).toMatch(/^HOME='\/box' /) + expect(command.endsWith(`'node' '/a b/cli.js' '--json'`)).toBe(true) + expect(command).not.toContain('||') + }) + + it.skipIf(process.platform === 'win32')('lets exit code 1 through only when asked', () => { + const box = mkdtempSync(join(tmpdir(), 'inup-bench-box-')) + try { + const status = (code: number, failsWhenOutdated: boolean) => + spawnSync('sh', [ + '-c', + shellCommand(['sh', '-c', `exit ${code}`], box, { failsWhenOutdated }), + ]).status + expect(status(0, false)).toBe(0) + expect(status(1, false)).toBe(1) + expect(status(1, true)).toBe(0) + expect(status(2, true)).not.toBe(0) + } finally { + rmSync(box, { recursive: true, force: true }) + } + }) + + it.skipIf(process.platform === 'win32')('runs the command with the sandbox environment', () => { + const box = mkdtempSync(join(tmpdir(), 'inup-bench-box-')) + try { + const printed = spawnSync( + 'sh', + ['-c', shellCommand(['sh', '-c', 'printf "%s|%s" "$HOME" "$TMPDIR"'], box)], + { encoding: 'utf8' } + ).stdout + expect(printed).toBe(`${box}|${join(box, 'tmp')}`) + } finally { + rmSync(box, { recursive: true, force: true }) + } + }) + + it.skipIf(process.platform === 'win32')('empties the box and recreates its temp folder', () => { + const box = mkdtempSync(join(tmpdir(), 'inup-bench-box-')) + try { + writeFileSync(join(box, 'cache.json'), '{}') + expect(spawnSync('sh', ['-c', resetBoxCommand(box)]).status).toBe(0) + expect(spawnSync('ls', ['-A', box], { encoding: 'utf8' }).stdout.trim()).toBe('tmp') + } finally { + rmSync(box, { recursive: true, force: true }) + } + }) +}) + +describe('countManifests', () => { + let dir = '' + afterEach(() => rmSync(dir, { recursive: true, force: true })) + + const write = (path: string, pkg: object) => { + mkdirSync(join(dir, path), { recursive: true }) + writeFileSync(join(dir, path, 'package.json'), JSON.stringify(pkg)) + } + + it('counts package.json files and the registry dependencies they name', () => { + dir = mkdtempSync(join(tmpdir(), 'inup-bench-manifests-')) + write('.', { devDependencies: { typescript: '^5.0.0', vitest: '^4.0.0' } }) + write('packages/a', { + dependencies: { react: '^19.0.0', b: 'workspace:*' }, + peerDependencies: { react: '>=18' }, + }) + write('packages/b', { + dependencies: { local: 'file:../local', linked: 'link:../x', yaml: 'catalog:' }, + }) + write('node_modules/react', { dependencies: { 'loose-envify': '^1.0.0' } }) + write('.cache/thing', { dependencies: { hidden: '^1.0.0' } }) + expect(countManifests(dir)).toEqual({ manifests: 3, dependencies: 4 }) + }) +}) diff --git a/test/unit/scripts/bench-report.test.ts b/test/unit/scripts/bench-report.test.ts new file mode 100644 index 00000000..4034141b --- /dev/null +++ b/test/unit/scripts/bench-report.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, it } from 'vitest' +import { + CHART_PATHS, + describeChart, + escapeXml, + formatSeconds, + renderChart, + renderDetails, + renderReadme, + renderTables, + replaceBlock, + sortResults, +} from '../../../scripts/bench-report.mjs' + +const timing = (median: number) => ({ + median, + mean: median, + stddev: 0.012, + min: median - 0.02, + max: median + 0.03, + times: [median], +}) + +const environment = { + os: 'Linux 6.8.0-1017-azure', + image: 'ubuntu24 20261012.1', + cpu: 'AMD EPYC 7763 64-Core Processor', + cores: 4, + memoryGb: 16, + node: 'v24.11.0', + hyperfine: '1.20.0', + run: 'https://github.com/donfear/inup/actions/runs/1', +} + +const tool = (id: string, name: string, display: string, updates: number | null) => ({ + id, + name, + version: id === 'inup' ? '1.8.6 (dcb4f0c)' : '1.0.0', + display, + updates, +}) + +const express = { + schema: 1, + date: '2026-10-21T12:00:00.000Z', + runs: 10, + fixture: { + id: 'express', + repo: 'expressjs/express', + sha: '0dee2c52c7c873498f8ed7d8ee4ddd0a04d3ada8', + packageManager: 'npm', + workspaces: false, + manifests: 1, + dependencies: 44, + }, + environment, + tools: [ + tool('inup', 'inup', 'inup --json', 22), + tool('ncu', 'npm-check-updates', 'ncu --jsonUpgraded', 21), + tool('npm-check', 'npm-check', 'npm-check --skip-unused', null), + tool('pm', 'npm outdated', 'npm outdated --json', 13), + ], + skipped: [], + results: { + cold: { inup: timing(0.389), ncu: timing(0.726), 'npm-check': timing(1.2), pm: timing(0.9) }, + warm: { inup: timing(0.38), ncu: timing(0.72), 'npm-check': timing(1.1), pm: timing(0.13) }, + }, +} + +const vue = { + ...express, + fixture: { + id: 'vue-core', + repo: 'vuejs/core', + sha: '65eae178383289b6a9f93505ab85017460f72395', + packageManager: 'pnpm@12.11.2', + workspaces: true, + manifests: 22, + dependencies: 69, + }, + tools: [ + tool('inup', 'inup', 'inup --json', 26), + tool('ncu', 'npm-check-updates', 'ncu --jsonUpgraded --workspaces', 25), + tool('pm', 'pnpm outdated', 'pnpm outdated --recursive --format json', 27), + ], + skipped: [{ id: 'npm-check', name: 'npm-check', reason: 'no workspace support' }], + results: { + cold: { inup: timing(1.68), ncu: timing(18.348), pm: timing(2.1) }, + warm: { inup: timing(0.9), ncu: timing(5.6), pm: timing(0.8) }, + }, +} + +const results = [express, vue] + +describe('formatSeconds', () => { + it('keeps two decimals under ten seconds and one above', () => { + expect(formatSeconds(0.389)).toBe('0.39 s') + expect(formatSeconds(1.6849)).toBe('1.68 s') + expect(formatSeconds(18.348)).toBe('18.3 s') + }) +}) + +describe('sortResults', () => { + it('orders repositories as the fixtures are declared, smallest first', () => { + const unknown = { ...express, fixture: { ...express.fixture, id: 'other' } } + expect(sortResults([unknown, vue, express]).map((r: typeof express) => r.fixture.id)).toEqual([ + 'express', + 'vue-core', + 'other', + ]) + }) +}) + +describe('renderTables', () => { + const tables = renderTables(results) + const [first, repeat] = tables.split('\n\n') + + it('heads each column with the pinned repository and its size', () => { + expect(first).toContain( + '[expressjs/express](https://github.com/expressjs/express/tree/0dee2c52c7c873498f8ed7d8ee4ddd0a04d3ada8)
1 package.json · 44 dependencies' + ) + expect(first).toContain('22 package.json files · 69 dependencies') + expect(repeat.split('\n')[0]).toBe( + '| Repeat run | [expressjs/express](https://github.com/expressjs/express/tree/0dee2c52c7c873498f8ed7d8ee4ddd0a04d3ada8) | [vuejs/core](https://github.com/vuejs/core/tree/65eae178383289b6a9f93505ab85017460f72395) |' + ) + }) + + it('gives each tool a row, a dash where it did not run, and bolds the fastest', () => { + expect(first).toContain('| inup | **0.39 s** | **1.68 s** |') + expect(first).toContain('| npm-check-updates | 0.73 s | 18.3 s |') + expect(first).toContain('| npm-check | 1.20 s | — |') + expect(first).toContain('| npm outdated / pnpm outdated | 0.90 s | 2.10 s |') + expect(repeat).toContain('| inup | 0.38 s | 0.90 s |') + expect(repeat).toContain('| npm outdated / pnpm outdated | **0.13 s** | **0.80 s** |') + }) + + it('says how, where and when it was measured', () => { + expect(tables).toContain('Median of 10 runs per tool') + expect(tables).toContain('hyperfine](https://github.com/sharkdp/hyperfine) 1.20.0') + expect(tables).toContain( + 'GitHub Actions ubuntu24 20261012.1, 4 CPUs, Node v24.11.0, 21 Oct 2026' + ) + expect(tables).toContain("inup's times include its vulnerability check") + }) + + it('falls back to the OS name off GitHub Actions', () => { + const local = { ...express, environment: { ...environment, image: null, cores: 1 } } + expect(renderTables([local])).toContain('on Linux 6.8.0-1017-azure, 1 CPU,') + }) +}) + +describe('renderReadme', () => { + const readme = renderReadme(results) + + it('opens with the heading and shows the chart for both themes', () => { + expect(readme.startsWith('## Speed\n\n')).toBe(true) + expect(readme).toContain( + `` + ) + expect(readme).toContain(`src="${CHART_PATHS.light}"`) + }) + + it('describes the chart in its alt text', () => { + expect(readme).toContain( + 'alt="First-run times in seconds. expressjs/express: inup 0.39 s, npm-check-updates 0.73 s, npm-check 1.20 s, npm outdated 0.90 s; vuejs/core: inup 1.68 s' + ) + }) + + it('links to the method', () => { + expect(readme).toContain( + "[How it's measured, tool versions and exact commands](docs/guide/benchmarks.md)" + ) + }) +}) + +describe('renderDetails', () => { + const details = renderDetails(results) + + it('names each pinned repository with its package manager and size', () => { + expect(details).toContain('### vuejs/core') + expect(details).toContain( + '[`vuejs/core@65eae17`](https://github.com/vuejs/core/tree/65eae178383289b6a9f93505ab85017460f72395) · pnpm@12.11.2 · 22 package.json files · 69 dependencies' + ) + }) + + it('lists version, spread and updates found for each tool, and why one did not run', () => { + expect(details).toContain( + '| inup | 1.8.6 (dcb4f0c) | **1.68 s** ± 0.01
1.66–1.71 | 0.90 s ± 0.01
0.88–0.93 | 26 |' + ) + expect(details).toContain('| npm-check | 1.0.0 | 1.20 s ± 0.01
1.18–1.23 |') + expect(details).toContain('| — |\n') + expect(details).toContain('| npm-check | — | not run: no workspace support | | |') + }) + + it('shows the exact commands and the machine', () => { + expect(details).toContain( + '```sh\ninup --json\nncu --jsonUpgraded --workspaces\npnpm outdated --recursive --format json\n```' + ) + expect(details).toContain( + 'Measured 21 Oct 2026 on GitHub Actions ubuntu24 20261012.1 · Linux 6.8.0-1017-azure · AMD EPYC 7763 64-Core Processor · 4 CPUs · 16 GB · Node v24.11.0 · hyperfine 1.20.0 · [workflow run](https://github.com/donfear/inup/actions/runs/1).' + ) + }) +}) + +describe('renderChart', () => { + it('draws one panel per repository, fastest first, inup in the accent color', () => { + const svg = renderChart(results, 'light') + expect(svg.startsWith('')).toBe(true) + expect(svg.match(/fill="#2a78d6"/g)).toHaveLength(2) + const labels = [...svg.matchAll(/]*dominant-baseline="central"[^>]*>([^<]+)<\/text>/g)] + .map((match) => match[1]) + .filter((label) => !label.endsWith(' s')) + expect(labels).toEqual([ + 'inup', + 'npm-check-updates', + 'npm outdated', + 'npm-check', + 'inup', + 'pnpm outdated', + 'npm-check-updates', + ]) + }) + + it('scales each panel to its slowest tool', () => { + const svg = renderChart(results, 'dark') + const widths = [...svg.matchAll(/ { + const odd = { ...vue, fixture: { ...vue.fixture, repo: 'a&b/' } } + expect(renderChart([odd], 'light')).toContain('a&b/<c>') + expect(escapeXml('"x"')).toBe('"x"') + }) + + it('describes every bar', () => { + expect(describeChart([vue])).toBe( + 'vuejs/core: inup 1.68 s, npm-check-updates 18.3 s, pnpm outdated 2.10 s' + ) + }) +}) + +describe('replaceBlock', () => { + const page = 'intro\n\nold\n\noutro\n' + + it('replaces only what sits between the markers', () => { + const updated = replaceBlock(page, 'BENCHMARK', 'new') + expect(updated).toBe('intro\n\nnew\n\noutro\n') + expect(replaceBlock(updated, 'BENCHMARK', 'new')).toBe(updated) + }) + + it('fails without both markers', () => { + expect(() => replaceBlock('intro\n', 'BENCHMARK', 'new')).toThrow('') + expect(() => + replaceBlock('\n', 'BENCHMARK', 'new') + ).toThrow() + }) +})