diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5c88821..84f3b48 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,10 +1,12 @@ name: Benchmark -# Times inup next to npm-check-updates, taze, npm-check and npm/pnpm outdated on -# public repositories pinned to a commit (scripts/bench-compare.mjs), then opens a -# pull request that updates the README's speed section, docs/guide/benchmarks.md -# and the chart (scripts/bench-report.mjs). Pull requests that change the -# benchmark get a short run, with the tables in the job summary only. +# Times inup next to npm-check-updates, taze, syncpack, npm-check and npm/pnpm/bun +# outdated, and their interactive pickers next to yarn upgrade-interactive and +# pnpm/bun update -i, on public repositories pinned to a commit +# (scripts/bench-compare.mjs). Then it opens a pull request that updates the +# README's speed section, docs/guide/benchmarks.md and the chart +# (scripts/bench-report.mjs). Pull requests that change the benchmark get a +# short run, with the tables in the job summary only. on: workflow_dispatch: @@ -19,6 +21,7 @@ on: pull_request: paths: - scripts/bench-compare.mjs + - scripts/bench-pty.py - scripts/bench-report.mjs - .github/workflows/benchmark.yml @@ -36,10 +39,10 @@ jobs: if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' # A fixed image, so successive results come from the same software. runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 120 strategy: matrix: - fixture: [express, vue-core, tanstack-query] + fixture: [express, vue-core, tanstack-query, vendure, backstage] steps: - name: Checkout repository diff --git a/docs/guide/benchmarks.md b/docs/guide/benchmarks.md index 478fbce..26cbcd2 100644 --- a/docs/guide/benchmarks.md +++ b/docs/guide/benchmarks.md @@ -1,15 +1,27 @@ # Benchmarks -How long inup takes to find outdated dependencies, next to the tools people use for the same job: [npm-check-updates](https://github.com/raineorshine/npm-check-updates), [taze](https://github.com/antfu-collective/taze), [npm-check](https://github.com/dylang/npm-check), and the package manager's own `npm outdated` or `pnpm outdated`. The [Benchmark workflow](../../.github/workflows/benchmark.yml) produces the results below and the chart in the README. +How long inup takes to find outdated dependencies, next to the tools people use for the same job: [npm-check-updates](https://github.com/raineorshine/npm-check-updates), [taze](https://github.com/antfu-collective/taze), [syncpack](https://github.com/JamieMason/syncpack), [npm-check](https://github.com/dylang/npm-check), and the package manager's own `npm outdated`, `pnpm outdated` or `bun outdated`. The [Benchmark workflow](../../.github/workflows/benchmark.yml) produces the results below and the chart in the README. ## What is measured +Two things: how long each tool takes to list every outdated dependency, and how long its interactive picker takes to load every update. + +### Listing outdated dependencies + Every tool answers the same question: which dependencies, in every `package.json` of the repository, have a newer version than the one declared? Each one runs headless with the flags that make it check every dependency against its latest version and change nothing (the exact commands are listed with each result). The time runs from launch to exit, with the output discarded. - **First run**: every cache the tool keeps is emptied before each run. Each run gets a fresh home folder, fresh XDG cache, config and data folders, and a fresh temp folder, so nothing is left from the run before. It matches a first run on a new machine, or a CI job without a cache, with the tool itself already installed. - **Repeat run**: the tool keeps its caches, filled by an untimed warmup run. This is the second run on your own machine. -The interactive pickers and installing the upgrades are not measured. inup's picker loads versions with the same lookups as the headless run. +### Opening the picker + +Each tool that has an interactive list of updates opens it in a terminal of 160 columns by 50 rows: inup, `yarn upgrade-interactive`, `pnpm update --interactive --latest`, `bun update --interactive`, taze, syncpack, npm-check and npm-check-updates. The time runs from launch until the picker has every update loaded, and nothing is selected or installed. + +"Loaded" is the last thing the picker draws on screen or writes to its cache before it stays quiet for 5 seconds. A picker can finish drawing the rows on screen and keep fetching the rest. `yarn upgrade-interactive` on backstage, for one, stops drawing after about 2.5 seconds and keeps fetching for several more. Counting cache writes includes that work. The 5 quiet seconds are not part of the time. + +First and repeat runs work as for listing. The pickers run in rounds, each tool once per round, so a slow spell on the network affects them all alike. + +Installing the upgrades is not measured. ## The repositories @@ -18,33 +30,40 @@ Public repositories, each pinned to a commit so every run checks the same manife - [expressjs/express](https://github.com/expressjs/express): one package, npm - [vuejs/core](https://github.com/vuejs/core): a pnpm workspace with about 20 packages - [TanStack/query](https://github.com/TanStack/query): a pnpm workspace with about 100 packages, examples included +- [vendurehq/vendure](https://github.com/vendurehq/vendure): a Bun workspace with about 50 packages +- [backstage/backstage](https://github.com/backstage/backstage): a Yarn 4 workspace with about 270 packages and 590 dependencies -Each repository is installed (`npm install` or `pnpm install --frozen-lockfile`, scripts off) before any timing, because `npm outdated`, `pnpm outdated` and npm-check read `node_modules`. pnpm runs at the version in the repository's `packageManager` field. +Each repository is installed with its own package manager, with install scripts off, before any timing, because `npm outdated`, `pnpm outdated`, `bun outdated` and npm-check read `node_modules`. pnpm runs at the version in the repository's `packageManager` field, Yarn as the release the repository commits, and Bun at its latest version. ## How it runs -- [hyperfine](https://github.com/sharkdp/hyperfine) times each tool: one warmup run, then the timed runs. The tables show the median, and the details below them show the standard deviation and the fastest and slowest runs. +- [hyperfine](https://github.com/sharkdp/hyperfine) times each tool's listing, and the benchmark script times the pickers: one warmup run, then the timed runs. The tables show the median, and the details below them show the standard deviation and the fastest and slowest runs. - Each repository runs as its own job on a GitHub-hosted Ubuntu runner, and the tools run one after another on that runner. Compare tools within a column. Two columns, or two workflow runs, may come from different machines and network conditions. - inup is built from the commit being tested. The other tools are the latest versions on npm when the workflow runs, and the results list each version. -- Before any timing, each tool runs once and must exit successfully and find updates. The **Updates found** column shows what each tool reported, as a check that they all did the full job. Counts differ because tools group results differently: `pnpm outdated` lists each package once for the whole workspace, while the others list it once for each `package.json` that declares it. npm-check prints only a table, so it has no count. +- Before any timing, each tool runs once and must exit successfully and find updates. The **Updates found** column shows what each tool reported, as a check that they all did the full job. Counts differ because tools group results differently: `pnpm outdated` lists each package once for the whole workspace, syncpack lists each version of a dependency once however many `package.json` files share it, and the others list a dependency once for each `package.json` that declares it. npm-check prints only a table, so it has no count. ## Where tools differ - inup also checks every dependency for known vulnerabilities during the run, and its times include that check. +- inup's first interactive run downloads its native core (about 1.5 MB) and checks it, so its first-run picker times include that download. Its listing runs never download it. - taze looks versions up through its own metadata service (`npm.antfu.dev`) by default. On a repeat run it reuses what it fetched in the last 30 minutes without contacting the registry. inup revalidates each package with the registry on every run, so a version published a minute ago is already in the report. - npm-check-updates keeps no cache by default, so its repeat runs take as long as its first runs. +- syncpack shows how old each version is, so it downloads each package's full registry document, publish dates included, which is several times larger than the version list the other tools ask for. On a repeat run it reuses what it fetched in the last 30 minutes. +- `bun outdated` reuses registry responses for as long as the registry's cache headers allow (5 minutes on registry.npmjs.org), so its repeat runs make no requests. - npm-check doesn't support workspaces, so it only runs on the single-package repository. +- With `--workspaces`, npm-check-updates' picker shows one `package.json` at a time and fetches the next after you answer, so its picker only runs on the single-package repository. +- Yarn 2 and later has no command that lists updates outside `yarn upgrade-interactive`, so the Yarn repository has no package-manager row in the listing tables. npm has no interactive update. ## Run it yourself -You need [hyperfine](https://github.com/sharkdp/hyperfine#installation) and network access: +You need [hyperfine](https://github.com/sharkdp/hyperfine#installation), `python3` (for the pickers' terminal) and network access, on macOS or Linux: ```bash pnpm bench:compare --fixture vue-core --runs 10 # writes bench-results/vue-core.json node scripts/bench-report.mjs bench-results/*.json # rewrites the README section, this page and the chart ``` -Fixtures: `express`, `vue-core`, `tanstack-query`. Times from your machine and network will differ from the runner's. +Fixtures: `express`, `vue-core`, `tanstack-query`, `vendure`, `backstage`. Times from your machine and network will differ from the runner's. ## Results diff --git a/scripts/bench-compare.mjs b/scripts/bench-compare.mjs index 2d9d5d3..e204532 100644 --- a/scripts/bench-compare.mjs +++ b/scripts/bench-compare.mjs @@ -13,12 +13,18 @@ // gives it a fresh HOME, XDG directories and TMPDIR // warm the tool keeps its caches; an untimed warmup run fills them // +// Then each tool's interactive picker, where it has one, runs in a terminal +// (scripts/bench-pty.py, so python3 is needed) and is timed until it has every +// update loaded: the last thing it draws or writes to its cache before going +// quiet. The same two modes apply. +// // Cloning and installing the fixture, and installing the tools, happen before // any timing. Before timing, each tool runs once to prove it works on the // fixture and found updates. The output is one JSON file per fixture, which // scripts/bench-report.mjs turns into the README section, docs page and chart. -import { execFileSync, spawnSync } from 'node:child_process' +import { execFileSync, spawn, spawnSync } from 'node:child_process' import { + appendFileSync, closeSync, existsSync, mkdirSync, @@ -26,6 +32,8 @@ import { readdirSync, readFileSync, rmSync, + statSync, + watch, writeFileSync, } from 'node:fs' import { cpus, release, tmpdir, totalmem, type } from 'node:os' @@ -44,24 +52,37 @@ export const FIXTURES = [ repo: 'TanStack/query', sha: 'ce77be5e15c695df9d4d9122caf9256d334433ba', }, + { id: 'vendure', repo: 'vendurehq/vendure', sha: '1dcaf0ee504675f3462d6302c23ff9f16beb2773' }, + { + id: 'backstage', + repo: 'backstage/backstage', + sha: '76b0c7a48ef737b1a46342c29269827715d70684', + }, ] /** Entries in a JSON report keyed by package (or by package.json, holding arrays). */ const countEntries = (json) => Object.values(json).reduce((n, value) => n + (Array.isArray(value) ? value.length : 1), 0) +/** Rows of a `| Package | Current | ... |` table, header excluded. */ +const tableRows = (text) => + Math.max(0, text.split('\n').filter((line) => /^\| \S/.test(line)).length - 1) + /** * The tools, each set up to report every dependency against its latest * version without writing anything. `fixture` is a described fixture: - * `{ pm, workspaces }`. `count` reads the number of updates from stdout; - * `failsWhenOutdated` marks the tools that exit 1 when they find updates. + * `{ pm, workspaces }`. `count` reads the number of updates from the output, + * `{ stdout, stderr }`; `failsWhenOutdated` marks the tools that exit 1 when + * they find updates. `picker` opens the tool's interactive list of updates + * instead, where it has one that shows every update at once. */ export const TOOLS = [ { id: 'inup', name: () => 'inup', args: () => ['--json'], - count: (stdout) => JSON.parse(stdout).summary.outdated, + count: ({ stdout }) => JSON.parse(stdout).summary.outdated, + picker: { args: () => [] }, }, { id: 'ncu', @@ -70,11 +91,17 @@ export const TOOLS = [ bin: 'ncu', args: (fixture) => ['--jsonUpgraded', ...(fixture.workspaces ? ['--workspaces'] : [])], // One object of upgrades, or one per package.json with --workspaces. - count: (stdout) => + count: ({ stdout }) => Object.values(JSON.parse(stdout)).reduce( (n, value) => n + (typeof value === 'object' ? Object.keys(value).length : 1), 0 ), + picker: { + args: () => ['--interactive'], + // With --workspaces it lists one package.json, and fetches the next after you answer. + unsupported: (fixture) => + fixture.workspaces ? 'asks about one package.json at a time' : null, + }, }, { id: 'taze', @@ -92,11 +119,34 @@ export const TOOLS = [ '--no-node-version', '--json', ], - count: (stdout) => + count: ({ stdout }) => JSON.parse(stdout).packages.reduce( (n, pkg) => n + pkg.resolved.filter((dep) => dep.update).length, 0 ), + picker: { + args: (fixture) => [ + 'major', + '--interactive', + ...(fixture.workspaces ? ['--recursive'] : []), + '--include-locked', + '--no-github-actions', + '--no-node-version', + ], + }, + }, + { + id: 'syncpack', + name: () => 'syncpack', + pkg: 'syncpack', + bin: 'syncpack', + // --check lists updates without writing; it reports on stderr, one line + // per dependency and version: " 1x react ^18.3.1 (~1.9y) → ^19.2.0 (~3mo)". + args: () => ['update', '--check'], + failsWhenOutdated: true, + count: ({ stderr }) => + stderr.split('\n').filter((line) => /^\s+\d+x \S.* → /.test(line)).length, + picker: { args: () => ['update', '--interactive'] }, }, { id: 'npm-check', @@ -108,27 +158,55 @@ export const TOOLS = [ failsWhenOutdated: true, unsupported: (fixture) => (fixture.workspaces ? 'no workspace support' : null), count: () => null, // prints a table only + picker: { + args: () => ['--update', '--skip-unused'], + unsupported: (fixture) => (fixture.workspaces ? 'no workspace support' : null), + }, }, { id: 'pm', - name: (fixture) => `${fixture.pm} outdated`, - args: (fixture) => - fixture.pm === 'pnpm' - ? ['outdated', ...(fixture.workspaces ? ['--recursive'] : []), '--format', 'json'] - : [ - 'outdated', - ...(fixture.workspaces ? ['--workspaces', '--include-workspace-root'] : []), - '--json', - ], + name: (fixture) => + fixture.pm === 'yarn' ? 'yarn upgrade-interactive' : `${fixture.pm} outdated`, + args: (fixture) => { + const workspaces = fixture.workspaces + if (fixture.pm === 'pnpm') { + return ['outdated', ...(workspaces ? ['--recursive'] : []), '--format', 'json'] + } + if (fixture.pm === 'bun') return ['outdated', ...(workspaces ? ['--recursive'] : [])] + return [ + 'outdated', + ...(workspaces ? ['--workspaces', '--include-workspace-root'] : []), + '--json', + ] + }, + // npm and pnpm exit 1 when they find updates; bun exits 0. failsWhenOutdated: true, - count: (stdout) => countEntries(JSON.parse(stdout)), + unsupported: (fixture) => + fixture.pm === 'yarn' ? 'Yarn 2+ lists updates only in this interactive picker' : null, + count: ({ stdout }, fixture) => + fixture.pm === 'bun' ? tableRows(stdout) : countEntries(JSON.parse(stdout)), + picker: { + name: (fixture) => + fixture.pm === 'yarn' ? 'yarn upgrade-interactive' : `${fixture.pm} update -i`, + args: (fixture) => { + const recursive = fixture.workspaces ? ['--recursive'] : [] + if (fixture.pm === 'yarn') return ['upgrade-interactive'] + if (fixture.pm === 'pnpm') return ['update', '--interactive', '--latest', ...recursive] + return ['update', '--interactive', ...recursive] + }, + unsupported: (fixture) => (fixture.pm === 'npm' ? 'npm has no interactive update' : null), + }, }, ] /** POSIX shell quoting. */ export const quote = (value) => `'${String(value).replaceAll("'", "'\\''")}'` -/** The environment that points every cache a tool might use into `box`. */ +/** + * The environment that points every cache a tool might use into `box`. Most + * follow HOME, XDG and TMPDIR; the rest have variables of their own that a + * developer's shell may set (BUN_INSTALL, for one). + */ export function sandboxEnv(box) { return { HOME: box, @@ -137,6 +215,10 @@ export function sandboxEnv(box) { XDG_DATA_HOME: join(box, '.local', 'share'), XDG_STATE_HOME: join(box, '.local', 'state'), TMPDIR: join(box, 'tmp'), + npm_config_cache: join(box, '.npm'), + BUN_INSTALL: join(box, '.bun'), + BUN_INSTALL_CACHE_DIR: join(box, '.bun', 'install', 'cache'), + YARN_GLOBAL_FOLDER: join(box, '.yarn', 'berry'), } } @@ -148,6 +230,122 @@ export function shellCommand(argv, box, { failsWhenOutdated = false } = {}) { return failsWhenOutdated ? `${command} || [ $? -eq 1 ]` : command } +const PTY_SCRIPT = join(dirname(fileURLToPath(import.meta.url)), 'bench-pty.py') +/** How long a picker must stay quiet to count as loaded. */ +const PICKER_IDLE_MS = 5000 +const PICKER_LIMIT_MS = 180_000 +/** Pickers refuse to open in CI; these are the variables they look at. */ +const CI_VARIABLES = ['CI', 'CONTINUOUS_INTEGRATION', 'GITHUB_ACTIONS', 'BUILD_NUMBER', 'RUN_ID'] + +/** A picker's environment: the sandbox, a color terminal, and no sign of CI. */ +export function pickerEnv(box, env = process.env) { + const result = { ...env, ...sandboxEnv(box), TERM: 'xterm-256color' } + for (const key of CI_VARIABLES) delete result[key] + return result +} + +/** + * Time a picker, in seconds: launch `argv` in a terminal (scripts/bench-pty.py) + * and wait until it goes quiet, with nothing drawn and nothing written under + * `box` for `idleMs`. The time is the last thing it drew or wrote: a picker + * may finish drawing the rows on screen and keep fetching the rest, writing + * each into its cache. One that exits first never opened; one that is still + * busy after `limitMs` fails too. + */ +export function timePicker(argv, { cwd, box, idleMs = PICKER_IDLE_MS, limitMs = PICKER_LIMIT_MS }) { + return new Promise((resolve, reject) => { + const start = Date.now() + let lastDrawn = 0 + let lastEvent = 0 + let lastWritten = 0 + let drawn = 0 + let stderr = '' + let exit = null + const watcher = watch(box, { recursive: true }, (_event, file) => { + lastEvent = Date.now() + let written = lastEvent + try { + if (file) written = statSync(join(box, file)).mtimeMs + } catch { + // renamed or removed already + } + lastWritten = Math.max(lastWritten, written) + }) + const child = spawn('python3', ['-I', PTY_SCRIPT, ...argv], { + cwd, + env: pickerEnv(box), + stdio: ['pipe', 'pipe', 'pipe'], + }) + child.stdout.on('data', (data) => { + drawn += data.length + lastDrawn = Date.now() + }) + child.stderr.on('data', (data) => { + stderr += data + }) + child.on('exit', (code, signal) => { + exit ??= code ?? signal + }) + child.on('error', (error) => { + exit ??= error.message + }) + // Settle once the picker is gone, so the next run starts on a quiet machine. + let outcome + let closed = false + const settle = () => { + if (outcome === undefined || !closed) return + outcome instanceof Error ? reject(outcome) : resolve(outcome) + } + child.on('close', () => { + closed = true + settle() + }) + const poll = setInterval(() => { + const now = Date.now() + const quiet = now - Math.max(start, lastDrawn, lastEvent) + if (exit === null && quiet < idleMs && now - start < limitMs) return + clearInterval(poll) + watcher.close() + const command = argv.join(' ') + if (exit !== null) { + outcome = new Error( + `${command} exited (${exit}) instead of waiting\n${stderr.slice(-2000)}` + ) + } else if (quiet < idleMs) { + outcome = new Error(`${command} was still busy after ${limitMs / 1000} s`) + } else if (drawn === 0) { + outcome = new Error(`${command} drew nothing`) + } else { + outcome = (Math.max(lastDrawn, lastWritten) - start) / 1000 + } + child.kill('SIGTERM') + settle() + }, 50) + }) +} + +/** Median, mean, sample standard deviation and range, as hyperfine reports them. */ +export function summarize(times) { + const sorted = [...times].sort((a, b) => a - b) + const middle = sorted.length / 2 + const median = Number.isInteger(middle) + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[Math.floor(middle)] + const mean = times.reduce((sum, time) => sum + time, 0) / times.length + const variance = + times.length > 1 + ? times.reduce((sum, time) => sum + (time - mean) ** 2, 0) / (times.length - 1) + : 0 + return { + median, + mean, + stddev: Math.sqrt(variance), + min: sorted[0], + max: sorted.at(-1), + times, + } +} + /** Empty `box` and recreate it, ready for a run. */ export const resetBoxCommand = (box) => `rm -rf ${quote(box)} && mkdir -p ${quote(join(box, 'tmp'))}` @@ -196,34 +394,54 @@ const run = (command, args, options = {}) => const log = (message) => console.error(`\n> ${message}`) function checkout(fixture, dir) { - if ( - existsSync(join(dir, '.git')) && - run('git', ['rev-parse', 'HEAD'], { cwd: dir }) === fixture.sha - ) - return + const git = (...args) => run('git', args, { cwd: dir }) + if (existsSync(join(dir, '.git')) && git('rev-parse', 'HEAD') === fixture.sha) return rmSync(dir, { recursive: true, force: true }) mkdirSync(dir, { recursive: true }) - run('git', ['init', '-q'], { cwd: dir }) - run( - 'git', - ['fetch', '-q', '--depth', '1', `https://github.com/${fixture.repo}.git`, fixture.sha], - { - cwd: dir, - } - ) - run('git', ['checkout', '-q', 'FETCH_HEAD'], { cwd: dir }) + git('init', '-q') + git('fetch', '-q', '--depth', '1', `https://github.com/${fixture.repo}.git`, fixture.sha) + git('checkout', '-q', 'FETCH_HEAD') } -/** The fixture's package manager (from `packageManager`) and whether it is a monorepo. */ -function inspectFixture(dir) { +const LOCKFILES = [ + ['pnpm-lock.yaml', 'pnpm'], + ['yarn.lock', 'yarn'], + ['bun.lock', 'bun'], + ['bun.lockb', 'bun'], +] + +/** + * The fixture's package manager - from `packageManager`, else its lockfile - + * and whether it is a monorepo. + */ +export function inspectFixture(dir) { const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) const spec = pkg.packageManager?.split('+')[0] - const [pm, version] = spec ? spec.split('@') : ['npm'] - if (pm !== 'npm' && pm !== 'pnpm') throw new Error(`unsupported package manager: ${pm}`) + const fromLockfile = LOCKFILES.find(([file]) => existsSync(join(dir, file)))?.[1] ?? 'npm' + const [pm, version] = spec ? spec.split('@') : [fromLockfile] + if (!['npm', 'pnpm', 'yarn', 'bun'].includes(pm)) { + throw new Error(`unsupported package manager: ${pm}`) + } + if (pm === 'yarn' && !(Number(version?.split('.')[0]) >= 2)) { + throw new Error('only Yarn 2+ projects with a packageManager version are supported') + } const workspaces = existsSync(join(dir, 'pnpm-workspace.yaml')) || pkg.workspaces !== undefined return { pm, pmVersion: version ?? null, workspaces } } +/** Arguments that install a fixture's dependencies without running its scripts. */ +const INSTALL = { + npm: (dir) => [ + existsSync(join(dir, 'package-lock.json')) ? 'ci' : 'install', + '--ignore-scripts', + '--no-audit', + '--no-fund', + ], + pnpm: () => ['install', '--frozen-lockfile', '--ignore-scripts'], + bun: () => ['install', '--frozen-lockfile', '--ignore-scripts'], + yarn: () => ['install', '--immutable', '--mode=skip-build'], +} + /** Install npm packages into `dir`; returns the directory holding node_modules. */ function npmInstall(dir, specs) { mkdirSync(dir, { recursive: true }) @@ -251,24 +469,47 @@ function nodeBin(prefix, name, bin) { } /** - * pnpm as people run it. Since pnpm 11 the `pnpm` bin is a native executable - * shipped in `@pnpm/exe.`; corepack starts it too. Older versions are - * a Node.js script. + * The native executable an npm package ships in a per-platform optional + * dependency (`@pnpm/exe.linux-x64/pnpm`, `@oven/bun-linux-x64/bin/bun`): what + * its bin runs once installed. npm installs only the host's platform package. */ -function pnpmBin(prefix, version) { - const scope = join(prefix, 'node_modules', '@pnpm') - const exe = existsSync(scope) - ? readdirSync(scope) - .filter((name) => name.startsWith('exe.')) - .map((name) => join(scope, name, 'pnpm')) - .find((path) => existsSync(path)) - : undefined - if (exe) return { argv: [exe], version } - return nodeBin(prefix, 'pnpm', 'pnpm') +function platformBinary(prefix, scope, packagePrefix, file) { + const dir = join(prefix, 'node_modules', scope) + if (!existsSync(dir)) return undefined + return readdirSync(dir) + .filter((name) => name.startsWith(packagePrefix)) + .map((name) => join(dir, name, file)) + .find((path) => existsSync(path)) +} + +/** The package manager, run as its own users run it. */ +function packageManagerBin(fixture, dir, workdir) { + const { pm, pmVersion } = fixture + if (pm === 'npm') return { argv: ['npm'], version: run('npm', ['--version']) } + if (pm === 'yarn') { + // Yarn projects usually commit their Yarn release and point yarnPath at it. + const rc = join(dir, '.yarnrc.yml') + const yarnPath = + existsSync(rc) && /^yarnPath:\s*['"]?([^'"\s]+)/m.exec(readFileSync(rc, 'utf8'))?.[1] + if (yarnPath) return { argv: [process.execPath, join(dir, yarnPath)], version: pmVersion } + const prefix = npmInstall(join(workdir, `yarn-${pmVersion}`), [ + `@yarnpkg/cli-dist@${pmVersion}`, + ]) + return nodeBin(prefix, '@yarnpkg/cli-dist', 'yarn') + } + const spec = `${pm}@${pmVersion ?? 'latest'}` + const prefix = npmInstall(join(workdir, spec), [spec]) + const version = readPackage(prefix, pm).version + // Since pnpm 11, and for bun, the bin is a native executable. + const exe = + pm === 'pnpm' + ? platformBinary(prefix, '@pnpm', 'exe.', 'pnpm') + : platformBinary(prefix, '@oven', 'bun-', join('bin', 'bun')) + return exe ? { argv: [exe], version } : nodeBin(prefix, pm, pm) } /** Where each tool's command comes from, and its version. */ -function resolveTools(fixture, workdir) { +function resolveTools(fixture, dir, workdir) { const prefix = npmInstall( join(workdir, 'tools'), TOOLS.filter((tool) => tool.pkg).map((tool) => `${tool.pkg}@latest`) @@ -280,13 +521,7 @@ function resolveTools(fixture, workdir) { argv: [process.execPath, join(ROOT, 'dist', 'cli.js')], version: `${inupVersion} (${commit})`, }, - pm: - fixture.pm === 'pnpm' - ? pnpmBin( - npmInstall(join(workdir, `pnpm-${fixture.pmVersion}`), [`pnpm@${fixture.pmVersion}`]), - fixture.pmVersion - ) - : { argv: ['npm'], version: run('npm', ['--version']) }, + pm: packageManagerBin(fixture, dir, workdir), } for (const tool of TOOLS) { if (tool.pkg) bins[tool.id] = nodeBin(prefix, tool.pkg, tool.bin) @@ -295,28 +530,30 @@ function resolveTools(fixture, workdir) { } /** Run each tool once, untimed: it must succeed and report updates. */ -function smokeTest(tool, argv, cwd, box) { +function smokeTest(tool, fixture, argv, cwd, box) { rmSync(box, { recursive: true, force: true }) mkdirSync(join(box, 'tmp'), { recursive: true }) - // Into a file, as hyperfine's /dev/null is: some tools exit before a pipe + // Into files, as hyperfine's /dev/null is: some tools exit before a pipe // drains, cutting their output off at 64 KB. const outFile = join(box, 'stdout') + const errFile = join(box, 'stderr') const out = openSync(outFile, 'w') + const err = openSync(errFile, 'w') const result = spawnSync(argv[0], argv.slice(1), { cwd, - encoding: 'utf8', env: { ...process.env, ...sandboxEnv(box) }, - stdio: ['ignore', out, 'pipe'], + stdio: ['ignore', out, err], }) closeSync(out) - const stdout = readFileSync(outFile, 'utf8') + closeSync(err) + const output = { stdout: readFileSync(outFile, 'utf8'), stderr: readFileSync(errFile, 'utf8') } const ok = result.status === 0 || (tool.failsWhenOutdated && result.status === 1) - if (!ok || !stdout.trim()) { + if (!ok || !`${output.stdout}${output.stderr}`.trim()) { throw new Error( - `${argv.join(' ')} exited with ${result.status ?? result.signal}\n${result.stderr.slice(-2000)}` + `${argv.join(' ')} exited with ${result.status ?? result.signal}\n${output.stderr.slice(-2000)}` ) } - const updates = tool.count(stdout) + const updates = tool.count(output, fixture) if (updates === 0) throw new Error(`${argv.join(' ')} reported no updates; it checked nothing`) return updates } @@ -358,6 +595,46 @@ function environment() { } } +/** + * Keep what installing left behind out of `git status`: inup's picker asks + * before it works on a tree with uncommitted changes. + */ +function hideFromGit(dir) { + const git = (...args) => run('git', args, { cwd: dir }) + for (const entry of git('status', '--porcelain', '-z').split('\0').filter(Boolean)) { + const path = entry.slice(3) + if (entry.startsWith('??')) appendFileSync(join(dir, '.git', 'info', 'exclude'), `/${path}\n`) + else git('update-index', '--assume-unchanged', path) + } + if (git('status', '--porcelain')) throw new Error(`could not clean the git status of ${dir}`) +} + +/** + * Time each picker `runs` times, in rounds: every tool once per round, so a + * slow spell on the network lands on all of them alike. An untimed first run + * proves each one opens and, for warm runs, fills its caches. + */ +async function timePickers(tools, { cwd, runs, mode, workdir }) { + const boxes = tools.map((tool) => join(workdir, 'boxes', `picker-${mode}`, tool.id)) + const reset = (box) => { + rmSync(box, { recursive: true, force: true }) + mkdirSync(join(box, 'tmp'), { recursive: true }) + } + const time = async (i) => { + if (mode === 'cold') reset(boxes[i]) + const seconds = await timePicker(tools[i].argv, { cwd, box: boxes[i] }) + console.error(` ${tools[i].name}: ${seconds.toFixed(2)} s`) + return seconds + } + boxes.forEach(reset) + for (const i of tools.keys()) await time(i) + const times = tools.map(() => []) + for (let round = 0; round < runs; round++) { + for (const i of tools.keys()) times[i].push(await time(i)) + } + return Object.fromEntries(tools.map((tool, i) => [tool.id, summarize(times[i])])) +} + export async function benchmark({ fixtureId, runs, out, workdir }) { const fixture = FIXTURES.find((candidate) => candidate.id === fixtureId) if (!fixture) { @@ -370,6 +647,9 @@ export async function benchmark({ fixtureId, runs, out, workdir }) { if (!existsSync(join(ROOT, 'dist', 'cli.js'))) { throw new Error('dist/cli.js not found; run pnpm build') } + if (spawnSync('python3', ['--version']).error) { + throw new Error('python3 not found; the picker timings run in a terminal it provides') + } const dir = join(workdir, 'fixtures', fixture.id) log(`Checking out ${fixture.repo}@${fixture.sha.slice(0, 7)}`) @@ -377,22 +657,15 @@ export async function benchmark({ fixtureId, runs, out, workdir }) { const described = { ...fixture, ...inspectFixture(dir) } log('Installing the tools') - const bins = resolveTools(described, workdir) + const bins = resolveTools(described, dir, workdir) log(`Installing ${fixture.repo} with ${described.pm}`) - const pm = bins.pm.argv - if (described.pm === 'pnpm') { - run(pm[0], [...pm.slice(1), 'install', '--frozen-lockfile', '--ignore-scripts'], { - cwd: dir, - stdio: ['ignore', 'ignore', 'inherit'], - }) - } else { - const install = existsSync(join(dir, 'package-lock.json')) ? 'ci' : 'install' - run('npm', [install, '--ignore-scripts', '--no-audit', '--no-fund'], { - cwd: dir, - stdio: ['ignore', 'ignore', 'inherit'], - }) - } + const [command, ...pmArgs] = bins.pm.argv + run(command, [...pmArgs, ...INSTALL[described.pm](dir)], { + cwd: dir, + stdio: ['ignore', 'ignore', 'inherit'], + }) + hideFromGit(dir) const tools = [] const skipped = [] @@ -406,7 +679,7 @@ export async function benchmark({ fixtureId, runs, out, workdir }) { const args = tool.args(described) const argv = [...bins[tool.id].argv, ...args] log(`Checking ${name} works on ${fixture.repo}`) - const updates = smokeTest(tool, argv, dir, join(workdir, 'boxes', 'smoke', tool.id)) + const updates = smokeTest(tool, described, argv, dir, join(workdir, 'boxes', 'smoke', tool.id)) tools.push({ id: tool.id, name, @@ -432,6 +705,30 @@ export async function benchmark({ fixtureId, runs, out, workdir }) { results[mode] = Object.fromEntries(timed.map((tool, i) => [tool.id, stats(measured[i])])) } + const pickers = [] + const pickersSkipped = [] + for (const tool of TOOLS) { + const name = tool.picker.name?.(described) ?? tool.name(described) + const reason = tool.picker.unsupported?.(described) + if (reason) { + pickersSkipped.push({ id: tool.id, name, reason }) + continue + } + const args = tool.picker.args(described) + pickers.push({ + id: tool.id, + name, + version: bins[tool.id].version, + display: [tool.bin ?? name.split(' ')[0], ...args].join(' '), + argv: [...bins[tool.id].argv, ...args], + }) + } + const picker = { tools: pickers.map(({ argv, ...tool }) => tool), skipped: pickersSkipped } + for (const mode of ['cold', 'warm']) { + log(`Timing ${mode} picker runs (${runs} per tool)`) + picker[mode] = await timePickers(pickers, { cwd: dir, runs, mode, workdir }) + } + const report = { schema: 1, date: new Date().toISOString(), @@ -454,6 +751,7 @@ export async function benchmark({ fixtureId, runs, out, workdir }) { })), skipped, results, + picker, } mkdirSync(dirname(out), { recursive: true }) writeFileSync(out, `${JSON.stringify(report, null, 2)}\n`) diff --git a/scripts/bench-pty.py b/scripts/bench-pty.py new file mode 100644 index 0000000..f6b0da6 --- /dev/null +++ b/scripts/bench-pty.py @@ -0,0 +1,58 @@ +"""Run a command in a pseudo-terminal, for scripts/bench-compare.mjs. + + python3 -I scripts/bench-pty.py [args...] + +Interactive pickers only draw when they see a terminal. This gives the command +one of 160 columns by 50 rows, copies everything it draws to stdout, and passes +stdin through as keystrokes. It exits when the command does, with its code. +SIGTERM stops the command too: it runs in a session of its own, which killing +this process alone would leave behind. +""" + +import fcntl +import os +import pty +import select +import signal +import struct +import sys +import termios + +ROWS, COLUMNS = 50, 160 + +pid, fd = pty.fork() +if pid == 0: + os.execvp(sys.argv[1], sys.argv[1:]) + + +def stop(signum, _frame): + try: + os.killpg(pid, signal.SIGKILL) + os.waitpid(pid, 0) # gone, not just dying, before this process exits + except (ProcessLookupError, ChildProcessError): + pass + os._exit(128 + signum) + + +signal.signal(signal.SIGTERM, stop) +fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("HHHH", ROWS, COLUMNS, 0, 0)) +inputs = [fd, sys.stdin.fileno()] +while True: + ready, _, _ = select.select(inputs, [], []) + if fd in ready: + try: + data = os.read(fd, 65536) + except OSError: # the command exited and closed the terminal + break + if not data: + break + os.write(sys.stdout.fileno(), data) + if sys.stdin.fileno() in ready: + data = os.read(sys.stdin.fileno(), 1024) + if data: + os.write(fd, data) + else: + inputs.remove(sys.stdin.fileno()) + +_, status = os.waitpid(pid, 0) +sys.exit(os.waitstatus_to_exitcode(status)) diff --git a/scripts/bench-report.mjs b/scripts/bench-report.mjs index 3f23432..58aa061 100644 --- a/scripts/bench-report.mjs +++ b/scripts/bench-report.mjs @@ -46,22 +46,54 @@ export function sortResults(results) { return [...results].sort((a, b) => order(a) - order(b)) } -/** One row per tool across all fixtures, in the order the tools ran. */ -function toolRows(results) { +/** + * Names that share a command after the tool's name read as one: + * "npm outdated", "pnpm outdated" as "npm / pnpm outdated". + */ +export function joinNames(names) { + const groups = new Map() + for (const name of names) { + const space = name.indexOf(' ') + const command = space === -1 ? '' : name.slice(space) + groups.set(command, [ + ...(groups.get(command) ?? []), + name.slice(0, name.length - command.length), + ]) + } + return [...groups].map(([command, tools]) => `${tools.join(' / ')}${command}`).join(' / ') +} + +/** + * What a table shows, as `{ tools, skipped, cold, warm }`: the tools listing + * outdated dependencies, or their pickers. + */ +const LISTING = (result) => ({ tools: result.tools, skipped: result.skipped, ...result.results }) +const PICKER = (result) => result.picker + +/** + * One row per tool across all fixtures, in the order the tools ran, named for + * the fixtures it ran on (a skipped tool names its row only if it ran nowhere). + */ +function toolRows(views) { const rows = new Map() - for (const result of results) { - for (const tool of [...result.tools, ...result.skipped]) { - const row = rows.get(tool.id) ?? { id: tool.id, names: new Set() } - row.names.add(tool.name) - rows.set(tool.id, row) - } + const add = (tool, kind) => { + const row = rows.get(tool.id) ?? { id: tool.id, ran: new Set(), skipped: new Set() } + row[kind].add(tool.name) + rows.set(tool.id, row) } - return [...rows.values()].map((row) => ({ id: row.id, name: [...row.names].join(' / ') })) + for (const view of views) { + for (const tool of view.tools) add(tool, 'ran') + for (const tool of view.skipped) add(tool, 'skipped') + } + return [...rows.values()].map((row) => ({ + id: row.id, + name: joinNames([...(row.ran.size > 0 ? row.ran : row.skipped)]), + })) } const fastest = (timings) => Math.min(...Object.values(timings).map((timing) => timing.median)) -function table(results, mode, { title, withSizes }) { +function table(results, view, mode, { title, withSizes }) { const header = results.map(({ fixture }) => { const link = `[${fixture.repo}](${repoUrl(fixture)})` return withSizes ? `${link}
${sizes(fixture)}` : link @@ -70,12 +102,13 @@ function table(results, mode, { title, withSizes }) { `| ${title} | ${header.join(' | ')} |`, `| :-- | ${results.map(() => '--:').join(' | ')} |`, ] - for (const row of toolRows(results)) { - const cells = results.map((result) => { - const timing = result.results[mode][row.id] + const views = results.map(view) + for (const row of toolRows(views)) { + const cells = views.map((timings) => { + const timing = timings[mode][row.id] if (!timing) return '—' const text = formatSeconds(timing.median) - return timing.median === fastest(result.results[mode]) ? `**${text}**` : text + return timing.median === fastest(timings[mode]) ? `**${text}**` : text }) lines.push(`| ${row.name} | ${cells.join(' | ')} |`) } @@ -87,18 +120,38 @@ function where(environment) { return `${machine}, ${plural(environment.cores, 'CPU', 'CPUs')}, Node ${environment.node}` } -/** The two tables (first run, repeat run) and the line under them. */ +/** + * The tables - listing outdated dependencies, then opening the picker, each + * as a first run and a repeat run - and the line under them. + */ export function renderTables(results) { const [first] = results + const pickers = results.every((result) => result.picker) const note = [ 'First run: every cache the tool keeps is emptied before each run. Repeat run: caches kept.', - `Median of ${first.runs} runs per tool with [hyperfine](https://github.com/sharkdp/hyperfine) ${first.environment.hyperfine} on ${where(first.environment)}, ${formatDate(first.date)}.`, + `Median of ${first.runs} runs per tool on ${where(first.environment)}, ${formatDate(first.date)}; listing timed with [hyperfine](https://github.com/sharkdp/hyperfine) ${first.environment.hyperfine}.`, + pickers + ? 'Pickers run in a 160×50 terminal and count as loaded at the last thing they draw or write to their cache before 5 s of quiet, so rows fetched off screen count too.' + : null, "inup's times include its vulnerability check, which the other tools don't run.", + pickers ? "inup's first picker run also downloads its native core (about 1.5 MB)." : null, 'Fastest in **bold**.', - ].join(' ') + ] + .filter(Boolean) + .join(' ') return [ - table(results, 'cold', { title: 'First run', withSizes: true }), - table(results, 'warm', { title: 'Repeat run', withSizes: false }), + '### Listing outdated dependencies', + 'Each tool prints every outdated dependency and exits.', + table(results, LISTING, 'cold', { title: 'First run', withSizes: true }), + table(results, LISTING, 'warm', { title: 'Repeat run', withSizes: false }), + ...(pickers + ? [ + '### Opening the picker', + "Each tool's interactive list of updates, timed until every update is loaded.", + table(results, PICKER, 'cold', { title: 'First run', withSizes: false }), + table(results, PICKER, 'warm', { title: 'Repeat run', withSizes: false }), + ] + : []), `${note}`, ].join('\n\n') } @@ -119,7 +172,7 @@ export function describeChart(results) { export function renderReadme(results) { return [ '## Speed', - 'How long each tool takes to list every outdated dependency, from launch to exit, on public repositories pinned to a commit. Lower is better.', + 'How long each tool takes, on public repositories pinned to a commit, to list every outdated dependency and to open its picker with every update loaded. Lower is better.', [ '', ` `, @@ -138,22 +191,40 @@ function timingCell(timing, best) { return `${timing.median === best ? `**${median}**` : median} ${spread}` } +/** One tool per row: version, first run, repeat run and any extra columns. */ +function detailTable({ tools, skipped, cold, warm }, extra = []) { + const bestCold = fastest(cold) + const bestWarm = fastest(warm) + const rows = tools.map((tool) => + [ + tool.name, + tool.version, + timingCell(cold[tool.id], bestCold), + timingCell(warm[tool.id], bestWarm), + ...extra.map(({ value }) => value(tool)), + ].join(' | ') + ) + for (const tool of skipped) { + rows.push([tool.name, '—', `not run: ${tool.reason}`, '', ...extra.map(() => '')].join(' | ')) + } + return [ + `| Tool | Version | First run | Repeat run |${extra.map(({ title }) => ` ${title} |`).join('')}`, + `| :-- | :-- | --: | --: |${extra.map(() => ' --: |').join('')}`, + ...rows.map((row) => `| ${row} |`), + ].join('\n') +} + +const commands = (tools) => ['```sh', ...tools.map((tool) => tool.display), '```'].join('\n') + /** The per-repository detail on docs/guide/benchmarks.md. */ export function renderDetails(results) { const sections = results.map((result) => { - const { fixture, environment, tools, skipped } = result - const bestCold = fastest(result.results.cold) - const bestWarm = fastest(result.results.warm) + const { fixture, environment, tools } = result const facts = [ `[\`${fixture.repo}@${fixture.sha.slice(0, 7)}\`](${repoUrl(fixture)})`, fixture.packageManager, sizes(fixture), ].join(' · ') - const rows = tools.map( - (tool) => - `| ${tool.name} | ${tool.version} | ${timingCell(result.results.cold[tool.id], bestCold)} | ${timingCell(result.results.warm[tool.id], bestWarm)} | ${tool.updates ?? '—'} |` - ) - for (const tool of skipped) rows.push(`| ${tool.name} | — | not run: ${tool.reason} | | |`) const machine = [ environment.image ? `GitHub Actions ${environment.image}` : null, environment.os, @@ -166,17 +237,23 @@ export function renderDetails(results) { .filter(Boolean) .join(' · ') const run = environment.run ? ` · [workflow run](${environment.run})` : '' + const updates = { title: 'Updates found', value: (tool) => tool.updates ?? '—' } return [ `### ${fixture.repo}`, facts, - [ - '| Tool | Version | First run | Repeat run | Updates found |', - '| :-- | :-- | --: | --: | --: |', - ...rows, - ].join('\n'), `Median, ± standard deviation, min–max, over ${result.runs} runs. Measured ${formatDate(result.date)} on ${machine}${run}.`, + '#### Listing outdated dependencies', + detailTable(LISTING(result), [updates]), 'Commands, run in the repository root:', - ['```sh', ...tools.map((tool) => tool.display), '```'].join('\n'), + commands(tools), + ...(result.picker + ? [ + '#### Opening the picker', + detailTable(result.picker), + 'Commands, run in the repository root in a 160×50 terminal:', + commands(result.picker.tools), + ] + : []), ].join('\n\n') }) return sections.join('\n\n') @@ -212,18 +289,24 @@ export function renderChart(results, theme) { const labelWidth = 150 const valueWidth = 64 const barArea = width - labelWidth - valueWidth - const rowHeight = 26 - const barHeight = 16 + const rowHeight = 22 + const barHeight = 14 + const panelGap = 16 const parts = [] const text = (x, y, content, attrs = '') => parts.push(`${escapeXml(content)}`) - text(0, 14, 'First run, empty caches · seconds, lower is better', ` fill="${color.muted}"`) - let y = 40 + text( + 0, + 14, + 'Listing outdated dependencies, first run (empty caches) · seconds, lower is better', + ` fill="${color.muted}"` + ) + let y = 36 for (const { fixture, tools, results: timings } of results) { text(0, y + 14, fixture.repo, ` fill="${color.ink}" font-weight="600"`) - text(0, y + 32, sizes(fixture), ` fill="${color.muted}"`) - y += 44 + text(0, y + 31, sizes(fixture), ` fill="${color.muted}"`) + y += 40 const rows = tools .map((tool) => ({ ...tool, median: timings.cold[tool.id].median })) .sort((a, b) => a.median - b.median) @@ -245,9 +328,9 @@ export function renderChart(results, theme) { ) y += rowHeight } - y += 20 + y += panelGap } - const height = y - 20 + const height = y - panelGap return [ ``, 'How long each tool takes to list outdated dependencies', diff --git a/test/unit/scripts/bench-compare.test.ts b/test/unit/scripts/bench-compare.test.ts index 4106ae8..887cedb 100644 --- a/test/unit/scripts/bench-compare.test.ts +++ b/test/unit/scripts/bench-compare.test.ts @@ -1,16 +1,20 @@ import { spawnSync } from 'node:child_process' -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import { countManifests, FIXTURES, + inspectFixture, + pickerEnv, quote, resetBoxCommand, sandboxEnv, shellCommand, + summarize, TOOLS, + timePicker, } from '../../../scripts/bench-compare.mjs' const tool = (id: string) => { @@ -22,6 +26,8 @@ const tool = (id: string) => { const single = { pm: 'npm', workspaces: false } const npmWorkspace = { pm: 'npm', workspaces: true } const pnpmWorkspace = { pm: 'pnpm', workspaces: true } +const bunWorkspace = { pm: 'bun', workspaces: true } +const yarnWorkspace = { pm: 'yarn', workspaces: true } describe('FIXTURES', () => { it('pins every repository to a full commit', () => { @@ -48,6 +54,7 @@ describe('TOOLS', () => { '--json', ]) expect(tool('taze').args(pnpmWorkspace)).toContain('--recursive') + expect(tool('syncpack').args(pnpmWorkspace)).toEqual(['update', '--check']) expect(tool('npm-check').args(single)).toEqual(['--skip-unused']) for (const { args } of TOOLS) { for (const flag of ['-u', '--upgrade', '-w', '--write', '--install', '--update']) { @@ -68,6 +75,15 @@ describe('TOOLS', () => { ]) expect(pm.name(pnpmWorkspace)).toBe('pnpm outdated') expect(pm.args(pnpmWorkspace)).toEqual(['outdated', '--recursive', '--format', 'json']) + expect(pm.name(bunWorkspace)).toBe('bun outdated') + expect(pm.args(bunWorkspace)).toEqual(['outdated', '--recursive']) + expect(pm.args({ pm: 'bun', workspaces: false })).toEqual(['outdated']) + expect(pm.unsupported(bunWorkspace)).toBeNull() + }) + + it('has no package-manager row for Yarn 2+, whose only update list is its picker', () => { + expect(tool('pm').name(yarnWorkspace)).toBe('yarn upgrade-interactive') + expect(tool('pm').unsupported(yarnWorkspace)).toMatch(/interactive picker/) }) it('skips npm-check on workspaces only', () => { @@ -79,43 +95,227 @@ describe('TOOLS', () => { const failing = TOOLS.filter( (candidate: { failsWhenOutdated?: boolean }) => candidate.failsWhenOutdated ) - expect(failing.map((candidate: { id: string }) => candidate.id)).toEqual(['npm-check', 'pm']) + expect(failing.map((candidate: { id: string }) => candidate.id)).toEqual([ + 'syncpack', + 'npm-check', + 'pm', + ]) }) - it('counts the updates in each JSON report', () => { - expect(tool('inup').count(JSON.stringify({ summary: { outdated: 22 } }))).toBe(22) - expect(tool('ncu').count(JSON.stringify({ a: '^2.0.0', b: '^3.0.0' }))).toBe(2) + const out = (stdout: string) => ({ stdout, stderr: '' }) + + it('counts the updates in each report', () => { + expect(tool('inup').count(out(JSON.stringify({ summary: { outdated: 22 } })))).toBe(22) + expect(tool('ncu').count(out(JSON.stringify({ a: '^2.0.0', b: '^3.0.0' })))).toBe(2) expect( tool('ncu').count( - JSON.stringify({ - 'package.json': { a: '^2.0.0' }, - 'packages/x/package.json': { a: '^2.0.0', b: '1.0.0' }, - }) + out( + JSON.stringify({ + 'package.json': { a: '^2.0.0' }, + 'packages/x/package.json': { a: '^2.0.0', b: '1.0.0' }, + }) + ) ) ).toBe(3) expect( tool('taze').count( - JSON.stringify({ - packages: [ - { resolved: [{ update: true }, { update: false }] }, - { resolved: [{ update: true }] }, - ], - }) + out( + JSON.stringify({ + packages: [ + { resolved: [{ update: true }, { update: false }] }, + { resolved: [{ update: true }] }, + ], + }) + ) ) ).toBe(2) - expect(tool('npm-check').count('table')).toBeNull() + expect(tool('npm-check').count(out('table'))).toBeNull() // npm lists a package once, or once per workspace as an array; pnpm keys by package. - expect(tool('pm').count(JSON.stringify({ a: {}, b: [{}, {}] }))).toBe(3) + expect(tool('pm').count(out(JSON.stringify({ a: {}, b: [{}, {}] })), single)).toBe(3) + }) + + it('counts the lines syncpack writes to stderr', () => { + const stderr = [ + '= Catalog Definitions ==========================================', + ' 1x @babel/parser ^7.29.9 (~1mo) → ^8.0.7 (~14d)', + '= Default Version Group ========================================', + ' 3x react ^18.3.1 (~1.9y) → ^19.2.0 (~3mo)', + '', + ].join('\n') + expect(tool('syncpack').count({ stdout: '', stderr })).toBe(2) + }) + + it("counts the rows of bun's table", () => { + const stdout = [ + 'bun outdated v1.3.5 (1e86cebd)', + '|----------------------------------------------|', + '| Package | Current | Update | Latest | Workspace |', + '|----------------------------------------------|', + '| @types/nodemailer | 8.0.1 | 8.0.2 | 8.0.2 | vendure |', + '|----------------------------------------------|', + '| typescript (dev) | 5.9.3 | 5.9.3 | 6.0.3 | core |', + '|----------------------------------------------|', + ].join('\n') + expect(tool('pm').count(out(stdout), bunWorkspace)).toBe(2) + expect(tool('pm').count(out('bun outdated v1.3.5\n'), bunWorkspace)).toBe(0) }) it('fails loudly on output it cannot read', () => { - expect(() => tool('inup').count('Error: something broke')).toThrow() - expect(() => tool('taze').count('{}')).toThrow() + expect(() => tool('inup').count(out('Error: something broke'))).toThrow() + expect(() => tool('taze').count(out('{}'))).toThrow() + }) +}) + +describe('pickers', () => { + const picker = (id: string) => tool(id).picker + const npmSingle = { pm: 'npm', workspaces: false } + + it('opens each interactive list, every update included', () => { + expect(picker('inup').args(pnpmWorkspace)).toEqual([]) + expect(picker('ncu').args(single)).toEqual(['--interactive']) + expect(picker('taze').args(pnpmWorkspace)).toEqual([ + 'major', + '--interactive', + '--recursive', + '--include-locked', + '--no-github-actions', + '--no-node-version', + ]) + expect(picker('syncpack').args(pnpmWorkspace)).toEqual(['update', '--interactive']) + expect(picker('npm-check').args(single)).toEqual(['--update', '--skip-unused']) + }) + + it("opens the package manager's own picker", () => { + const pm = picker('pm') + expect(pm.name(yarnWorkspace)).toBe('yarn upgrade-interactive') + expect(pm.args(yarnWorkspace)).toEqual(['upgrade-interactive']) + expect(pm.name(pnpmWorkspace)).toBe('pnpm update -i') + expect(pm.args(pnpmWorkspace)).toEqual(['update', '--interactive', '--latest', '--recursive']) + expect(pm.args(bunWorkspace)).toEqual(['update', '--interactive', '--recursive']) + expect(pm.unsupported(npmSingle)).toBe('npm has no interactive update') + expect(pm.unsupported(yarnWorkspace)).toBeNull() + }) + + it('skips pickers that cannot show a whole workspace at once', () => { + expect(picker('ncu').unsupported(pnpmWorkspace)).toBe('asks about one package.json at a time') + expect(picker('ncu').unsupported(single)).toBeNull() + expect(picker('npm-check').unsupported(pnpmWorkspace)).toBe('no workspace support') + }) + + it('runs them sandboxed, in a color terminal, outside CI', () => { + const env = pickerEnv('/box', { + PATH: '/bin', + CI: 'true', + GITHUB_ACTIONS: 'true', + CONTINUOUS_INTEGRATION: '1', + }) + expect(env).toMatchObject({ PATH: '/bin', HOME: '/box', TERM: 'xterm-256color' }) + expect(env).not.toHaveProperty('CI') + expect(env).not.toHaveProperty('GITHUB_ACTIONS') + expect(env).not.toHaveProperty('CONTINUOUS_INTEGRATION') + }) +}) + +describe('summarize', () => { + it('reports what hyperfine reports', () => { + expect(summarize([3, 1, 2])).toEqual({ + median: 2, + mean: 2, + stddev: 1, + min: 1, + max: 3, + times: [3, 1, 2], + }) + expect(summarize([4, 1, 2, 3]).median).toBe(2.5) + expect(summarize([5]).stddev).toBe(0) + }) +}) + +describe.skipIf(process.platform === 'win32')('timePicker', () => { + let box = '' + afterEach(() => rmSync(box, { recursive: true, force: true })) + const start = (script: string, options: { idleMs?: number; limitMs?: number } = {}) => { + box = mkdtempSync(join(tmpdir(), 'inup-bench-picker-')) + return timePicker(['sh', '-c', script], { cwd: box, box, idleMs: 300, ...options }) + } + + it('times a picker until it goes quiet, in a terminal', async () => { + const seconds = await start('[ -t 1 ] && printf ready; sleep 30') + expect(seconds).toBeGreaterThanOrEqual(0) + expect(seconds).toBeLessThan(2) + }) + + it('counts cache writes after the last thing drawn', async () => { + const seconds = await start('printf ready; sleep 0.5; touch "$HOME/fetched"; sleep 30', { + idleMs: 1000, + }) + expect(seconds).toBeGreaterThanOrEqual(0.45) + }) + + it('stops the picker once timed', async () => { + await start('echo $$ > "$HOME/pid"; printf ready; sleep 30') + const pid = Number(readFileSync(join(box, 'pid'), 'utf8')) + expect(() => process.kill(pid, 0)).toThrow() + }) + + it('fails a picker that exits instead of waiting', async () => { + await expect(start('printf oops; exit 3')).rejects.toThrow('exited (3) instead of waiting') + }) + + it('fails a picker that draws nothing', async () => { + await expect(start('sleep 30')).rejects.toThrow('drew nothing') + }) + + it('fails a picker that never goes quiet', async () => { + await expect( + start('while true; do printf x; sleep 0.05; done', { limitMs: 800 }) + ).rejects.toThrow('still busy') + }) +}) + +describe('inspectFixture', () => { + let dir = '' + afterEach(() => rmSync(dir, { recursive: true, force: true })) + + const fixture = (pkg: object, ...files: string[]) => { + dir = mkdtempSync(join(tmpdir(), 'inup-bench-fixture-')) + writeFileSync(join(dir, 'package.json'), JSON.stringify(pkg)) + for (const file of files) writeFileSync(join(dir, file), '') + return inspectFixture(dir) + } + + it('reads the package manager and version from packageManager', () => { + expect(fixture({ packageManager: 'pnpm@12.11.2+sha512.abc' }, 'pnpm-workspace.yaml')).toEqual({ + pm: 'pnpm', + pmVersion: '12.11.2', + workspaces: true, + }) + expect(fixture({ packageManager: 'yarn@4.18.0', workspaces: ['packages/*'] })).toEqual({ + pm: 'yarn', + pmVersion: '4.18.0', + workspaces: true, + }) + }) + + it('falls back to the lockfile, then npm', () => { + expect(fixture({ workspaces: ['packages/*'] }, 'bun.lock')).toEqual({ + pm: 'bun', + pmVersion: null, + workspaces: true, + }) + expect(fixture({}, 'bun.lockb').pm).toBe('bun') + expect(fixture({})).toEqual({ pm: 'npm', pmVersion: null, workspaces: false }) + }) + + it('refuses what it cannot run', () => { + expect(() => fixture({ packageManager: 'yarn@1.22.22' })).toThrow('Yarn 2+') + expect(() => fixture({}, 'yarn.lock')).toThrow('Yarn 2+') + expect(() => fixture({ packageManager: 'deno@2.0.0' })).toThrow('unsupported') }) }) describe('sandbox', () => { - it('points home, the XDG folders and the temp folder into the box', () => { + it('points home, the XDG folders, the temp folder and every known cache into the box', () => { const box = join('/bench', 'box') expect(sandboxEnv(box)).toEqual({ HOME: box, @@ -124,7 +324,12 @@ describe('sandbox', () => { XDG_DATA_HOME: join(box, '.local', 'share'), XDG_STATE_HOME: join(box, '.local', 'state'), TMPDIR: join(box, 'tmp'), + npm_config_cache: join(box, '.npm'), + BUN_INSTALL: join(box, '.bun'), + BUN_INSTALL_CACHE_DIR: join(box, '.bun', 'install', 'cache'), + YARN_GLOBAL_FOLDER: join(box, '.yarn', 'berry'), }) + for (const path of Object.values(sandboxEnv(box))) expect(path.startsWith(box)).toBe(true) }) it('quotes for a POSIX shell', () => { diff --git a/test/unit/scripts/bench-report.test.ts b/test/unit/scripts/bench-report.test.ts index 4034141..9aef673 100644 --- a/test/unit/scripts/bench-report.test.ts +++ b/test/unit/scripts/bench-report.test.ts @@ -4,6 +4,7 @@ import { describeChart, escapeXml, formatSeconds, + joinNames, renderChart, renderDetails, renderReadme, @@ -65,6 +66,15 @@ const express = { cold: { inup: timing(0.389), ncu: timing(0.726), 'npm-check': timing(1.2), pm: timing(0.9) }, warm: { inup: timing(0.38), ncu: timing(0.72), 'npm-check': timing(1.1), pm: timing(0.13) }, }, + picker: { + tools: [ + tool('inup', 'inup', 'inup', null), + tool('ncu', 'npm-check-updates', 'ncu --interactive', null), + ], + skipped: [{ id: 'pm', name: 'npm update -i', reason: 'npm has no interactive update' }], + cold: { inup: timing(0.95), ncu: timing(0.86) }, + warm: { inup: timing(0.5), ncu: timing(0.84) }, + }, } const vue = { @@ -88,10 +98,70 @@ const vue = { cold: { inup: timing(1.68), ncu: timing(18.348), pm: timing(2.1) }, warm: { inup: timing(0.9), ncu: timing(5.6), pm: timing(0.8) }, }, + picker: { + tools: [ + tool('inup', 'inup', 'inup', null), + tool('pm', 'pnpm update -i', 'pnpm update --interactive --latest --recursive', null), + ], + skipped: [ + { id: 'ncu', name: 'npm-check-updates', reason: 'asks about one package.json at a time' }, + ], + cold: { inup: timing(1.81), pm: timing(2.41) }, + warm: { inup: timing(0.62), pm: timing(0.7) }, + }, } const results = [express, vue] +/** The tables in rendered Markdown, in order. */ +const tablesIn = (text: string) => text.split('\n\n').filter((block) => block.startsWith('| ')) + +describe('joinNames', () => { + it('shares a common last word', () => { + expect(joinNames(['npm outdated', 'pnpm outdated', 'bun outdated'])).toBe( + 'npm / pnpm / bun outdated' + ) + expect(joinNames(['inup'])).toBe('inup') + expect(joinNames(['npm outdated', 'yarn upgrade-interactive'])).toBe( + 'npm outdated / yarn upgrade-interactive' + ) + expect(joinNames(['pnpm update -i', 'bun update -i', 'yarn upgrade-interactive'])).toBe( + 'pnpm / bun update -i / yarn upgrade-interactive' + ) + }) +}) + +describe('a repository where a tool did not run', () => { + const yarn = { + ...vue, + fixture: { ...vue.fixture, id: 'backstage', repo: 'backstage/backstage' }, + tools: vue.tools.filter((t) => t.id !== 'pm'), + skipped: [ + ...vue.skipped, + { + id: 'pm', + name: 'yarn upgrade-interactive', + reason: 'Yarn 2+ lists updates only in this interactive picker', + }, + ], + results: { + cold: { inup: timing(6), ncu: timing(21) }, + warm: { inup: timing(1), ncu: timing(20) }, + }, + } + + it('keeps the row named for where it ran and shows a dash', () => { + const [first] = tablesIn(renderTables([express, vue, yarn])) + expect(first).toContain('| npm / pnpm outdated | 0.90 s | 2.10 s | — |') + }) + + it('says why on the details page', () => { + expect(renderDetails([yarn])).toContain( + '| yarn upgrade-interactive | — | not run: Yarn 2+ lists updates only in this interactive picker | | |' + ) + }) +}) + describe('formatSeconds', () => { it('keeps two decimals under ten seconds and one above', () => { expect(formatSeconds(0.389)).toBe('0.39 s') @@ -113,7 +183,7 @@ describe('sortResults', () => { describe('renderTables', () => { const tables = renderTables(results) - const [first, repeat] = tables.split('\n\n') + const [first, repeat, pickerFirst, pickerRepeat] = tablesIn(tables) it('heads each column with the pinned repository and its size', () => { expect(first).toContain( @@ -129,9 +199,29 @@ describe('renderTables', () => { expect(first).toContain('| inup | **0.39 s** | **1.68 s** |') expect(first).toContain('| npm-check-updates | 0.73 s | 18.3 s |') expect(first).toContain('| npm-check | 1.20 s | — |') - expect(first).toContain('| npm outdated / pnpm outdated | 0.90 s | 2.10 s |') + expect(first).toContain('| npm / pnpm outdated | 0.90 s | 2.10 s |') expect(repeat).toContain('| inup | 0.38 s | 0.90 s |') - expect(repeat).toContain('| npm outdated / pnpm outdated | **0.13 s** | **0.80 s** |') + expect(repeat).toContain('| npm / pnpm outdated | **0.13 s** | **0.80 s** |') + }) + + it('times the pickers in tables of their own', () => { + expect(tables).toContain('### Listing outdated dependencies') + expect(tables).toContain('### Opening the picker') + expect(pickerFirst.split('\n')[0]).toMatch(/^\| First run \| \[expressjs\/express\]/) + expect(pickerFirst).toContain('| inup | 0.95 s | **1.81 s** |') + expect(pickerFirst).toContain('| npm-check-updates | **0.86 s** | — |') + expect(pickerFirst).toContain('| pnpm update -i | — | 2.41 s |') + expect(pickerRepeat).toContain('| inup | **0.50 s** | **0.62 s** |') + expect(tables).toContain('Pickers run in a 160×50 terminal') + expect(tables).toContain('downloads its native core') + }) + + it('leaves the pickers out when a result has none', () => { + const { picker: _, ...listingOnly } = express + const tables = renderTables([listingOnly]) + expect(tablesIn(tables)).toHaveLength(2) + expect(tables).not.toContain('picker') + expect(tables).not.toContain('Picker') }) it('says how, where and when it was measured', () => { @@ -189,7 +279,20 @@ describe('renderDetails', () => { ) expect(details).toContain('| npm-check | 1.0.0 | 1.20 s ± 0.01
1.18–1.23 |') expect(details).toContain('| — |\n') - expect(details).toContain('| npm-check | — | not run: no workspace support | | |') + expect(details).toContain('| npm-check | — | not run: no workspace support | | |') + }) + + it('lists the pickers with their commands', () => { + expect(details).toContain('#### Opening the picker') + expect(details).toContain( + '| pnpm update -i | 1.0.0 | 2.41 s ± 0.01
2.39–2.44 | 0.70 s ± 0.01
0.68–0.73 |' + ) + expect(details).toContain( + '| npm-check-updates | — | not run: asks about one package.json at a time | |' + ) + expect(details).toContain( + 'Commands, run in the repository root in a 160×50 terminal:\n\n```sh\ninup\npnpm update --interactive --latest --recursive\n```' + ) }) it('shows the exact commands and the machine', () => {