From 991215afcc08735bf50e25cef36b96a7169e7e0c Mon Sep 17 00:00:00 2001 From: allan Date: Mon, 21 Sep 2026 14:45:18 +0200 Subject: [PATCH 1/2] chore(roadmap): v1.20.0 with server metrics, GUI/CLI enhancements, and advanced reporting --- .github/roadmap.json | 3985 +++++++++++++++++++++--------------------- 1 file changed, 2029 insertions(+), 1956 deletions(-) diff --git a/.github/roadmap.json b/.github/roadmap.json index aae9cad..35e7027 100644 --- a/.github/roadmap.json +++ b/.github/roadmap.json @@ -1,1958 +1,2031 @@ { - "_comment": "Roadmap source of truth. `scripts/sync-roadmap.sh` turns this file into GitHub milestones, labels and issues, and the roadmap-sync workflow replays it on every push to main. The sync is idempotent and additive: it creates what is missing and re-aligns milestone and labels, it never closes an issue and never rewrites a body somebody edited by hand. Each issue is matched by its `key`, not by its title, so a title can be reworded here without opening a duplicate. Consequence worth remembering: editing a `body` here does NOT update an issue that already exists — put the change in a comment on the issue as well.", - "labels": [ - { - "name": "area:cli", - "color": "1d76db", - "description": "bin/crowdsim, the bash driver" - }, - { - "name": "area:generator", - "color": "0e8a16", - "description": "k6/live-event.js and k6/lib" - }, - { - "name": "area:gui", - "color": "5319e7", - "description": "gui/server and gui/ui" - }, - { - "name": "area:tests", - "color": "fbca04", - "description": "unit, CLI, GUI and e2e suites" - }, - { - "name": "area:ci", - "color": "c2e0c6", - "description": "CI, image builds, release automation" - }, - { - "name": "area:cache-ab", - "color": "bfd4f2", - "description": "the two-leg reverse-proxy A/B harness" - }, - { - "name": "area:release", - "color": "d4c5f9", - "description": "versioning, tags, CHANGELOG" - }, - { - "name": "safety", - "color": "b60205", - "description": "touches a safety gate — review with care" - }, - { - "name": "validity", - "color": "e99695", - "description": "affects whether a run's numbers mean anything" - } - ], - "milestones": [ - { - "title": "v1.1.0", - "description": "SHIPPED in release 1.1.0. Modular generator (k6/lib) and a test suite that generates no load: unit tests over the mix, cache classification and summary verdicts; bats tests over the safety gates and the exit-code contract." - }, - { - "title": "v1.2.0", - "description": "SHIPPED EARLY, inside release 1.1.0. A browser front-end for the driver: profile editor with live validation, run launcher, live log, run archive. The GUI is a client of bin/crowdsim and holds no second copy of the safety gates. Careful with the numbering: this milestone's scope and the 1.2.0 RELEASE are different things — release 1.2.0 ships the single container image (driver + generator + GUI) on GHCR. The next planned milestone is v1.3.0." - }, - { - "title": "v1.3.0", - "description": "Fewer ways to get a wrong answer, and no manual steps left in the loop. Prove the brake aborts a run end to end, warn before a run the generator cannot sustain, stop using URL pools nobody verified, turn a browser recording into a journey, and collapse the two profile validators into one — plus the GUI refinements that keep a click from being a decision made blind." - }, - { - "title": "v1.4.0", - "description": "Two answers that exist only in a terminal, and one number that is still a guess. `compare` refuses to compare runs that were not the same experiment — that judgement belongs in the page that shows the archive, not only in the CLI. And `safety.generator_mbps` is declared by hand: the estimate that warns you before a run comes back generator_ok:false is only as good as a number somebody typed once and copied between profiles, so measure it instead." - }, - { - "title": "v1.5.0", - "description": "The GUI, audited. Its one job is to let somebody watch load being generated and read the result honestly, and the audit found four ways it fails at that quietly: it goes silent when it loses the stream, it dies with a stack trace on a busy port, it spends more time rendering the log than the run spends producing it, and half of it cannot be reached without a mouse. Every item here came with evidence — a killed server, a measured 760 MB of string churn, a DOM with no focusable row — not from a wish list." - }, - { - "title": "v1.6.0", - "description": "Test-first for the front end, which currently has no tests at all: 1415 lines of it, nothing importing a component, no runner, and `make test` not touching it. Every other part of this repo starts from a failing test — the driver, the generator, the API, the manifests — and the rule 'every fixed bug starts with a test that reproduces it' has been honoured everywhere except the one place where there is nowhere to put such a test. It shows: the UI bugs found in 1.7.0 and 1.10.0 were all caught by screenshotting the page and reading the DOM, one of them a reload that silently discarded a finished run. This milestone builds the place, moves the decisions somewhere they can be tested, and writes the rule down." - }, - { - "title": "v1.7.0", - "description": "Four features, one theme: the tool knows things it does not yet hand over. It measures a page, verifies a pool and records a fan-out, then asks you to write the profile from scratch. It judges an HTML document and a static asset against one threshold. It reports a cold cache's numbers as the system's. And it produces a result that somebody retypes into an incident document, losing every caveat on the way. Each item closes one of those gaps without loosening anything: no default allowlist, no weaker brake, no invented number." - }, - { - "title": "v1.8.0", - "description": "The gap between what is tested and what is delivered. The image pins k6 0.52.0 while every test in this repository has been run against 2.1.0 — two majors apart, and the e2e suite installs whatever is newest on the runner, so the generator that ships is never the generator that was proven. The documentation still tells people to pull :1.2.0, eleven releases old, and calls it the exact version to use. Both were found by looking rather than by anything failing, which is the point: nothing in CI is watching the distance between the artefact and the evidence for it." - }, - { - "title": "v1.9.0", - "description": "Two numbers the tool still averages away. Every result it produces is one p95 for a whole ramp — 30 req/s and 120 req/s inside the same figure — so the knee, the thing this tool is named after, can only be found by running four times and comparing. And the one input it insists must be measured, the class mix, is the one it refuses to help measure: the documentation says on every page that weights come from your own edge log, and `init` writes them as a TODO. This milestone makes a single run show where the knee is, and turns the mix from a guess with a decimal point into a measurement. Nothing here loosens a gate, invents a number, or asks for access to anything privileged." - }, - { - "title": "v1.10.0", - "description": "Everything this tool knows how to do, it does well the second time. The first time you have to already know it: `crowdsim --help` opens with the line `!/usr/bin/env bash`, `crowdsim load --help` answers with the same sixty lines as every other subcommand, nothing completes a subcommand or a flag in a shell, `report` and `compare` want a 16-character UTC timestamp copied out of another command's output, `history` prints fourteen columns for every run ever recorded and takes no arguments at all, and the path to a first profile is six commands with hand-editing in between and nothing that says which one you are on. None of that is a missing feature — it is the cost of using the features that exist, paid on every run. Every item here was found by running the commands, not by reading the code, and none of them touches a gate: no default allowlist, no interactive prompt, no brake that is easier to get past." - }, - { - "title": "v1.11.0", - "description": "A result somebody else can read. Every number this tool produces is a curve — rate against latency, per step — and the only place that curve was ever drawn is the GUI: the moment a result left the page it went back to being a table of eight rows, and the reader was asked to draw the ramp in their head. One milestone, one feature, because the feature is small and the rules around it are not: a chart is the most persuasive thing this tool can produce, so the interesting half of the work is what it refuses to draw." - }, - { - "title": "v1.12.0", - "description": "The four things a real campaign needed and this tool could not say. Measured against a real campaign of 2026-09 — 7.000+ concurrent users required, the web tier clean past 8.800, the ceiling found in the login at ~150 login/s — which was run with the other generator because crowdsim cannot express half of that sentence. It answers in requests per second and the requirement is in concurrent users; it splits one peak by weight and the finding is a rate for ONE class; its think time is hard-coded where the whole method rests on measured reading pauses; and it creates real accounts it keeps no record of. None of these is a new kind of tool: each one is this tool saying what it already generates in the unit somebody asked the question in." - }, - { - "title": "Backlog", - "description": "Accepted but unscheduled. Nothing here blocks a release." - }, - { - "title": "v1.14.0" - }, - { - "title": "v1.15.0" - }, - { - "title": "v1.16.0" - }, - { - "title": "v1.17.0" - }, - { - "title": "v1.18.0" - }, - { - "title": "v1.19.0" - } - ], - "issues": [ - { - "key": "release-tag-1-0-0", - "title": "Tag the 1.0.0 release commit", - "milestone": "v1.1.0", - "labels": [ - "area:release", - "documentation" - ], - "body": [ - "The repository has no tags. `CHANGELOG.md` documents a 1.0.0 release, but the commit that shipped it was never tagged, so `git describe`, the GitHub release list and any consumer pinning a version all see an unreleased repository.", - "", - "**Done when**", - "- An annotated tag `v1.0.0` points at the initial release commit.", - "- Every later release keeps the one-commit-one-tag rule the project already follows." - ] - }, - { - "key": "e2e-suite", - "title": "tests/e2e/run.sh: the one suite allowed to generate load", - "milestone": "v1.1.0", - "labels": [ - "area:tests", - "area:cli" - ], - "body": [ - "`make test-e2e` is documented as the only suite that starts a real generator, and only against a container on the loopback address. The target exists; `tests/e2e/run.sh` does not, so the target fails.", - "", - "The value of this suite is that it is the only place where a real k6 process, a real profile and a real HTTP origin meet — it is what catches an argument the wrapper composes but k6 rejects.", - "", - "**Done when**", - "- `tests/e2e/run.sh` brings up a local origin, runs a short `load` against it, and asserts on the resulting `out/summary-.json`.", - "- The run is discarded, loudly, when `generator_ok` is false: a generator-bound run proves nothing.", - "- The profile it uses allowlists loopback only, so the suite cannot be pointed at somebody's infrastructure by editing one variable.", - "- It skips with a clear message, rather than failing, when docker or k6 is missing." - ] - }, - { - "key": "ci-lint-test", - "title": "CI: run make lint and make test on push and pull request", - "milestone": "v1.3.0", - "labels": [ - "area:ci", - "area:tests" - ], - "body": [ - "The unit, CLI and GUI suites run entirely against fakes — a stub k6 on PATH, a temporary profile, a loopback HTTP server — which is exactly what makes them safe to run on a shared CI runner. Nothing runs them today, so the suite only protects whoever remembers to type `make test`.", - "", - "**Done when**", - "- A workflow runs `make lint` and `make test` on push to main and on pull requests.", - "- `make test-e2e` is **not** part of it: it generates load and needs docker plus k6.", - "- The job fails on a `bash -n` or `node --check` error, so a syntax break in the driver cannot land.", - "- It passes on a clean checkout after `npm install` alone: no globally installed bats, no globally installed k6." - ] - }, - { - "key": "bats-invocation-mismatch", - "title": "Align the bats invocation between Makefile and package.json", - "milestone": "v1.1.0", - "labels": [ - "area:tests" - ], - "body": [ - "`make test-cli` runs `npx bats tests/cli` while `npm run test:cli` runs `bats tests/cli`. The second only works if bats is also installed globally, so the two entry points disagree on a machine that has only the devDependency.", - "", - "**Done when**", - "- Both entry points resolve bats the same way, and a clean clone passes after `npm install` alone." - ] - }, - { - "key": "document-test-layout", - "title": "README: document the test layout and which suite generates load", - "milestone": "v1.1.0", - "labels": [ - "documentation", - "area:tests" - ], - "body": [ - "The split between suites is a safety property, not a convention: `make test` must stay runnable on any machine at any time because it touches nothing outside the checkout, and `make test-e2e` must stay separate because it does not.", - "", - "**Done when**", - "- README explains the four suites, what each covers, and states plainly that only the e2e suite generates traffic.", - "- The `bin/crowdsim` usage header and `AGENTS.md` agree with it." - ] - }, - { - "key": "gui-server-profiles", - "title": "gui/server: profile browsing API", - "milestone": "v1.2.0", - "labels": [ - "area:gui", - "safety" - ], - "body": [ - "`package.json` declares the `gui/server` workspace and `make gui` serves it, but the directory is empty.", - "", - "First slice: read-only. List the profiles the server is allowed to see, return one for inspection, and report whether it parses and carries an allowlist.", - "", - "**Done when**", - "- The server resolves profiles under one configured root and rejects any path that escapes it — a profile path is user input and a profile is a map of somebody's infrastructure.", - "- A profile with no `safety.allow_hosts` is reported as such rather than silently listed as usable.", - "- The server binds the loopback interface by default.", - "- Profile contents are never logged." - ] - }, - { - "key": "gui-server-run", - "title": "gui/server: launch a run without weakening the safety gates", - "milestone": "v1.2.0", - "labels": [ - "area:gui", - "safety" - ], - "body": [ - "The GUI must launch runs by invoking `bin/crowdsim`, not by re-implementing argument composition. The driver is where the two gates live, and a second code path that builds k6 arguments is a second place for them to be wrong.", - "", - "**Done when**", - "- A run is started as a child process of `bin/crowdsim`; the GUI never calls k6 directly.", - "- The allowlist gate keeps working end to end: a target outside it fails the run with the driver's own exit code 3, surfaced to the browser.", - "- Only known flags with validated values are composed: no free-form argument passthrough, and no shell.", - "- `--i-know-this-breaks-production` is never inferred and never remembered. It is sent only when, for that single run, the operator ticks the box AND types the profile name by hand; the confirmation is not stored, not defaulted, and not part of any saved configuration. (Decided 2026-08-05, superseding the original 'never sent by the server': somebody who needs the knee will run it either way, and pushing them to hand-assemble a command line buys a copy-paste window, not consent.)", - "- The driver's exit codes are propagated verbatim: a tripped brake is an outcome (0), not an error." - ] - }, - { - "key": "gui-server-stream", - "title": "gui/server: follow a running load test", - "milestone": "v1.2.0", - "labels": [ - "area:gui" - ], - "body": [ - "Reading a run means watching it: the knee is visible while it happens, and a summary read afterwards tells you the run tripped the brake without telling you where.", - "", - "**Done when**", - "- The server streams the run log to the browser as it is written.", - "- When the run ends, the summary is delivered and `generator_ok` is displayed prominently — a false value invalidates the run and the UI must say so, not bury it.", - "- Closing the browser does not kill the run, and reopening it reattaches." - ] - }, - { - "key": "gui-ui", - "title": "gui/ui: run form and history view", - "milestone": "v1.2.0", - "labels": [ - "area:gui" - ], - "body": [ - "The `gui/ui` workspace is declared and `make gui-dev` / `make gui-build` target it, but it does not exist.", - "", - "**Done when**", - "- A form picks a profile, a target and a shape, and shows what the requested peak means per class before it runs — no hidden defaults.", - "- The history view reads `out/history.tsv` and marks runs with `generator_ok: false` as unusable rather than plotting them next to valid ones.", - "- The UI states that two runs are only comparable when they used the same URL pool." - ] - }, - { - "key": "tests-gui", - "title": "tests/gui: API tests for profiles, run launching and gate propagation", - "milestone": "v1.2.0", - "labels": [ - "area:tests", - "area:gui", - "safety" - ], - "body": [ - "`make test-gui` and `npm run test:gui` both point at `tests/gui/`, which does not exist yet.", - "", - "**Done when**", - "- Path traversal against the profile endpoints is covered by a test, not by a code review.", - "- A test asserts that a target outside the allowlist is refused, and that the safe-peak override is impossible without the per-run confirmation.", - "- The suite runs against a stub `crowdsim` on PATH and a loopback server, so it generates no load and needs no k6." - ] - }, - { - "key": "gui-docs", - "title": "Document the GUI: what it is, what it deliberately cannot do", - "milestone": "v1.2.0", - "labels": [ - "documentation", - "area:gui" - ], - "body": [ - "**Done when**", - "- README covers `crowdsim serve` and the build steps, and states that the server binds loopback by default and refuses any other bind address without `CROWDSIM_GUI_TOKEN` — a local console first, a shared dashboard only deliberately.", - "- It says what the GUI cannot do: no scheduler, no user accounts, no second copy of the gates.", - "- It says how the safe-peak override works from the GUI (the box ticked plus the profile name typed, per run) and why it is not a setting." - ] - }, - { - "key": "e2e-brake-trips", - "title": "e2e: prove the brake actually aborts a run", - "milestone": "v1.3.0", - "labels": [ - "area:tests", - "validity", - "safety" - ], - "body": [ - "The emergency brake is what makes it defensible to point this tool at anything. Today `brakeTripped()` is unit-tested against synthetic metric trees, and the e2e suite asserts the *opposite* case — that a static nginx does **not** trip it. Nothing proves that a real k6 run against a real slow target aborts, that the summary says `aborted: true`, and that the wrapper still exits 0 and still writes its history row.", - "", - "The gap matters more than it looks: the brake is a k6 threshold with `abortOnFail` and `delayAbortEval`. A typo in a threshold expression, or a metric renamed by a k6 upgrade, silently produces a run that no longer stops — and the first time anybody notices is during the outage the brake existed to cut short.", - "", - "**Done when**", - "- The e2e suite adds a leg whose target is deliberately slow enough to cross the SLO (a delay in the origin, on loopback, at a rate that hurts nothing).", - "- It asserts `aborted: true`, a non-zero `over_guillotine_rate`, exit code 0 from `crowdsim load`, and a history row for the aborted run.", - "- It asserts the run stopped EARLY: wall-clock well below the full ramp plus hold, so a brake that never fires cannot pass by simply finishing.", - "- `target_unreachable` is false: a target that refuses connections trips nothing and must not be mistaken for a knee." - ] - }, - { - "key": "e2e-skip-missing-deps", - "title": "e2e: skip cleanly when docker or k6 is missing", - "milestone": "v1.3.0", - "labels": [ - "area:tests" - ], - "body": [ - "`tests/e2e/run.sh` exits 1 when docker or k6 is absent. That is indistinguishable from a real failure, which trains people to ignore a red e2e run — and the suite is legitimately skipped on most machines.", - "", - "**Done when**", - "- A missing prerequisite prints what is missing and exits 0 with a clear SKIPPED line, distinct from a failure.", - "- An assertion that actually fails still exits non-zero.", - "- `make test-e2e` stays excluded from `make test`: this must not become a reason to fold it in." - ] - }, - { - "key": "doctor-bandwidth-check", - "title": "doctor/load: warn before a run the generator cannot sustain", - "milestone": "v1.3.0", - "labels": [ - "area:cli", - "validity" - ], - "body": [ - "`generator_ok: false` is diagnosed after the fact — after the window was agreed, the alerting silenced and the run burned. Most of those runs were predictable beforehand: a page weight of ~45 KB at 380 req/s is ~17 MB/s sustained, which a generator behind a slow link cannot deliver no matter how many VUs k6 provisions.", - "", - "`probe` already measures the weight of one page. The arithmetic is available before the run, and nothing does it.", - "", - "**Done when**", - "- `probe` records the observed page weight in a machine-readable form, not only in prose.", - "- `load` estimates the sustained throughput the requested peak implies and warns, loudly, when it exceeds what was measured as available; the estimate and its assumptions are printed, not hidden.", - "- `doctor --profile ` reports the same estimate for the profile's peak, so it can be checked without generating anything.", - "- It is a warning, not a gate: the number is an estimate, and a wrong estimate must never stop a run somebody needs. The one thing it must not do is stay silent." - ] - }, - { - "key": "discover-verify", - "title": "discover --verify: drop the URLs that do not render", - "milestone": "v1.3.0", - "labels": [ - "area:cli", - "validity" - ], - "body": [ - "`discover` writes a pool from the sitemap and the README then tells you to verify the URLs by hand before using them. Nobody verifies 400 URLs by hand. The consequence is specific and quiet: a 404 is cheap for the app tier and a 307 measures a redirect, so a pool full of either yields a flattering capacity number for a load that never reached the renderer.", - "", - "**Done when**", - "- `discover --verify` checks the discovered paths, keeps those answering 2xx, and reports how many were dropped and why (404, redirect, timeout) instead of silently shrinking the pool.", - "- Verification is sequential and rate-limited: building a pool must not itself be a load test.", - "- It goes through the same allowlist gate as everything else that talks to the target.", - "- The written pool records when it was verified — a pool contains build hashes and goes stale at the next deploy." - ] - }, - { - "key": "record-journey-from-har", - "title": "crowdsim record : build a journey from a browser recording", - "milestone": "v1.3.0", - "labels": [ - "area:cli", - "area:generator", - "enhancement" - ], - "body": [ - "`--shape journey` is the mode that reproduces a visitor session — one document plus the fan-out the browser actually performs — and it needs a `journey.json` of `{path, rsc[], static[]}` per page. The README says to record it with a real browser, and offers no way to turn that recording into the file. In practice the mode goes unused and the mix shape carries load nobody clicks.", - "", - "A HAR export is exactly the recording needed, and every browser produces one.", - "", - "**Done when**", - "- `crowdsim record ` writes a journey file: documents become pages, each carrying the navigation requests and static assets the browser fetched with it.", - "- Third-party hosts are dropped: analytics and fonts are not your capacity problem, and generating them would point load at someone else's infrastructure.", - "- Per-request cache-busting query parameters are normalised, so the recording does not become a pool of unique cold URLs.", - "- The output records the origin it came from, and `record` never writes into `profiles/` by accident — a journey is data about a site, like a pool." - ] - }, - { - "key": "validate-subcommand", - "title": "One profile validator, reachable from the CLI", - "milestone": "v1.3.0", - "labels": [ - "enhancement", - "area:cli", - "validity" - ], - "body": [ - "Profile validation exists, and only in JavaScript: `gui/server/lib/validate.js` catches an empty pool, an allowlist of `*`, a brake class that is not in the mix, a read timeout below the p95 SLO. The CLI cannot use any of it — so the operator at a terminal, the primary user of this tool, still learns about a broken profile from a k6 stack trace in the init context, after deciding to generate load.", - "", - "Two validators would be worse than one late validator. This has to end with a single implementation.", - "", - "**Done when**", - "- `crowdsim validate ` reports every problem at once, not the first, and exits 2 on a malformed profile without generating traffic.", - "- `doctor` and `load` reuse the same checks, so validation cannot drift from what a run actually requires.", - "- There is exactly one implementation of the rules — either the CLI calls it (node when available, with a documented fallback) or the JS side calls the CLI — and the choice is written down with its cost.", - "- The rules are tested once, and both entry points are asserted to agree." - ] - }, - { - "key": "compare-subcommand", - "title": "crowdsim compare : a delta between two runs", - "milestone": "v1.3.0", - "labels": [ - "enhancement", - "area:cli", - "validity" - ], - "body": [ - "The tool measures deltas honestly and absolutes optimistically, so comparison is the operation that carries meaning. Today it is done by reading two summary files side by side.", - "", - "**Done when**", - "- `crowdsim compare` prints per-class latency, error and cache deltas between two runs from `out/`.", - "- It refuses to compare, rather than quietly comparing, when either run has `generator_ok: false`, or when the two runs did not use the same URL pool — a pool identity is recorded with the run so this can be checked instead of assumed.", - "- The GUI's comparison view uses the same rules, and says the same thing when they are not met." - ] - }, - { - "key": "gui-command-preview", - "title": "GUI: show the resolved command before it runs", - "milestone": "v1.3.0", - "labels": [ - "area:gui", - "safety" - ], - "body": [ - "The GUI composes a `crowdsim load` command line and shows it only afterwards, in the run record. Before the click the operator sees form fields, which means the defaults the form fills in are invisible at the moment the decision is made. For a tool that generates real traffic, the command should be readable before it is run, not after.", - "", - "**Done when**", - "- The exact argv is rendered live as the form changes, in a form that can be copied into a terminal and produce the same run.", - "- It is visibly the same string the server will execute, not a reconstruction assembled separately in the UI.", - "- The safe-peak override appears in that preview when it is armed, so nobody launches it without having seen it." - ] - }, - { - "key": "gui-runs-survive-restart", - "title": "GUI: a restart must not lose a run that is still going", - "milestone": "v1.3.0", - "labels": [ - "area:gui" - ], - "body": [ - "The run list lives in the server's memory. Restart the server — a crash, a rebuild, a closed laptop — and the page shows no runs while the k6 process it started keeps generating load. The archive in `out/` survives; the knowledge that something is in flight does not.", - "", - "**Done when**", - "- A run in progress is discoverable after a server restart, and the page says a generator is running rather than showing an empty list.", - "- Reattaching gives back the log from the driver's own run log file — the GUI still stores no results of its own.", - "- Stop still works after a restart, or says plainly that it cannot and how to stop it by hand.", - "- The one-run-at-a-time rule survives too: a restart must not make it possible to start a second generator." - ] - }, - { - "key": "gui-probe-tables", - "title": "GUI: read probe and discover output as data, not as a log", - "milestone": "v1.3.0", - "labels": [ - "area:gui" - ], - "body": [ - "`probe` answers the question that decides whether a run is worth doing — which layer answered, what it said about caching, how heavy the page is — and the GUI shows it as raw terminal output. `discover` reports what it found and dropped, also as text.", - "", - "**Done when**", - "- The probe result is shown per layer: the header seen, whether it counts as a hit under the profile's pattern, and the page weight.", - "- A declared cache header that never appeared is called out as such, since that is usually a wrong header name in the profile rather than a cold cache.", - "- The discover result shows the pool it would write, with the counts kept and dropped, before a run uses it." - ] - }, - { - "key": "per-step-metrics", - "title": "One p95 for a whole ramp is a number for a rate that never happened", - "milestone": "v1.9.0", - "labels": [ - "area:generator", - "validity" - ], - "body": [ - "A run climbs from `--start` to `--peak` in `--steps` steps and then holds. The summary reports one p50/p95/p99 over all of it — so the figure people quote describes a mixture of rates, most of them below the one they think they measured. A run that ramps 30 → 120 reports a p95 that is neither the p95 at 30 nor the p95 at 120, and it is the *only* latency number the tool hands over.", - "", - "It is also why finding the knee takes four runs and a `compare` between each: the shape of the curve is already inside a single run, and gets averaged out before anybody sees it.", - "", - "**Done when**", - "- Every request is tagged with the ramp step it belongs to, and the summary carries a per-step block: requested rate, achieved rate, p50/p95/p99, failed rate, share past `guillotine_ms`, and the same per class.", - "- The panel prints it as a table — rate against p95 — because that table *is* the knee.", - "- The step boundaries come from the same `stages()` the ramp is built from, not from a second calculation that can drift. Computed twice, they will disagree once.", - "- A run stopped by the brake reports the steps it completed and marks the last one partial. A partial step is not a measurement of its rate.", - "- `--hold` is its own step, labelled as the one at peak: it is the only part of the run where the requested rate was sustained rather than passed through.", - "- Unit tests in `k6/lib/`: attribution at the boundaries, `--steps 1`, a run aborted mid-step, and the invariant that the aggregate numbers do not change." - ] - }, - { - "key": "knee-estimate", - "title": "Name the knee: the highest rate that stayed inside the SLO, and the one that did not", - "milestone": "v1.9.0", - "labels": [ - "area:cli", - "validity" - ], - "body": [ - "With per-step numbers the answer everybody actually came for is computable: *clean up to 90 req/s, crossed at 120*. Today the tool reports \"completed\" or \"aborted\" and the reader converts that into a capacity figure by hand — usually by rounding up to `--peak`, which is the one rate nobody measured the system surviving.", - "", - "**Done when**", - "- The summary carries a knee: the highest step whose p95 and failed rate stayed inside the SLO, and the first step that crossed, with both rates named.", - "- It **refuses** instead of estimating when the run cannot support one: fewer than two completed steps, a step shorter than the abort delay, `generator_ok: false`, an unreachable target. A knee from one step is a straight line through one point.", - "- The refusal names the condition and what to change (`--steps`, `--step-dur`), in the same voice as `compare`'s refusals.", - "- `report` states it as the headline claim, with the caveat that it is a knee at *this* URL pool.", - "- `history.tsv` and the GUI's knee plot carry it, so the plot stops being one dot per run and becomes a curve." - ] - }, - { - "key": "weights-from-access-log", - "title": "The mix is the one input the tool demands and refuses to help measure", - "milestone": "v1.9.0", - "labels": [ - "area:cli", - "validity" - ], - "body": [ - "Every page of the documentation says the class weights must come from your own edge access log, and `crowdsim init` writes them as a `TODO` for exactly that reason. Nothing in the tool reads a log. So the single most important input — the one that decides whether a run reproduces your traffic or a shape nobody has ever served — is left to somebody counting lines in a terminal, which in practice means the weights are a guess with a decimal point.", - "", - "The tool must not go and fetch the log itself: that would mean privileged access to production edges, which this project deliberately does not want. The log arrives on **stdin, or as a file the operator hands over**.", - "", - "**Done when**", - "- `crowdsim weights ` (or stdin) classifies each request line with the profile's own class rules and prints the mix — count, share, and the weight to paste — plus what it could not classify, which is the interesting part.", - "- Unclassified lines are reported as a share, never silently dropped: a mix computed from 40% of the log is a mix of something else.", - "- It reads the combined format and a `--format` field list, and says what it could not parse rather than guessing a column.", - "- It never writes into a profile by itself: it prints the `classes` block, and `init` uses it when a log is available.", - "- Its own output states that this is your traffic during *that* window — a mix measured in a quiet hour does not reproduce a spike.", - "- Nothing from the log reaches `out/`. An access log holds URLs, addresses and user agents, and this repository is not where any of that goes." - ] - }, - { - "key": "gui-warmup-and-report", - "title": "The GUI cannot express two flags the CLI has, and cannot hand over a result", - "milestone": "v1.9.0", - "labels": [ - "area:gui" - ], - "body": [ - "`--warmup` and `--warmup-peak` exist on the command line and not in `gui/server/lib/args.js`, so every run launched from the page folds its own cold start into the numbers — and the page is where the people least likely to know that are launching runs. Meanwhile `crowdsim report` writes exactly the artefact somebody reading a finished run wants next, and the GUI has no way to produce it.", - "", - "**Done when**", - "- The form offers a warm-up (duration and rate), `args.js` validates both as known flags with validated values, and the command preview shows them — the preview is the contract.", - "- The result view offers the run's report as a file, produced by spawning `crowdsim report` like every other action. The GUI still re-implements nothing.", - "- The safe-peak gate applies to the warm-up rate exactly as to the peak, typed-profile-name override included. A warm-up is load.", - "- Tests: `tests/gui/` for the argv and the refusals, `tests/ui/` for the form's decisions, and the report download in the e2e page pass." - ] - }, - { - "key": "docs-quoted-output", - "title": "The docs quote the tool's own output, and nothing checks the quotes", - "milestone": "v1.9.0", - "labels": [ - "area:ci", - "area:tests" - ], - "body": [ - "`scripts/check-doc-commands.sh` proves the documented commands run and that every documented flag is one the driver parses. What it cannot see is the **output** those commands produce, and the documentation is full of it: validator refusals, panel lines, report sections. Those blocks are the part a reader compares against their own terminal, so a stale one is worse than a missing one.", - "", - "Not hypothetical: while writing the v1.7.0 documentation a quoted `validate` refusal turned out never to have existed in that wording. It was caught by running the command by hand one more time than usual, which is not a control.", - "", - "**Done when**", - "- A checker extracts the distinctive sentences from output blocks in `docs/` and `README.md` and asserts each is still produced by the tool, against the sources those messages live in (`lib/validate.mjs`, `k6/lib/`, `bin/crowdsim`, `gui/ui/src/lib/messages.js`).", - "- It runs in CI, and it can fail: the change that adds it also plants a wrong quote and shows the checker catching it.", - "- A block that cannot be checked mechanically is marked illustrative, explicitly, rather than skipped in silence.", - "- Numbers inside quoted output are exempt by design — they came from a real run on somebody's machine and will never match again. The wording is what is asserted." - ] - }, - { - "key": "cache-ab-third-leg", - "title": "cache-ab: make the third leg a first-class option", - "milestone": "Backlog", - "labels": [ - "enhancement", - "area:cache-ab" - ], - "body": [ - "`cache-ab/README.md` explains that the useful third leg is the narrow subset of a fix that can actually ship this week, measured in the same run — but adding it means hand-copying a compose service block and a template.", - "", - "**Done when**", - "- A third leg can be added without editing the compose file by hand.", - "- The warning on `candidate.conf.template` about ignoring the origin's `Cache-Control` is carried onto any new template, unchanged." - ] - }, - { - "key": "ci-image-smoke-test", - "title": "CI: build the container image and smoke-test it", - "milestone": "Backlog", - "labels": [ - "area:ci" - ], - "body": [ - "The image pins `grafana/k6` and resets its entrypoint so crowdsim drives k6. Nothing checks that the image still builds and still starts after a change to the driver or to the `k6/` layout — and `k6/` now contains `lib/`, which the generator imports at init time.", - "", - "**Done when**", - "- CI builds the image and runs `crowdsim --help` and `crowdsim doctor` inside it.", - "- The smoke test asserts that the image refuses to run without an allowlist — the absent default is a feature, and a regression there is the one that matters most.", - "- No load is generated: the image is never pointed at a network target in CI.", - "- Nothing reaches the registry before the smoke test has passed." - ] - }, - { - "key": "release-helper", - "title": "A release helper, so one-commit-one-tag stops depending on memory", - "milestone": "Backlog", - "labels": [ - "area:release", - "area:ci" - ], - "body": [ - "The rule is that every commit is a tagged release with its own CHANGELOG section. It held for 1.0.0 only because somebody remembered — the tag was created after the fact, in a later session.", - "", - "**Done when**", - "- A helper prepares the CHANGELOG section and the annotated tag for `minor`, `patch` or an explicit version, and does not push.", - "- It refuses to tag a dirty tree, or a version that is already tagged.", - "- The commits exempt from the rule (roadmap, CI plumbing) are named in one place instead of being argued about per commit.", - "- A pushed tag publishes a GitHub Release whose notes are that CHANGELOG section, and fails rather than improvising when the section is missing." - ] - }, - { - "key": "gui-compare-runs", - "title": "GUI: compare two runs, with the same refusals as the CLI", - "milestone": "v1.4.0", - "labels": [ - "area:gui", - "validity" - ], - "body": [ - "History compares a selected run against previous runs at the same profile, target and shape. Choosing two runs by hand and reading the delta — per class, per cache layer — is `crowdsim compare` in a terminal, and the page that already displays the whole archive is where somebody looks first.", - "", - "The point is not the table. It is that the refusals travel with it: two runs at different URL pools are two different experiments, and a run with `generator_ok: false` has no numbers at all. A page that renders a delta those two cases would have blocked is a confident percentage with nothing behind it — the exact failure the CLI command was written to prevent.", - "", - "**Done when**", - "- Two runs can be selected in History and compared, showing the same overall, per-class and per-layer deltas the CLI prints.", - "- The comparison is refused, with the reason, in every case the CLI refuses it — and the refusal is as visible as a result would have been.", - "- A different target or peak is allowed and labelled exactly as in the CLI: a comparison between two targets, not a before/after of one.", - "- The rules are not re-implemented in the page. Whatever produces the verdict is shared with `crowdsim compare`, or the two will disagree and the wrong one will be the one on screen." - ] - }, - { - "key": "doctor-bench-generator", - "title": "doctor --bench: measure what this generator can sustain, instead of declaring it", - "milestone": "v1.4.0", - "labels": [ - "area:cli", - "validity" - ], - "body": [ - "`load` and `doctor` state the bandwidth a peak implies and compare it against `safety.generator_mbps` — a number typed into the profile by hand. So the warning that exists to predict `generator_ok: false` is only as good as somebody's guess about a laptop, a runner or a VM, and in practice it is copied between profiles once and never revisited.", - "", - "The machine can answer this itself, without touching the target.", - "", - "**Done when**", - "- `crowdsim doctor --bench` measures what this host can actually push, against a local endpoint it starts itself — never against the profile's target, and never as a side effect of a plain `doctor` run.", - "- The measurement is written where the bandwidth estimate can read it, so the warning stops depending on a declared value. A declared `safety.generator_mbps` still wins: somebody who knows the uplink is narrower than loopback is right.", - "- It states what it measured and what it did not. Loopback is an upper bound; the path to a real target is narrower than the path to yourself, and a benchmark that let anyone forget that would be worse than none.", - "- It stays a warning and never becomes a gate: a wrong estimate must not be able to stop a run somebody needs." - ] - }, - { - "key": "cache-ab-run-and-compare", - "title": "cache-ab: run the legs and print the comparison, instead of printing two commands", - "milestone": "Backlog", - "labels": [ - "area:cache-ab", - "area:cli" - ], - "body": [ - "`cache-ab` brings the legs up and then prints the `crowdsim load` commands to run by hand, leaving the comparison to whoever remembers to make it. The whole reason for two legs is the delta between them, and `crowdsim compare` now exists.", - "", - "**Done when**", - "- One command loads each leg with the same pool, in the same window, and prints the delta.", - "- The legs are still loaded one at a time unless asked otherwise: two generators at once on one host measure the host.", - "- It refuses to report a comparison in every case `crowdsim compare` refuses one." - ] - }, - { - "key": "refuse-docker-generator-on-laptop", - "title": "Refuse (or loudly warn about) generating load from Docker on a macOS/Windows host", - "milestone": "Backlog", - "labels": [ - "area:cli", - "validity" - ], - "body": [ - "That the Docker network layer on macOS and Windows saturates before the target does is measured, and documented in three places — and the tool still lets the run happen. The result is `generator_ok: false` diagnosed after the window was agreed and the run burned, which is precisely the failure the bandwidth estimate was added to prevent.", - "", - "The driver can tell: it knows when it is in a container, and the host kernel is visible from inside one.", - "", - "**Done when**", - "- A load run started from a container on a non-Linux host says so before generating anything, and says what to do instead: native k6 locally, or the container on a Linux host near the target.", - "- It does not fire for the GUI, which is a page and not a generator, nor for a container on Linux, which is the intended way to run it.", - "- Whether it refuses or only warns follows from how reliably the host can be detected: a detection that can be wrong must not become a gate." - ] - }, - { - "key": "gui-stream-loss-visible", - "title": "GUI: say when the live log has been lost, instead of going quiet", - "milestone": "v1.5.0", - "labels": [ - "area:gui", - "validity" - ], - "body": [ - "`streamRun` ends with `es.onerror = () => es.close()`. The stream closes, nothing is retried, and nothing is said. Reproduced by killing the server during a run: the log stops mid-line, the state pill still reads *running*, Stop is still offered, and the page looks exactly like a run that has gone quiet on its own.", - "", - "That is the one confusion this page cannot afford. It exists so somebody can watch load being generated against a real system; \"nothing is happening\" and \"I can no longer see what is happening\" have to look different, because the second one may mean a generator is still running and nobody is watching it.", - "", - "**Done when**", - "- Losing the event stream is visible in the page, immediately, in the same place the log is.", - "- It reconnects on its own where that is possible, and says it is doing so; a reconnect that recovers must leave no doubt about what was missed in between.", - "- When the server is simply gone, the page says that rather than implying the run went quiet — and the run's state stops being asserted, since it is no longer known.", - "- Whatever it claims about the run after a reconnect comes from the server, not from what the page remembers." - ] - }, - { - "key": "gui-log-render-cost", - "title": "GUI: the run log costs more to render than the run costs to produce", - "milestone": "v1.5.0", - "labels": [ - "area:gui" - ], - "body": [ - "The live log is `{log.join('\\n')}` inside a `
`, re-evaluated on every appended line. Measured over the 4000 lines the server keeps: **760 MB of strings built and 215 ms of pure join time**, before React reconciles a 371 KB text node — once per line. Joining once at the end is 0 ms.",
-    "",
-    "A load test at a few hundred req/s produces log lines steadily for minutes, and the page is normally open on the same laptop that is generating the load. Spending the CPU there is not free: it competes with the thing being measured.",
-    "",
-    "**Done when**",
-    "- Appending a line costs the same whether it is the tenth or the four-thousandth.",
-    "- The log still follows the tail while it is running, and can still be scrolled back without fighting it.",
-    "- The truncation notice the server already emits at 4000 lines stays visible rather than being scrolled into a virtualised void."
-   ]
-  },
-  {
-   "key": "gui-keyboard-access",
-   "title": "GUI: half the archive cannot be reached without a mouse",
-   "milestone": "v1.5.0",
-   "labels": [
-    "area:gui"
-   ],
-   "body": [
-    "In History, the table rows and the knee-plot points carry `onClick` and nothing else: no `tabindex`, no role, no key handler. Selecting a run to read its result is mouse-only. The comparison checkboxes next to them *are* focusable, so the same panel is half usable from the keyboard and half not, which is worse than either.",
-    "",
-    "This is a console people open next to a terminal, often while typing, sometimes over a share on a call.",
-    "",
-    "**Done when**",
-    "- A run can be selected, and a comparison chosen and run, using only the keyboard.",
-    "- Whatever is focused is visibly focused, in a page that is mostly dark and mostly grey.",
-    "- The clickable points in the knee plot are reachable too, or their information is reachable another way — a chart nobody can tab into is not a reason to leave the data unreachable.",
-    "- The form controls keep their current labelling: every one of them is already wrapped in a `