perf(go): measure Go parsers the way the Rust ones are measured

benches/go/cmd/parse-n-cobra/main.go

@@ -0,0 +1,38 @@
+// Command parse-n-cobra resolves the same command line N times, N coming from the
+// environment, against cobra's command tree.
+//
+// The counterpart of `go/internal/bench/parse-n` for one of the frameworks usage-go
+// is compared against, and the same protocol: differencing two runs of one binary
+// separates what the resolves cost from what the Go runtime costs to start, without
+// subtracting a second binary whose startup is not the same size.
+//
+// What this answers that `cmd/sweep` cannot: instruction counts, which are
+// deterministic where wall clock is not, and what a whole process costs — the number
+// an adopter feels, most of which is the runtime rather than the parser.
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	misecobra "github.com/jdx/usage/benches/go/mise-cobra"
+)
+
+func main() {
+	n := 1
+	if v, err := strconv.Atoi(os.Getenv("PARSE_N")); err == nil {
+		n = v
+	}
+
+	// Printed at the end, and it is what keeps the measurement honest: a rejected
+	// command line is cheap to resolve, so a harness that did not check would happily
+	// report the cost of failing early.
+	seen := 0
+	for i := 0; i < n; i++ {
+		if misecobra.Resolve(os.Args[1:]) {
+			seen = 1
+		}
+	}
+	fmt.Println(seen)
+}

benches/go/cmd/parse-n-kong/main.go

@@ -0,0 +1,38 @@
+// Command parse-n-kong resolves the same command line N times, N coming from the
+// environment, against kong's reflected grammar.
+//
+// The counterpart of `go/internal/bench/parse-n` for one of the frameworks usage-go
+// is compared against, and the same protocol: differencing two runs of one binary
+// separates what the resolves cost from what the Go runtime costs to start, without
+// subtracting a second binary whose startup is not the same size.
+//
+// What this answers that `cmd/sweep` cannot: instruction counts, which are
+// deterministic where wall clock is not, and what a whole process costs — the number
+// an adopter feels, most of which is the runtime rather than the parser.
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	misekong "github.com/jdx/usage/benches/go/mise-kong"
+)
+
+func main() {
+	n := 1
+	if v, err := strconv.Atoi(os.Getenv("PARSE_N")); err == nil {
+		n = v
+	}
+
+	// Printed at the end, and it is what keeps the measurement honest: a rejected
+	// command line is cheap to resolve, so a harness that did not check would happily
+	// report the cost of failing early.
+	seen := 0
+	for i := 0; i < n; i++ {
+		if misekong.Resolve(os.Args[1:]) {
+			seen = 1
+		}
+	}
+	fmt.Println(seen)
+}

benches/go/cmd/parse-n-urfave/main.go

@@ -0,0 +1,38 @@
+// Command parse-n-urfave resolves the same command line N times, N coming from the
+// environment, against urfave's command tree.
+//
+// The counterpart of `go/internal/bench/parse-n` for one of the frameworks usage-go
+// is compared against, and the same protocol: differencing two runs of one binary
+// separates what the resolves cost from what the Go runtime costs to start, without
+// subtracting a second binary whose startup is not the same size.
+//
+// What this answers that `cmd/sweep` cannot: instruction counts, which are
+// deterministic where wall clock is not, and what a whole process costs — the number
+// an adopter feels, most of which is the runtime rather than the parser.
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	miseurfave "github.com/jdx/usage/benches/go/mise-urfave"
+)
+
+func main() {
+	n := 1
+	if v, err := strconv.Atoi(os.Getenv("PARSE_N")); err == nil {
+		n = v
+	}
+
+	// Printed at the end, and it is what keeps the measurement honest: a rejected
+	// command line is cheap to resolve, so a harness that did not check would happily
+	// report the cost of failing early.
+	seen := 0
+	for i := 0; i < n; i++ {
+		if miseurfave.Resolve(os.Args[1:]) {
+			seen = 1
+		}
+	}
+	fmt.Println(seen)
+}

benches/go/cmd/sweep/main.go

@@ -0,0 +1,177 @@
+// Command sweep is wall clock for the four Go parsers, measured so it survives a
+// loaded machine.
+//
+// This is `benches/gate/src/bin/time-sweep.rs` in Go, deliberately: the two cards on
+// the landing page should be answering the same question with the same estimator, and
+// what a parse costs is a question about parsing rather than about how long a Go
+// process takes to start. A whole-process measurement cannot answer it — 0.95 ms of a
+// Go process is the runtime coming up, which is three orders of magnitude larger than
+// the thing being compared and varies run to run by more than the thing being
+// compared costs.
+//
+// So each parser runs repeatedly in one process and the report is the *fastest*
+// per-parse time from many short rounds. Noise from other tenants is additive —
+// nothing another process does can make this one faster — so the minimum is the
+// estimator to want, and short rounds are the ones an interruption can only spoil
+// individually. Each framework gets rounds sized to about the same wall time rather
+// than the same iteration count, so a parser 200x slower than another is not asked
+// for 200x the work.
+//
+// Two things this does not measure, both reported elsewhere by `tasks/perf-go.sh`:
+// what a whole process costs, which is the number an adopter feels, and the garbage
+// collection an allocating parser causes, which a minimum over short rounds mostly
+// steps around. The median column is printed beside the minimum because that is where
+// collection shows up.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/jdx/usage/benches/go/mise"
+	misecobra "github.com/jdx/usage/benches/go/mise-cobra"
+	misekong "github.com/jdx/usage/benches/go/mise-kong"
+	miseurfave "github.com/jdx/usage/benches/go/mise-urfave"
+	"github.com/jdx/usage/go/argv"
+)
+
+// The one command line every row is measured against, the same one the Rust shadows
+// use, so the two cards describe the same work.
+var words = []string{"use", "-g", "node@20"}
+
+// Rounds, and per-round iteration counts chosen so a round is ~0.5-3ms of work.
+const rounds = 4000
+
+// sink keeps a parse from being optimized away. Go has no `black_box`, and a compiler
+// that can see the result is unused is within its rights to skip producing it. A store to
+// a package-level variable is a side effect it has to perform, and `main` reads the last
+// one before exiting, so the parse is observed as well as performed.
+var sink bool
+
+type stats struct {
+	min, p01, p10, median float64
+}
+
+// sweep times iters calls of f, rounds times, and describes the distribution per call.
+//
+// The collector is off for the duration and asked to run between rounds, outside the
+// timed interval. Three frameworks here build a model per parse and drop it, so left to
+// itself the collector runs inside most rounds and lands unevenly: two runs of this
+// program read urfave's minimum as 266 µs and 546 µs, which is not a measurement of
+// anything. Off, every round measures the same work.
+//
+// Excluding it is also the closer answer for a CLI. A process that parses one command
+// line and gets on with it usually exits before the collector was ever going to run; what
+// the garbage costs shows up in the whole-process table instead, where a real process pays
+// for it.
+func sweep(rounds, iters int, f func() bool) stats {
+	defer debug.SetGCPercent(debug.SetGCPercent(-1))
+
+	// Warm the allocator, the caches, the branch predictors and — for the frameworks
+	// that build a model per call — the heap they will keep reusing. Whatever the first
+	// call pays for is not what a parse costs on the millionth.
+	warm := iters
+	if warm < 200 {
+		warm = 200
+	}
+	for i := 0; i < warm; i++ {
+		sink = f()
+	}
+
+	perCall := make([]float64, 0, rounds)
+	for r := 0; r < rounds; r++ {
+		// Between rounds, never inside one: with the collector off, whatever the last
+		// round allocated is still on the heap, and a round that has to grow it is
+		// measuring the allocator's bad day rather than the parser.
+		runtime.GC()
+		start := time.Now()
+		for i := 0; i < iters; i++ {
+			sink = f()
+		}
+		perCall = append(perCall, float64(time.Since(start).Nanoseconds())/float64(iters))
+	}
+	sort.Float64s(perCall)
+	at := func(q float64) float64 {
+		return perCall[int(float64(len(perCall)-1)*q)]
+	}
+	return stats{min: perCall[0], p01: at(0.01), p10: at(0.10), median: at(0.50)}
+}
+
+// row is one framework, and how much work to ask it for.
+type row struct {
+	label  string
+	rounds int
+	iters  int
+	f      func() bool
+}
+
+func main() {
+	tsv := flag.Bool("tsv", false, "one tab-separated row per parser, for a script to read")
+	flag.Parse()
+
+	rows := []row{
+		{"usage-go, argv -> struct", rounds, 2_000, func() bool {
+			cli, err := mise.Parse(words)
+			return err == nil && cli.Use != nil
+		}},
+		// The binder alone, without the post-binding rules or the structs it fills.
+		// Reported because it is the part that is comparable to nothing else here: no
+		// other framework has a stage that answers "which token is which" and stops.
+		{"usage-go, argv -> events", rounds, 2_000, func() bool {
+			p := argv.New(mise.Root, words)
+			reached := false
+			for p.Next() {
+				if ev := p.Event(); ev.Kind == argv.KindCommand {
+					reached = true
+				}
+			}
+			return p.Err() == nil && reached
+		}},
+		{"urfave/cli v3, build tree + run", rounds / 8, 4, func() bool {
+			return miseurfave.Resolve(words)
+		}},
+		{"cobra, build tree + resolve", rounds / 8, 4, func() bool {
+			return misecobra.Resolve(words)
+		}},
+		{"kong, reflect over structs + parse", rounds / 40, 1, func() bool {
+			return misekong.Resolve(words)
+		}},
+	}
+
+	// Every row is checked before any row is timed. A parser that rejected this command
+	// line would be cheap for the wrong reason, and a table that reported it anyway
+	// would be measuring how fast a framework can fail.
+	for _, r := range rows {
+		if !r.f() {
+			fmt.Fprintf(os.Stderr,
+				"sweep: %s did not reach a subcommand on `%s`, so there is nothing worth measuring\n",
+				r.label, strings.Join(words, " "))
+			os.Exit(1)
+		}
+	}
+
+	if !*tsv {
+		fmt.Printf("%-40s%9s %9s %9s %9s\n", "", "min", "p01", "p10", "median")
+	}
+	for _, r := range rows {
+		s := sweep(r.rounds, r.iters, r.f)
+		// The last parse this row made, read after it was timed: the guard above proves each
+		// parser can reach a subcommand once, and this proves the millions in between were
+		// the same parse rather than a cheap failure the timing loop never looked at.
+		if !sink {
+			fmt.Fprintf(os.Stderr, "sweep: %s stopped reaching a subcommand mid-round\n", r.label)
+			os.Exit(1)
+		}
+		if *tsv {
+			fmt.Printf("%s\t%.0f\t%.0f\t%.0f\t%.0f\n", r.label, s.min, s.p01, s.p10, s.median)
+			continue
+		}
+		fmt.Printf("%-40s%9.0f %9.0f %9.0f %9.0f  ns\n", r.label, s.min, s.p01, s.p10, s.median)
+	}
+}

benches/go/go.mod

@@ -0,0 +1,29 @@
+// One module for every mise-scale Go shadow, and the only place in the repository
+// where another CLI framework is a dependency.
+//
+// `github.com/jdx/usage/go` has none, deliberately: an adopter's binary carries the
+// tables and nothing else. A benchmark that put cobra, urfave/cli or kong in *that*
+// go.mod would be measuring the thing it is comparing against while claiming to have
+// no dependencies. So the four shadows live here, together, and the sweep that times
+// them links all four into one binary — the Go counterpart of `benches/gate`, which
+// does the same for the Rust four.
+module github.com/jdx/usage/benches/go
+
+go 1.24
+
+require (
+	github.com/alecthomas/kong v1.16.1
+	github.com/jdx/usage/go v0.0.0
+	github.com/spf13/cobra v1.10.2
+	github.com/urfave/cli/v3 v3.11.0
+)
+
+require (
+	github.com/expr-lang/expr v1.17.8 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.9 // indirect
+)
+
+// The Go module is unreleased, and what is being measured is this checkout of it
+// rather than a published version.
+replace github.com/jdx/usage/go => ../../go

benches/go/go.sum

@@ -0,0 +1,30 @@
+github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
+github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/kong v1.16.1 h1:ixhCt93XkJ98kGposQ54+bl0IK6XwqB40AsMynU7Z8E=
+github.com/alecthomas/kong v1.16.1/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
+github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
+github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM=
+github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/urfave/cli/v3 v3.11.0 h1:P/euJp99kb9p0tlVY+iYTLYYTAQlfl0hR2gUO1Img1Q=
+github.com/urfave/cli/v3 v3.11.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=