diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..8ae2588 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,357 @@ +name: bench +# Portability and benchmark matrix: pbit built, tested and measured on each standard GitHub runner OS; static and cross +# builds; the installers (install.sh, install.ps1), the npm wrapper and the docs/agents.md recipes, each against a local +# server (no release needed). Runs on pushes to its own branch and by hand only, never on pull requests or main: macOS +# minutes bill at 10x on a private repo. Standard runners only. Results: BENCHMARK-MATRIX.md, PORTABILITY.md. +# The macOS Intel row (macos-15-intel: the slowest runner, at 10x) runs only when asked: "[intel]" in the pushed commit +# message, or the `intel` input of a manual run. It ran once, in run 36969399069. +on: + push: + branches: [r22-portability-matrix] + paths-ignore: ["**.md", "scripts/bench/results/**"] + workflow_dispatch: + inputs: + intel: + description: also run the macOS Intel row (macos-15-intel) + type: boolean + default: false +permissions: + contents: read +env: + CARGO_TERM_COLOR: never + TAG: v0.2.0 + +jobs: + bench: + name: bench ${{ matrix.label }} + strategy: + fail-fast: false + matrix: + include: >- + ${{ fromJSON(format('[{0}, {1}, {2}{3}]', + '{"os": "ubuntu-latest", "label": "linux-x86_64", "target": "x86_64-unknown-linux-gnu", "py": "python3", "native": true, "timeout": 15}', + '{"os": "macos-latest", "label": "macos-arm64", "target": "aarch64-apple-darwin", "py": "python3", "native": false, "timeout": 15}', + '{"os": "windows-latest", "label": "windows-x86_64", "target": "x86_64-pc-windows-msvc", "py": "python", "native": true, "timeout": 15}', + (inputs.intel || contains(github.event.head_commit.message, '[intel]')) && ', {"os": "macos-15-intel", "label": "macos-x86_64", "target": "x86_64-apple-darwin", "py": "python3", "native": true, "timeout": 12}' || '')) }} + runs-on: ${{ matrix.os }} + timeout-minutes: ${{ matrix.timeout }} + defaults: + run: + shell: bash + env: + PY: ${{ matrix.py }} + LABEL: ${{ matrix.label }} + TARGET: ${{ matrix.target }} + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: build (portable, as release.yml) + id: build + run: | + mkdir -p out + start=$(date +%s) + cargo build --release --locked --workspace + echo "build_s=$(( $(date +%s) - start ))" >> "$GITHUB_OUTPUT" + cargo build --release --locked -p pbit-core --example kernels + ls -l target/release/ + - name: build (-C target-cpu=native, for bench/portable_vs_native.py) + id: native + if: matrix.native + continue-on-error: true + env: + RUSTFLAGS: -C target-cpu=native + run: | + cargo build --release --locked -p pbit-cli --target-dir target/native + cargo build --release --locked -p pbit-core --example kernels --target-dir target/native + + - name: callers (shell pipe, python/pbit.py tests, examples/node/decide.mjs) + id: callers + continue-on-error: true + run: | + B="$PWD/target/release/pbit" + if [ "$RUNNER_OS" = Windows ]; then B="$(cygpath -w "$B.exe")"; fi + export PBIT_BIN="$B" + mkdir -p onpath && cp target/release/pbit* onpath/ 2> /dev/null || true + export PATH="$PWD/onpath:$PATH" + { + echo "\$ pbit demo --tasks 12 | pbit decide --pretty ($(command -v pbit))" + pbit demo --tasks 12 | pbit decide --pretty > d12.json + echo "exit $?"; sed -n 1,8p d12.json + pbit demo --tasks 24 > router.json + echo "\$ pbit decide --budget-ms 200 < router.json > decision.json; case \$? in ..." + pbit decide --budget-ms 200 < router.json > decision.json && rc=0 || rc=$? + case $rc in 0) echo answer ;; 1) echo infeasible ;; 2) echo "bad input" ;; 3) echo "refused: escalate" ;; *) echo "exit $rc"; exit 1 ;; esac + $PY -c 'import json,sys; print(json.load(sys.stdin)["verdict"])' < decision.json + echo "\$ $PY python/test_pbit.py" + $PY python/test_pbit.py + echo "\$ $PY python/examples/route_agents.py" + $PY python/examples/route_agents.py + echo "\$ node examples/node/decide.mjs (node $(node --version))" + node examples/node/decide.mjs + } 2>&1 | tee "out/callers-$LABEL.txt" + + - name: install.sh (release.yml layout, served on 127.0.0.1) + id: install_sh + if: runner.os != 'Windows' + continue-on-error: true + run: | + for s in sh bash dash; do + command -v "$s" > /dev/null || continue + SH=$s sh scripts/bench/test_install_sh.sh target/release/pbit "$TARGET" "$TAG" + done 2>&1 | tee "out/install-sh-$LABEL.txt" + + - name: npm wrapper (npm pack, npm install -g from 127.0.0.1, npm uninstall -g) + id: npm + if: runner.os == 'Linux' + continue-on-error: true + run: sh scripts/bench/test_npm.sh target/release/pbit "$TARGET" "$TAG" 2>&1 | tee "out/npm-$LABEL.txt" + - name: npm wrapper on Windows (informational) + id: npm_windows + if: runner.os == 'Windows' + continue-on-error: true + run: sh scripts/bench/test_npm.sh target/release/pbit.exe "$TARGET" "$TAG" 2>&1 | tee "out/npm-$LABEL.txt" + + # PowerShell scripts are started from bash (pwsh/powershell -File, exit code 1 on a throw): redirecting a native + # command's stderr inside a pwsh step can itself raise under the runner's ErrorActionPreference = stop. + - name: pack a release-layout .zip for install.ps1 + id: pack_zip + if: runner.os == 'Windows' + continue-on-error: true + run: | + sh scripts/bench/package_like_release.sh target/release/pbit.exe "$TARGET" "$TAG" "srv/good/$TAG" + cp -R srv/good srv/bad + printf '%064d %s\n' 0 "pbit-$TAG-$TARGET.zip" > "srv/bad/$TAG/pbit-$TAG-$TARGET.zip.sha256" + ls -l "srv/good/$TAG"; cat "srv/good/$TAG/pbit-$TAG-$TARGET.zip.sha256" + - name: install.ps1 under PowerShell 7 + id: install_ps1_pwsh + if: runner.os == 'Windows' + continue-on-error: true + run: pwsh -NoProfile -File scripts/bench/test_install_ps1.ps1 -Srv srv -Tag "$TAG" -AddToPath 2>&1 | tee "out/install-ps1-pwsh-$LABEL.txt" + - name: install.ps1 under Windows PowerShell 5.1 + id: install_ps1_ps51 + if: runner.os == 'Windows' + continue-on-error: true + run: powershell -NoProfile -ExecutionPolicy Bypass -File scripts/bench/test_install_ps1.ps1 -Srv srv -Tag "$TAG" 2>&1 | tee "out/install-ps1-ps51-$LABEL.txt" + - name: callers (PowerShell 7 and Windows PowerShell 5.1) + id: ps_callers + if: runner.os == 'Windows' + continue-on-error: true + run: | + export PATH="$PWD/target/release:$PATH" + { pwsh -NoProfile -File scripts/bench/callers.ps1 + powershell -NoProfile -ExecutionPolicy Bypass -File scripts/bench/callers.ps1; } 2>&1 | tee "out/callers-powershell-$LABEL.txt" + - name: peak RSS via Get-Process (cross-check of the harness value) + id: rss_ps + if: runner.os == 'Windows' + continue-on-error: true + run: pwsh -NoProfile -File scripts/bench/peak_rss_windows.ps1 -Exe target/release/pbit.exe -Out out/peak-rss-powershell.json + + - name: cross build x86_64-apple-darwin on the arm64 runner (Rosetta smoke if present) + id: cross_darwin + if: matrix.label == 'macos-arm64' + continue-on-error: true + run: | + { + echo "\$ rustup target add x86_64-apple-darwin"; rustup target add x86_64-apple-darwin + echo "\$ cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin" + cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin 2>&1 | tail -1 + B=target/x86_64-apple-darwin/release/pbit + echo "\$ file $B"; file "$B" + echo "\$ lipo -archs $B"; lipo -archs "$B" + echo "\$ otool -L $B"; otool -L "$B" + echo "size $(stat -f %z "$B") bytes" + if /usr/bin/arch -x86_64 /usr/bin/true 2> /dev/null; then + echo "rosetta: present" + echo "\$ arch -x86_64 $B version"; /usr/bin/arch -x86_64 "$B" version + /usr/bin/arch -x86_64 "$B" demo --tasks 12 | /usr/bin/arch -x86_64 "$B" decide > d12-x86.json + echo "\$ arch -x86_64 pbit demo --tasks 12 | arch -x86_64 pbit decide: exit $?, verdict $(python3 -c 'import json; print(json.load(open("d12-x86.json"))["verdict"])')" + echo "\$ arch -x86_64 pbit stats (self-test under Rosetta)" + /usr/bin/arch -x86_64 "$B" stats | python3 -c 'import json,sys; s=json.load(sys.stdin)["self_test"]; print("1 thread", s["site_updates_per_s_1_thread"], "| threads", s["threads"], s["site_updates_per_s"])' + else + echo "rosetta: absent, build only" + fi + } 2>&1 | tee out/cross-x86_64-apple-darwin.txt + + - name: record check outcomes + if: always() + run: | + mkdir -p out + cat > "out/checks-$LABEL.json" << EOF + {"build": "${{ steps.build.outcome }}", "native_build": "${{ steps.native.outcome }}", "pack_zip": "${{ steps.pack_zip.outcome }}", "callers": "${{ steps.callers.outcome }}", "install_sh": "${{ steps.install_sh.outcome }}", + "npm": "${{ steps.npm.outcome }}", "npm_windows": "${{ steps.npm_windows.outcome }}", + "install_ps1_pwsh": "${{ steps.install_ps1_pwsh.outcome }}", "install_ps1_ps51": "${{ steps.install_ps1_ps51.outcome }}", + "ps_callers": "${{ steps.ps_callers.outcome }}", "rss_powershell": "${{ steps.rss_ps.outcome }}", + "cross_darwin": "${{ steps.cross_darwin.outcome }}"} + EOF + cat "out/checks-$LABEL.json" + - name: measure (scripts/bench/matrix_bench.py, N = 5) + id: measure + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: | + set -- --attach "checks=out/checks-$LABEL.json" --note "build_s=${{ steps.build.outputs.build_s }}" --note "runner=${{ matrix.os }}" + if [ "${{ steps.native.outcome }}" = success ]; then set -- "$@" --native-dir target/native/release; fi + if [ -f out/peak-rss-powershell.json ]; then set -- "$@" --attach peak_rss_powershell=out/peak-rss-powershell.json; fi + if [ -f out/cross-x86_64-apple-darwin.txt ]; then set -- "$@" --attach cross_x86_64_apple_darwin=out/cross-x86_64-apple-darwin.txt; fi + $PY scripts/bench/matrix_bench.py --bin target/release/pbit --label "$LABEL" --out "out/bench-$LABEL.json" "$@" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: bench-${{ matrix.label }} + path: out/ + if-no-files-found: warn + - name: fail the job if a required check failed + if: always() + run: | + fail=0 + for kv in callers=${{ steps.callers.outcome }} install_sh=${{ steps.install_sh.outcome }} npm=${{ steps.npm.outcome }} \ + install_ps1_pwsh=${{ steps.install_ps1_pwsh.outcome }} install_ps1_ps51=${{ steps.install_ps1_ps51.outcome }} \ + ps_callers=${{ steps.ps_callers.outcome }} measure=${{ steps.measure.outcome }}; do + case "$kv" in *=failure | *=cancelled) echo "required check failed: $kv"; fail=1 ;; esac + done + exit $fail + + test: + name: test ${{ matrix.label }} + strategy: + fail-fast: false + matrix: + include: >- + ${{ fromJSON(format('[{0}, {1}, {2}{3}]', + '{"os": "ubuntu-latest", "label": "linux-x86_64", "py": "python3", "timeout": 15}', + '{"os": "macos-latest", "label": "macos-arm64", "py": "python3", "timeout": 15}', + '{"os": "windows-latest", "label": "windows-x86_64", "py": "python", "timeout": 15}', + (inputs.intel || contains(github.event.head_commit.message, '[intel]')) && ', {"os": "macos-15-intel", "label": "macos-x86_64", "py": "python3", "timeout": 12}' || '')) }} + runs-on: ${{ matrix.os }} + timeout-minutes: ${{ matrix.timeout }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: cargo test --release --locked --workspace --no-fail-fast + run: | + mkdir -p out + start=$(date +%s) + set +e + cargo test --release --locked --workspace --no-fail-fast 2>&1 | tee "out/test-${{ matrix.label }}.log" + rc=${PIPESTATUS[0]} + set -e + ${{ matrix.py }} scripts/bench/test_summary.py "out/test-${{ matrix.label }}.log" "$rc" "$(( $(date +%s) - start ))" \ + "${{ matrix.label }}" > "out/test-${{ matrix.label }}.json" || echo "test_summary.py failed" + grep -E '"(passed|seconds|exit)"' "out/test-${{ matrix.label }}.json" || true + exit "$rc" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: test-${{ matrix.label }} + path: out/ + if-no-files-found: warn + + cross: + name: static and cross builds (ubuntu-latest) + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: x86_64-unknown-linux-musl (static) + id: musl_x86 + continue-on-error: true + run: | + mkdir -p out + { + echo "\$ rustup target add x86_64-unknown-linux-musl"; rustup target add x86_64-unknown-linux-musl + echo "\$ cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl" + cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl 2>&1 | tail -1 + B=target/x86_64-unknown-linux-musl/release/pbit + echo "\$ file $B"; file "$B" + echo "\$ ldd $B"; ldd "$B" 2>&1 || true + echo "size $(stat -c %s "$B") bytes" + echo "\$ $B version"; "$B" version + "$B" demo --tasks 12 | "$B" decide > d12.json + echo "\$ pbit demo --tasks 12 | pbit decide: exit $?, verdict $(python3 -c 'import json; print(json.load(open("d12.json"))["verdict"])')" + } 2>&1 | tee out/musl-x86_64.txt + - name: glibc build in the same VM (for the musl vs glibc comparison) + run: cargo build --release --locked -p pbit-cli 2>&1 | tail -1 + - name: aarch64-unknown-linux-musl (cross-linked; qemu smoke if installable within 120 s) + id: musl_arm + continue-on-error: true + run: | + { + echo "\$ sudo apt-get install -y gcc-aarch64-linux-gnu" + t0=$(date +%s); sudo apt-get update -qq && sudo apt-get install -y -qq gcc-aarch64-linux-gnu > /dev/null + echo "linker installed in $(( $(date +%s) - t0 )) s" + echo "\$ rustup target add aarch64-unknown-linux-musl"; rustup target add aarch64-unknown-linux-musl + echo "\$ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl" + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl 2>&1 | tail -1 + B=target/aarch64-unknown-linux-musl/release/pbit + echo "\$ file $B"; file "$B" + echo "size $(stat -c %s "$B") bytes" + t0=$(date +%s) + if timeout 120 sudo apt-get install -y -qq qemu-user-static > /dev/null 2>&1; then + echo "qemu-user-static installed in $(( $(date +%s) - t0 )) s" + Q=$(command -v qemu-aarch64-static || command -v qemu-aarch64) + echo "\$ $(basename "$Q") $B version"; "$Q" "$B" version + "$Q" "$B" demo --tasks 12 | "$Q" "$B" decide > d12-arm.json + echo "\$ qemu pbit demo --tasks 12 | qemu pbit decide: exit $?, verdict $(python3 -c 'import json; print(json.load(open("d12-arm.json"))["verdict"])')" + else + echo "qemu-user-static: not installed within 120 s, build only" + fi + } 2>&1 | tee out/musl-aarch64.txt + - name: wasm32-wasip1 feasibility probe (cargo check, nothing fixed) + id: wasm + continue-on-error: true + run: | + { + echo "\$ rustup target add wasm32-wasip1"; rustup target add wasm32-wasip1 + echo "\$ cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide" + set +e; cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide > wasm.log 2>&1; rc=$?; set -e + echo "exit $rc"; echo "first error: $(grep -m1 -E '^error' wasm.log || echo none)"; tail -3 wasm.log + } 2>&1 | tee out/wasm32-wasip1.txt + - name: minimum Rust version probes (debug builds of pbit-cli) + id: msrv + continue-on-error: true + run: | + { + for v in 1.72 1.73 1.77 1.78; do + rustup toolchain install "$v" --profile minimal --no-self-update > /dev/null 2>&1 || { echo "rust $v: toolchain install failed"; continue; } + flags=--locked; how="--locked (Cargo.lock as committed, lockfile v4)" + if [ "$v" = 1.72 ] || [ "$v" = 1.73 ]; then mv Cargo.lock Cargo.lock.v4; flags=""; how="Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4)"; fi + set +e; cargo "+$v" build $flags -p pbit-cli --target-dir "target/msrv-$v" > "msrv-$v.log" 2>&1; rc=$?; set -e + if [ -f Cargo.lock.v4 ]; then rm -f Cargo.lock; mv Cargo.lock.v4 Cargo.lock; fi + if [ "$rc" = 0 ]; then echo "rust $v: build ok, $how"; else echo "rust $v: build FAILED, $how: $(grep -m1 -E '^error' "msrv-$v.log")"; fi + done + echo "Cargo.lock restored: $(git status --short Cargo.lock | wc -l | tr -d ' ') changes" + } 2>&1 | tee out/msrv.txt + - name: measure the static (musl) and glibc binaries in this VM (N = 5) + id: measure + continue-on-error: true + run: | + python3 scripts/bench/matrix_bench.py --bin target/x86_64-unknown-linux-musl/release/pbit --label linux-x86_64-musl --out out/bench-linux-x86_64-musl.json --note "static musl build, same VM as bench-linux-x86_64-gnu-samevm" + python3 scripts/bench/matrix_bench.py --bin target/release/pbit --label linux-x86_64-gnu-samevm --out out/bench-linux-x86_64-gnu-samevm.json --note "glibc build, same VM as the musl row" + - name: one JSON for the job + if: always() + env: + OUTCOMES: '{"musl_x86_64": "${{ steps.musl_x86.outcome }}", "musl_aarch64": "${{ steps.musl_arm.outcome }}", "wasm32_wasip1": "${{ steps.wasm.outcome }}", "msrv": "${{ steps.msrv.outcome }}", "measure": "${{ steps.measure.outcome }}"}' + run: | + python3 - << 'EOF' + import glob, json, os, sys, time + sys.path.insert(0, "scripts/bench") + from matrix_bench import machine, runner, toolchain + out = {"schema": "pbit-cross/1", "label": "cross-linux-x86_64", "date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "outcomes": json.loads(os.environ["OUTCOMES"]), "runner": runner(), "machine": machine(), "toolchain": toolchain(), + "transcripts": {os.path.basename(p): open(p).read().splitlines() for p in sorted(glob.glob("out/*.txt"))}, + "bench": {os.path.basename(p)[6:-5]: json.load(open(p)) for p in sorted(glob.glob("out/bench-*.json"))}} + json.dump(out, open("out/cross-linux-x86_64.json", "w"), indent=1) + print(json.dumps(out["outcomes"])) + EOF + - uses: actions/upload-artifact@v7 + if: always() + with: + name: cross-linux-x86_64 + path: out/ + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68d1b89..6f61d7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,7 @@ jobs: cp "target/${{ matrix.target }}/release/pbit" "dist/$NAME/" fi cp README.md LICENSE CHANGELOG.md "dist/$NAME/" + cp -R examples python docs "dist/$NAME/" # JSON examples, the Python wrapper, the IR spec and docs/agents.md cd dist if [ "${{ matrix.archive }}" = "zip" ]; then ASSET="$NAME.zip"; 7z a "$ASSET" "$NAME" > /dev/null diff --git a/BENCHMARK-MATRIX.md b/BENCHMARK-MATRIX.md new file mode 100644 index 0000000..5bc01cc --- /dev/null +++ b/BENCHMARK-MATRIX.md @@ -0,0 +1,184 @@ +# pbit benchmark matrix (0.2.0, measured 2026-10-01) + +pbit 0.2.0 (main `d33e781`; this branch changes no Rust source), built and measured on every platform reachable on +2026-10-01 (PDT): three GitHub-hosted standard runners twice, the macOS Intel runner once, a static (musl) Linux build, +and one local Mac mini. Every cell below comes from a run made that night and traces to a JSON file in +[scripts/bench/results/2026-10-01/](scripts/bench/results/2026-10-01/) (named in the first table; the other tables keep +its row order). `python3 scripts/bench/render_matrix.py scripts/bench/results/2026-10-01` prints the tables from those +files. No number here comes from an older build. Format as in [BENCHMARKS.md](BENCHMARKS.md): median [interquartile +range] (`statistics.quantiles(n=4)`), N = 5 runs unless a header says otherwise. What builds and links where: +[PORTABILITY.md](PORTABILITY.md). + +**Runs.** GitHub Actions `bench` run [36970631022](https://github.com/BitmapAsset/pbit/actions/runs/36970631022) +(2026-10-02 05:49 UTC) gives the Linux, macOS arm64, Windows and static-musl rows; run +[36969399069](https://github.com/BitmapAsset/pbit/actions/runs/36969399069) (05:32 UTC, same Rust source) gives the +macOS Intel row, which ran once (a slow runner billed at 10x; the workflow now runs it only on request), and the first +half of the run-to-run table. The runners compiled with rustc 1.99.0. The M4 row ran locally (rustc 1.98.1) while a +second worker was compiling and running pbit on the same machine: 1-minute load averages 8.5 at the start, 6.4 at the +end, recorded per block in its JSON. Runner VMs are shared; their load is not known. + +**What runs** (`scripts/bench/matrix_bench.py`, stdlib Python, the same script on every OS): + +```sh +cargo build --release --locked --workspace +cargo build --release --locked -p pbit-core --example kernels +RUSTFLAGS="-C target-cpu=native" cargo build --release --locked -p pbit-cli --target-dir target/native # x86_64 rows, M4 +RUSTFLAGS="-C target-cpu=native" cargo build --release --locked -p pbit-core --example kernels --target-dir target/native +python3 scripts/bench/matrix_bench.py --bin target/release/pbit --label --out bench-.json \ + [--native-dir target/native/release] +``` + +- `pbit stats`: `self_test.site_updates_per_s_1_thread` and `self_test.site_updates_per_s` at the default thread count + (`min(cores, 4)`). +- `pbit-core/examples/kernels`: the BENCHMARKS.md §1 kernel rows, portable build. +- `pbit demo --tasks 12 | pbit decide` and `pbit demo --tasks 300 | pbit decide --budget-ms 300`: the wall clock of both + processes, from Python (`time.perf_counter`), next to the `ms` pbit reports itself; released tasks, violations, + verdicts and exit codes of every run. +- Peak memory of `pbit decide --budget-ms 300 < demo300.json` (N = 3): `/usr/bin/time -l` on macOS, `/usr/bin/time -v` + on Linux, and on Windows `PeakWorkingSetSize` from `K32GetProcessMemoryInfo` on the exited process's handle, + cross-checked by polling PowerShell `Get-Process` (`scripts/bench/peak_rss_windows.ps1`). +- `bench/portable_vs_native.py` (unchanged) where a native build exists: the native / portable ratio column. +- `bench/decide_threads.py` (unchanged). +- Tests: `cargo test --release --locked --workspace` in a separate job per OS (run 2 with `--no-fail-fast`). + +The workflow is `.github/workflows/bench.yml` (pushes to its branch or a manual run; `[intel]` in the commit message +adds the Intel row). + +## Tables + +#### Machines, build and tests + +| platform | CPU | logical cores | memory | OS | rustc | release build | binary | cargo test --release | source | +|---|---|---|---|---|---|---|---|---|---| +| macOS arm64, Apple M4 (local) | Apple M4 | 10 (4P+6E) | 16 GiB | 26.5.2 | 1.98.1 | n/a | 1.16 MB | not run in this job | `local-m4/bench-macos-arm64-m4-local.json` | +| Linux x86_64 (`ubuntu-latest`) | AMD EPYC 9V45 96-Core Processor | 2 | 8 GiB | Ubuntu 24.04.5 LTS | 1.99.0 | 13 s | 1.40 MB | pass: 127 passed, 0 failed, 1 ignored, 201 s | `run-36970631022/bench-linux-x86_64.json` | +| Linux x86_64, static musl (`ubuntu-latest`) | AMD EPYC 7763 64-Core Processor | 2 | 8 GiB | Ubuntu 24.04.5 LTS | 1.99.0 | n/a | 1.55 MB | not run in this job | `run-36970631022/cross-linux-x86_64.json → bench.linux-x86_64-musl` | +| Linux x86_64, glibc, same VM as musl | AMD EPYC 7763 64-Core Processor | 2 | 8 GiB | Ubuntu 24.04.5 LTS | 1.99.0 | n/a | 1.40 MB | not run in this job | `run-36970631022/cross-linux-x86_64.json → bench.linux-x86_64-gnu-samevm` | +| macOS arm64 (`macos-latest`) | Apple M1 (Virtual) | 3 | 7 GiB | 26.6.2 | 1.99.0 | 27 s | 1.15 MB | pass: 127 passed, 0 failed, 1 ignored, 195 s | `run-36970631022/bench-macos-arm64.json` | +| Windows x86_64 (`windows-latest`) | AMD EPYC 7763 64-Core Processor | 2 | 8 GiB | Microsoft Windows Server 2025 Datacenter (build 26100) | 1.99.0 | 37 s | 0.97 MB | pass: 127 passed, 0 failed, 1 ignored, 572 s | `run-36970631022/bench-windows-x86_64.json` | +| macOS x86_64 (`macos-15-intel`, run 1 only) | Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz | 4 | 14 GiB | 15.7.9 | 1.99.0 | 86 s | 1.27 MB | FAIL: 47 passed, 1 failed, 0 ignored, 309 s | `run-36969399069/bench-macos-x86_64.json` | +| Linux x86_64, WSL2 on a Windows desktop with an RTX 4070 | pending: machine offline | | | | | | | | | +| Linux x86_64, VirtualBox VM (CPU only) | pending: machine offline | | | | | | | | | + +#### `pbit stats` self-test: 400-spin ring, IR sampler, site updates/s (N = 5) + +| platform | 1 thread | default threads | threads (= min(cores, 4)) | +|---|---|---|---| +| macOS arm64, Apple M4 (local) | 2.23e7 [2.16e7-2.24e7] | 7.3e7 [6.82e7-7.4e7] | 4 | +| Linux x86_64 (`ubuntu-latest`) | 1.76e7 [1.71e7-1.78e7] | 2.33e7 [2.31e7-2.37e7] | 2 | +| Linux x86_64, static musl (`ubuntu-latest`) | 1.12e7 [1.09e7-1.13e7] | 1.42e7 [1.4e7-1.42e7] | 2 | +| Linux x86_64, glibc, same VM as musl | 1.1e7 [1.09e7-1.1e7] | 1.4e7 [1.39e7-1.4e7] | 2 | +| macOS arm64 (`macos-latest`) | 1.32e7 [1.31e7-1.36e7] | 2.63e7 [2.57e7-2.64e7] | 3 | +| Windows x86_64 (`windows-latest`) | 1.03e7 [9.97e6-1.1e7] | 1.46e7 [1.28e7-1.46e7] | 2 | +| macOS x86_64 (`macos-15-intel`, run 1 only) | 5.66e6 [4.27e6-6.62e6] | 1.67e7 [1.34e7-1.86e7] | 4 | +| Linux x86_64, WSL2 on a Windows desktop with an RTX 4070 | pending: machine offline | | | +| Linux x86_64, VirtualBox VM (CPU only) | pending: machine offline | | | + +#### Lattice kernels (`pbit-core/examples/kernels`, portable build), updates/s (N = 5) + +| platform | multispin fast, 1 thread | multispin fast, 4 threads | multispin fast, 10 threads | heat-bath f32 fast, 1 thread | native / portable (multispin 1t / 4t / heat-bath) | +|---|---|---|---|---|---| +| macOS arm64, Apple M4 (local) | 3.64e10 [3.63e10-3.67e10] | 1.07e11 [1.06e11-1.29e11] | 1.22e11 [1.19e11-1.24e11] | 8.96e8 [8.89e8-8.98e8] | 0.99 / 0.94 / 1.00 | +| Linux x86_64 (`ubuntu-latest`) | 2.47e10 [2.43e10-2.6e10] | 2.22e10 [2.12e10-2.32e10] | 2.22e10 [2.18e10-2.26e10] | 5.03e8 [5.01e8-5.3e8] | 1.06 / 1.07 / 1.05 | +| Linux x86_64, static musl (`ubuntu-latest`) | n/a | n/a | n/a | n/a | n/a: no --native-dir | +| Linux x86_64, glibc, same VM as musl | n/a | n/a | n/a | n/a | n/a: no --native-dir | +| macOS arm64 (`macos-latest`) | 2e10 [1.62e10-2.1e10] | 4.6e10 [4.1e10-5.47e10] | 5.77e10 [4.83e10-5.87e10] | 4.95e8 [4.4e8-5.13e8] | n/a: no --native-dir | +| Windows x86_64 (`windows-latest`) | 1.32e10 [1.17e10-1.33e10] | 1.2e10 [1.08e10-1.26e10] | 1.21e10 [1.17e10-1.25e10] | 2.37e8 [1.44e8-2.43e8] | 0.98 / 1.04 / 1.00 | +| macOS x86_64 (`macos-15-intel`, run 1 only) | 7.83e9 [3.59e9-8.97e9] | 2.05e10 [1.89e10-2.36e10] | 2.35e10 [1.66e10-2.53e10] | 1.3e8 [7.83e7-1.48e8] | 1.00 / 0.85 / 1.12 | +| Linux x86_64, WSL2 on a Windows desktop with an RTX 4070 | pending: machine offline | | | | | +| Linux x86_64, VirtualBox VM (CPU only) | pending: machine offline | | | | | + +#### `pbit demo --tasks 12 | pbit decide`, ms (N = 5) + +| platform | wall, both processes | pbit's own `ms` | verdict | exit | +|---|---|---|---|---| +| macOS arm64, Apple M4 (local) | 10.9 [10.9-11.0] | 8.2 [8.2-8.3] | exact | 0 | +| Linux x86_64 (`ubuntu-latest`) | 10.5 [10.4-10.5] | 9.1 [9.1-9.1] | exact | 0 | +| Linux x86_64, static musl (`ubuntu-latest`) | 19.0 [19.0-19.1] | 17.6 [17.5-17.7] | exact | 0 | +| Linux x86_64, glibc, same VM as musl | 19.6 [19.5-19.7] | 17.9 [17.8-18.0] | exact | 0 | +| macOS arm64 (`macos-latest`) | 17.8 [17.7-18.2] | 13.0 [12.9-13.1] | exact | 0 | +| Windows x86_64 (`windows-latest`) | 30.8 [30.6-41.8] | 18.0 [17.9-21.5] | exact | 0 | +| macOS x86_64 (`macos-15-intel`, run 1 only) | 69.6 [63.2-78.3] | 47.8 [43.4-53.8] | exact | 0 | +| Linux x86_64, WSL2 on a Windows desktop with an RTX 4070 | pending: machine offline | | | | +| Linux x86_64, VirtualBox VM (CPU only) | pending: machine offline | | | | + +#### `pbit demo --tasks 300 | pbit decide --budget-ms 300`, ms (N = 5) + +| platform | wall, both processes | pbit's own `ms` | released (min-max) | violations (max) | verdict | sampler site updates/s | +|---|---|---|---|---|---|---| +| macOS arm64, Apple M4 (local) | 423.6 [422.1-426.9] | 418.0 [417.2-421.2] | 300-300 of 300 | 0 | diagnostics_passed | 4.5e7 [3.85e7-4.53e7] | +| Linux x86_64 (`ubuntu-latest`) | 422.5 [421.7-424.0] | 417.9 [417.3-419.5] | 300-300 of 300 | 0 | diagnostics_passed | 1.49e7 [1.47e7-1.51e7] | +| Linux x86_64, static musl (`ubuntu-latest`) | 478.5 [477.3-483.3] | 470.3 [469.2-475.2] | 300-300 of 300 | 0 | diagnostics_passed | 9.7e6 [9.61e6-9.8e6] | +| Linux x86_64, glibc, same VM as musl | 471.8 [469.7-476.0] | 465.5 [463.6-469.8] | 300-300 of 300 | 0 | diagnostics_passed | 9.37e6 [9.36e6-9.38e6] | +| macOS arm64 (`macos-latest`) | 439.2 [437.8-444.7] | 431.2 [429.7-436.0] | 300-300 of 300 | 0 | diagnostics_passed | 1.47e7 [1.34e7-1.48e7] | +| Windows x86_64 (`windows-latest`) | 493.0 [490.8-507.8] | 470.1 [467.7-490.4] | 296-300 of 300 | 0 | diagnostics_passed, partial | 8.69e6 [8.44e6-8.84e6] | +| macOS x86_64 (`macos-15-intel`, run 1 only) | 642.3 [576.8-667.2] | 616.3 [551.0-638.1] | 300-300 of 300 | 0 | diagnostics_passed | 1.06e7 [9.39e6-1.2e7] | +| Linux x86_64, WSL2 on a Windows desktop with an RTX 4070 | pending: machine offline | | | | | | +| Linux x86_64, VirtualBox VM (CPU only) | pending: machine offline | | | | | | + +#### Peak memory of the 300-task `pbit decide --budget-ms 300`, MiB (N = 3) + +| platform | peak RSS | how | pbit's own `peak_rss_mb` | Get-Process polling (Windows, lower bound) | +|---|---|---|---|---| +| macOS arm64, Apple M4 (local) | 47.6 [47.0-48.0] | /usr/bin/time -l (maximum resident set size, bytes) | 47.0, 47.6, 48.0 | - | +| Linux x86_64 (`ubuntu-latest`) | 21.0 [21.0-21.3] | /usr/bin/time -v (Maximum resident set size, KiB x 1024) | 21.0, 21.3, 21.0 | - | +| Linux x86_64, static musl (`ubuntu-latest`) | 17.5 [17.0-17.9] | /usr/bin/time -v (Maximum resident set size, KiB x 1024) | 17.5, 17.9, 17.0 | - | +| Linux x86_64, glibc, same VM as musl | 20.1 [19.7-21.1] | /usr/bin/time -v (Maximum resident set size, KiB x 1024) | 20.1, 19.7, 21.1 | - | +| macOS arm64 (`macos-latest`) | 29.8 [28.8-30.6] | /usr/bin/time -l (maximum resident set size, bytes) | 28.7, 29.7, 30.6 | - | +| Windows x86_64 (`windows-latest`) | 19.2 [19.2-20.3] | K32GetProcessMemoryInfo PeakWorkingSetSize (process handle, after exit) | n/a (Windows: `sys::usage()` is Unix-only) | 19.3, 19.9, 20.2 | +| macOS x86_64 (`macos-15-intel`, run 1 only) | 25.1 [23.4-25.5] | /usr/bin/time -l (maximum resident set size, bytes) | 23.4, 25.0, 25.4 | - | +| Linux x86_64, WSL2 on a Windows desktop with an RTX 4070 | pending: machine offline | | | | +| Linux x86_64, VirtualBox VM (CPU only) | pending: machine offline | | | | + +#### `bench/decide_threads.py`: 300-task decide, fixed work (4 chains x 400 sweeps, polish 0), wall ms (N = 5) + +| platform | --threads 1 | --threads 2 | --threads 4 | verdict | +|---|---|---|---|---| +| macOS arm64, Apple M4 (local) | 84.1 [83.1-85.8] | 65.2 [64.9-65.4] | 55.7 [55.6-55.8] | refused | +| Linux x86_64 (`ubuntu-latest`) | 99.1 [99.1-99.8] | 84.1 [82.4-85.1] | 83.2 [83.1-83.6] | refused | +| Linux x86_64, static musl (`ubuntu-latest`) | 176.4 [176.2-186.1] | 153.5 [152.4-154.6] | 156.6 [155.2-159.9] | refused | +| Linux x86_64, glibc, same VM as musl | 172.9 [172.3-173.7] | 153.6 [150.3-155.3] | 152.0 [150.6-152.6] | refused | +| macOS arm64 (`macos-latest`) | 135.6 [134.7-139.9] | 109.4 [106.1-133.6] | 119.4 [105.3-131.7] | refused | +| Windows x86_64 (`windows-latest`) | n/a: exit 1, TypeError: '<' not supported between instances of 'NoneType' and 'NoneType' | | | | +| macOS x86_64 (`macos-15-intel`, run 1 only) | 295.4 [272.9-298.7] | 258.0 [225.4-310.7] | 304.1 [249.8-363.4] | refused | +| Linux x86_64, WSL2 on a Windows desktop with an RTX 4070 | pending: machine offline | | | | +| Linux x86_64, VirtualBox VM (CPU only) | pending: machine offline | | | | + +#### Run to run: run 1 / run 2 medians, same source, two VMs per runner label + +| platform | CPU (run 1 / run 2) | `pbit stats` 1 thread | multispin fast, 1 thread | 12-task wall ms | 300-task wall ms | peak RSS MiB | +|---|---|---|---|---|---|---| +| Linux x86_64 (`ubuntu-latest`) | both AMD EPYC 9V45 96-Core Processor | 1.91e7 / 1.76e7 (-8%) | 2.59e10 / 2.47e10 (-5%) | 9.6 / 10.5 (+9%) | 418.8 / 422.5 (+1%) | 21.1 / 21.0 (-0%) | +| macOS arm64 (`macos-latest`) | both Apple M1 (Virtual) | 1.25e7 / 1.32e7 (+6%) | 2.17e10 / 2e10 (-8%) | 15.5 / 17.8 (+15%) | 442.3 / 439.2 (-1%) | 30.5 / 29.8 (-3%) | +| Windows x86_64 (`windows-latest`) | AMD EPYC 9V74 80-Core Processor / AMD EPYC 7763 64-Core Processor | 1.26e7 / 1.03e7 (-19%) | 1.37e10 / 1.32e10 (-4%) | 20.6 / 30.8 (+49%) | 457.9 / 493.0 (+8%) | 20.2 / 19.2 (-5%) | + +**What this lets us claim.** One source tree builds with `--locked` and gives the same answers on macOS arm64 and +x86_64, Linux x86_64 (glibc and static musl) and Windows x86_64. The 12-task decision is `exact` on every row; the +300-task decision ended with 0 violations on every one of the 35 timed runs and released all 300 tasks on every row +except two Windows runs (`partial`, 299 and 296 released; the Windows runner had 2 vCPUs and the fewest sampler updates +per second). `cargo test --release` passes in full (127 passed, 1 ignored) on the Linux, macOS arm64 and Windows runners +in run 2; the two failures seen (Intel in run 1, Windows in run 1 but not in run 2) are wall-clock bounds, not wrong +answers (PORTABILITY.md, Findings 4). A small exact decision costs 10-31 ms of wall clock including two process starts +on the standard runners (pbit's own `ms`: 9-18), 70 ms on the Intel runner; the 300-task decision with `--budget-ms 300` +returns in 420-495 ms on every standard runner and the M4 (the budget, then the gate and the 50 ms polish). Its peak +memory is 17-48 MiB and grew with the sweeps run inside the budget (median 45,008 sweeps and 47.6 MiB on the M4 with 4 +threads; 8,720-14,960 sweeps and 17-30 MiB on the runners; `telemetry.sweeps` in each JSON). Binaries are 0.97-1.55 MB. +On x86_64, `-C target-cpu=native` moved the three kernel cells by -2% to +7% on the Linux and Windows runners in run 2 +(-14% to +9% across both runs) and by -15% to +12% on the noisier Intel runner, the same size as the run-to-run noise: +the portable release builds give up little. The static musl build was within 4% of the glibc build on the same VM in run +2, and within 7% either way in run 1 (`run-36969399069/cross-linux-x86_64.json`), using 2.5-3.2 MiB less memory. + +**What it does not let us claim.** A speed ranking of platforms: the runners are shared VMs with 2 (Linux, Windows), 3 +(macOS arm64) or 4 (Intel) vCPUs, the same runner label landed on different CPUs from job to job (the two Linux jobs of +run 2: AMD EPYC 9V45 and 7763; Windows: EPYC 9V74 in run 1, 7763 in run 2), and the same cell moved by up to 19% in +`pbit stats` and 49% in the 12-task wall clock between the two runs (table above). Thread scaling: with 2-3 vCPUs the 4- +and 10-thread kernel rows oversubscribe the runner, and `pbit stats`' default thread count differs by row (last column). +Anything about the Intel runner beyond one run (its lower quartiles reach 54% below the median). The 12-task wall clocks +compare process start-up as much as pbit (Windows starts processes slowest). The M4 row ran under load 6-8. GPU, Linux +on aarch64 hardware (only a qemu smoke test, PORTABILITY.md), WSL2 and VM rows: not measured, machines offline. Rosetta +2 ran one `pbit stats` per run (1 thread 1.10e7 and 9.09e6 updates/s, 3 threads 2.00e7 and 1.42e7, runs 1 and 2; +`transcripts/cross-x86_64-apple-darwin.txt`), not enough for a row. + +**Not run on every platform.** `bench/decide_threads.py` stops on Windows (`process_cpu_ms` is `null` there), and +`bench/portable_vs_native.py` needs extensionless copies of the Windows binaries, which the harness makes +(PORTABILITY.md, Findings 6). The kernels and native rows were not measured for the musl build (release binary only). \ No newline at end of file diff --git a/PORTABILITY.md b/PORTABILITY.md new file mode 100644 index 0000000..8a62e61 --- /dev/null +++ b/PORTABILITY.md @@ -0,0 +1,148 @@ +# pbit portability (0.2.0) + +What builds, what runs and what each binary links, per platform. Measured on 2026-10-01 (PDT) from the 0.2.0 source +(main `d33e781`; this branch adds scripts and docs, no Rust source change) in GitHub Actions bench runs +[36969399069](https://github.com/BitmapAsset/pbit/actions/runs/36969399069) (run 1) and +[36970631022](https://github.com/BitmapAsset/pbit/actions/runs/36970631022) (run 2) on standard runners, plus one +local Apple M4. +Evidence: [scripts/bench/results/2026-10-01/](scripts/bench/results/2026-10-01/) (one JSON per job, and the transcripts). +Speed and memory per platform: [BENCHMARK-MATRIX.md](BENCHMARK-MATRIX.md). Workflow: `.github/workflows/bench.yml`. + +## Targets + +| target | built on | `cargo test --release` | `pbit demo --tasks 12 \| pbit decide` | links | binary | +|---|---|---|---|---|---| +| `aarch64-apple-darwin` | `macos-latest` (M1 VM), local M4 | pass: 127, 1 ignored (`macos-latest`, runs 1 and 2) | exit 0, `exact` | `libSystem.B.dylib` only | 1.15 MB | +| `x86_64-apple-darwin` | `macos-15-intel` (native); cross-built on `macos-latest` | 1 failure in 48 (a timing bound, see Findings 4; run 1; the test run stopped at that binary) | exit 0 natively on the Intel runner, and under Rosetta 2 on the M1 runner | `libSystem.B.dylib` only | 1.27 MB | +| `x86_64-unknown-linux-gnu` | `ubuntu-latest` (Ubuntu 24.04) | pass: 127, 1 ignored (runs 1 and 2) | exit 0 | `libc.so.6`, `libm.so.6`, `libgcc_s.so.1`; needs **glibc >= 2.34** | 1.40 MB | +| `x86_64-unknown-linux-musl` | `ubuntu-latest` | not run (release build only) | exit 0 | none: `static-pie linked`, `ldd`: statically linked | 1.55 MB | +| `aarch64-unknown-linux-musl` | `ubuntu-latest`, linked with `gcc-aarch64-linux-gnu` | not run | exit 0 under `qemu-aarch64-static` | none: `statically linked` | 1.48 MB | +| `x86_64-pc-windows-msvc` | `windows-latest` (Server 2025) | pass: 127, 1 ignored (run 2, `--no-fail-fast`); run 1: 1 failure, a timing bound (Findings 4) | exit 0 in Git Bash and PowerShell 7.6; Windows PowerShell 5.1 adds a byte-order mark to the pipe (Findings 1): pipe through `cmd` | `KERNEL32`, `ntdll`, `bcryptprimitives`, **`VCRUNTIME140`** and the UCRT (`api-ms-win-crt-*`) | 0.97 MB | +| `wasm32-wasip1` | `ubuntu-latest` | n/a | not built or run | `cargo check -p pbit-ir -p pbit-decide`: pass | n/a | + +Not reached tonight: Linux x86_64 on WSL2 (a Windows desktop with an RTX 4070) and a Linux VirtualBox VM (CPU only): +pending, both machines offline. No Linux aarch64 machine was run natively (only `qemu-aarch64-static`), and no large or +ARM GitHub runners were used (billed on a private repository). + +## What each binary links + +Exact tool output from the runs above (`file`, `otool -L`, `ldd`, `objdump -T`, `dumpbin /dependents`). + +```text +aarch64-apple-darwin (macos-latest) Mach-O 64-bit executable arm64 + otool -L: /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0) +x86_64-apple-darwin (macos-15-intel) Mach-O 64-bit executable x86_64 + otool -L: /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1351.0.0) +x86_64-unknown-linux-gnu (ubuntu-latest) + file: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, + interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 3.2.0, not stripped + ldd: linux-vdso.so.1, libgcc_s.so.1, libm.so.6, libc.so.6, /lib64/ld-linux-x86-64.so.2 + GLIBC symbol versions (objdump -T): 2.2.5 2.3 2.3.4 2.14 2.16 2.17 2.18 2.25 2.28 2.29 2.30 2.32 2.33 2.34 +x86_64-unknown-linux-musl (ubuntu-latest) + file: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), static-pie linked, not stripped + ldd: statically linked +aarch64-unknown-linux-musl (ubuntu-latest, cross-linked) + file: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), statically linked, not stripped +x86_64-pc-windows-msvc (windows-latest) PE32+ executable for MS Windows 6.00 (console), x86-64, 5 sections + dumpbin /dependents (dumpbin is not on the runner's PATH; found with vswhere): + bcryptprimitives.dll api-ms-win-core-synch-l1-2-0.dll KERNEL32.dll ntdll.dll VCRUNTIME140.dll + api-ms-win-crt-{math,string,runtime,stdio,locale,heap}-l1-1-0.dll +``` + +No binary needs anything outside the OS except two cases: the glibc build needs glibc 2.34 or later (Ubuntu 22.04+, +Debian 12+, RHEL 9+; not Ubuntu 20.04, Debian 11, RHEL 8 or Amazon Linux 2), and the Windows build needs +`VCRUNTIME140.dll` from the Visual C++ 2015-2022 redistributable (present on most machines, absent on a clean install). +The musl builds have no runtime dependency at all. + +## Building for each target + +```sh +# Linux x86_64, static (no glibc requirement): +rustup target add x86_64-unknown-linux-musl +cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl + +# Linux aarch64, static, cross-linked from x86_64 Linux: +sudo apt-get install -y gcc-aarch64-linux-gnu +rustup target add aarch64-unknown-linux-musl +CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc \ + cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl +sudo apt-get install -y qemu-user-static # to run it on x86_64: qemu-aarch64-static ./pbit version + +# macOS Intel from Apple silicon (runs under Rosetta 2 with `arch -x86_64`): +rustup target add x86_64-apple-darwin +cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin + +# WebAssembly (WASI), feasibility check only: +rustup target add wasm32-wasip1 +cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide +``` + +Build times on the runners: 12-28 s for `cargo build --release --locked --workspace` (fat LTO, one codegen unit), 86 s on +the Intel runner; the musl and cross builds took 18-19 s each. `apt-get install gcc-aarch64-linux-gnu` took 13 s and +`qemu-user-static` 4 s on `ubuntu-latest`. + +## Minimum Rust version + +Checked on `ubuntu-latest` with debug builds of `pbit-cli` (run 1, `transcripts/msrv.txt`): + +| toolchain | result | +|---|---| +| 1.72 | fails: `error[E0658]: use of unstable library feature 'int_roundings'` (`div_ceil`) | +| 1.73 | builds (with `Cargo.lock` set aside: cargo before 1.78 cannot read lockfile version 4) | +| 1.77 | fails: `failed to parse lock file` (the committed lockfile is version 4) | +| 1.78 | builds with `--locked` | + +So the source needs Rust 1.73 and the repository as committed needs 1.78. `Cargo.toml` declares no `rust-version`; tests +and examples were not built on these toolchains. The runners used 1.99.0, the local M4 1.98.1. + +## WebAssembly + +`cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide` passes (exit 0, no warnings shown, 0.69 s). A check does +not run anything. Both crates use `std::thread` (the chains and the gate's worker pool) and `std::time::Instant` (budgets +and deadlines): on `wasm32-wasip1` a thread spawn fails at run time, and on `wasm32-unknown-unknown` (a browser) +`Instant::now()` panics. A playground needs a single-threaded path and a clock passed in. Inferred from the source, not +run; nothing was changed. + +## Installers and packages + +No release exists yet, so every installer was tested against a local server that serves an archive packed the way +`release.yml` packs it (`scripts/bench/package_like_release.sh`: same names, layout and `.sha256`). + +| what | where it passed | checks | +|---|---|---| +| `install.sh` (POSIX sh) | `ubuntu-latest` (`sh` = dash, `bash`, `dash`), `macos-latest` and `macos-15-intel` (`sh`, `bash`, `dash`), local M4 | piped into the shell as `curl \| sh` does; default directory under a scratch `HOME`; a wrong `.sha256` fails and installs nothing; `PBIT_DOWNLOAD_BASE` without `PBIT_VERSION` fails | +| `install.ps1` | `windows-latest`: PowerShell 7.6 (all checks, runs 1 and 2); Windows PowerShell 5.1 (download, checksum, install, `pbit version`; the test then stopped at the demo pipeline, Findings 1, and now pipes it through `cmd`) | `-DownloadBase`, `-AddToPath` (user PATH), the `irm \| iex` form with `PBIT_*` variables, a wrong `.sha256` fails and installs nothing, `-DownloadBase` without `-Version` fails | +| npm wrapper (`npm/`) | `ubuntu-latest`, `windows-latest`, local M4 | `npm pack`, `npm install -g` (scratch prefix), postinstall fetch and SHA-256 check, `which pbit`, exit codes 0 / 1 / 2 / 3 passed through, `npm uninstall -g`; an `--ignore-scripts` install fetches on first run; `PBIT_BINARY`; a wrong `.sha256` fails the install | +| `docs/agents.md` recipes | shell, Python (`python/test_pbit.py`), Node (`examples/node/decide.mjs`): Linux, macOS arm64 and x86_64, Windows (Git Bash); PowerShell 7.6: Windows | each recipe as written, plus one call per exit code | + +Install locations: `install.sh` writes `/usr/local/bin` when it can (every runner image above: their user can write it) +and `~/.local/bin` otherwise (the local M4, where `/usr/local/bin` belongs to root), and never uses sudo; `install.ps1` +writes `$HOME\.local\bin`, adds it to the session's PATH, and to the user PATH only with `-AddToPath`. Names: `pbit` is +free on npm and `pbit`, `pbit-core`, `pbit-ir`, `pbit-decide` and `pbit-cli` are free on crates.io (checked 2026-10-01 +22:31 PDT); nothing was published. + +## Findings that need a source, test or release change (not made here) + +1. **A UTF-8 byte-order mark on stdin is rejected** (`bad JSON: unexpected character 'ï' at byte 0`, exit 2). Windows + PowerShell 5.1 on the `windows-latest` image adds one to text it pipes into a native program even with + `$OutputEncoding` at its `us-ascii` default (runs 1 and 2), so `pbit demo | pbit decide` exits 2 there; PowerShell 7 + adds one when a profile sets `$OutputEncoding = [Text.Encoding]::UTF8` (seen with 7.6 locally). RFC 8259 lets a parser + ignore a leading BOM: `read_stdin()` in `pbit-cli/src/main.rs` could strip `U+FEFF`. Today: PowerShell 7, or pipe + through `cmd /c` (docs/agents.md). +2. **The Linux release binary needs glibc 2.34.** `release.yml` builds `x86_64-unknown-linux-gnu` on `ubuntu-latest`. + The static musl build above runs on any x86_64 Linux and was within 4% (run 2) and 7% (run 1) of the glibc build on + the same VM, with 2.5-3.2 MiB less peak memory (BENCHMARK-MATRIX.md). Shipping it, plus `aarch64-unknown-linux-musl`, + would also give `install.sh` something for Alpine and arm64 Linux, which it refuses today. +3. **The Windows binary needs `VCRUNTIME140.dll`.** Linking the CRT statically (`-C target-feature=+crt-static` for the + msvc release build) would remove that; not built or measured here. +4. **Two wall-clock test bounds failed on shared runners** in run 1: `run_deadline_ms_bounds_the_whole_call` on + `macos-15-intel` (990 ms for `--deadline-ms 600`, bound 900 ms) and `acc1_tv_le_001_within_10ms` on `windows-latest` + (p50 13.1 ms, bound 10 ms; one seed's maximum 37 ms). In run 2 (`--no-fail-fast`) all 127 tests passed on Linux, + macOS arm64 and Windows; the Intel row was not re-run, and its run-1 test job stopped after the failing binary (48 of + the 128 tests ran). The same class as the `d33e781` CI tolerance. +5. **`--deadline-ms` is not a hard bound on a slow CPU**: 990 ms for a 600 ms deadline on the Intel runner (65% over). +6. **`bench/*.py` assume a Unix machine.** `bench/decide_threads.py` stops with a `TypeError` on Windows + (`process_cpu_ms` is `null` there: `sys::usage()` is Unix-only); `bench/portable_vs_native.py` runs `/pbit` + without `.exe` (the harness adds extensionless copies on Windows); both print a hard-coded "Apple M4" header line. +7. **npm 11.17 warns about install scripts** (`npm warn allow-scripts ... not yet covered by allowScripts`). They still + ran; if a future npm blocks them, the wrapper fetches the binary on first run (tested with `--ignore-scripts`). diff --git a/docs/agents.md b/docs/agents.md new file mode 100644 index 0000000..e676165 --- /dev/null +++ b/docs/agents.md @@ -0,0 +1,108 @@ +# Calling pbit from anything + +`pbit` is one binary with a JSON contract: a document on stdin, one JSON document on stdout, and an exit code that says +what kind of answer it is. Anything that can start a process can use it: a shell, Python, Node, PowerShell, a CI job, or +an agent harness with a shell tool. No server, no bindings, no network. The document formats are in the README +("Use it from anything": the router document of `pbit decide`) and in [pbit-ir-json.md](pbit-ir-json.md) (programs for +`pbit run`). + +| exit | meaning | what to do | +|---|---|---| +| 0 | an answer: verdict `exact`, `diagnostics_passed` or `partial` | act on `released`; escalate `escalated` (empty unless `partial`) | +| 1 | `infeasible`: no plan satisfies the rules (a proof) | relax a rule or a cap; it is an answer, not a crash | +| 2 | bad input: one `{"error": {"code", "path", "message"}}` object on stdout; a bad flag prints one line on stderr instead | fix the document or the flag | +| 3 | `refused` / `declined` (the gate or a cap said no), or `{"error": {"code": "numeric"}}` | escalate the whole decision; the best-effort plan is still in the output | + +Every recipe below was run in the cross-platform matrix (`.github/workflows/bench.yml`; transcripts in +`scripts/bench/results/`), on the OSes named under each. + +## Shell (bash, zsh, sh) + +Run on Linux and macOS (and in Git Bash on Windows). + +```sh +pbit demo --tasks 12 | pbit decide --pretty # a synthetic 12-task routing document, decided +pbit decide --budget-ms 200 < router.json > decision.json # your document +case $? in 0) echo answer ;; 1) echo infeasible ;; 2) echo "bad input" ;; 3) echo "refused: escalate" ;; esac +``` + +Read fields with any JSON tool, e.g. `python3 -c 'import json,sys; print(json.load(sys.stdin)["verdict"])' < decision.json`. + +## Python + +Run on Linux, macOS and Windows: `python/pbit.py` is stdlib only (Python >= 3.9); `python3 python/test_pbit.py` is its test. + +```python +import pbit # python/pbit.py (copy it next to your code, or add python/ to sys.path) +decision = pbit.decide(pbit.demo(tasks=12), budget_ms=200) # exit 1 and 3 come back as answers +answer = pbit.run(program, deadline_ms=1000) # a pbit-ir program; bad input raises pbit.PbitInputError +``` + +The binary is found via `binary=`, `$PBIT_BIN`, `pbit` on `PATH`, or `../target/release/pbit` next to `pbit.py` +(on Windows set `PBIT_BIN` to the full path of `pbit.exe`). + +## Node + +Run on Linux, macOS and Windows: [examples/node/decide.mjs](../examples/node/decide.mjs), no dependencies (Node >= 18). +It exports `pbit(args, input)`, which spawns the binary, writes the JSON, and resolves to +`{kind: 'answer' | 'infeasible' | 'bad_input' | 'refused', exitCode, output, stderr}`. + +```sh +node examples/node/decide.mjs # a 12-task demo, then one call per exit code (0, 1, 2, 3) +node examples/node/decide.mjs router.json # decide your document +``` + +```js +import { pbit } from './decide.mjs'; +const r = await pbit(['decide', '--budget-ms', '200'], problem); // problem: an object or JSON text +if (r.kind === 'answer') act(r.output.plan, r.output.released); else escalate(r); +``` + +The npm package (`npm/`) installs the binary and a `pbit` command with exit codes passed through. On Windows, spawn +`pbit.exe` itself (set `PBIT_BIN`): Node cannot spawn npm's `pbit.cmd` shim without a shell. + +## PowerShell + +Run on Windows (`windows-latest`) with PowerShell 7.6 as written, and with Windows PowerShell 5.1 through `cmd` (below). + +```powershell +pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms +Get-Content router.json -Raw | pbit decide --budget-ms 200 | ConvertFrom-Json; $LASTEXITCODE # 0 / 1 / 2 / 3 +``` + +Windows PowerShell 5.1 (the `powershell.exe` that ships with Windows) adds a UTF-8 byte-order mark to text it pipes into +a program, even with `$OutputEncoding` set to `us-ascii` (the image default), and pbit 0.2.0 rejects that input: exit 2, +`bad JSON: unexpected character 'ï' at byte 0`. Use PowerShell 7 (`pwsh`), or let `cmd` do the piping and the +redirection; PowerShell then only reads pbit's output: + +```powershell +cmd /c "pbit demo --tasks 12 | pbit decide" | ConvertFrom-Json | Select-Object verdict, violations, ms +cmd /c "pbit decide --budget-ms 200 < router.json" | ConvertFrom-Json; $LASTEXITCODE +``` + +PowerShell 7 writes a byte-order mark too if a profile sets `$OutputEncoding = [Text.Encoding]::UTF8` (a string piped +into pbit then exits 2; seen with PowerShell 7.6); `$OutputEncoding = [System.Text.UTF8Encoding]::new($false)` fixes that. + +## For agents + +Coding agents with a shell tool (Claude Code, Codex, and other harnesses that can run commands) call pbit the same way a +script does: they run the command, read the JSON, and branch on the exit code. Nothing needs to be registered. A +paragraph like this one in the project's agent instructions (`CLAUDE.md`, `AGENTS.md`) is enough: + +> To route tasks to workers under hard rules (allowed workers, quotas) with odds, write a router document (README, "Use it +> from anything") and run `pbit decide --budget-ms 200 < doc.json`. Exit 0: act on `plan` for the ids in `released` and +> hand the ids in `escalated` to a human. Exit 1: no plan satisfies the rules. Exit 2: fix the document (the `error` +> object says where). Exit 3: escalate everything. Never read stderr as data. `pbit --help` lists every flag. + +Practical notes for agent use: + +- Bound the call: `--budget-ms` caps the sampler (with `--budget-ms 300` the 300-task demo answered in 420-495 ms on + every machine in BENCHMARK-MATRIX.md: the budget, then the gate and the polish), and `pbit run --deadline-ms N` bounds + a whole `run` call (a target, not a hard limit on a slow CPU: PORTABILITY.md). Small documents (about 12 tasks by 6 + workers) are answered exactly. +- Fixed work is reproducible: `--sweeps N --polish-ms 0` makes the answer a pure function of the document and `--seed`, + which is what a test or a replayed agent step wants. +- `pbit stats` prints the machine, the effective controls and a measured self-test; `--threads`, `--cpu-limit` and + (on Linux and macOS) `--priority low` keep it from crowding the agent's own process. +- An MCP server is planned for a later release and is not built yet; until then the shell command above is the + integration. diff --git a/examples/node/decide.mjs b/examples/node/decide.mjs new file mode 100644 index 0000000..1c979a2 --- /dev/null +++ b/examples/node/decide.mjs @@ -0,0 +1,87 @@ +// Call pbit from Node with no dependencies: one child process per call, JSON on stdin, JSON on stdout, the exit code mapped. +// node examples/node/decide.mjs # decides a 12-task demo document, then shows each exit code once +// node examples/node/decide.mjs router.json # decides your router document (README, "Use it from anything") +// The binary: $PBIT_BIN, else `pbit` on PATH. On Windows point PBIT_BIN at pbit.exe: Node cannot spawn the npm `pbit.cmd` +// shim without a shell. +import { execFileSync, spawn } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +const KIND = { 0: 'answer', 1: 'infeasible', 2: 'bad_input', 3: 'refused' }; + +/** + * Run `pbit ` with `input` (an object or JSON text; omit for none) on stdin. Resolves to {kind, exitCode, output, stderr}: + * 'answer' exit 0: verdict exact | diagnostics_passed | partial (act on output.released, escalate output.escalated) + * 'infeasible' exit 1: no plan satisfies the rules (a proof, not an error) + * 'bad_input' exit 2: output.error = {code, path, message} for a bad document; a bad flag leaves only a stderr line + * 'refused' exit 3: verdict refused / declined (escalate), or output.error.code === 'numeric' + * Rejects on a missing binary, a crash or signal, unparseable output, or after `timeoutMs`. + */ +export function pbit(args, input, { bin = process.env.PBIT_BIN || 'pbit', timeoutMs } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(bin, args, { stdio: ['pipe', 'pipe', 'pipe'] }); + const out = []; + const err = []; + const timer = timeoutMs && setTimeout(() => { child.kill(); reject(new Error(`pbit ${args[0]} passed ${timeoutMs} ms`)); }, timeoutMs); + child.stdout.on('data', (b) => out.push(b)); + child.stderr.on('data', (b) => err.push(b)); + child.on('error', (e) => { clearTimeout(timer); reject(e); }); // ENOENT: no binary + child.on('close', (code, signal) => { + clearTimeout(timer); + const stdout = Buffer.concat(out).toString('utf8'); + const stderr = Buffer.concat(err).toString('utf8').trim(); + if (!(code in KIND)) return reject(new Error(`pbit exited ${code ?? signal}: ${stderr.slice(0, 300)}`)); + let output = null; + try { + if (stdout.trim()) output = JSON.parse(stdout); + } catch { + return reject(new Error(`pbit ${args[0]}: unparseable output (exit ${code})`)); + } + resolve({ kind: KIND[code], exitCode: code, output, stderr }); + }); + child.stdin.on('error', () => {}); // EPIPE when pbit exits before reading all of it (a bad flag) + child.stdin.end(input === undefined ? undefined : typeof input === 'string' || Buffer.isBuffer(input) ? input : JSON.stringify(input)); + }); +} + +function describe(r) { + const o = r.output || {}; + if (r.kind === 'bad_input') return o.error ? `${o.error.code} at ${o.error.path}: ${o.error.message}` : r.stderr; + const plan = o.plan ? Object.entries(o.plan).slice(0, 3).map(([t, w]) => `${t}->${w}`).join(', ') : ''; + const released = Array.isArray(o.released) ? `, ${o.released.length} of ${o.tasks} released` : ''; + return `verdict ${o.verdict}${released}${plan ? `, plan ${plan}, ...` : ''}${o.reason ? ` (${o.reason})` : ''}`; +} + +async function main() { + const bin = process.env.PBIT_BIN || 'pbit'; + try { + console.log(`${execFileSync(bin, ['version'], { encoding: 'utf8' }).trim()} (${bin})`); + } catch (e) { + console.error(`cannot run ${bin} (${e.message}); install pbit or set PBIT_BIN`); + process.exit(127); + } + if (process.argv[2]) { + const r = await pbit(['decide', '--budget-ms', '200'], readFileSync(process.argv[2])); + console.log(`exit ${r.exitCode} ${r.kind}: ${describe(r)}`); + process.exit(r.exitCode); + } + const demo = (await pbit(['demo', '--tasks', '12'])).output; + const cases = [ + ['answer', 'a 12-task demo document', ['decide'], demo], + ['infeasible', 'two tasks, one worker with room for one', ['decide'], + { workers: [{ id: 'a', cap: 1 }], tasks: [{ id: 't1', allowed: ['a'], scores: { a: 1 } }, { id: 't2', allowed: ['a'], scores: { a: 1 } }] }], + ['bad_input', 'a malformed document', ['decide'], { workers: 1 }], + ['refused', 'exact enumeration of a 96-variable program, capped at 50 ms', ['run', '--op', 'exact', '--exact-ms', '50'], + readFileSync(new URL('../denoise-8x12.json', import.meta.url))], + ]; + let ok = true; + for (const [want, what, args, input] of cases) { + const r = await pbit(args, input); + ok &&= r.kind === want; + console.log(`exit ${r.exitCode} ${r.kind.padEnd(10)} ${what}: ${describe(r)}`); + } + console.log(ok ? 'all four exit codes mapped as expected' : 'UNEXPECTED exit-code mapping'); + process.exit(ok ? 0 : 1); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) main(); diff --git a/install.ps1 b/install.ps1 new file mode 100644 index 0000000..7873ba5 --- /dev/null +++ b/install.ps1 @@ -0,0 +1,100 @@ +# pbit installer for Windows (Windows PowerShell 5.1 or PowerShell 7): fetches a prebuilt release archive, checks its SHA-256 +# and installs pbit.exe. No administrator rights needed. +# +# irm https://raw.githubusercontent.com/BitmapAsset/pbit/main/install.ps1 | iex +# & ([scriptblock]::Create((irm https://raw.githubusercontent.com/BitmapAsset/pbit/main/install.ps1))) -Version v0.2.0 +# +# Parameters (each also read from the environment variable named after it): +# -Version PBIT_VERSION release tag, e.g. v0.2.0 (default: the latest release) +# -InstallDir PBIT_INSTALL_DIR where pbit.exe goes (default: $HOME\.local\bin) +# -DownloadBase PBIT_DOWNLOAD_BASE the archive is fetched from //pbit--.zip +# (default: https://github.com/BitmapAsset/pbit/releases/download); needs -Version +# -Target PBIT_TARGET Rust target triple to fetch instead of the detected one +# -AddToPath PBIT_ADD_TO_PATH=1 also add InstallDir to your user PATH (otherwise only this session's PATH) +param( + [string]$Version = $env:PBIT_VERSION, + [string]$InstallDir = $env:PBIT_INSTALL_DIR, + [string]$DownloadBase = $env:PBIT_DOWNLOAD_BASE, + [string]$Target = $env:PBIT_TARGET, + [switch]$AddToPath = ($env:PBIT_ADD_TO_PATH -eq '1') +) +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' # the progress bar makes Invoke-WebRequest many times slower on 5.1 +$Repo = 'BitmapAsset/pbit' +if ($PSVersionTable.PSVersion.Major -lt 6) { + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 +} + +if (-not $Target) { + $arch = if ($env:PROCESSOR_ARCHITEW6432) { $env:PROCESSOR_ARCHITEW6432 } else { $env:PROCESSOR_ARCHITECTURE } + switch ($arch) { + 'AMD64' { $Target = 'x86_64-pc-windows-msvc' } + 'ARM64' { $Target = 'x86_64-pc-windows-msvc'; Write-Host 'pbit install: no native arm64 build yet; installing the x86_64 build (Windows runs it under emulation)' } + default { throw "pbit install: no prebuilt pbit for Windows on $arch" } + } +} +$customBase = [bool]$DownloadBase +if (-not $DownloadBase) { $DownloadBase = "https://github.com/$Repo/releases/download" } +$DownloadBase = $DownloadBase.TrimEnd('/') +if (-not $Version) { + if ($customBase) { throw 'pbit install: set -Version (or PBIT_VERSION), e.g. v0.2.0, together with -DownloadBase' } + $Version = (Invoke-RestMethod -UseBasicParsing -Uri "https://api.github.com/repos/$Repo/releases/latest").tag_name + if (-not $Version) { throw 'pbit install: no published release found (set -Version)' } +} +if (-not $Version.StartsWith('v')) { $Version = "v$Version" } +if (-not $InstallDir) { $InstallDir = Join-Path $HOME '.local\bin' } + +$name = "pbit-$Version-$Target" +$asset = "$name.zip" +$url = "$DownloadBase/$Version/$asset" +$tmp = Join-Path ([IO.Path]::GetTempPath()) ("pbit-install-" + [guid]::NewGuid().ToString('N')) +New-Item -ItemType Directory -Path $tmp | Out-Null +try { + Write-Host "pbit install: $Version for $Target" + Write-Host " fetching $url" + $zip = Join-Path $tmp $asset + Invoke-WebRequest -UseBasicParsing -Uri $url -OutFile $zip + Invoke-WebRequest -UseBasicParsing -Uri "$url.sha256" -OutFile "$zip.sha256" + $want = ((Get-Content -LiteralPath "$zip.sha256" -Raw).Trim() -split '\s+')[0].ToLowerInvariant() + $got = (Get-FileHash -Algorithm SHA256 -LiteralPath $zip).Hash.ToLowerInvariant() + if ($want -notmatch '^[0-9a-f]{64}$') { throw "pbit install: unreadable checksum file $asset.sha256" } + if ($want -ne $got) { throw "pbit install: checksum mismatch for ${asset}: expected $want, got $got (nothing was installed)" } + Write-Host " sha256 ok $got" + + Expand-Archive -LiteralPath $zip -DestinationPath (Join-Path $tmp 'x') -Force + $src = Join-Path $tmp "x\$name\pbit.exe" + if (-not (Test-Path -LiteralPath $src)) { throw "pbit install: $asset has no $name\pbit.exe" } + New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + $dest = Join-Path $InstallDir 'pbit.exe' + Copy-Item -LiteralPath $src -Destination $dest -Force + $ran = & $dest version + if ($LASTEXITCODE -ne 0) { throw "pbit install: installed $dest, but it does not run here: $ran" } + Write-Host " installed $dest ($ran)" +} finally { + Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue +} + +$full = (Resolve-Path -LiteralPath $InstallDir).Path.TrimEnd('\') +$onPath = { param($p) @($p -split ';' | Where-Object { $_ } | ForEach-Object { $_.TrimEnd('\') }) -contains $full } +if (-not (& $onPath $env:Path)) { $env:Path = "$full;$env:Path" } # this session +$userPath = [Environment]::GetEnvironmentVariable('Path', 'User') +if (-not (& $onPath $userPath)) { + if ($AddToPath) { + [Environment]::SetEnvironmentVariable('Path', ($(if ($userPath) { "$full;$userPath" } else { $full })), 'User') + Write-Host " added $full to your user PATH (new terminals see it)" + } else { + Write-Host '' + Write-Host "$full is on this session's PATH only. To keep it, run (or re-run this installer with -AddToPath):" + Write-Host " [Environment]::SetEnvironmentVariable('Path', '$full;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User')" + } +} +Write-Host '' +Write-Host 'Next:' +Write-Host ' pbit version' +if ($PSVersionTable.PSEdition -ne 'Core') { + # Windows PowerShell 5.1 adds a byte-order mark to text it pipes into a program, and pbit 0.2.0 rejects it + Write-Host ' cmd /c "pbit demo --tasks 12 | pbit decide --pretty" (Windows PowerShell 5.1: let cmd pipe; or use pwsh)' +} else { + Write-Host ' pbit demo --tasks 12 | pbit decide --pretty' +} +Write-Host ' pbit stats --pretty' diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..ebd2b70 --- /dev/null +++ b/install.sh @@ -0,0 +1,144 @@ +#!/bin/sh +# pbit installer: fetches a prebuilt release archive, checks its SHA-256 and installs the `pbit` binary. Never uses sudo. +# +# curl -fsSL https://raw.githubusercontent.com/BitmapAsset/pbit/main/install.sh | sh +# curl -fsSL https://raw.githubusercontent.com/BitmapAsset/pbit/main/install.sh | PBIT_VERSION=v0.2.0 sh +# +# Environment (all optional): +# PBIT_VERSION release tag, e.g. v0.2.0 (default: the latest release) +# PBIT_INSTALL_DIR where `pbit` goes (default: /usr/local/bin if you can write there, else ~/.local/bin) +# PBIT_DOWNLOAD_BASE the archive is fetched from $PBIT_DOWNLOAD_BASE//pbit--.tar.gz +# (default: https://github.com/BitmapAsset/pbit/releases/download); needs PBIT_VERSION +# PBIT_TARGET Rust target triple to fetch instead of the detected one +# Needs curl or wget, tar, and one of sha256sum, shasum or openssl. +set -eu + +REPO="BitmapAsset/pbit" + +say() { printf '%s\n' "$*"; } +die() { printf 'pbit install: error: %s\n' "$*" >&2; exit 1; } +has() { command -v "$1" > /dev/null 2>&1; } + +fetch() { # fetch URL FILE + if has curl; then + case "$1" in + https://*) curl --proto '=https' --proto-redir '=https' -fsSL "$1" -o "$2" ;; + *) curl -fsSL "$1" -o "$2" ;; + esac + elif has wget; then + wget -q -O "$2" "$1" + else + die "need curl or wget" + fi +} + +sha256() { + if has sha256sum; then sha256sum "$1" | cut -d ' ' -f 1 + elif has shasum; then shasum -a 256 "$1" | cut -d ' ' -f 1 + elif has openssl; then openssl dgst -sha256 "$1" | sed 's/^.*= *//' + else die "need sha256sum, shasum or openssl to check the download" + fi +} + +detect_target() { + os=$(uname -s) + arch=$(uname -m) + case "$os" in + Darwin) + # a shell running under Rosetta reports x86_64 on Apple silicon: take the native build + if [ "$arch" = x86_64 ] && [ "$(sysctl -n sysctl.proc_translated 2> /dev/null || echo 0)" = 1 ]; then arch=arm64; fi + case "$arch" in + arm64 | aarch64) echo aarch64-apple-darwin ;; + x86_64) echo x86_64-apple-darwin ;; + *) die "no prebuilt pbit for macOS on $arch" ;; + esac + ;; + Linux) + if (ldd --version 2>&1 || true) | grep -qi musl; then + die "this system uses musl libc (Alpine?) and the release ships a glibc build only; build from source: cargo install --git https://github.com/$REPO pbit-cli" + fi + case "$arch" in + x86_64 | amd64) echo x86_64-unknown-linux-gnu ;; + *) die "no prebuilt pbit for Linux on $arch yet; build from source: cargo install --git https://github.com/$REPO pbit-cli" ;; + esac + ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) die "on Windows use install.ps1 (PowerShell)" ;; + *) die "no prebuilt pbit for $os; build from source: cargo install --git https://github.com/$REPO pbit-cli" ;; + esac +} + +latest_tag() { + api="https://api.github.com/repos/$REPO/releases/latest" + fetch "$api" "$tmp/latest.json" || die "could not look up the latest release at $api (set PBIT_VERSION)" + sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p' "$tmp/latest.json" | head -n 1 +} + +main() { + base="${PBIT_DOWNLOAD_BASE:-https://github.com/$REPO/releases/download}" + base="${base%/}" + target="${PBIT_TARGET:-}" + if [ -z "$target" ]; then target=$(detect_target) || exit 1; fi + + tmp=$(mktemp -d 2> /dev/null || mktemp -d -t pbit) + trap 'rm -rf "$tmp"' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + version="${PBIT_VERSION:-}" + if [ -z "$version" ]; then + [ -z "${PBIT_DOWNLOAD_BASE:-}" ] || die "set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE" + version=$(latest_tag) || exit 1 + [ -n "$version" ] || die "no published release found (set PBIT_VERSION)" + fi + case "$version" in v*) ;; *) version="v$version" ;; esac + + name="pbit-$version-$target" + asset="$name.tar.gz" + url="$base/$version/$asset" + say "pbit install: $version for $target" + say " fetching $url" + fetch "$url" "$tmp/$asset" || die "download failed: $url" + fetch "$url.sha256" "$tmp/$asset.sha256" || die "download failed: $url.sha256" + + want=$(cut -d ' ' -f 1 < "$tmp/$asset.sha256" | tr 'ABCDEF' 'abcdef') + got=$(sha256 "$tmp/$asset" | tr 'ABCDEF' 'abcdef') + case "$want" in *[!0-9a-f]* | '') die "unreadable checksum file $asset.sha256" ;; esac + [ "${#want}" -eq 64 ] || die "unreadable checksum file $asset.sha256" + [ "$want" = "$got" ] || die "checksum mismatch for $asset: expected $want, got $got (nothing was installed)" + say " sha256 ok $got" + + tar -xzf "$tmp/$asset" -C "$tmp" || die "could not unpack $asset" + [ -f "$tmp/$name/pbit" ] || die "$asset has no $name/pbit" + + if [ -n "${PBIT_INSTALL_DIR:-}" ]; then + dir="$PBIT_INSTALL_DIR" + elif [ -d /usr/local/bin ] && [ -w /usr/local/bin ]; then + dir=/usr/local/bin + else + dir="${HOME:?HOME is not set; set PBIT_INSTALL_DIR}/.local/bin" + fi + mkdir -p "$dir" 2> /dev/null || die "cannot create $dir (set PBIT_INSTALL_DIR to a directory you can write)" + [ -w "$dir" ] || die "cannot write to $dir (set PBIT_INSTALL_DIR to a directory you can write; this script never uses sudo)" + cp "$tmp/$name/pbit" "$dir/.pbit.$$" && chmod 755 "$dir/.pbit.$$" && mv -f "$dir/.pbit.$$" "$dir/pbit" \ + || die "could not install into $dir" + ran=$("$dir/pbit" version 2>&1) || die "installed $dir/pbit, but it does not run here: $ran" + say " installed $dir/pbit ($ran)" + + cmd=pbit + case ":$PATH:" in + *":$dir:"*) ;; + *) + cmd="$dir/pbit" + say "" + say "$dir is not on your PATH. Add it (e.g. in ~/.profile):" + say " export PATH=\"$dir:\$PATH\"" + ;; + esac + say "" + say "Next:" + say " $cmd version" + say " $cmd demo --tasks 12 | $cmd decide --pretty" + say " $cmd stats --pretty" +} + +main "$@" diff --git a/npm/LICENSE b/npm/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/npm/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/npm/README.md b/npm/README.md new file mode 100644 index 0000000..c90b3e7 --- /dev/null +++ b/npm/README.md @@ -0,0 +1,29 @@ +# pbit (npm) + +Installs the prebuilt `pbit` command-line binary: a virtual p-bit processor that returns joint decisions under hard +rules, with gated odds. JSON in, JSON out. Source, documentation and every benchmark: +. + +```sh +npm install -g pbit +pbit demo --tasks 12 | pbit decide --pretty +pbit stats --pretty +``` + +The install step fetches the release archive for your platform from GitHub Releases (macOS arm64 and x86_64, Linux +x86_64 with glibc, Windows x86_64), checks it against the `.sha256` file published beside it and keeps the binary inside +this package. No dependencies; Node >= 18. If the install step could not run (`--ignore-scripts`, offline), the first +`pbit` call fetches the binary. A checksum mismatch fails the install and installs nothing. + +| variable | effect | +|---|---| +| `PBIT_VERSION` | release tag to fetch (default: `v` + this package's version) | +| `PBIT_DOWNLOAD_BASE` | fetch `$PBIT_DOWNLOAD_BASE//pbit--.tar.gz` (`.zip` on Windows) instead of GitHub | +| `PBIT_TARGET` | Rust target triple to fetch instead of the detected one | +| `PBIT_BINARY` | at install: copy this binary instead of downloading; at run time: run this binary | + +`pbit`'s exit code passes through unchanged: 0 answer (`exact`, `diagnostics_passed`, `partial`), 1 `infeasible` +(a proof), 2 bad input or flag, 3 `refused` / `declined`. Calling it from Node code (spawn, JSON on stdin, exit codes +mapped): `examples/node/decide.mjs` in the repository; from other languages and agent harnesses: `docs/agents.md`. + +License: Apache-2.0. diff --git a/npm/bin/pbit.js b/npm/bin/pbit.js new file mode 100755 index 0000000..6b98aec --- /dev/null +++ b/npm/bin/pbit.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node +'use strict'; +// `pbit` for npm installs: runs the native binary with this process's stdin/stdout/stderr and exits with its exit code +// (0 answer, 1 infeasible, 2 bad input, 3 refused / declined; see `pbit decide --help`). PBIT_BINARY=/path/to/pbit +// runs that binary instead. A missing binary (an install with --ignore-scripts) is fetched on first run. +const fs = require('fs'); +const os = require('os'); +const { spawn } = require('child_process'); +const { install, binaryPath } = require('../install.js'); + +async function main() { + let bin = process.env.PBIT_BINARY || binaryPath(); + if (!process.env.PBIT_BINARY && !fs.existsSync(bin)) { + process.stderr.write('pbit: the binary is not installed yet, fetching it now\n'); + bin = await install(); + } + const child = spawn(bin, process.argv.slice(2), { stdio: 'inherit' }); + const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'].filter((s) => process.platform !== 'win32' || s !== 'SIGHUP'); + const forward = (s) => { try { child.kill(s); } catch (e) { /* already gone */ } }; + for (const s of signals) process.on(s, forward); + child.on('error', (e) => { + process.stderr.write(`pbit: cannot run ${bin}: ${e.message}\n`); + process.exit(127); + }); + child.on('exit', (code, signal) => { + if (signal) { + for (const s of signals) process.removeListener(s, forward); + process.exitCode = 128 + (os.constants.signals[signal] || 0); // if the signal is ignored here (SIGPIPE) + process.kill(process.pid, signal); // die the same way, so the shell sees 128 + n + return; + } + process.exit(code === null ? 1 : code); + }); +} + +main().catch((e) => { + process.stderr.write(`pbit: ${e.message}\n`); + process.exit(127); +}); diff --git a/npm/install.js b/npm/install.js new file mode 100644 index 0000000..7ab460c --- /dev/null +++ b/npm/install.js @@ -0,0 +1,113 @@ +'use strict'; +// Fetches the prebuilt pbit binary for this platform from the GitHub release that matches this package's version, checks its +// SHA-256 against the .sha256 file beside it and puts it in vendor/. Runs as `postinstall`; bin/pbit.js calls it again on +// first use if the binary is missing (installs with --ignore-scripts). No dependencies: Node >= 18 (global fetch) and the +// system `tar` (Windows 10+ ships tar.exe, which also unpacks .zip). +// +// Environment (same names as install.sh): +// PBIT_VERSION release tag to fetch (default: v) +// PBIT_DOWNLOAD_BASE archive URL = $PBIT_DOWNLOAD_BASE//pbit--. +// (default: https://github.com/BitmapAsset/pbit/releases/download) +// PBIT_TARGET Rust target triple to fetch instead of the detected one +// PBIT_BINARY path to an existing pbit binary: copied into vendor/ instead of downloading +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const { execFileSync } = require('child_process'); +const pkg = require('./package.json'); + +const REPO = 'BitmapAsset/pbit'; +const TARGETS = { + 'darwin-arm64': 'aarch64-apple-darwin', + 'darwin-x64': 'x86_64-apple-darwin', + 'linux-x64': 'x86_64-unknown-linux-gnu', + 'win32-x64': 'x86_64-pc-windows-msvc', + 'win32-arm64': 'x86_64-pc-windows-msvc', // x64 emulation on Windows on Arm +}; +const EXE = process.platform === 'win32' ? 'pbit.exe' : 'pbit'; + +function binaryPath() { + return path.join(__dirname, 'vendor', EXE); +} + +class ChecksumError extends Error {} + +async function get(url) { + const res = await fetch(url, { redirect: 'follow' }); + if (!res.ok) throw new Error(`GET ${url}: HTTP ${res.status}`); + return Buffer.from(await res.arrayBuffer()); +} + +// copy then rename, so a concurrent first run never sees a half-written binary +function place(src, dest) { + fs.mkdirSync(path.dirname(dest), { recursive: true }); + const tmp = `${dest}.${process.pid}.tmp`; + fs.copyFileSync(src, tmp); + fs.chmodSync(tmp, 0o755); + try { + fs.renameSync(tmp, dest); + } catch (e) { + fs.rmSync(tmp, { force: true }); + if (!fs.existsSync(dest)) throw e; // another process put it there first + } +} + +async function install(log = (m) => process.stderr.write(`${m}\n`)) { + const env = process.env; + const dest = binaryPath(); + if (env.PBIT_BINARY) { + const src = path.resolve(env.PBIT_BINARY); + place(src, dest); + log(`pbit: using ${src} (PBIT_BINARY)`); + return dest; + } + const target = env.PBIT_TARGET || TARGETS[`${process.platform}-${process.arch}`]; + if (!target) { + throw new Error(`no prebuilt pbit for ${process.platform}-${process.arch}; build one (cargo build --release -p pbit-cli) ` + + 'and reinstall with PBIT_BINARY=/path/to/pbit'); + } + let tag = env.PBIT_VERSION || `v${pkg.version}`; + if (!tag.startsWith('v')) tag = `v${tag}`; + const base = (env.PBIT_DOWNLOAD_BASE || `https://github.com/${REPO}/releases/download`).replace(/\/+$/, ''); + const zip = target.includes('windows'); + const name = `pbit-${tag}-${target}`; + const asset = `${name}.${zip ? 'zip' : 'tar.gz'}`; + const url = `${base}/${tag}/${asset}`; + log(`pbit: fetching ${url}`); + const [archive, sums] = await Promise.all([get(url), get(`${url}.sha256`)]); + const want = sums.toString('utf8').trim().split(/\s+/)[0].toLowerCase(); + const got = crypto.createHash('sha256').update(archive).digest('hex'); + if (!/^[0-9a-f]{64}$/.test(want) || want !== got) { + throw new ChecksumError(`checksum mismatch for ${asset}: expected ${want}, got ${got}`); + } + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pbit-npm-')); + try { + const file = path.join(tmp, asset); + fs.writeFileSync(file, archive); + // Windows: call System32\tar.exe (bsdtar, reads .zip) by full path; a GNU tar earlier on PATH (Git) cannot + const tar = process.platform === 'win32' ? path.join(env.SystemRoot || 'C:\\Windows', 'System32', 'tar.exe') : 'tar'; + execFileSync(tar, [zip ? '-xf' : '-xzf', file, '-C', tmp], { stdio: ['ignore', 'ignore', 'inherit'] }); + const src = path.join(tmp, name, EXE); + if (!fs.existsSync(src)) throw new Error(`${asset} has no ${name}/${EXE}`); + place(src, dest); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + log(`pbit: installed ${tag} (${target}), sha256 ${got}`); + return dest; +} + +module.exports = { install, binaryPath, ChecksumError }; + +if (require.main === module) { + install().catch((e) => { + if (e instanceof ChecksumError) { + process.stderr.write(`pbit: ${e.message}; nothing was installed\n`); + process.exit(1); + } + // offline or no release yet: do not fail the npm install; bin/pbit.js retries on first run + process.stderr.write(`pbit: could not fetch the binary now (${e.message}); it will be fetched on first run, ` + + 'or reinstall with PBIT_BINARY=/path/to/pbit\n'); + }); +} diff --git a/npm/package.json b/npm/package.json new file mode 100644 index 0000000..d15e369 --- /dev/null +++ b/npm/package.json @@ -0,0 +1,24 @@ +{ + "name": "pbit", + "version": "0.2.0", + "description": "pbit, a virtual p-bit processor for joint decisions under hard rules: installs the prebuilt pbit CLI (JSON in, JSON out).", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/BitmapAsset/pbit.git", + "directory": "npm" + }, + "homepage": "https://github.com/BitmapAsset/pbit#readme", + "bugs": "https://github.com/BitmapAsset/pbit/issues", + "keywords": ["p-bit", "probabilistic", "decision", "constraints", "sampling", "cli"], + "bin": { + "pbit": "bin/pbit.js" + }, + "scripts": { + "postinstall": "node install.js" + }, + "files": ["bin/", "install.js", "README.md", "LICENSE"], + "engines": { + "node": ">=18" + } +} diff --git a/pbit-cli/Cargo.toml b/pbit-cli/Cargo.toml index d5f40b8..26025bf 100644 --- a/pbit-cli/Cargo.toml +++ b/pbit-cli/Cargo.toml @@ -5,6 +5,9 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +readme = "../README.md" +keywords = ["p-bit", "cli", "decision", "constraints", "probabilistic"] +categories = ["command-line-utilities", "algorithms", "science"] [[bin]] name = "pbit" diff --git a/pbit-core/Cargo.toml b/pbit-core/Cargo.toml index 0550fe6..da68e65 100644 --- a/pbit-core/Cargo.toml +++ b/pbit-core/Cargo.toml @@ -5,5 +5,8 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +readme = "../README.md" +keywords = ["p-bit", "ising", "monte-carlo", "philox", "probabilistic"] +categories = ["algorithms", "science", "simulation"] [dependencies] diff --git a/pbit-decide/Cargo.toml b/pbit-decide/Cargo.toml index 6ac47df..c732e42 100644 --- a/pbit-decide/Cargo.toml +++ b/pbit-decide/Cargo.toml @@ -5,6 +5,9 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +readme = "../README.md" +keywords = ["p-bit", "assignment", "routing", "constraints", "probabilistic"] +categories = ["algorithms", "science"] [dependencies] pbit-core = { path = "../pbit-core" } diff --git a/pbit-ir/Cargo.toml b/pbit-ir/Cargo.toml index 7dfe68f..52a4858 100644 --- a/pbit-ir/Cargo.toml +++ b/pbit-ir/Cargo.toml @@ -5,6 +5,9 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +readme = "../README.md" +keywords = ["p-bit", "constraints", "gibbs-sampling", "probabilistic", "ising"] +categories = ["algorithms", "science", "mathematics"] [dependencies] pbit-core = { path = "../pbit-core" } diff --git a/scripts/bench/callers.ps1 b/scripts/bench/callers.ps1 new file mode 100644 index 0000000..2c2a4cd --- /dev/null +++ b/scripts/bench/callers.ps1 @@ -0,0 +1,62 @@ +# The PowerShell recipes of docs/agents.md, run as written, plus one call per exit code. `pbit` must be on PATH. +# pwsh -File scripts/bench/callers.ps1 powershell -ExecutionPolicy Bypass -File scripts/bench/callers.ps1 +# (No 2>$null on native calls: Windows PowerShell 5.1 turns redirected stderr into errors.) +$ErrorActionPreference = 'Continue' +"PowerShell $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition)), pbit at $((Get-Command pbit).Source)" +"`$OutputEncoding $($OutputEncoding.WebName) ($($OutputEncoding.GetPreamble().Length)-byte preamble), " + +"[Console]::InputEncoding $([Console]::InputEncoding.WebName) ($([Console]::InputEncoding.GetPreamble().Length)-byte preamble)" +# Windows PowerShell 5.1 adds a UTF-8 byte-order mark to text it pipes into a native program (runs 36969399069 and +# 36970631022, with $OutputEncoding us-ascii) and pbit 0.2.0 rejects it (exit 2). docs/agents.md: let cmd.exe pipe. +$desktop = $PSVersionTable.PSEdition -ne 'Core' +if ($desktop) { + pbit demo --tasks 12 | pbit decide | Out-Null + "as is: pbit demo --tasks 12 | pbit decide -> exit $LASTEXITCODE (2 = the byte-order mark was rejected)" +} + +if ($desktop) { + '> cmd /c "pbit demo --tasks 12 | pbit decide" | ConvertFrom-Json | Select-Object verdict, violations, ms' + cmd /c "pbit demo --tasks 12 | pbit decide" | ConvertFrom-Json | Select-Object verdict, violations, ms | Format-List | Out-String -Width 200 +} else { + '> pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms' + pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms | Format-List | Out-String -Width 200 +} +if ($LASTEXITCODE -ne 0) { throw "demo pipeline: exit $LASTEXITCODE" } + +pbit demo --tasks 24 | Set-Content -Encoding ascii router.json +if ($desktop) { + '> cmd /c "pbit decide --budget-ms 200 < router.json" | ConvertFrom-Json; $LASTEXITCODE' + $d = cmd /c "pbit decide --budget-ms 200 < router.json" | ConvertFrom-Json +} else { + '> Get-Content router.json -Raw | pbit decide --budget-ms 200 | ConvertFrom-Json; $LASTEXITCODE' + $d = Get-Content router.json -Raw | pbit decide --budget-ms 200 | ConvertFrom-Json +} +$code = $LASTEXITCODE +"verdict $($d.verdict), released $(@($d.released).Count) of $($d.tasks), violations $($d.violations); exit $code" +if ($code -ne 0) { throw "router.json: exit $code" } + +'> one call per exit code (infeasible, bad input, declined)' +'{"workers":[{"id":"a","cap":1}],"tasks":[{"id":"t1","allowed":["a"],"scores":{"a":1}},{"id":"t2","allowed":["a"],"scores":{"a":1}}]}' | Set-Content -Encoding ascii infeasible.json +'{"workers": 1}' | Set-Content -Encoding ascii bad.json +if ($desktop) { + cmd /c "pbit decide < infeasible.json" | Out-Null; $r1 = $LASTEXITCODE + cmd /c "pbit decide < bad.json" | Out-Null; $r2 = $LASTEXITCODE + cmd /c "pbit run --op exact --exact-ms 50 < examples\denoise-8x12.json" | Out-Null; $r3 = $LASTEXITCODE +} else { + Get-Content infeasible.json -Raw | pbit decide | Out-Null; $r1 = $LASTEXITCODE + Get-Content bad.json -Raw | pbit decide | Out-Null; $r2 = $LASTEXITCODE + Get-Content examples/denoise-8x12.json -Raw | pbit run --op exact --exact-ms 50 | Out-Null; $r3 = $LASTEXITCODE +} +"exit codes: infeasible $r1 (want 1), bad input $r2 (want 2), declined $r3 (want 3)" +if ($r1 -ne 1 -or $r2 -ne 2 -or $r3 -ne 3) { throw 'unexpected exit codes' } + +if ($desktop) { + # informational: does a BOM-free console input encoding stop 5.1 from adding the mark? + try { + [Console]::InputEncoding = [System.Text.UTF8Encoding]::new($false) + pbit demo --tasks 12 | pbit decide | Out-Null + "probe (informational): [Console]::InputEncoding = UTF8Encoding(`$false), then pbit demo --tasks 12 | pbit decide -> exit $LASTEXITCODE" + } catch { + "probe (informational): [Console]::InputEncoding cannot be set here: $($_.Exception.Message)" + } +} +'PowerShell recipes: all passed' diff --git a/scripts/bench/matrix_bench.py b/scripts/bench/matrix_bench.py new file mode 100644 index 0000000..0866d19 --- /dev/null +++ b/scripts/bench/matrix_bench.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""One row of BENCHMARK-MATRIX.md: machine facts, build facts and pbit timings for one OS/arch, written as one JSON file. + + python3 scripts/bench/matrix_bench.py --bin target/release/pbit --label linux-x86_64 --out bench-linux-x86_64.json \ + [--native-dir target-native/release] [--runs 5] [--attach KEY=FILE ...] + +Stdlib only, Python >= 3.8, Linux / macOS / Windows. Run from the repo root after +`cargo build --release --locked --workspace` and `cargo build --release --locked -p pbit-core --example kernels`. +`--native-dir` names a `-C target-cpu=native` build of the same two targets; with it, bench/portable_vs_native.py runs too. + +Every timing is N runs (default 5): median [IQR] (statistics.quantiles, n=4, as in bench/*.py) plus every raw value. +Wall clock is taken outside pbit (time.perf_counter around the processes); `pbit_ms` is what pbit reports itself. +Peak RSS of the 300-task decide: /usr/bin/time -l (macOS, "maximum resident set size", bytes), /usr/bin/time -v (Linux, +"Maximum resident set size", KiB), PeakWorkingSetSize from K32GetProcessMemoryInfo on the process handle (Windows; the +handle is still open after the wait). Absolute paths of this machine are scrubbed from the output. +""" +import argparse, json, os, platform, re, shutil, statistics, subprocess, sys, tempfile, time + +KLINE = re.compile(r"^(.+?)\s*:\s*([0-9.]+e[+-]?[0-9]+|[0-9.]+) updates/s") +NUM = r"([0-9.]+(?:e[+-]?[0-9]+)?)" +PVN = re.compile(r"^(.+?): portable " + NUM + r" \[" + NUM + "-" + NUM + r"\] \| native " + NUM + r" \[" + NUM + "-" + NUM + + r"\] \| ratio ([0-9.]+)$") +DT = re.compile(r"^--threads (\d+): wall ms ([0-9.]+) \[([0-9.]+)-([0-9.]+)\], CPU ms ([0-9.]+), site updates/s ([0-9.]+) M;" + r" answer identical: yes; verdict (\S+)$") +DD = re.compile(r"^default path \(--budget-ms 300, 4 chains, threads (\d+)\): wall ms ([0-9.]+) \[([0-9.]+)-([0-9.]+)\]," + r" sweeps (\d+), verdict (\S+)$") +# BENCHMARKS.md §1 rows, from pbit-core/examples/kernels.rs +KERNEL_CELLS = {"kernel_multispin_fast_4t": "fast multispin splitmix x 4 thr", "kernel_multispin_fast_10t": "fast multispin splitmix x10 thr", + "kernel_multispin_fast_1t": "fast T0 multispin, splitmix", "kernel_heatbath_fast_1t": "fast T1 f32 LUT, splitmix"} + + +def sh(args, input=None, env=None, timeout=900): + try: + p = subprocess.run(args, input=input, capture_output=True, text=True, env=env, timeout=timeout) + return p.returncode, p.stdout, p.stderr + except Exception as e: # tool missing, timeout + return None, "", "%s: %s" % (type(e).__name__, e) + + +def summary(xs): + xs = [x for x in xs if isinstance(x, (int, float))] + if not xs: + return None + q = statistics.quantiles(xs, n=4) if len(xs) > 1 else [xs[0]] * 3 + return {"median": statistics.median(xs), "q1": q[0], "q3": q[2], "min": min(xs), "max": max(xs), "n": len(xs), "values": xs} + + +def load(): + up = sh(["uptime"])[1].strip() if shutil.which("uptime") else None + return {"loadavg": list(os.getloadavg()) if hasattr(os, "getloadavg") else None, "uptime": up, "t": time.strftime("%H:%M:%S")} + + +def exe(path): + return path + ".exe" if platform.system() == "Windows" and not path.endswith(".exe") else path + + +def machine(): + m = {"system": platform.system(), "os_release": platform.release(), "arch": platform.machine(), + "python": platform.python_version(), "logical_cores": os.cpu_count()} + s = m["system"] + if s == "Darwin": + rc, out, _ = sh(["sysctl", "-n", "machdep.cpu.brand_string", "hw.ncpu", "hw.memsize"]) + v = out.strip().splitlines() + m["cpu_cmd"] = "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize" + m["cpu_model"] = v[0] if v else None + m["hw_ncpu"] = int(v[1]) if len(v) > 1 else None + m["memory_bytes"] = int(v[2]) if len(v) > 2 else None + rc, out, _ = sh(["sysctl", "-n", "hw.perflevel0.logicalcpu", "hw.perflevel1.logicalcpu"]) + if rc == 0: + v = out.split(); m["perf_cores"], m["efficiency_cores"] = int(v[0]), (int(v[1]) if len(v) > 1 else 0) + rc, out, _ = sh(["sysctl", "-n", "machdep.cpu.features", "machdep.cpu.leaf7_features"]) + if rc == 0: + f = set(out.lower().split()); m["cpu_flags"] = {k: k in f for k in ["sse4.2", "avx1.0", "avx2", "fma", "bmi2", "avx512f"]} + rc, out, _ = sh(["sysctl", "-n", "sysctl.proc_translated"]) + m["rosetta_translated"] = out.strip() == "1" if rc == 0 else None + m["os_version"] = sh(["sw_vers", "-productVersion"])[1].strip() + elif s == "Linux": + rc, out, _ = sh(["lscpu"]) + kv = dict((a.strip(), b.strip()) for a, b in (l.split(":", 1) for l in out.splitlines() if ":" in l)) + m["cpu_cmd"] = "lscpu" + m["cpu_model"] = kv.get("Model name") + m["lscpu"] = {k: kv.get(k) for k in ["Architecture", "CPU(s)", "Thread(s) per core", "Core(s) per socket", "Socket(s)", + "Vendor ID", "Model name", "Hypervisor vendor", "L3"] if k in kv} + f = set(kv.get("Flags", "").split()) + m["cpu_flags"] = {k: k in f for k in ["sse4_2", "avx", "avx2", "fma", "bmi2", "avx512f", "avx512bw", "avx512vl"]} + try: + with open("/proc/meminfo") as fh: + m["memory_bytes"] = int(re.search(r"MemTotal:\s+(\d+) kB", fh.read()).group(1)) * 1024 + with open("/etc/os-release") as fh: + m["os_version"] = re.search(r'PRETTY_NAME="([^"]*)"', fh.read()).group(1) + except Exception as e: + m["meminfo_error"] = str(e) + m["libc"] = " ".join(platform.libc_ver()) + elif s == "Windows": + ps = ("$p = Get-CimInstance Win32_Processor | Select-Object -First 1; $c = Get-CimInstance Win32_ComputerSystem; " + "$o = Get-CimInstance Win32_OperatingSystem; [pscustomobject]@{Name = $p.Name; Cores = $p.NumberOfCores; " + "Logical = $p.NumberOfLogicalProcessors; Mem = $c.TotalPhysicalMemory; Os = $o.Caption; Build = $o.BuildNumber} | ConvertTo-Json") + rc, out, err = sh(["powershell", "-NoProfile", "-NonInteractive", "-Command", ps]) + m["cpu_cmd"] = "Get-CimInstance Win32_Processor" + try: + j = json.loads(out) + m.update(cpu_model=(j["Name"] or "").strip(), physical_cores=j["Cores"], cim_logical=j["Logical"], + memory_bytes=int(j["Mem"]), os_version="%s (build %s)" % (j["Os"], j["Build"])) + except Exception as e: + m["cim_error"] = "%s %s" % (e, err[:300]) + return m + + +def runner(): + keys = ["RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "GITHUB_JOB", "GITHUB_SHA", + "GITHUB_REF_NAME", "GITHUB_WORKFLOW"] + return {k: os.environ[k] for k in keys if k in os.environ} or None + + +def toolchain(): + rc, out, _ = sh(["rustc", "-vV"]) + t = dict((a.strip(), b.strip()) for a, b in (l.split(":", 1) for l in out.splitlines() if ":" in l)) + t["rustc"] = out.splitlines()[0] if out else None + t["cargo"] = sh(["cargo", "-V"])[1].strip() + return t + + +def linkage(bin_): + s = platform.system(); r = {} + if shutil.which("file"): + r["file"] = sh(["file", "-b", bin_])[1].strip() + if s == "Darwin": + r["otool_L"] = sh(["otool", "-L", bin_])[1].strip().splitlines()[1:] + r["lipo_archs"] = sh(["lipo", "-archs", bin_])[1].strip() + elif s == "Linux": + rc, out, err = sh(["ldd", bin_]); r["ldd"] = (out + err).strip().splitlines() + rc, out, _ = sh(["objdump", "-T", bin_]) + vers = sorted(set(re.findall(r"GLIBC_([0-9.]+)", out)), key=lambda v: [int(x) for x in v.split(".")]) + r["glibc_symbol_versions"] = vers + r["min_glibc"] = vers[-1] if vers else None + elif s == "Windows": + dump, how = shutil.which("dumpbin"), "PATH" + if not dump: + vsw = os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), "Microsoft Visual Studio", "Installer", "vswhere.exe") + if os.path.exists(vsw): + rc, out, _ = sh([vsw, "-latest", "-products", "*", "-find", r"VC\Tools\MSVC\**\bin\Hostx64\x64\dumpbin.exe"]) + cand = [l.strip() for l in out.splitlines() if l.strip().lower().endswith("dumpbin.exe")] + dump, how = (cand[0], "vswhere (not on PATH)") if cand else (None, None) + r["dumpbin_found_via"] = how if dump else "not found: dumpbin is not on PATH and vswhere found none" + if dump: + rc, out, err = sh([dump, "/nologo", "/dependents", bin_]) + r["dll_imports"] = [l.strip() for l in out.splitlines() if l.strip().lower().endswith(".dll")] + r["dumpbin_rc"] = rc + return r + + +def run_json(args, input=None): + rc, out, err = sh(args, input=input) + try: + return rc, json.loads(out), err + except Exception: + return rc, None, (err or "")[-500:] + " | stdout: " + (out or "")[:300] + + +def pipeline(bin_, demo_args, decide_args): + t0 = time.perf_counter() + p1 = subprocess.Popen([bin_, "demo"] + demo_args, stdout=subprocess.PIPE) + p2 = subprocess.Popen([bin_, "decide"] + decide_args, stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p1.stdout.close() + out, err = p2.communicate() + rc1 = p1.wait() + wall = (time.perf_counter() - t0) * 1e3 + try: + d = json.loads(out.decode("utf-8")) + except Exception: + d = None + return wall, rc1, p2.returncode, d, err.decode("utf-8", "replace")[-300:] + + +def win_peak(handle): + import ctypes + from ctypes import wintypes + + class PMC(ctypes.Structure): + _fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD)] + [(n, ctypes.c_size_t) for n in ( + "PeakWorkingSetSize", "WorkingSetSize", "QuotaPeakPagedPoolUsage", "QuotaPagedPoolUsage", "QuotaPeakNonPagedPoolUsage", + "QuotaNonPagedPoolUsage", "PagefileUsage", "PeakPagefileUsage")] + k32 = ctypes.WinDLL("kernel32", use_last_error=True) + k32.K32GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(PMC), wintypes.DWORD] + k32.K32GetProcessMemoryInfo.restype = wintypes.BOOL + pmc = PMC(); pmc.cb = ctypes.sizeof(PMC) + if not k32.K32GetProcessMemoryInfo(int(handle), ctypes.byref(pmc), pmc.cb): + raise OSError("K32GetProcessMemoryInfo failed, error %d" % ctypes.get_last_error()) + return pmc.PeakWorkingSetSize, pmc.PeakPagefileUsage + + +def peak_rss(bin_, doc, args): + """One run of `pbit decide args < doc` under the OS tool: (bytes, method, pbit's own peak_rss_mb, verdict, extra).""" + s = platform.system(); extra = {} + with open(doc, "rb") as f: + if s == "Windows": + p = subprocess.Popen([bin_, "decide"] + args, stdin=f, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + peak, peak_commit = win_peak(p._handle) + extra["peak_pagefile_bytes"] = peak_commit + method = "K32GetProcessMemoryInfo PeakWorkingSetSize (process handle, after exit)" + val = peak + elif s == "Darwin" or os.path.exists("/usr/bin/time"): + flag = "-l" if s == "Darwin" else "-v" + p = subprocess.run(["/usr/bin/time", flag, bin_, "decide"] + args, stdin=f, capture_output=True) + out, err = p.stdout, p.stderr + if s == "Darwin": + m = re.search(rb"(\d+)\s+maximum resident set size", err); val = int(m.group(1)) if m else None + fp = re.search(rb"(\d+)\s+peak memory footprint", err) + if fp: extra["peak_memory_footprint_bytes"] = int(fp.group(1)) + method = "/usr/bin/time -l (maximum resident set size, bytes)" + else: + m = re.search(rb"Maximum resident set size \(kbytes\): (\d+)", err); val = int(m.group(1)) * 1024 if m else None + method = "/usr/bin/time -v (Maximum resident set size, KiB x 1024)" + else: + p = subprocess.run([bin_, "decide"] + args, stdin=f, capture_output=True) + out, val, method = p.stdout, None, "none: /usr/bin/time missing" + try: + d = json.loads(out.decode("utf-8")) + except Exception: + d = {} + tel = d.get("telemetry") or {} + return val, method, tel.get("peak_rss_mb"), d.get("verdict"), extra + + +def parse_kernels(text): + r = {} + for line in text.splitlines(): + m = KLINE.match(line) + if m: + r[m.group(1).strip()] = float(m.group(2)) + if line.startswith("chk"): + r["chk"] = line.split()[1] + return r + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--bin", required=True); ap.add_argument("--label", required=True); ap.add_argument("--out", required=True) + ap.add_argument("--native-dir"); ap.add_argument("--runs", type=int, default=5) + ap.add_argument("--attach", action="append", default=[], help="KEY=FILE: embed FILE (JSON, else text) under attached.KEY") + ap.add_argument("--note", action="append", default=[]) + a = ap.parse_args() + bin_ = exe(a.bin); pdir = os.path.dirname(bin_) or "." + N = a.runs; t_start = time.time() + R = {"schema": "pbit-bench-matrix/1", "label": a.label, "date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "git_sha": sh(["git", "rev-parse", "HEAD"])[1].strip() or None, "runner": runner(), "notes": a.note, + "machine": machine(), "toolchain": toolchain(), "load": {"start": load()}, "errors": []} + print("machine:", R["machine"].get("cpu_model"), "| logical cores", R["machine"]["logical_cores"], "| load", R["load"]["start"]["loadavg"], flush=True) + + R["binary"] = {"path": a.bin, "size_bytes": os.path.getsize(bin_), "version": sh([bin_, "version"])[1].strip(), "linkage": linkage(bin_)} + + # pbit stats: the IR self-test (400-spin ring, fixed work), 1 thread and default threads + st = [] + for _ in range(N): + rc, j, err = run_json([bin_, "stats"]) + if j is None: + R["errors"].append("stats rc=%s %s" % (rc, err)); continue + st.append(j) + if st: + R["stats_first"] = {k: st[0].get(k) for k in ("engine", "machine", "build", "controls")} + R["stats"] = {"site_updates_per_s_1_thread": summary([j["self_test"]["site_updates_per_s_1_thread"] for j in st]), + "site_updates_per_s": summary([j["self_test"]["site_updates_per_s"] for j in st]), + "threads": sorted(set(j["self_test"]["threads"] for j in st)), + "speedup": summary([j["self_test"]["speedup"] for j in st])} + R["load"]["after_stats"] = load() + + # pbit demo --tasks 12 | pbit decide, and pbit demo --tasks 300 | pbit decide --budget-ms 300 + for key, demo, dec in [("decide_12", ["--tasks", "12"], []), ("decide_300", ["--tasks", "300"], ["--budget-ms", "300"])]: + runs = [] + for _ in range(N): + wall, rc1, rc2, d, err = pipeline(bin_, demo, dec) + d = d or {}; tel = d.get("telemetry") or {} + runs.append({"wall_ms": wall, "demo_exit": rc1, "decide_exit": rc2, "pbit_ms": d.get("ms"), "verdict": d.get("verdict"), + "tier": d.get("tier"), "released": len(d.get("released") or []), "escalated": len(d.get("escalated") or []), + "violations": d.get("violations"), "sweeps": tel.get("sweeps"), "site_updates_per_s": tel.get("site_updates_per_s"), + "threads": tel.get("threads"), "peak_rss_mb_self": tel.get("peak_rss_mb"), "stderr": err or None}) + R[key] = {"cmd": "pbit demo %s | pbit decide %s" % (" ".join(demo), " ".join(dec)), "runs": runs, + "wall_ms": summary([r["wall_ms"] for r in runs]), "pbit_ms": summary([r["pbit_ms"] for r in runs]), + "released": summary([r["released"] for r in runs]), "violations_max": max([r["violations"] or 0 for r in runs]), + "verdicts": sorted(set(str(r["verdict"]) for r in runs)), "exit_codes": sorted(set(r["decide_exit"] for r in runs)), + "site_updates_per_s": summary([r["site_updates_per_s"] for r in runs])} + R["load"]["after_decide"] = load() + + # peak RSS of the 300-task decide (3 runs; the document is generated once) + tmp = tempfile.mkdtemp(prefix="pbit-bench-") + doc = os.path.join(tmp, "demo300.json") + with open(doc, "w") as f: + f.write(sh([bin_, "demo", "--tasks", "300"])[1]) + rss = [] + for _ in range(3): + try: + rss.append(peak_rss(bin_, doc, ["--budget-ms", "300"])) + except Exception as e: + R["errors"].append("peak_rss: %s: %s" % (type(e).__name__, e)) + if rss: + R["peak_rss_300"] = {"cmd": "pbit decide --budget-ms 300 < demo300.json", "method": rss[0][1], + "bytes": summary([r[0] for r in rss]), "self_reported_peak_rss_mb": [r[2] for r in rss], + "verdicts": [r[3] for r in rss], "extra": [r[4] for r in rss]} + + # lattice kernels (pbit-core/examples/kernels.rs), portable build, N runs + kbin = exe(os.path.join(pdir, "examples", "kernels")) + if os.path.exists(kbin): + ks = [] + for _ in range(N): + rc, out, err = sh([kbin]) + if rc == 0: + ks.append(parse_kernels(out)) + else: + R["errors"].append("kernels rc=%s %s" % (rc, err[-300:])) + if ks: + R["kernels"] = {"cmd": os.path.relpath(kbin).replace(os.sep, "/"), "checksums": sorted(set(k.get("chk") for k in ks)), + "lines": {name: summary([k.get(name) for k in ks]) for name in ks[0] if name != "chk"}} + else: + R["kernels"] = {"skipped": "no kernels example at %s (build it with -p pbit-core --example kernels)" % os.path.relpath(kbin)} + R["load"]["after_kernels"] = load() + + # bench/portable_vs_native.py (unchanged; its header line names the M4 whatever the machine) + if a.native_dir: + if platform.system() == "Windows": + # the script runs /pbit and /examples/kernels; CreateProcess appends no .exe to a name with a path + R["windows_shim"] = [] + for d in (pdir, a.native_dir): + for name in ("pbit", os.path.join("examples", "kernels")): + src, dst = os.path.join(d, name + ".exe"), os.path.join(d, name) + if os.path.exists(src) and not os.path.exists(dst): + shutil.copyfile(src, dst); R["windows_shim"].append(os.path.relpath(dst).replace(os.sep, "/")) + env = dict(os.environ, PBIT_DIR=pdir, PBIT_NATIVE_DIR=a.native_dir) + rc, out, err = sh([sys.executable, "bench/portable_vs_native.py"], env=env, timeout=1800) + rows = {} + for line in out.splitlines(): + m = PVN.match(line) + if m: + g = m.groups() + rows[g[0]] = {"portable": float(g[1]), "portable_iqr": [float(g[2]), float(g[3])], "native": float(g[4]), + "native_iqr": [float(g[5]), float(g[6])], "ratio": float(g[7])} + R["portable_vs_native"] = {"cmd": "PBIT_DIR=%s PBIT_NATIVE_DIR=%s python3 bench/portable_vs_native.py" % ( + os.path.relpath(pdir).replace(os.sep, "/"), os.path.relpath(a.native_dir).replace(os.sep, "/")), "exit": rc, + "rows": rows, "stdout": out.splitlines(), "stderr_tail": err[-1500:] or None, + "note": "the script's first line is hard-coded to the M4; the machine is the one in `machine`"} + else: + R["portable_vs_native"] = {"skipped": "no --native-dir"} + R["load"]["after_pvn"] = load() + + # bench/decide_threads.py (unchanged; same hard-coded header) + rc, out, err = sh([sys.executable, "bench/decide_threads.py"], env=dict(os.environ, PBIT=bin_), timeout=1800) + th = {} + for line in out.splitlines(): + m = DT.match(line) + if m: + g = m.groups(); th["threads_%s" % g[0]] = {"wall_ms": float(g[1]), "wall_iqr": [float(g[2]), float(g[3])], "cpu_ms": float(g[4]), + "site_updates_per_s_M": float(g[5]), "verdict": g[6]} + m = DD.match(line) + if m: + g = m.groups(); th["default"] = {"threads": int(g[0]), "wall_ms": float(g[1]), "wall_iqr": [float(g[2]), float(g[3])], + "sweeps": int(g[4]), "verdict": g[5]} + R["decide_threads"] = {"cmd": "PBIT=%s python3 bench/decide_threads.py" % a.bin, "exit": rc, "rows": th, "stdout": out.splitlines(), + "stderr_tail": err[-1500:] or None} + R["load"]["end"] = load() + + for kv in a.attach: + k, _, path = kv.partition("=") + try: + with open(path) as f: + txt = f.read() + try: + R.setdefault("attached", {})[k] = json.loads(txt) + except ValueError: + R.setdefault("attached", {})[k] = txt.splitlines() + except OSError as e: + R.setdefault("attached", {})[k] = "missing: %s" % e + + # the cells BENCHMARK-MATRIX.md shows (each one also lives in full above) + C = {"cpu_model": R["machine"].get("cpu_model"), "logical_cores": R["machine"]["logical_cores"], + "binary_size_bytes": R["binary"]["size_bytes"]} + if "stats" in R: + C["stats_1t"] = R["stats"]["site_updates_per_s_1_thread"]; C["stats_default"] = R["stats"]["site_updates_per_s"] + C["stats_default_threads"] = R["stats"]["threads"] + for name, line in KERNEL_CELLS.items(): + C[name] = (R.get("kernels", {}).get("lines") or {}).get(line) + for key in ("decide_12", "decide_300"): + C[key + "_wall_ms"] = R[key]["wall_ms"]; C[key + "_pbit_ms"] = R[key]["pbit_ms"]; C[key + "_verdicts"] = R[key]["verdicts"] + C[key + "_exit_codes"] = R[key]["exit_codes"] + C["decide_300_released"] = R["decide_300"]["released"]; C["decide_300_violations_max"] = R["decide_300"]["violations_max"] + C["decide_300_site_updates_per_s"] = R["decide_300"]["site_updates_per_s"] + if "peak_rss_300" in R: + C["decide_300_peak_rss_bytes"] = R["peak_rss_300"]["bytes"]; C["decide_300_peak_rss_method"] = R["peak_rss_300"]["method"] + pv = R["portable_vs_native"].get("rows") or {} + C["native_over_portable"] = {k: v["ratio"] for k, v in pv.items()} or R["portable_vs_native"].get("skipped") or "n/a: exit %s" % R["portable_vs_native"].get("exit") + C["threads_rows"] = th or "n/a: exit %s, %s" % (rc, (err.strip().splitlines() or [""])[-1][:200]) + R["cells"] = C + R["elapsed_s"] = round(time.time() - t_start, 1) + + text = json.dumps(R, indent=1) + home = os.path.expanduser("~") + for p, rep in ((os.getcwd(), "."), (home, "~")): # no absolute paths of this machine in the output + if len(p) > 3: + text = text.replace(json.dumps(p)[1:-1], rep).replace(p, rep) + with open(a.out, "w") as f: + f.write(text + "\n") + print(json.dumps(C, indent=1)) + print("wrote", a.out, "in", R["elapsed_s"], "s;", len(R["errors"]), "errors", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/bench/package_like_release.sh b/scripts/bench/package_like_release.sh new file mode 100755 index 0000000..32a1047 --- /dev/null +++ b/scripts/bench/package_like_release.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Pack a local pbit binary the way .github/workflows/release.yml's "package" step does (same names, layout and .sha256), +# so install.sh, install.ps1 and the npm wrapper can be tested before any release exists. Keep in sync with release.yml. +# scripts/bench/package_like_release.sh +# Writes /pbit--.tar.gz (.zip for *-windows-*, with 7z as release.yml does) and the .sha256 beside it. +set -eu +BIN=$1 TARGET=$2 TAG=$3 OUT=$4 +NAME="pbit-$TAG-$TARGET" +STAGE=$(mktemp -d 2> /dev/null || mktemp -d -t pbitpkg) +mkdir -p "$STAGE/$NAME" "$OUT" +cp "$BIN" "$STAGE/$NAME/" +cp README.md LICENSE CHANGELOG.md "$STAGE/$NAME/" +cp -R examples python docs "$STAGE/$NAME/" +rm -rf "$STAGE/$NAME/python/__pycache__" +case "$TARGET" in + *windows*) ASSET="$NAME.zip"; (cd "$STAGE" && 7z a "$ASSET" "$NAME" > /dev/null) ;; + *) ASSET="$NAME.tar.gz"; (cd "$STAGE" && tar czf "$ASSET" "$NAME") ;; +esac +mv "$STAGE/$ASSET" "$OUT/" +rm -rf "$STAGE" +cd "$OUT" +if command -v sha256sum > /dev/null; then sha256sum "$ASSET" > "$ASSET.sha256"; else shasum -a 256 "$ASSET" > "$ASSET.sha256"; fi +echo "$OUT/$ASSET" diff --git a/scripts/bench/peak_rss_windows.ps1 b/scripts/bench/peak_rss_windows.ps1 new file mode 100644 index 0000000..9e151cd --- /dev/null +++ b/scripts/bench/peak_rss_windows.ps1 @@ -0,0 +1,29 @@ +# Peak working set of `pbit decide --budget-ms 300` on the 300-task demo, read with Get-Process every ~5 ms until the process +# exits: a lower bound (the last sample before exit). Cross-check of the exact value matrix_bench.py reads with +# K32GetProcessMemoryInfo on the process handle. +# pwsh -File scripts/bench/peak_rss_windows.ps1 [-Exe target\release\pbit.exe] [-Out peak-rss-powershell.json] [-Runs 3] +param([string]$Exe = 'target\release\pbit.exe', [string]$Out = 'peak-rss-powershell.json', [int]$Runs = 3) +$ErrorActionPreference = 'Stop' +$exePath = (Resolve-Path -LiteralPath $Exe).Path +$doc = Join-Path ([IO.Path]::GetTempPath()) 'pbit-demo300.json' +& $exePath demo --tasks 300 | Set-Content -Encoding ascii -LiteralPath $doc +$rows = foreach ($i in 1..$Runs) { + $o = Join-Path ([IO.Path]::GetTempPath()) "pbit-d300-$i.json" + $p = Start-Process -FilePath $exePath -ArgumentList 'decide', '--budget-ms', '300' -RedirectStandardInput $doc ` + -RedirectStandardOutput $o -NoNewWindow -PassThru + $peak = [long]0; $samples = 0 + while (-not $p.HasExited) { + $g = Get-Process -Id $p.Id -ErrorAction SilentlyContinue + if ($g) { $samples++; if ($g.PeakWorkingSet64 -gt $peak) { $peak = $g.PeakWorkingSet64 } } + Start-Sleep -Milliseconds 5 + } + $verdict = (Get-Content -Raw -LiteralPath $o | ConvertFrom-Json).verdict + [pscustomobject]@{ peak_working_set_bytes = $peak; samples = $samples; verdict = $verdict } +} +[pscustomobject]@{ + method = 'Get-Process PeakWorkingSet64, polled every ~5 ms until exit (lower bound)' + cmd = 'pbit decide --budget-ms 300 < demo300.json' + powershell = "$($PSVersionTable.PSVersion)" + runs = @($rows) +} | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $Out +Get-Content -LiteralPath $Out diff --git a/scripts/bench/render_matrix.py b/scripts/bench/render_matrix.py new file mode 100644 index 0000000..f8189d9 --- /dev/null +++ b/scripts/bench/render_matrix.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Render the BENCHMARK-MATRIX.md tables from the JSON files in a results directory (every cell comes from one of them). + + python3 scripts/bench/render_matrix.py scripts/bench/results/2026-10-01 > /tmp/tables.md +""" +import json, os, sys + +D = sys.argv[1] +RUN1 = "run-36969399069/" # first run: the only one with the macOS Intel row +RUN2 = "run-36970631022/" # second run, same source: the standard rows and the cross job +ROWS = [ # (row label, bench JSON, test JSON or None, key inside cross-linux-x86_64.json "bench" when nested); paths relative to D + ("macOS arm64, Apple M4 (local)", "local-m4/bench-macos-arm64-m4-local.json", None, None), + ("Linux x86_64 (`ubuntu-latest`)", RUN2 + "bench-linux-x86_64.json", RUN2 + "test-linux-x86_64.json", None), + ("Linux x86_64, static musl (`ubuntu-latest`)", RUN2 + "cross-linux-x86_64.json", None, "linux-x86_64-musl"), + ("Linux x86_64, glibc, same VM as musl", RUN2 + "cross-linux-x86_64.json", None, "linux-x86_64-gnu-samevm"), + ("macOS arm64 (`macos-latest`)", RUN2 + "bench-macos-arm64.json", RUN2 + "test-macos-arm64.json", None), + ("Windows x86_64 (`windows-latest`)", RUN2 + "bench-windows-x86_64.json", RUN2 + "test-windows-x86_64.json", None), + ("macOS x86_64 (`macos-15-intel`, run 1 only)", RUN1 + "bench-macos-x86_64.json", RUN1 + "test-macos-x86_64.json", None), +] +PENDING = ["Linux x86_64, WSL2 on a Windows desktop with an RTX 4070", "Linux x86_64, VirtualBox VM (CPU only)"] + + +def load(name, key=None): + p = os.path.join(D, name) + if not os.path.exists(p): + return None + j = json.load(open(p)) + return j.get("bench", {}).get(key) if key else j + + +def e(x): # 2.227e+07 -> 2.23e7 + if x is None: + return "n/a" + m, _, ex = ("%.3g" % x).partition("e") + return m + ("e" + str(int(ex)) if ex else "") + + +def s(v, f=e): + if not isinstance(v, dict): + return "n/a" + return "%s [%s-%s]" % (f(v["median"]), f(v["q1"]), f(v["q3"])) + + +def ms(x): + return "%.1f" % x + + +def mib(x): + return "%.1f" % (x / 1048576) + + +def table(title, head, rows): + out = ["#### " + title, "", "| platform | " + " | ".join(head) + " |", "|---|" + "---|" * len(head)] + out += ["| %s | %s |" % (r[0], " | ".join(r[1:])) for r in rows] + out += ["| %s | %s |" % (p, " | ".join(["pending: machine offline"] + [""] * (len(head) - 1))) for p in PENDING] + return "\n".join(out) + "\n" + + +data = [(lab, load(b, k), load(t) if t else None, b if not k else "%s → bench.%s" % (b, k)) for lab, b, t, k in ROWS] +data = [d for d in data if d[1]] + +rows = [] +for lab, b, t, src in data: + m, tc = b["machine"], b["toolchain"] + mem = "%.0f GiB" % (m["memory_bytes"] / 2 ** 30) if m.get("memory_bytes") else "n/a" + cores = str(m["logical_cores"]) + (" (%dP+%dE)" % (m["perf_cores"], m["efficiency_cores"]) if m.get("efficiency_cores") else "") + build = next((n.split("=", 1)[1] + " s" for n in b.get("notes", []) if n.startswith("build_s=")), "n/a") + if t: + tests = "%s: %d passed, %d failed, %d ignored, %d s" % ("pass" if t["passed"] else "FAIL", t["totals"]["passed"], + t["totals"]["failed"], t["totals"]["ignored"], t["seconds"]) + else: + tests = "not run in this job" + rows.append([lab, m.get("cpu_model") or "n/a", cores, mem, m.get("os_version") or m.get("os_release") or "n/a", + tc.get("release", "n/a"), build, "%.2f MB" % (b["binary"]["size_bytes"] / 1e6), tests, "`%s`" % src]) +print(table("Machines, build and tests", ["CPU", "logical cores", "memory", "OS", "rustc", "release build", "binary", "cargo test --release", "source"], rows)) + +rows = [] +for lab, b, t, src in data: + c = b["cells"] + rows.append([lab, s(c.get("stats_1t")), s(c.get("stats_default")), ", ".join(map(str, c.get("stats_default_threads", [])))]) +print(table("`pbit stats` self-test: 400-spin ring, IR sampler, site updates/s (N = 5)", ["1 thread", "default threads", "threads (= min(cores, 4))"], rows)) + +rows = [] +for lab, b, t, src in data: + c = b["cells"]; r = c.get("native_over_portable") + if isinstance(r, dict): + nat = "%.2f / %.2f / %.2f" % (r.get("fast T0 multispin, splitmix", 0), r.get("fast multispin splitmix x 4 thr", 0), r.get("fast T1 f32 LUT, splitmix", 0)) + else: + nat = "n/a: " + str(r) + rows.append([lab, s(c.get("kernel_multispin_fast_1t")), s(c.get("kernel_multispin_fast_4t")), s(c.get("kernel_multispin_fast_10t")), + s(c.get("kernel_heatbath_fast_1t")), nat]) +print(table("Lattice kernels (`pbit-core/examples/kernels`, portable build), updates/s (N = 5)", + ["multispin fast, 1 thread", "multispin fast, 4 threads", "multispin fast, 10 threads", "heat-bath f32 fast, 1 thread", + "native / portable (multispin 1t / 4t / heat-bath)"], rows)) + +rows = [] +for lab, b, t, src in data: + c = b["cells"] + rows.append([lab, s(c.get("decide_12_wall_ms"), ms), s(c.get("decide_12_pbit_ms"), ms), ", ".join(c.get("decide_12_verdicts", [])), + ", ".join(map(str, c.get("decide_12_exit_codes", [])))]) +print(table("`pbit demo --tasks 12 | pbit decide`, ms (N = 5)", ["wall, both processes", "pbit's own `ms`", "verdict", "exit"], rows)) + +rows = [] +for lab, b, t, src in data: + c = b["cells"]; rel = c.get("decide_300_released") or {} + rows.append([lab, s(c.get("decide_300_wall_ms"), ms), s(c.get("decide_300_pbit_ms"), ms), + "%d-%d of 300" % (rel.get("min", 0), rel.get("max", 0)) if rel else "n/a", str(c.get("decide_300_violations_max")), + ", ".join(c.get("decide_300_verdicts", [])), s(c.get("decide_300_site_updates_per_s"))]) +print(table("`pbit demo --tasks 300 | pbit decide --budget-ms 300`, ms (N = 5)", + ["wall, both processes", "pbit's own `ms`", "released (min-max)", "violations (max)", "verdict", "sampler site updates/s"], rows)) + +rows = [] +for lab, b, t, src in data: + c = b["cells"]; v = c.get("decide_300_peak_rss_bytes") + selfr = [x for x in b.get("peak_rss_300", {}).get("self_reported_peak_rss_mb", []) if x is not None] + ps = (b.get("attached") or {}).get("peak_rss_powershell") + psv = ", ".join(mib(r["peak_working_set_bytes"]) for r in ps["runs"]) if isinstance(ps, dict) else "" + rows.append([lab, s(v, mib) if v else "n/a", c.get("decide_300_peak_rss_method", "n/a"), + ", ".join("%.1f" % x for x in selfr) or "n/a (Windows: `sys::usage()` is Unix-only)", psv or "-"]) +print(table("Peak memory of the 300-task `pbit decide --budget-ms 300`, MiB (N = 3)", + ["peak RSS", "how", "pbit's own `peak_rss_mb`", "Get-Process polling (Windows, lower bound)"], rows)) + +rows = [] +for lab, b, t, src in data: + th = b["cells"].get("threads_rows") + if isinstance(th, dict) and th: + rows.append([lab] + ["%.1f [%.1f-%.1f]" % (th[k]["wall_ms"], th[k]["wall_iqr"][0], th[k]["wall_iqr"][1]) if k in th else "n/a" + for k in ("threads_1", "threads_2", "threads_4")] + [", ".join(sorted({th[k]["verdict"] for k in th if k.startswith("threads_")}))]) + else: + rows.append([lab, str(th)[:120], "", "", ""]) +print(table("`bench/decide_threads.py`: 300-task decide, fixed work (4 chains x 400 sweeps, polish 0), wall ms (N = 5)", + ["--threads 1", "--threads 2", "--threads 4", "verdict"], rows)) + +# run to run: the same source measured by run 1 and run 2 on two VMs of the same runner label +rows = [] +for lab, f in [("Linux x86_64 (`ubuntu-latest`)", "bench-linux-x86_64.json"), ("macOS arm64 (`macos-latest`)", "bench-macos-arm64.json"), + ("Windows x86_64 (`windows-latest`)", "bench-windows-x86_64.json")]: + a, b = load(RUN1 + f), load(RUN2 + f) + if not (a and b): + continue + cell = lambda k, j: (j["cells"].get(k) or {}).get("median") + def pair(k, f=e): + x, y = cell(k, a), cell(k, b) + return "%s / %s (%+.0f%%)" % (f(x), f(y), 100 * (y / x - 1)) if x and y else "n/a" + cpus = a["machine"].get("cpu_model"), b["machine"].get("cpu_model") + rows.append([lab, "%s / %s" % cpus if cpus[0] != cpus[1] else "both " + str(cpus[0]), pair("stats_1t"), pair("kernel_multispin_fast_1t"), + pair("decide_12_wall_ms", ms), pair("decide_300_wall_ms", ms), pair("decide_300_peak_rss_bytes", mib)]) +out = ["#### Run to run: run 1 / run 2 medians, same source, two VMs per runner label", "", + "| platform | CPU (run 1 / run 2) | `pbit stats` 1 thread | multispin fast, 1 thread | 12-task wall ms | 300-task wall ms | peak RSS MiB |", + "|---|---|---|---|---|---|---|"] + ["| %s |" % " | ".join(r) for r in rows] +print("\n".join(out)) diff --git a/scripts/bench/results/2026-10-01/local-m4/bench-macos-arm64-m4-local.json b/scripts/bench/results/2026-10-01/local-m4/bench-macos-arm64-m4-local.json new file mode 100644 index 0000000..1aaff63 --- /dev/null +++ b/scripts/bench/results/2026-10-01/local-m4/bench-macos-arm64-m4-local.json @@ -0,0 +1,1290 @@ +{ + "schema": "pbit-bench-matrix/1", + "label": "macos-arm64-m4-local", + "date_utc": "2026-10-02T05:33:07Z", + "git_sha": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "runner": null, + "notes": [ + "local Mac mini, Apple M4 (4P+6E), 16 GB; a second worker was compiling and running pbit concurrently: see load", + "build: cargo build --release --locked --workspace (rustc 1.98.1 Homebrew), same commit as the CI run" + ], + "machine": { + "system": "Darwin", + "os_release": "25.5.0", + "arch": "arm64", + "python": "3.9.6", + "logical_cores": 10, + "cpu_cmd": "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize", + "cpu_model": "Apple M4", + "hw_ncpu": 10, + "memory_bytes": 17179869184, + "perf_cores": 4, + "efficiency_cores": 6, + "rosetta_translated": false, + "os_version": "26.5.2" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "48a229ceaefd4985c50990b14116b6d856af0985", + "commit-date": "2026-09-01", + "host": "aarch64-apple-darwin", + "release": "1.98.1", + "LLVM version": "22.1.8", + "rustc": "rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew)", + "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05) (Homebrew)" + }, + "load": { + "start": { + "loadavg": [ + 8.501953125, + 8.39697265625, + 8.078125 + ], + "uptime": "22:33 up 27 days, 5:04, 1 user, load averages: 8.50 8.40 8.08", + "t": "22:33:07" + }, + "after_stats": { + "loadavg": [ + 8.501953125, + 8.39697265625, + 8.078125 + ], + "uptime": "22:33 up 27 days, 5:04, 1 user, load averages: 8.50 8.40 8.08", + "t": "22:33:09" + }, + "after_decide": { + "loadavg": [ + 8.38134765625, + 8.37353515625, + 8.07177734375 + ], + "uptime": "22:33 up 27 days, 5:04, 1 user, load averages: 8.38 8.37 8.07", + "t": "22:33:11" + }, + "after_kernels": { + "loadavg": [ + 7.48388671875, + 8.1748046875, + 8.005859375 + ], + "uptime": "22:33 up 27 days, 5:04, 1 user, load averages: 7.48 8.17 8.01", + "t": "22:33:27" + }, + "after_pvn": { + "loadavg": [ + 6.35400390625, + 7.82080078125, + 7.8828125 + ], + "uptime": "22:34 up 27 days, 5:05, 1 user, load averages: 6.35 7.82 7.88", + "t": "22:34:00" + }, + "end": { + "loadavg": [ + 6.35400390625, + 7.82080078125, + 7.8828125 + ], + "uptime": "22:34 up 27 days, 5:05, 1 user, load averages: 6.35 7.82 7.88", + "t": "22:34:04" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 1161392, + "version": "pbit 0.2.0", + "linkage": { + "file": "Mach-O 64-bit executable arm64", + "otool_L": [ + "\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0)" + ], + "lipo_archs": "arm64" + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "macos", + "arch": "aarch64", + "logical_cpus": 10 + }, + "build": { + "target_features": [ + "neon", + "dotprod" + ], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 4, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 22273291, + "q1": 21605145.0, + "q3": 22377794.5, + "min": 21397103, + "max": 22407055, + "n": 5, + "values": [ + 21813187, + 22273291, + 22348534, + 21397103, + 22407055 + ] + }, + "site_updates_per_s": { + "median": 72973558, + "q1": 68241880.0, + "q3": 74031019.5, + "min": 66749639, + "max": 74041796, + "n": 5, + "values": [ + 66749639, + 69734121, + 72973558, + 74041796, + 74020243 + ] + }, + "threads": [ + 4 + ], + "speedup": { + "median": 3.27, + "q1": 3.0949999999999998, + "q3": 3.38, + "min": 3.06, + "max": 3.46, + "n": 5, + "values": [ + 3.06, + 3.13, + 3.27, + 3.46, + 3.3 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 11.139958000000005, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.294708, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 10.899624999999968, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.203167, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 10.858291000000131, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.125167, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 10.949874999999887, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.280042, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 10.914666999999989, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.1885, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 10.914666999999989, + "q1": 10.87895800000005, + "q3": 11.044916499999946, + "min": 10.858291000000131, + "max": 11.139958000000005, + "n": 5, + "values": [ + 11.139958000000005, + 10.899624999999968, + 10.858291000000131, + 10.949874999999887, + 10.914666999999989 + ] + }, + "pbit_ms": { + "median": 8.203167, + "q1": 8.1568335, + "q3": 8.287375, + "min": 8.125167, + "max": 8.294708, + "n": 5, + "values": [ + 8.294708, + 8.203167, + 8.125167, + 8.280042, + 8.1885 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 422.04720899999984, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 417.360959, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 41776, + "site_updates_per_s": 41734573, + "threads": 4, + "peak_rss_mb_self": 46.078125, + "stderr": null + }, + { + "wall_ms": 422.16154099999994, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 417.023792, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 45040, + "site_updates_per_s": 45008032, + "threads": 4, + "peak_rss_mb_self": 48.046875, + "stderr": null + }, + { + "wall_ms": 424.41275000000013, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 419.079333, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 45008, + "site_updates_per_s": 44978133, + "threads": 4, + "peak_rss_mb_self": 48.234375, + "stderr": null + }, + { + "wall_ms": 423.645166, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 417.989958, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 45656, + "site_updates_per_s": 45622227, + "threads": 4, + "peak_rss_mb_self": 48.484375, + "stderr": null + }, + { + "wall_ms": 429.36420799999996, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 423.392708, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 35208, + "site_updates_per_s": 35172490, + "threads": 4, + "peak_rss_mb_self": 42.375, + "stderr": null + } + ], + "wall_ms": { + "median": 423.645166, + "q1": 422.1043749999999, + "q3": 426.8884790000001, + "min": 422.04720899999984, + "max": 429.36420799999996, + "n": 5, + "values": [ + 422.04720899999984, + 422.16154099999994, + 424.41275000000013, + 423.645166, + 429.36420799999996 + ] + }, + "pbit_ms": { + "median": 417.989958, + "q1": 417.1923755, + "q3": 421.2360205, + "min": 417.023792, + "max": 423.392708, + "n": 5, + "values": [ + 417.360959, + 417.023792, + 419.079333, + 417.989958, + 423.392708 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 44978133, + "q1": 38453531.5, + "q3": 45315129.5, + "min": 35172490, + "max": 45622227, + "n": 5, + "values": [ + 41734573, + 45008032, + 44978133, + 45622227, + 35172490 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -l (maximum resident set size, bytes)", + "bytes": { + "median": 49922048, + "q1": 49266688.0, + "q3": 50364416.0, + "min": 49266688, + "max": 50364416, + "n": 3, + "values": [ + 49266688, + 49922048, + 50364416 + ] + }, + "self_reported_peak_rss_mb": [ + 46.953125, + 47.609375, + 48.03125 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + { + "peak_memory_footprint_bytes": 48284080 + }, + { + "peak_memory_footprint_bytes": 48939440 + }, + { + "peak_memory_footprint_bytes": 49381808 + } + ] + }, + "kernels": { + "cmd": "target/release/examples/kernels", + "checksums": [ + "8520834" + ], + "lines": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "median": 525600000.0, + "q1": 467850000.0, + "q3": 526500000.0, + "min": 410600000.0, + "max": 527100000.0, + "n": 5, + "values": [ + 410600000.0, + 525100000.0, + 525600000.0, + 527100000.0, + 525900000.0 + ] + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "median": 392600000.0, + "q1": 392400000.0, + "q3": 396750000.0, + "min": 392200000.0, + "max": 400900000.0, + "n": 5, + "values": [ + 400900000.0, + 392600000.0, + 392600000.0, + 392200000.0, + 392600000.0 + ] + }, + "T0 multispin 64-lane, splitmix": { + "median": 23460000000.0, + "q1": 22465000000.0, + "q3": 23820000000.0, + "min": 22090000000.0, + "max": 24090000000.0, + "n": 5, + "values": [ + 24090000000.0, + 23550000000.0, + 22090000000.0, + 23460000000.0, + 22840000000.0 + ] + }, + "T0 multispin 64-lane, Philox": { + "median": 18570000000.0, + "q1": 18345000000.0, + "q3": 18785000000.0, + "min": 18340000000.0, + "max": 18950000000.0, + "n": 5, + "values": [ + 18950000000.0, + 18620000000.0, + 18340000000.0, + 18570000000.0, + 18350000000.0 + ] + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "median": 3154000000.0, + "q1": 3093500000.0, + "q3": 3171500000.0, + "min": 3076000000.0, + "max": 3172000000.0, + "n": 5, + "values": [ + 3171000000.0, + 3154000000.0, + 3076000000.0, + 3111000000.0, + 3172000000.0 + ] + }, + "T0 multispin splitmix x 4 threads": { + "median": 76270000000.0, + "q1": 73395000000.0, + "q3": 79245000000.0, + "min": 70730000000.0, + "max": 81680000000.0, + "n": 5, + "values": [ + 76060000000.0, + 76270000000.0, + 70730000000.0, + 76810000000.0, + 81680000000.0 + ] + }, + "T0 multispin splitmix x10 threads": { + "median": 107500000000.0, + "q1": 104300000000.0, + "q3": 112300000000.0, + "min": 103500000000.0, + "max": 116900000000.0, + "n": 5, + "values": [ + 103500000000.0, + 107500000000.0, + 105100000000.0, + 107700000000.0, + 116900000000.0 + ] + }, + "fast multispin splitmix x 4 thr": { + "median": 107100000000.0, + "q1": 105900000000.0, + "q3": 129350000000.0, + "min": 104800000000.0, + "max": 144200000000.0, + "n": 5, + "values": [ + 104800000000.0, + 114500000000.0, + 107000000000.0, + 107100000000.0, + 144200000000.0 + ] + }, + "fast multispin splitmix x10 thr": { + "median": 121600000000.0, + "q1": 118750000000.0, + "q3": 123600000000.0, + "min": 117900000000.0, + "max": 125400000000.0, + "n": 5, + "values": [ + 119600000000.0, + 121600000000.0, + 117900000000.0, + 121800000000.0, + 125400000000.0 + ] + }, + "fast T1 f32 LUT, splitmix": { + "median": 896000000.0, + "q1": 889350000.0, + "q3": 898250000.0, + "min": 887600000.0, + "max": 900100000.0, + "n": 5, + "values": [ + 887600000.0, + 891100000.0, + 896000000.0, + 896400000.0, + 900100000.0 + ] + }, + "fast T1 f32 LUT, Philox": { + "median": 563500000.0, + "q1": 562150000.0, + "q3": 568100000.0, + "min": 561000000.0, + "max": 570800000.0, + "n": 5, + "values": [ + 563500000.0, + 561000000.0, + 565400000.0, + 563300000.0, + 570800000.0 + ] + }, + "fast T0 multispin, splitmix": { + "median": 36360000000.0, + "q1": 36285000000.0, + "q3": 36730000000.0, + "min": 36240000000.0, + "max": 36960000000.0, + "n": 5, + "values": [ + 36240000000.0, + 36330000000.0, + 36360000000.0, + 36500000000.0, + 36960000000.0 + ] + }, + "fast T0 multispin, Philox": { + "median": 23890000000.0, + "q1": 23540000000.0, + "q3": 24080000000.0, + "min": 23220000000.0, + "max": 24230000000.0, + "n": 5, + "values": [ + 23220000000.0, + 23930000000.0, + 23860000000.0, + 23890000000.0, + 24230000000.0 + ] + } + } + }, + "portable_vs_native": { + "cmd": "PBIT_DIR=target/release PBIT_NATIVE_DIR=target/native/release python3 bench/portable_vs_native.py", + "exit": 0, + "rows": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "portable": 518200000.0, + "portable_iqr": [ + 440200000.0, + 532700000.0 + ], + "native": 532900000.0, + "native_iqr": [ + 482100000.0, + 534600000.0 + ], + "ratio": 1.028 + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "portable": 394700000.0, + "portable_iqr": [ + 348000000.0, + 396600000.0 + ], + "native": 397500000.0, + "native_iqr": [ + 394600000.0, + 397900000.0 + ], + "ratio": 1.007 + }, + "T0 multispin 64-lane, splitmix": { + "portable": 23690000000.0, + "portable_iqr": [ + 22820000000.0, + 23950000000.0 + ], + "native": 23940000000.0, + "native_iqr": [ + 23790000000.0, + 23940000000.0 + ], + "ratio": 1.011 + }, + "T0 multispin 64-lane, Philox": { + "portable": 18760000000.0, + "portable_iqr": [ + 18050000000.0, + 18850000000.0 + ], + "native": 18860000000.0, + "native_iqr": [ + 18720000000.0, + 18870000000.0 + ], + "ratio": 1.005 + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "portable": 3156000000.0, + "portable_iqr": [ + 3114000000.0, + 3186000000.0 + ], + "native": 3167000000.0, + "native_iqr": [ + 3138000000.0, + 3181000000.0 + ], + "ratio": 1.003 + }, + "T0 multispin splitmix x 4 threads": { + "portable": 90140000000.0, + "portable_iqr": [ + 78420000000.0, + 92120000000.0 + ], + "native": 90070000000.0, + "native_iqr": [ + 73030000000.0, + 90440000000.0 + ], + "ratio": 0.999 + }, + "T0 multispin splitmix x10 threads": { + "portable": 109500000000.0, + "portable_iqr": [ + 98580000000.0, + 114300000000.0 + ], + "native": 111900000000.0, + "native_iqr": [ + 89820000000.0, + 113400000000.0 + ], + "ratio": 1.022 + }, + "fast multispin splitmix x 4 thr": { + "portable": 114100000000.0, + "portable_iqr": [ + 100800000000.0, + 135200000000.0 + ], + "native": 106800000000.0, + "native_iqr": [ + 90760000000.0, + 131600000000.0 + ], + "ratio": 0.936 + }, + "fast multispin splitmix x10 thr": { + "portable": 124800000000.0, + "portable_iqr": [ + 120200000000.0, + 128000000000.0 + ], + "native": 124000000000.0, + "native_iqr": [ + 111300000000.0, + 126000000000.0 + ], + "ratio": 0.994 + }, + "fast T1 f32 LUT, splitmix": { + "portable": 903600000.0, + "portable_iqr": [ + 889000000.0, + 931300000.0 + ], + "native": 903400000.0, + "native_iqr": [ + 695000000.0, + 904000000.0 + ], + "ratio": 1.0 + }, + "fast T1 f32 LUT, Philox": { + "portable": 567800000.0, + "portable_iqr": [ + 553400000.0, + 569600000.0 + ], + "native": 568800000.0, + "native_iqr": [ + 532800000.0, + 569800000.0 + ], + "ratio": 1.002 + }, + "fast T0 multispin, splitmix": { + "portable": 36760000000.0, + "portable_iqr": [ + 35950000000.0, + 36820000000.0 + ], + "native": 36460000000.0, + "native_iqr": [ + 35740000000.0, + 36740000000.0 + ], + "ratio": 0.992 + }, + "fast T0 multispin, Philox": { + "portable": 23970000000.0, + "portable_iqr": [ + 23620000000.0, + 24100000000.0 + ], + "native": 23790000000.0, + "native_iqr": [ + 22520000000.0, + 24010000000.0 + ], + "ratio": 0.992 + }, + "pbit stats IR ring, 1 thread": { + "portable": 22070000.0, + "portable_iqr": [ + 21410000.0, + 22230000.0 + ], + "native": 22090000.0, + "native_iqr": [ + 19510000.0, + 22180000.0 + ], + "ratio": 1.001 + }, + "pbit stats IR ring, 4 threads": { + "portable": 74710000.0, + "portable_iqr": [ + 66570000.0, + 75780000.0 + ], + "native": 72300000.0, + "native_iqr": [ + 57440000.0, + 73070000.0 + ], + "ratio": 0.968 + }, + "pbit decide 300 tasks, 4 thr (sampler)": { + "portable": 44080000.0, + "portable_iqr": [ + 40500000.0, + 44550000.0 + ], + "native": 44610000.0, + "native_iqr": [ + 30400000.0, + 45120000.0 + ], + "ratio": 1.012 + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB, rustc 1.98.1; 5 alternating runs; medians [IQR]; ratio = native / portable", + "T1 SoA f32 heat-bath LUT, splitmix: portable 5.182e+08 [4.402e+08-5.327e+08] | native 5.329e+08 [4.821e+08-5.346e+08] | ratio 1.028", + "T1 SoA f32 heat-bath LUT, Philox: portable 3.947e+08 [3.48e+08-3.966e+08] | native 3.975e+08 [3.946e+08-3.979e+08] | ratio 1.007", + "T0 multispin 64-lane, splitmix: portable 2.369e+10 [2.282e+10-2.395e+10] | native 2.394e+10 [2.379e+10-2.394e+10] | ratio 1.011", + "T0 multispin 64-lane, Philox: portable 1.876e+10 [1.805e+10-1.885e+10] | native 1.886e+10 [1.872e+10-1.887e+10] | ratio 1.005", + "T0 PT-in-a-word prefix-mask, splitm: portable 3.156e+09 [3.114e+09-3.186e+09] | native 3.167e+09 [3.138e+09-3.181e+09] | ratio 1.003", + "T0 multispin splitmix x 4 threads: portable 9.014e+10 [7.842e+10-9.212e+10] | native 9.007e+10 [7.303e+10-9.044e+10] | ratio 0.999", + "T0 multispin splitmix x10 threads: portable 1.095e+11 [9.858e+10-1.143e+11] | native 1.119e+11 [8.982e+10-1.134e+11] | ratio 1.022", + "fast multispin splitmix x 4 thr: portable 1.141e+11 [1.008e+11-1.352e+11] | native 1.068e+11 [9.076e+10-1.316e+11] | ratio 0.936", + "fast multispin splitmix x10 thr: portable 1.248e+11 [1.202e+11-1.28e+11] | native 1.24e+11 [1.113e+11-1.26e+11] | ratio 0.994", + "fast T1 f32 LUT, splitmix: portable 9.036e+08 [8.89e+08-9.313e+08] | native 9.034e+08 [6.95e+08-9.04e+08] | ratio 1.000", + "fast T1 f32 LUT, Philox: portable 5.678e+08 [5.534e+08-5.696e+08] | native 5.688e+08 [5.328e+08-5.698e+08] | ratio 1.002", + "fast T0 multispin, splitmix: portable 3.676e+10 [3.595e+10-3.682e+10] | native 3.646e+10 [3.574e+10-3.674e+10] | ratio 0.992", + "fast T0 multispin, Philox: portable 2.397e+10 [2.362e+10-2.41e+10] | native 2.379e+10 [2.252e+10-2.401e+10] | ratio 0.992", + "pbit stats IR ring, 1 thread: portable 2.207e+07 [2.141e+07-2.223e+07] | native 2.209e+07 [1.951e+07-2.218e+07] | ratio 1.001", + "pbit stats IR ring, 4 threads: portable 7.471e+07 [6.657e+07-7.578e+07] | native 7.23e+07 [5.744e+07-7.307e+07] | ratio 0.968", + "pbit decide 300 tasks, 4 thr (sampler): portable 4.408e+07 [4.05e+07-4.455e+07] | native 4.461e+07 [3.04e+07-4.512e+07] | ratio 1.012", + "kernels checksum identical across builds: True | decide answer (odds+plan+gate) identical across builds: True" + ], + "stderr_tail": null, + "note": "the script's first line is hard-coded to the M4; the machine is the one in `machine`" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 84.1, + "wall_iqr": [ + 83.1, + 85.8 + ], + "cpu_ms": 87.4, + "site_updates_per_s_M": 12.17, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 65.2, + "wall_iqr": [ + 64.9, + 65.4 + ], + "cpu_ms": 89.7, + "site_updates_per_s_M": 23.04, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 55.7, + "wall_iqr": [ + 55.6, + 55.8 + ], + "cpu_ms": 94.0, + "site_updates_per_s_M": 42.33, + "verdict": "refused" + }, + "default": { + "threads": 4, + "wall_ms": 418.9, + "wall_iqr": [ + 417.6, + 424.7 + ], + "sweeps": 40288, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 84.1 [83.1-85.8], CPU ms 87.4, site updates/s 12.17 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 65.2 [64.9-65.4], CPU ms 89.7, site updates/s 23.04 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 55.7 [55.6-55.8], CPU ms 94.0, site updates/s 42.33 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 4): wall ms 418.9 [417.6-424.7], sweeps 40288, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "cells": { + "cpu_model": "Apple M4", + "logical_cores": 10, + "binary_size_bytes": 1161392, + "stats_1t": { + "median": 22273291, + "q1": 21605145.0, + "q3": 22377794.5, + "min": 21397103, + "max": 22407055, + "n": 5, + "values": [ + 21813187, + 22273291, + 22348534, + 21397103, + 22407055 + ] + }, + "stats_default": { + "median": 72973558, + "q1": 68241880.0, + "q3": 74031019.5, + "min": 66749639, + "max": 74041796, + "n": 5, + "values": [ + 66749639, + 69734121, + 72973558, + 74041796, + 74020243 + ] + }, + "stats_default_threads": [ + 4 + ], + "kernel_multispin_fast_4t": { + "median": 107100000000.0, + "q1": 105900000000.0, + "q3": 129350000000.0, + "min": 104800000000.0, + "max": 144200000000.0, + "n": 5, + "values": [ + 104800000000.0, + 114500000000.0, + 107000000000.0, + 107100000000.0, + 144200000000.0 + ] + }, + "kernel_multispin_fast_10t": { + "median": 121600000000.0, + "q1": 118750000000.0, + "q3": 123600000000.0, + "min": 117900000000.0, + "max": 125400000000.0, + "n": 5, + "values": [ + 119600000000.0, + 121600000000.0, + 117900000000.0, + 121800000000.0, + 125400000000.0 + ] + }, + "kernel_multispin_fast_1t": { + "median": 36360000000.0, + "q1": 36285000000.0, + "q3": 36730000000.0, + "min": 36240000000.0, + "max": 36960000000.0, + "n": 5, + "values": [ + 36240000000.0, + 36330000000.0, + 36360000000.0, + 36500000000.0, + 36960000000.0 + ] + }, + "kernel_heatbath_fast_1t": { + "median": 896000000.0, + "q1": 889350000.0, + "q3": 898250000.0, + "min": 887600000.0, + "max": 900100000.0, + "n": 5, + "values": [ + 887600000.0, + 891100000.0, + 896000000.0, + 896400000.0, + 900100000.0 + ] + }, + "decide_12_wall_ms": { + "median": 10.914666999999989, + "q1": 10.87895800000005, + "q3": 11.044916499999946, + "min": 10.858291000000131, + "max": 11.139958000000005, + "n": 5, + "values": [ + 11.139958000000005, + 10.899624999999968, + 10.858291000000131, + 10.949874999999887, + 10.914666999999989 + ] + }, + "decide_12_pbit_ms": { + "median": 8.203167, + "q1": 8.1568335, + "q3": 8.287375, + "min": 8.125167, + "max": 8.294708, + "n": 5, + "values": [ + 8.294708, + 8.203167, + 8.125167, + 8.280042, + 8.1885 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 423.645166, + "q1": 422.1043749999999, + "q3": 426.8884790000001, + "min": 422.04720899999984, + "max": 429.36420799999996, + "n": 5, + "values": [ + 422.04720899999984, + 422.16154099999994, + 424.41275000000013, + 423.645166, + 429.36420799999996 + ] + }, + "decide_300_pbit_ms": { + "median": 417.989958, + "q1": 417.1923755, + "q3": 421.2360205, + "min": 417.023792, + "max": 423.392708, + "n": 5, + "values": [ + 417.360959, + 417.023792, + 419.079333, + 417.989958, + 423.392708 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 44978133, + "q1": 38453531.5, + "q3": 45315129.5, + "min": 35172490, + "max": 45622227, + "n": 5, + "values": [ + 41734573, + 45008032, + 44978133, + 45622227, + 35172490 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 49922048, + "q1": 49266688.0, + "q3": 50364416.0, + "min": 49266688, + "max": 50364416, + "n": 3, + "values": [ + 49266688, + 49922048, + 50364416 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -l (maximum resident set size, bytes)", + "native_over_portable": { + "T1 SoA f32 heat-bath LUT, splitmix": 1.028, + "T1 SoA f32 heat-bath LUT, Philox": 1.007, + "T0 multispin 64-lane, splitmix": 1.011, + "T0 multispin 64-lane, Philox": 1.005, + "T0 PT-in-a-word prefix-mask, splitm": 1.003, + "T0 multispin splitmix x 4 threads": 0.999, + "T0 multispin splitmix x10 threads": 1.022, + "fast multispin splitmix x 4 thr": 0.936, + "fast multispin splitmix x10 thr": 0.994, + "fast T1 f32 LUT, splitmix": 1.0, + "fast T1 f32 LUT, Philox": 1.002, + "fast T0 multispin, splitmix": 0.992, + "fast T0 multispin, Philox": 0.992, + "pbit stats IR ring, 1 thread": 1.001, + "pbit stats IR ring, 4 threads": 0.968, + "pbit decide 300 tasks, 4 thr (sampler)": 1.012 + }, + "threads_rows": { + "threads_1": { + "wall_ms": 84.1, + "wall_iqr": [ + 83.1, + 85.8 + ], + "cpu_ms": 87.4, + "site_updates_per_s_M": 12.17, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 65.2, + "wall_iqr": [ + 64.9, + 65.4 + ], + "cpu_ms": 89.7, + "site_updates_per_s_M": 23.04, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 55.7, + "wall_iqr": [ + 55.6, + 55.8 + ], + "cpu_ms": 94.0, + "site_updates_per_s_M": 42.33, + "verdict": "refused" + }, + "default": { + "threads": 4, + "wall_ms": 418.9, + "wall_iqr": [ + 417.6, + 424.7 + ], + "sweeps": 40288, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 56.4 +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/bench-linux-x86_64.json b/scripts/bench/results/2026-10-01/run-36969399069/bench-linux-x86_64.json new file mode 100644 index 0000000..57a7da0 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/bench-linux-x86_64.json @@ -0,0 +1,1345 @@ +{ + "schema": "pbit-bench-matrix/1", + "label": "linux-x86_64", + "date_utc": "2026-10-02T05:33:29Z", + "git_sha": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "bench", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "build_s=12", + "runner=ubuntu-latest" + ], + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "AMD EPYC 9V45 96-Core Processor", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "AuthenticAMD", + "Model name": "AMD EPYC 9V45 96-Core Processor", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": true, + "avx512bw": true, + "avx512vl": true + }, + "memory_bytes": 8323969024, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 0.86767578125, + 0.2587890625, + 0.09130859375 + ], + "uptime": "05:33:29 up 1 min, 0 user, load average: 0.87, 0.26, 0.09", + "t": "05:33:29" + }, + "after_stats": { + "loadavg": [ + 0.86767578125, + 0.2587890625, + 0.09130859375 + ], + "uptime": "05:33:31 up 1 min, 0 user, load average: 0.87, 0.26, 0.09", + "t": "05:33:31" + }, + "after_decide": { + "loadavg": [ + 1.03857421875, + 0.3046875, + 0.10693359375 + ], + "uptime": "05:33:33 up 1 min, 0 user, load average: 1.04, 0.30, 0.11", + "t": "05:33:33" + }, + "after_kernels": { + "loadavg": [ + 4.580078125, + 1.31494140625, + 0.45703125 + ], + "uptime": "05:34:13 up 1 min, 0 user, load average: 4.58, 1.31, 0.46", + "t": "05:34:13" + }, + "after_pvn": { + "loadavg": [ + 4.76611328125, + 2.21875, + 0.85400390625 + ], + "uptime": "05:35:33 up 3 min, 0 user, load average: 4.77, 2.22, 0.85", + "t": "05:35:33" + }, + "end": { + "loadavg": [ + 4.76611328125, + 2.21875, + 0.85400390625 + ], + "uptime": "05:35:36 up 3 min, 0 user, load average: 4.77, 2.22, 0.85", + "t": "05:35:36" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 1401248, + "version": "pbit 0.2.0", + "linkage": { + "file": "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 3.2.0, BuildID[sha1]=3a4791ff347b29ab20fc5b1ff998dd05660b1d15, not stripped", + "ldd": [ + "linux-vdso.so.1 (0x00007f2b7c5dc000)", + "\tlibgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f2b7c46e000)", + "\tlibm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f2b7c385000)", + "\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f2b7c000000)", + "\t/lib64/ld-linux-x86-64.so.2 (0x00007f2b7c5de000)" + ], + "glibc_symbol_versions": [ + "2.2.5", + "2.3", + "2.3.4", + "2.14", + "2.16", + "2.17", + "2.18", + "2.25", + "2.28", + "2.29", + "2.30", + "2.32", + "2.33", + "2.34" + ], + "min_glibc": "2.34" + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "linux", + "arch": "x86_64", + "logical_cpus": 2 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 2, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 19098193, + "q1": 18914071.0, + "q3": 19115245.0, + "min": 18740473, + "max": 19125553, + "n": 5, + "values": [ + 19098193, + 18740473, + 19087669, + 19125553, + 19104937 + ] + }, + "site_updates_per_s": { + "median": 25104302, + "q1": 24808846.0, + "q3": 25332779.5, + "min": 24711700, + "max": 25349348, + "n": 5, + "values": [ + 25349348, + 24905992, + 25316211, + 24711700, + 25104302 + ] + }, + "threads": [ + 2 + ], + "speedup": { + "median": 1.33, + "q1": 1.3, + "q3": 1.33, + "min": 1.29, + "max": 1.33, + "n": 5, + "values": [ + 1.33, + 1.33, + 1.33, + 1.29, + 1.31 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 9.767631000002552, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.503811, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 9.6203829999979, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.460717, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 9.565105999996604, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.471288, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 9.642544999991287, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.514556, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 9.505326999999397, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 8.401315, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 9.6203829999979, + "q1": 9.535216499998, + "q3": 9.70508799999692, + "min": 9.505326999999397, + "max": 9.767631000002552, + "n": 5, + "values": [ + 9.767631000002552, + 9.6203829999979, + 9.565105999996604, + 9.642544999991287, + 9.505326999999397 + ] + }, + "pbit_ms": { + "median": 8.471288, + "q1": 8.431016, + "q3": 8.5091835, + "min": 8.401315, + "max": 8.514556, + "n": 5, + "values": [ + 8.503811, + 8.460717, + 8.471288, + 8.514556, + 8.401315 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 418.0547519999891, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 414.227307, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15512, + "site_updates_per_s": 15494023, + "threads": 2, + "peak_rss_mb_self": 21.359375, + "stderr": null + }, + { + "wall_ms": 418.7893220000092, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 414.561453, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15224, + "site_updates_per_s": 15209250, + "threads": 2, + "peak_rss_mb_self": 21.070313, + "stderr": null + }, + { + "wall_ms": 418.20365299999196, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 413.981702, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15000, + "site_updates_per_s": 14983209, + "threads": 2, + "peak_rss_mb_self": 21.011719, + "stderr": null + }, + { + "wall_ms": 418.7795170000044, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 414.491188, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15136, + "site_updates_per_s": 15123166, + "threads": 2, + "peak_rss_mb_self": 21.21875, + "stderr": null + }, + { + "wall_ms": 420.25527799999907, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 415.775394, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15592, + "site_updates_per_s": 15573318, + "threads": 2, + "peak_rss_mb_self": 21.34375, + "stderr": null + } + ], + "wall_ms": { + "median": 418.7795170000044, + "q1": 418.1292024999905, + "q3": 419.5223000000041, + "min": 418.0547519999891, + "max": 420.25527799999907, + "n": 5, + "values": [ + 418.0547519999891, + 418.7893220000092, + 418.20365299999196, + 418.7795170000044, + 420.25527799999907 + ] + }, + "pbit_ms": { + "median": 414.491188, + "q1": 414.10450449999996, + "q3": 415.1684235, + "min": 413.981702, + "max": 415.775394, + "n": 5, + "values": [ + 414.227307, + 414.561453, + 413.981702, + 414.491188, + 415.775394 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 15209250, + "q1": 15053187.5, + "q3": 15533670.5, + "min": 14983209, + "max": 15573318, + "n": 5, + "values": [ + 15494023, + 15209250, + 14983209, + 15123166, + 15573318 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "bytes": { + "median": 22093824, + "q1": 22093824.0, + "q3": 22417408.0, + "min": 22093824, + "max": 22417408, + "n": 3, + "values": [ + 22093824, + 22093824, + 22417408 + ] + }, + "self_reported_peak_rss_mb": [ + 21.070313, + 21.070313, + 21.378906 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + {}, + {}, + {} + ] + }, + "kernels": { + "cmd": "target/release/examples/kernels", + "checksums": [ + "8520834" + ], + "lines": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "median": 340600000.0, + "q1": 337250000.0, + "q3": 340750000.0, + "min": 334500000.0, + "max": 340900000.0, + "n": 5, + "values": [ + 340000000.0, + 340600000.0, + 334500000.0, + 340600000.0, + 340900000.0 + ] + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "median": 240700000.0, + "q1": 240450000.0, + "q3": 241850000.0, + "min": 240200000.0, + "max": 241900000.0, + "n": 5, + "values": [ + 240700000.0, + 241800000.0, + 241900000.0, + 240700000.0, + 240200000.0 + ] + }, + "T0 multispin 64-lane, splitmix": { + "median": 18070000000.0, + "q1": 17790000000.0, + "q3": 18205000000.0, + "min": 17530000000.0, + "max": 18210000000.0, + "n": 5, + "values": [ + 18210000000.0, + 17530000000.0, + 18200000000.0, + 18050000000.0, + 18070000000.0 + ] + }, + "T0 multispin 64-lane, Philox": { + "median": 13590000000.0, + "q1": 13420000000.0, + "q3": 13630000000.0, + "min": 13370000000.0, + "max": 13630000000.0, + "n": 5, + "values": [ + 13630000000.0, + 13590000000.0, + 13470000000.0, + 13370000000.0, + 13630000000.0 + ] + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "median": 2414000000.0, + "q1": 2379500000.0, + "q3": 2435500000.0, + "min": 2353000000.0, + "max": 2440000000.0, + "n": 5, + "values": [ + 2406000000.0, + 2353000000.0, + 2414000000.0, + 2440000000.0, + 2431000000.0 + ] + }, + "T0 multispin splitmix x 4 threads": { + "median": 17590000000.0, + "q1": 17455000000.0, + "q3": 17740000000.0, + "min": 17340000000.0, + "max": 17750000000.0, + "n": 5, + "values": [ + 17750000000.0, + 17590000000.0, + 17570000000.0, + 17340000000.0, + 17730000000.0 + ] + }, + "T0 multispin splitmix x10 threads": { + "median": 16140000000.0, + "q1": 16105000000.0, + "q3": 16275000000.0, + "min": 16090000000.0, + "max": 16290000000.0, + "n": 5, + "values": [ + 16120000000.0, + 16290000000.0, + 16090000000.0, + 16140000000.0, + 16260000000.0 + ] + }, + "fast multispin splitmix x 4 thr": { + "median": 23630000000.0, + "q1": 23365000000.0, + "q3": 23765000000.0, + "min": 23300000000.0, + "max": 23790000000.0, + "n": 5, + "values": [ + 23790000000.0, + 23630000000.0, + 23300000000.0, + 23430000000.0, + 23740000000.0 + ] + }, + "fast multispin splitmix x10 thr": { + "median": 23120000000.0, + "q1": 22710000000.0, + "q3": 23245000000.0, + "min": 22660000000.0, + "max": 23260000000.0, + "n": 5, + "values": [ + 22660000000.0, + 23120000000.0, + 23230000000.0, + 23260000000.0, + 22760000000.0 + ] + }, + "fast T1 f32 LUT, splitmix": { + "median": 526700000.0, + "q1": 525700000.0, + "q3": 536550000.0, + "min": 525700000.0, + "max": 536600000.0, + "n": 5, + "values": [ + 525700000.0, + 536600000.0, + 525700000.0, + 526700000.0, + 536500000.0 + ] + }, + "fast T1 f32 LUT, Philox": { + "median": 330800000.0, + "q1": 299850000.0, + "q3": 332350000.0, + "min": 273600000.0, + "max": 333800000.0, + "n": 5, + "values": [ + 330900000.0, + 326100000.0, + 333800000.0, + 330800000.0, + 273600000.0 + ] + }, + "fast T0 multispin, splitmix": { + "median": 25880000000.0, + "q1": 25090000000.0, + "q3": 25910000000.0, + "min": 24480000000.0, + "max": 25920000000.0, + "n": 5, + "values": [ + 25920000000.0, + 25900000000.0, + 24480000000.0, + 25880000000.0, + 25700000000.0 + ] + }, + "fast T0 multispin, Philox": { + "median": 17390000000.0, + "q1": 17080000000.0, + "q3": 17460000000.0, + "min": 17000000000.0, + "max": 17510000000.0, + "n": 5, + "values": [ + 17000000000.0, + 17160000000.0, + 17410000000.0, + 17390000000.0, + 17510000000.0 + ] + } + } + }, + "portable_vs_native": { + "cmd": "PBIT_DIR=target/release PBIT_NATIVE_DIR=target/native/release python3 bench/portable_vs_native.py", + "exit": 0, + "rows": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "portable": 330700000.0, + "portable_iqr": [ + 327400000.0, + 339600000.0 + ], + "native": 379600000.0, + "native_iqr": [ + 371000000.0, + 382400000.0 + ], + "ratio": 1.148 + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "portable": 240100000.0, + "portable_iqr": [ + 233800000.0, + 241200000.0 + ], + "native": 252500000.0, + "native_iqr": [ + 248000000.0, + 253300000.0 + ], + "ratio": 1.052 + }, + "T0 multispin 64-lane, splitmix": { + "portable": 17870000000.0, + "portable_iqr": [ + 17600000000.0, + 18220000000.0 + ], + "native": 18060000000.0, + "native_iqr": [ + 17770000000.0, + 18240000000.0 + ], + "ratio": 1.011 + }, + "T0 multispin 64-lane, Philox": { + "portable": 13550000000.0, + "portable_iqr": [ + 13330000000.0, + 13630000000.0 + ], + "native": 13220000000.0, + "native_iqr": [ + 12550000000.0, + 13420000000.0 + ], + "ratio": 0.976 + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "portable": 2421000000.0, + "portable_iqr": [ + 2401000000.0, + 2429000000.0 + ], + "native": 3908000000.0, + "native_iqr": [ + 3845000000.0, + 3924000000.0 + ], + "ratio": 1.614 + }, + "T0 multispin splitmix x 4 threads": { + "portable": 17580000000.0, + "portable_iqr": [ + 17450000000.0, + 17760000000.0 + ], + "native": 17650000000.0, + "native_iqr": [ + 17520000000.0, + 17900000000.0 + ], + "ratio": 1.004 + }, + "T0 multispin splitmix x10 threads": { + "portable": 16220000000.0, + "portable_iqr": [ + 15940000000.0, + 16340000000.0 + ], + "native": 16100000000.0, + "native_iqr": [ + 15560000000.0, + 16240000000.0 + ], + "ratio": 0.993 + }, + "fast multispin splitmix x 4 thr": { + "portable": 23370000000.0, + "portable_iqr": [ + 23020000000.0, + 23630000000.0 + ], + "native": 24630000000.0, + "native_iqr": [ + 24020000000.0, + 26030000000.0 + ], + "ratio": 1.054 + }, + "fast multispin splitmix x10 thr": { + "portable": 23100000000.0, + "portable_iqr": [ + 22860000000.0, + 23320000000.0 + ], + "native": 25500000000.0, + "native_iqr": [ + 25000000000.0, + 25630000000.0 + ], + "ratio": 1.104 + }, + "fast T1 f32 LUT, splitmix": { + "portable": 523300000.0, + "portable_iqr": [ + 519500000.0, + 538600000.0 + ], + "native": 570900000.0, + "native_iqr": [ + 515200000.0, + 578100000.0 + ], + "ratio": 1.091 + }, + "fast T1 f32 LUT, Philox": { + "portable": 318000000.0, + "portable_iqr": [ + 292000000.0, + 326400000.0 + ], + "native": 349900000.0, + "native_iqr": [ + 338700000.0, + 355000000.0 + ], + "ratio": 1.1 + }, + "fast T0 multispin, splitmix": { + "portable": 25740000000.0, + "portable_iqr": [ + 25170000000.0, + 25850000000.0 + ], + "native": 27100000000.0, + "native_iqr": [ + 26500000000.0, + 27300000000.0 + ], + "ratio": 1.053 + }, + "fast T0 multispin, Philox": { + "portable": 16740000000.0, + "portable_iqr": [ + 15550000000.0, + 17430000000.0 + ], + "native": 17690000000.0, + "native_iqr": [ + 17040000000.0, + 17820000000.0 + ], + "ratio": 1.057 + }, + "pbit stats IR ring, 1 thread": { + "portable": 18410000.0, + "portable_iqr": [ + 18300000.0, + 18660000.0 + ], + "native": 18510000.0, + "native_iqr": [ + 18370000.0, + 18880000.0 + ], + "ratio": 1.005 + }, + "pbit stats IR ring, 4 threads": { + "portable": 24770000.0, + "portable_iqr": [ + 24270000.0, + 25050000.0 + ], + "native": 25040000.0, + "native_iqr": [ + 24600000.0, + 25270000.0 + ], + "ratio": 1.011 + }, + "pbit decide 300 tasks, 4 thr (sampler)": { + "portable": 15240000.0, + "portable_iqr": [ + 15080000.0, + 15340000.0 + ], + "native": 15540000.0, + "native_iqr": [ + 15130000.0, + 15600000.0 + ], + "ratio": 1.02 + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB, rustc 1.98.1; 5 alternating runs; medians [IQR]; ratio = native / portable", + "T1 SoA f32 heat-bath LUT, splitmix: portable 3.307e+08 [3.274e+08-3.396e+08] | native 3.796e+08 [3.71e+08-3.824e+08] | ratio 1.148", + "T1 SoA f32 heat-bath LUT, Philox: portable 2.401e+08 [2.338e+08-2.412e+08] | native 2.525e+08 [2.48e+08-2.533e+08] | ratio 1.052", + "T0 multispin 64-lane, splitmix: portable 1.787e+10 [1.76e+10-1.822e+10] | native 1.806e+10 [1.777e+10-1.824e+10] | ratio 1.011", + "T0 multispin 64-lane, Philox: portable 1.355e+10 [1.333e+10-1.363e+10] | native 1.322e+10 [1.255e+10-1.342e+10] | ratio 0.976", + "T0 PT-in-a-word prefix-mask, splitm: portable 2.421e+09 [2.401e+09-2.429e+09] | native 3.908e+09 [3.845e+09-3.924e+09] | ratio 1.614", + "T0 multispin splitmix x 4 threads: portable 1.758e+10 [1.745e+10-1.776e+10] | native 1.765e+10 [1.752e+10-1.79e+10] | ratio 1.004", + "T0 multispin splitmix x10 threads: portable 1.622e+10 [1.594e+10-1.634e+10] | native 1.61e+10 [1.556e+10-1.624e+10] | ratio 0.993", + "fast multispin splitmix x 4 thr: portable 2.337e+10 [2.302e+10-2.363e+10] | native 2.463e+10 [2.402e+10-2.603e+10] | ratio 1.054", + "fast multispin splitmix x10 thr: portable 2.31e+10 [2.286e+10-2.332e+10] | native 2.55e+10 [2.5e+10-2.563e+10] | ratio 1.104", + "fast T1 f32 LUT, splitmix: portable 5.233e+08 [5.195e+08-5.386e+08] | native 5.709e+08 [5.152e+08-5.781e+08] | ratio 1.091", + "fast T1 f32 LUT, Philox: portable 3.18e+08 [2.92e+08-3.264e+08] | native 3.499e+08 [3.387e+08-3.55e+08] | ratio 1.100", + "fast T0 multispin, splitmix: portable 2.574e+10 [2.517e+10-2.585e+10] | native 2.71e+10 [2.65e+10-2.73e+10] | ratio 1.053", + "fast T0 multispin, Philox: portable 1.674e+10 [1.555e+10-1.743e+10] | native 1.769e+10 [1.704e+10-1.782e+10] | ratio 1.057", + "pbit stats IR ring, 1 thread: portable 1.841e+07 [1.83e+07-1.866e+07] | native 1.851e+07 [1.837e+07-1.888e+07] | ratio 1.005", + "pbit stats IR ring, 4 threads: portable 2.477e+07 [2.427e+07-2.505e+07] | native 2.504e+07 [2.46e+07-2.527e+07] | ratio 1.011", + "pbit decide 300 tasks, 4 thr (sampler): portable 1.524e+07 [1.508e+07-1.534e+07] | native 1.554e+07 [1.513e+07-1.56e+07] | ratio 1.020", + "kernels checksum identical across builds: True | decide answer (odds+plan+gate) identical across builds: True" + ], + "stderr_tail": null, + "note": "the script's first line is hard-coded to the M4; the machine is the one in `machine`" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 102.1, + "wall_iqr": [ + 99.9, + 102.9 + ], + "cpu_ms": 104.1, + "site_updates_per_s_M": 9.79, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 83.5, + "wall_iqr": [ + 82.2, + 85.9 + ], + "cpu_ms": 117.3, + "site_updates_per_s_M": 15.13, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 85.1, + "wall_iqr": [ + 84.2, + 86.3 + ], + "cpu_ms": 118.7, + "site_updates_per_s_M": 14.51, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 415.2, + "wall_iqr": [ + 413.5, + 416.4 + ], + "sweeps": 13736, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 102.1 [99.9-102.9], CPU ms 104.1, site updates/s 9.79 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 83.5 [82.2-85.9], CPU ms 117.3, site updates/s 15.13 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 85.1 [84.2-86.3], CPU ms 118.7, site updates/s 14.51 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 2): wall ms 415.2 [413.5-416.4], sweeps 13736, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "attached": { + "checks": { + "build": "success", + "native_build": "success", + "pack_zip": "skipped", + "callers": "success", + "install_sh": "success", + "npm": "success", + "npm_windows": "skipped", + "install_ps1_pwsh": "skipped", + "install_ps1_ps51": "skipped", + "ps_callers": "skipped", + "rss_powershell": "skipped", + "cross_darwin": "skipped" + } + }, + "cells": { + "cpu_model": "AMD EPYC 9V45 96-Core Processor", + "logical_cores": 2, + "binary_size_bytes": 1401248, + "stats_1t": { + "median": 19098193, + "q1": 18914071.0, + "q3": 19115245.0, + "min": 18740473, + "max": 19125553, + "n": 5, + "values": [ + 19098193, + 18740473, + 19087669, + 19125553, + 19104937 + ] + }, + "stats_default": { + "median": 25104302, + "q1": 24808846.0, + "q3": 25332779.5, + "min": 24711700, + "max": 25349348, + "n": 5, + "values": [ + 25349348, + 24905992, + 25316211, + 24711700, + 25104302 + ] + }, + "stats_default_threads": [ + 2 + ], + "kernel_multispin_fast_4t": { + "median": 23630000000.0, + "q1": 23365000000.0, + "q3": 23765000000.0, + "min": 23300000000.0, + "max": 23790000000.0, + "n": 5, + "values": [ + 23790000000.0, + 23630000000.0, + 23300000000.0, + 23430000000.0, + 23740000000.0 + ] + }, + "kernel_multispin_fast_10t": { + "median": 23120000000.0, + "q1": 22710000000.0, + "q3": 23245000000.0, + "min": 22660000000.0, + "max": 23260000000.0, + "n": 5, + "values": [ + 22660000000.0, + 23120000000.0, + 23230000000.0, + 23260000000.0, + 22760000000.0 + ] + }, + "kernel_multispin_fast_1t": { + "median": 25880000000.0, + "q1": 25090000000.0, + "q3": 25910000000.0, + "min": 24480000000.0, + "max": 25920000000.0, + "n": 5, + "values": [ + 25920000000.0, + 25900000000.0, + 24480000000.0, + 25880000000.0, + 25700000000.0 + ] + }, + "kernel_heatbath_fast_1t": { + "median": 526700000.0, + "q1": 525700000.0, + "q3": 536550000.0, + "min": 525700000.0, + "max": 536600000.0, + "n": 5, + "values": [ + 525700000.0, + 536600000.0, + 525700000.0, + 526700000.0, + 536500000.0 + ] + }, + "decide_12_wall_ms": { + "median": 9.6203829999979, + "q1": 9.535216499998, + "q3": 9.70508799999692, + "min": 9.505326999999397, + "max": 9.767631000002552, + "n": 5, + "values": [ + 9.767631000002552, + 9.6203829999979, + 9.565105999996604, + 9.642544999991287, + 9.505326999999397 + ] + }, + "decide_12_pbit_ms": { + "median": 8.471288, + "q1": 8.431016, + "q3": 8.5091835, + "min": 8.401315, + "max": 8.514556, + "n": 5, + "values": [ + 8.503811, + 8.460717, + 8.471288, + 8.514556, + 8.401315 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 418.7795170000044, + "q1": 418.1292024999905, + "q3": 419.5223000000041, + "min": 418.0547519999891, + "max": 420.25527799999907, + "n": 5, + "values": [ + 418.0547519999891, + 418.7893220000092, + 418.20365299999196, + 418.7795170000044, + 420.25527799999907 + ] + }, + "decide_300_pbit_ms": { + "median": 414.491188, + "q1": 414.10450449999996, + "q3": 415.1684235, + "min": 413.981702, + "max": 415.775394, + "n": 5, + "values": [ + 414.227307, + 414.561453, + 413.981702, + 414.491188, + 415.775394 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 15209250, + "q1": 15053187.5, + "q3": 15533670.5, + "min": 14983209, + "max": 15573318, + "n": 5, + "values": [ + 15494023, + 15209250, + 14983209, + 15123166, + 15573318 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 22093824, + "q1": 22093824.0, + "q3": 22417408.0, + "min": 22093824, + "max": 22417408, + "n": 3, + "values": [ + 22093824, + 22093824, + 22417408 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "native_over_portable": { + "T1 SoA f32 heat-bath LUT, splitmix": 1.148, + "T1 SoA f32 heat-bath LUT, Philox": 1.052, + "T0 multispin 64-lane, splitmix": 1.011, + "T0 multispin 64-lane, Philox": 0.976, + "T0 PT-in-a-word prefix-mask, splitm": 1.614, + "T0 multispin splitmix x 4 threads": 1.004, + "T0 multispin splitmix x10 threads": 0.993, + "fast multispin splitmix x 4 thr": 1.054, + "fast multispin splitmix x10 thr": 1.104, + "fast T1 f32 LUT, splitmix": 1.091, + "fast T1 f32 LUT, Philox": 1.1, + "fast T0 multispin, splitmix": 1.053, + "fast T0 multispin, Philox": 1.057, + "pbit stats IR ring, 1 thread": 1.005, + "pbit stats IR ring, 4 threads": 1.011, + "pbit decide 300 tasks, 4 thr (sampler)": 1.02 + }, + "threads_rows": { + "threads_1": { + "wall_ms": 102.1, + "wall_iqr": [ + 99.9, + 102.9 + ], + "cpu_ms": 104.1, + "site_updates_per_s_M": 9.79, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 83.5, + "wall_iqr": [ + 82.2, + 85.9 + ], + "cpu_ms": 117.3, + "site_updates_per_s_M": 15.13, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 85.1, + "wall_iqr": [ + 84.2, + 86.3 + ], + "cpu_ms": 118.7, + "site_updates_per_s_M": 14.51, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 415.2, + "wall_iqr": [ + 413.5, + 416.4 + ], + "sweeps": 13736, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 127.6 +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/bench-macos-arm64.json b/scripts/bench/results/2026-10-01/run-36969399069/bench-macos-arm64.json new file mode 100644 index 0000000..db70022 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/bench-macos-arm64.json @@ -0,0 +1,1087 @@ +{ + "schema": "pbit-bench-matrix/1", + "label": "macos-arm64", + "date_utc": "2026-10-02T05:35:28Z", + "git_sha": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "runner": { + "RUNNER_OS": "macOS", + "RUNNER_ARCH": "ARM64", + "ImageOS": "macos26", + "ImageVersion": "20260907.0351.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "bench", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "build_s=18", + "runner=macos-latest" + ], + "machine": { + "system": "Darwin", + "os_release": "25.6.0", + "arch": "arm64", + "python": "3.14.7", + "logical_cores": 3, + "cpu_cmd": "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize", + "cpu_model": "Apple M1 (Virtual)", + "hw_ncpu": 3, + "memory_bytes": 7516192768, + "perf_cores": 3, + "efficiency_cores": 0, + "rosetta_translated": false, + "os_version": "26.6.2" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "aarch64-apple-darwin", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 12.2509765625, + 27.529296875, + 15.13525390625 + ], + "uptime": "5:35 up 4 mins, 1 user, load averages: 12.25 27.53 15.14", + "t": "05:35:28" + }, + "after_stats": { + "loadavg": [ + 12.2509765625, + 27.529296875, + 15.13525390625 + ], + "uptime": "5:35 up 4 mins, 1 user, load averages: 12.25 27.53 15.14", + "t": "05:35:31" + }, + "after_decide": { + "loadavg": [ + 13.271484375, + 27.4873046875, + 15.19287109375 + ], + "uptime": "5:35 up 4 mins, 1 user, load averages: 13.27 27.49 15.19", + "t": "05:35:33" + }, + "after_kernels": { + "loadavg": [ + 12.71337890625, + 26.23681640625, + 15.0986328125 + ], + "uptime": "5:36 up 5 mins, 1 user, load averages: 12.71 26.24 15.10", + "t": "05:36:01" + }, + "after_pvn": { + "loadavg": [ + 12.71337890625, + 26.23681640625, + 15.0986328125 + ], + "uptime": "5:36 up 5 mins, 1 user, load averages: 12.71 26.24 15.10", + "t": "05:36:01" + }, + "end": { + "loadavg": [ + 13.9375, + 26.265625, + 15.173828125 + ], + "uptime": "5:36 up 5 mins, 1 user, load averages: 13.94 26.27 15.17", + "t": "05:36:05" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 1146960, + "version": "pbit 0.2.0", + "linkage": { + "file": "Mach-O 64-bit executable arm64", + "otool_L": [ + "\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0)" + ], + "lipo_archs": "arm64" + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "macos", + "arch": "aarch64", + "logical_cpus": 3 + }, + "build": { + "target_features": [ + "neon", + "dotprod" + ], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 3, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 12463979, + "q1": 9767181.0, + "q3": 13381267.0, + "min": 9406207, + "max": 13858931, + "n": 5, + "values": [ + 12903603, + 13858931, + 12463979, + 9406207, + 10128155 + ] + }, + "site_updates_per_s": { + "median": 25825231, + "q1": 22095427.5, + "q3": 26603051.5, + "min": 19310092, + "max": 27013442, + "n": 5, + "values": [ + 26192661, + 19310092, + 25825231, + 24880763, + 27013442 + ] + }, + "threads": [ + 3 + ], + "speedup": { + "median": 2.07, + "q1": 1.71, + "q3": 2.66, + "min": 1.39, + "max": 2.67, + "n": 5, + "values": [ + 2.03, + 1.39, + 2.07, + 2.65, + 2.67 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 15.10804199998006, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 11.344709, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 15.050082999977121, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 11.4415, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 19.817458999995097, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 11.442583, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 15.493166999988262, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 11.642625, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 16.73816699999975, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.139, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 15.493166999988262, + "q1": 15.07906249997859, + "q3": 18.277812999997423, + "min": 15.050082999977121, + "max": 19.817458999995097, + "n": 5, + "values": [ + 15.10804199998006, + 15.050082999977121, + 19.817458999995097, + 15.493166999988262, + 16.73816699999975 + ] + }, + "pbit_ms": { + "median": 11.442583, + "q1": 11.3931045, + "q3": 11.8908125, + "min": 11.344709, + "max": 12.139, + "n": 5, + "values": [ + 11.344709, + 11.4415, + 11.442583, + 11.642625, + 12.139 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 432.3906250000107, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 425.390833, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 16136, + "site_updates_per_s": 16123822, + "threads": 3, + "peak_rss_mb_self": 31.1875, + "stderr": null + }, + { + "wall_ms": 443.55099999998515, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 432.115291, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15344, + "site_updates_per_s": 15334131, + "threads": 3, + "peak_rss_mb_self": 30.625, + "stderr": null + }, + { + "wall_ms": 436.02708299999904, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 428.499583, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 13032, + "site_updates_per_s": 13018051, + "threads": 3, + "peak_rss_mb_self": 29.40625, + "stderr": null + }, + { + "wall_ms": 442.2595000000058, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 430.249042, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 13784, + "site_updates_per_s": 13767238, + "threads": 3, + "peak_rss_mb_self": 29.71875, + "stderr": null + }, + { + "wall_ms": 448.6665839999944, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 435.776417, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15248, + "site_updates_per_s": 15236972, + "threads": 3, + "peak_rss_mb_self": 30.546875, + "stderr": null + } + ], + "wall_ms": { + "median": 442.2595000000058, + "q1": 434.20885400000486, + "q3": 446.10879199998976, + "min": 432.3906250000107, + "max": 448.6665839999944, + "n": 5, + "values": [ + 432.3906250000107, + 443.55099999998515, + 436.02708299999904, + 442.2595000000058, + 448.6665839999944 + ] + }, + "pbit_ms": { + "median": 430.249042, + "q1": 426.945208, + "q3": 433.945854, + "min": 425.390833, + "max": 435.776417, + "n": 5, + "values": [ + 425.390833, + 432.115291, + 428.499583, + 430.249042, + 435.776417 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 15236972, + "q1": 13392644.5, + "q3": 15728976.5, + "min": 13018051, + "max": 16123822, + "n": 5, + "values": [ + 16123822, + 15334131, + 13018051, + 13767238, + 15236972 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -l (maximum resident set size, bytes)", + "bytes": { + "median": 32014336, + "q1": 31948800.0, + "q3": 32243712.0, + "min": 31948800, + "max": 32243712, + "n": 3, + "values": [ + 32014336, + 32243712, + 31948800 + ] + }, + "self_reported_peak_rss_mb": [ + 30.484375, + 30.734375, + 30.453125 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + { + "peak_memory_footprint_bytes": 31016128 + }, + { + "peak_memory_footprint_bytes": 31245504 + }, + { + "peak_memory_footprint_bytes": 30950592 + } + ] + }, + "kernels": { + "cmd": "target/release/examples/kernels", + "checksums": [ + "8520834" + ], + "lines": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "median": 344900000.0, + "q1": 306000000.0, + "q3": 347000000.0, + "min": 304400000.0, + "max": 347800000.0, + "n": 5, + "values": [ + 347800000.0, + 346200000.0, + 307600000.0, + 304400000.0, + 344900000.0 + ] + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "median": 226000000.0, + "q1": 221450000.0, + "q3": 250250000.0, + "min": 220800000.0, + "max": 252700000.0, + "n": 5, + "values": [ + 252700000.0, + 247800000.0, + 226000000.0, + 222100000.0, + 220800000.0 + ] + }, + "T0 multispin 64-lane, splitmix": { + "median": 12970000000.0, + "q1": 12380000000.0, + "q3": 14295000000.0, + "min": 12030000000.0, + "max": 14300000000.0, + "n": 5, + "values": [ + 14300000000.0, + 14290000000.0, + 12970000000.0, + 12030000000.0, + 12730000000.0 + ] + }, + "T0 multispin 64-lane, Philox": { + "median": 11320000000.0, + "q1": 11230000000.0, + "q3": 12135000000.0, + "min": 11220000000.0, + "max": 12560000000.0, + "n": 5, + "values": [ + 12560000000.0, + 11710000000.0, + 11220000000.0, + 11320000000.0, + 11240000000.0 + ] + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "median": 2088000000.0, + "q1": 2061500000.0, + "q3": 2185000000.0, + "min": 2037000000.0, + "max": 2243000000.0, + "n": 5, + "values": [ + 2088000000.0, + 2127000000.0, + 2086000000.0, + 2037000000.0, + 2243000000.0 + ] + }, + "T0 multispin splitmix x 4 threads": { + "median": 39080000000.0, + "q1": 38805000000.0, + "q3": 39385000000.0, + "min": 38780000000.0, + "max": 39650000000.0, + "n": 5, + "values": [ + 39080000000.0, + 39120000000.0, + 38830000000.0, + 38780000000.0, + 39650000000.0 + ] + }, + "T0 multispin splitmix x10 threads": { + "median": 37660000000.0, + "q1": 32530000000.0, + "q3": 38985000000.0, + "min": 30200000000.0, + "max": 39100000000.0, + "n": 5, + "values": [ + 39100000000.0, + 30200000000.0, + 37660000000.0, + 34860000000.0, + 38870000000.0 + ] + }, + "fast multispin splitmix x 4 thr": { + "median": 64230000000.0, + "q1": 61225000000.0, + "q3": 66495000000.0, + "min": 58460000000.0, + "max": 66680000000.0, + "n": 5, + "values": [ + 66680000000.0, + 63990000000.0, + 64230000000.0, + 58460000000.0, + 66310000000.0 + ] + }, + "fast multispin splitmix x10 thr": { + "median": 65240000000.0, + "q1": 61865000000.0, + "q3": 65755000000.0, + "min": 60620000000.0, + "max": 65780000000.0, + "n": 5, + "values": [ + 65780000000.0, + 65240000000.0, + 63110000000.0, + 60620000000.0, + 65730000000.0 + ] + }, + "fast T1 f32 LUT, splitmix": { + "median": 532500000.0, + "q1": 523800000.0, + "q3": 534850000.0, + "min": 516400000.0, + "max": 535900000.0, + "n": 5, + "values": [ + 532500000.0, + 516400000.0, + 531200000.0, + 535900000.0, + 533800000.0 + ] + }, + "fast T1 f32 LUT, Philox": { + "median": 321200000.0, + "q1": 317000000.0, + "q3": 322700000.0, + "min": 314800000.0, + "max": 323600000.0, + "n": 5, + "values": [ + 321800000.0, + 321200000.0, + 319200000.0, + 323600000.0, + 314800000.0 + ] + }, + "fast T0 multispin, splitmix": { + "median": 21670000000.0, + "q1": 21120000000.0, + "q3": 21845000000.0, + "min": 20680000000.0, + "max": 21910000000.0, + "n": 5, + "values": [ + 21560000000.0, + 21910000000.0, + 21780000000.0, + 21670000000.0, + 20680000000.0 + ] + }, + "fast T0 multispin, Philox": { + "median": 14510000000.0, + "q1": 14320000000.0, + "q3": 15130000000.0, + "min": 14220000000.0, + "max": 15740000000.0, + "n": 5, + "values": [ + 15740000000.0, + 14520000000.0, + 14420000000.0, + 14220000000.0, + 14510000000.0 + ] + } + } + }, + "portable_vs_native": { + "skipped": "no --native-dir" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 116.5, + "wall_iqr": [ + 116.4, + 118.4 + ], + "cpu_ms": 121.1, + "site_updates_per_s_M": 8.36, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 90.5, + "wall_iqr": [ + 89.3, + 95.9 + ], + "cpu_ms": 125.3, + "site_updates_per_s_M": 15.56, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 87.0, + "wall_iqr": [ + 82.9, + 89.5 + ], + "cpu_ms": 131.1, + "site_updates_per_s_M": 20.78, + "verdict": "refused" + }, + "default": { + "threads": 3, + "wall_ms": 429.5, + "wall_iqr": [ + 425.9, + 430.0 + ], + "sweeps": 15216, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 116.5 [116.4-118.4], CPU ms 121.1, site updates/s 8.36 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 90.5 [89.3-95.9], CPU ms 125.3, site updates/s 15.56 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 87.0 [82.9-89.5], CPU ms 131.1, site updates/s 20.78 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 3): wall ms 429.5 [425.9-430.0], sweeps 15216, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "attached": { + "checks": { + "build": "success", + "native_build": "skipped", + "pack_zip": "skipped", + "callers": "success", + "install_sh": "success", + "npm": "skipped", + "npm_windows": "skipped", + "install_ps1_pwsh": "skipped", + "install_ps1_ps51": "skipped", + "ps_callers": "skipped", + "rss_powershell": "skipped", + "cross_darwin": "success" + }, + "cross_x86_64_apple_darwin": [ + "$ rustup target add x86_64-apple-darwin", + "info: downloading component rust-std", + "$ cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin", + " Finished `release` profile [optimized] target(s) in 17.81s", + "$ file target/x86_64-apple-darwin/release/pbit", + "target/x86_64-apple-darwin/release/pbit: Mach-O 64-bit executable x86_64", + "$ lipo -archs target/x86_64-apple-darwin/release/pbit", + "x86_64", + "$ otool -L target/x86_64-apple-darwin/release/pbit", + "target/x86_64-apple-darwin/release/pbit:", + "\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0)", + "size 1270368 bytes", + "rosetta: present", + "$ arch -x86_64 target/x86_64-apple-darwin/release/pbit version", + "pbit 0.2.0", + "$ arch -x86_64 pbit demo --tasks 12 | arch -x86_64 pbit decide: exit 0, verdict exact", + "$ arch -x86_64 pbit stats (self-test under Rosetta)", + "1 thread 10951089 | threads 3 19953406" + ] + }, + "cells": { + "cpu_model": "Apple M1 (Virtual)", + "logical_cores": 3, + "binary_size_bytes": 1146960, + "stats_1t": { + "median": 12463979, + "q1": 9767181.0, + "q3": 13381267.0, + "min": 9406207, + "max": 13858931, + "n": 5, + "values": [ + 12903603, + 13858931, + 12463979, + 9406207, + 10128155 + ] + }, + "stats_default": { + "median": 25825231, + "q1": 22095427.5, + "q3": 26603051.5, + "min": 19310092, + "max": 27013442, + "n": 5, + "values": [ + 26192661, + 19310092, + 25825231, + 24880763, + 27013442 + ] + }, + "stats_default_threads": [ + 3 + ], + "kernel_multispin_fast_4t": { + "median": 64230000000.0, + "q1": 61225000000.0, + "q3": 66495000000.0, + "min": 58460000000.0, + "max": 66680000000.0, + "n": 5, + "values": [ + 66680000000.0, + 63990000000.0, + 64230000000.0, + 58460000000.0, + 66310000000.0 + ] + }, + "kernel_multispin_fast_10t": { + "median": 65240000000.0, + "q1": 61865000000.0, + "q3": 65755000000.0, + "min": 60620000000.0, + "max": 65780000000.0, + "n": 5, + "values": [ + 65780000000.0, + 65240000000.0, + 63110000000.0, + 60620000000.0, + 65730000000.0 + ] + }, + "kernel_multispin_fast_1t": { + "median": 21670000000.0, + "q1": 21120000000.0, + "q3": 21845000000.0, + "min": 20680000000.0, + "max": 21910000000.0, + "n": 5, + "values": [ + 21560000000.0, + 21910000000.0, + 21780000000.0, + 21670000000.0, + 20680000000.0 + ] + }, + "kernel_heatbath_fast_1t": { + "median": 532500000.0, + "q1": 523800000.0, + "q3": 534850000.0, + "min": 516400000.0, + "max": 535900000.0, + "n": 5, + "values": [ + 532500000.0, + 516400000.0, + 531200000.0, + 535900000.0, + 533800000.0 + ] + }, + "decide_12_wall_ms": { + "median": 15.493166999988262, + "q1": 15.07906249997859, + "q3": 18.277812999997423, + "min": 15.050082999977121, + "max": 19.817458999995097, + "n": 5, + "values": [ + 15.10804199998006, + 15.050082999977121, + 19.817458999995097, + 15.493166999988262, + 16.73816699999975 + ] + }, + "decide_12_pbit_ms": { + "median": 11.442583, + "q1": 11.3931045, + "q3": 11.8908125, + "min": 11.344709, + "max": 12.139, + "n": 5, + "values": [ + 11.344709, + 11.4415, + 11.442583, + 11.642625, + 12.139 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 442.2595000000058, + "q1": 434.20885400000486, + "q3": 446.10879199998976, + "min": 432.3906250000107, + "max": 448.6665839999944, + "n": 5, + "values": [ + 432.3906250000107, + 443.55099999998515, + 436.02708299999904, + 442.2595000000058, + 448.6665839999944 + ] + }, + "decide_300_pbit_ms": { + "median": 430.249042, + "q1": 426.945208, + "q3": 433.945854, + "min": 425.390833, + "max": 435.776417, + "n": 5, + "values": [ + 425.390833, + 432.115291, + 428.499583, + 430.249042, + 435.776417 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 15236972, + "q1": 13392644.5, + "q3": 15728976.5, + "min": 13018051, + "max": 16123822, + "n": 5, + "values": [ + 16123822, + 15334131, + 13018051, + 13767238, + 15236972 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 32014336, + "q1": 31948800.0, + "q3": 32243712.0, + "min": 31948800, + "max": 32243712, + "n": 3, + "values": [ + 32014336, + 32243712, + 31948800 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -l (maximum resident set size, bytes)", + "native_over_portable": "no --native-dir", + "threads_rows": { + "threads_1": { + "wall_ms": 116.5, + "wall_iqr": [ + 116.4, + 118.4 + ], + "cpu_ms": 121.1, + "site_updates_per_s_M": 8.36, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 90.5, + "wall_iqr": [ + 89.3, + 95.9 + ], + "cpu_ms": 125.3, + "site_updates_per_s_M": 15.56, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 87.0, + "wall_iqr": [ + 82.9, + 89.5 + ], + "cpu_ms": 131.1, + "site_updates_per_s_M": 20.78, + "verdict": "refused" + }, + "default": { + "threads": 3, + "wall_ms": 429.5, + "wall_iqr": [ + 425.9, + 430.0 + ], + "sweeps": 15216, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 36.4 +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/bench-macos-x86_64.json b/scripts/bench/results/2026-10-01/run-36969399069/bench-macos-x86_64.json new file mode 100644 index 0000000..53a943c --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/bench-macos-x86_64.json @@ -0,0 +1,1322 @@ +{ + "schema": "pbit-bench-matrix/1", + "label": "macos-x86_64", + "date_utc": "2026-10-02T05:39:01Z", + "git_sha": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "runner": { + "RUNNER_OS": "macOS", + "RUNNER_ARCH": "X64", + "ImageOS": "macos15", + "ImageVersion": "20260824.0482.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "bench", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "build_s=86", + "runner=macos-15-intel" + ], + "machine": { + "system": "Darwin", + "os_release": "24.6.0", + "arch": "x86_64", + "python": "3.14.7", + "logical_cores": 4, + "cpu_cmd": "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize", + "cpu_model": "Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz", + "hw_ncpu": 4, + "memory_bytes": 15032385536, + "perf_cores": 4, + "efficiency_cores": 0, + "cpu_flags": { + "sse4.2": true, + "avx1.0": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": false + }, + "rosetta_translated": null, + "os_version": "15.7.9" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-apple-darwin", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 4.96337890625, + 17.12841796875, + 15.03369140625 + ], + "uptime": "5:39 up 10 mins, 1 user, load averages: 4.96 17.13 15.03", + "t": "05:39:01" + }, + "after_stats": { + "loadavg": [ + 4.98779296875, + 16.7294921875, + 14.916015625 + ], + "uptime": "5:39 up 10 mins, 1 user, load averages: 4.99 16.73 14.92", + "t": "05:39:11" + }, + "after_decide": { + "loadavg": [ + 5.46923828125, + 16.63427734375, + 14.89306640625 + ], + "uptime": "5:39 up 10 mins, 1 user, load averages: 5.47 16.63 14.89", + "t": "05:39:14" + }, + "after_kernels": { + "loadavg": [ + 7.55517578125, + 14.8935546875, + 14.37939453125 + ], + "uptime": "5:40 up 11 mins, 1 user, load averages: 7.56 14.89 14.38", + "t": "05:40:32" + }, + "after_pvn": { + "loadavg": [ + 10.02294921875, + 13.12158203125, + 13.75341796875 + ], + "uptime": "5:43 up 14 mins, 1 user, load averages: 10.02 13.12 13.75", + "t": "05:43:08" + }, + "end": { + "loadavg": [ + 9.30029296875, + 12.919921875, + 13.67822265625 + ], + "uptime": "5:43 up 14 mins, 1 user, load averages: 9.30 12.92 13.68", + "t": "05:43:16" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 1270104, + "version": "pbit 0.2.0", + "linkage": { + "file": "Mach-O 64-bit executable x86_64", + "otool_L": [ + "\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1351.0.0)" + ], + "lipo_archs": "x86_64" + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "macos", + "arch": "x86_64", + "logical_cpus": 4 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 4, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 5663274, + "q1": 4270034.0, + "q3": 6624280.5, + "min": 4227391, + "max": 6855806, + "n": 5, + "values": [ + 6392755, + 6855806, + 5663274, + 4312677, + 4227391 + ] + }, + "site_updates_per_s": { + "median": 16654846, + "q1": 13430895.0, + "q3": 18558171.5, + "min": 12828345, + "max": 19632520, + "n": 5, + "values": [ + 19632520, + 17483823, + 14033445, + 16654846, + 12828345 + ] + }, + "threads": [ + 4 + ], + "speedup": { + "median": 3.03, + "q1": 2.5149999999999997, + "q3": 3.465, + "min": 2.48, + "max": 3.86, + "n": 5, + "values": [ + 3.07, + 2.55, + 2.48, + 3.86, + 3.03 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 84.27907800000867, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 56.069897, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 72.26737600001343, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 47.83686, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 69.60024600004999, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 51.523844, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 64.46624400007295, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 45.110387, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 61.85951199995543, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 41.723932, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 69.60024600004999, + "q1": 63.16287800001419, + "q3": 78.27322700001105, + "min": 61.85951199995543, + "max": 84.27907800000867, + "n": 5, + "values": [ + 84.27907800000867, + 72.26737600001343, + 69.60024600004999, + 64.46624400007295, + 61.85951199995543 + ] + }, + "pbit_ms": { + "median": 47.83686, + "q1": 43.4171595, + "q3": 53.7968705, + "min": 41.723932, + "max": 56.069897, + "n": 5, + "values": [ + 56.069897, + 47.83686, + 51.523844, + 45.110387, + 41.723932 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 672.9500769999959, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 641.962333, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9128, + "site_updates_per_s": 9067606, + "threads": 4, + "peak_rss_mb_self": 23.429688, + "stderr": null + }, + { + "wall_ms": 661.5303419999918, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 634.213611, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9760, + "site_updates_per_s": 9705592, + "threads": 4, + "peak_rss_mb_self": 23.929688, + "stderr": null + }, + { + "wall_ms": 579.4944149999992, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 554.316603, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 12488, + "site_updates_per_s": 12438879, + "threads": 4, + "peak_rss_mb_self": 25.492188, + "stderr": null + }, + { + "wall_ms": 574.0179809999972, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 547.5859, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 11560, + "site_updates_per_s": 11506432, + "threads": 4, + "peak_rss_mb_self": 24.863281, + "stderr": null + }, + { + "wall_ms": 642.3366230000056, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 616.291193, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 10664, + "site_updates_per_s": 10627692, + "threads": 4, + "peak_rss_mb_self": 24.46875, + "stderr": null + } + ], + "wall_ms": { + "median": 642.3366230000056, + "q1": 576.7561979999982, + "q3": 667.2402094999939, + "min": 574.0179809999972, + "max": 672.9500769999959, + "n": 5, + "values": [ + 672.9500769999959, + 661.5303419999918, + 579.4944149999992, + 574.0179809999972, + 642.3366230000056 + ] + }, + "pbit_ms": { + "median": 616.291193, + "q1": 550.9512515, + "q3": 638.087972, + "min": 547.5859, + "max": 641.962333, + "n": 5, + "values": [ + 641.962333, + 634.213611, + 554.316603, + 547.5859, + 616.291193 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 10627692, + "q1": 9386599.0, + "q3": 11972655.5, + "min": 9067606, + "max": 12438879, + "n": 5, + "values": [ + 9067606, + 9705592, + 12438879, + 11506432, + 10627692 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -l (maximum resident set size, bytes)", + "bytes": { + "median": 26288128, + "q1": 24580096.0, + "q3": 26697728.0, + "min": 24580096, + "max": 26697728, + "n": 3, + "values": [ + 24580096, + 26288128, + 26697728 + ] + }, + "self_reported_peak_rss_mb": [ + 23.351563, + 25.011719, + 25.371094 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + { + "peak_memory_footprint_bytes": 18599936 + }, + { + "peak_memory_footprint_bytes": 18862080 + }, + { + "peak_memory_footprint_bytes": 20873216 + } + ] + }, + "kernels": { + "cmd": "target/release/examples/kernels", + "checksums": [ + "8520834" + ], + "lines": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "median": 73940000.0, + "q1": 45340000.0, + "q3": 96530000.0, + "min": 40400000.0, + "max": 99690000.0, + "n": 5, + "values": [ + 50280000.0, + 93370000.0, + 40400000.0, + 73940000.0, + 99690000.0 + ] + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "median": 53550000.0, + "q1": 50300000.0, + "q3": 78885000.0, + "min": 49340000.0, + "max": 88490000.0, + "n": 5, + "values": [ + 49340000.0, + 69280000.0, + 51260000.0, + 53550000.0, + 88490000.0 + ] + }, + "T0 multispin 64-lane, splitmix": { + "median": 5493000000.0, + "q1": 4704000000.0, + "q3": 5664500000.0, + "min": 4243000000.0, + "max": 5707000000.0, + "n": 5, + "values": [ + 5165000000.0, + 5493000000.0, + 4243000000.0, + 5707000000.0, + 5622000000.0 + ] + }, + "T0 multispin 64-lane, Philox": { + "median": 5443000000.0, + "q1": 3543000000.0, + "q3": 5752000000.0, + "min": 3424000000.0, + "max": 5859000000.0, + "n": 5, + "values": [ + 5443000000.0, + 3662000000.0, + 3424000000.0, + 5859000000.0, + 5645000000.0 + ] + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "median": 1003000000.0, + "q1": 822150000.0, + "q3": 1114500000.0, + "min": 656600000.0, + "max": 1203000000.0, + "n": 5, + "values": [ + 1003000000.0, + 656600000.0, + 987700000.0, + 1203000000.0, + 1026000000.0 + ] + }, + "T0 multispin splitmix x 4 threads": { + "median": 15770000000.0, + "q1": 10730000000.0, + "q3": 16350000000.0, + "min": 8760000000.0, + "max": 16380000000.0, + "n": 5, + "values": [ + 8760000000.0, + 12700000000.0, + 16320000000.0, + 15770000000.0, + 16380000000.0 + ] + }, + "T0 multispin splitmix x10 threads": { + "median": 12340000000.0, + "q1": 10785000000.0, + "q3": 14185000000.0, + "min": 9830000000.0, + "max": 14730000000.0, + "n": 5, + "values": [ + 13640000000.0, + 9830000000.0, + 14730000000.0, + 11740000000.0, + 12340000000.0 + ] + }, + "fast multispin splitmix x 4 thr": { + "median": 20540000000.0, + "q1": 18865000000.0, + "q3": 23575000000.0, + "min": 18540000000.0, + "max": 25570000000.0, + "n": 5, + "values": [ + 21580000000.0, + 19190000000.0, + 20540000000.0, + 18540000000.0, + 25570000000.0 + ] + }, + "fast multispin splitmix x10 thr": { + "median": 23460000000.0, + "q1": 16620000000.0, + "q3": 25290000000.0, + "min": 13630000000.0, + "max": 25530000000.0, + "n": 5, + "values": [ + 19610000000.0, + 13630000000.0, + 25050000000.0, + 23460000000.0, + 25530000000.0 + ] + }, + "fast T1 f32 LUT, splitmix": { + "median": 129800000.0, + "q1": 78315000.0, + "q3": 147500000.0, + "min": 70350000.0, + "max": 163400000.0, + "n": 5, + "values": [ + 70350000.0, + 86280000.0, + 131600000.0, + 129800000.0, + 163400000.0 + ] + }, + "fast T1 f32 LUT, Philox": { + "median": 109700000.0, + "q1": 62180000.0, + "q3": 115800000.0, + "min": 53590000.0, + "max": 120800000.0, + "n": 5, + "values": [ + 70770000.0, + 53590000.0, + 109700000.0, + 110800000.0, + 120800000.0 + ] + }, + "fast T0 multispin, splitmix": { + "median": 7829000000.0, + "q1": 3594500000.0, + "q3": 8973000000.0, + "min": 3367000000.0, + "max": 9724000000.0, + "n": 5, + "values": [ + 3367000000.0, + 7829000000.0, + 3822000000.0, + 8222000000.0, + 9724000000.0 + ] + }, + "fast T0 multispin, Philox": { + "median": 6549000000.0, + "q1": 4478500000.0, + "q3": 7506500000.0, + "min": 3820000000.0, + "max": 7800000000.0, + "n": 5, + "values": [ + 5137000000.0, + 7213000000.0, + 3820000000.0, + 6549000000.0, + 7800000000.0 + ] + } + } + }, + "portable_vs_native": { + "cmd": "PBIT_DIR=target/release PBIT_NATIVE_DIR=target/native/release python3 bench/portable_vs_native.py", + "exit": 0, + "rows": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "portable": 73190000.0, + "portable_iqr": [ + 60000000.0, + 100800000.0 + ], + "native": 88800000.0, + "native_iqr": [ + 70040000.0, + 94630000.0 + ], + "ratio": 1.213 + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "portable": 68150000.0, + "portable_iqr": [ + 47920000.0, + 91160000.0 + ], + "native": 77750000.0, + "native_iqr": [ + 57380000.0, + 81980000.0 + ], + "ratio": 1.141 + }, + "T0 multispin 64-lane, splitmix": { + "portable": 5384000000.0, + "portable_iqr": [ + 4326000000.0, + 6368000000.0 + ], + "native": 5950000000.0, + "native_iqr": [ + 4162000000.0, + 6476000000.0 + ], + "ratio": 1.105 + }, + "T0 multispin 64-lane, Philox": { + "portable": 4283000000.0, + "portable_iqr": [ + 3870000000.0, + 5440000000.0 + ], + "native": 4495000000.0, + "native_iqr": [ + 3164000000.0, + 4682000000.0 + ], + "ratio": 1.049 + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "portable": 1083000000.0, + "portable_iqr": [ + 920200000.0, + 1245000000.0 + ], + "native": 1173000000.0, + "native_iqr": [ + 1002000000.0, + 1219000000.0 + ], + "ratio": 1.083 + }, + "T0 multispin splitmix x 4 threads": { + "portable": 13310000000.0, + "portable_iqr": [ + 11170000000.0, + 17050000000.0 + ], + "native": 13270000000.0, + "native_iqr": [ + 10880000000.0, + 15460000000.0 + ], + "ratio": 0.997 + }, + "T0 multispin splitmix x10 threads": { + "portable": 12890000000.0, + "portable_iqr": [ + 11230000000.0, + 16420000000.0 + ], + "native": 13370000000.0, + "native_iqr": [ + 12120000000.0, + 16330000000.0 + ], + "ratio": 1.037 + }, + "fast multispin splitmix x 4 thr": { + "portable": 19870000000.0, + "portable_iqr": [ + 14960000000.0, + 23000000000.0 + ], + "native": 16960000000.0, + "native_iqr": [ + 12660000000.0, + 21520000000.0 + ], + "ratio": 0.854 + }, + "fast multispin splitmix x10 thr": { + "portable": 20830000000.0, + "portable_iqr": [ + 16540000000.0, + 25220000000.0 + ], + "native": 17860000000.0, + "native_iqr": [ + 15520000000.0, + 21010000000.0 + ], + "ratio": 0.857 + }, + "fast T1 f32 LUT, splitmix": { + "portable": 114200000.0, + "portable_iqr": [ + 95050000.0, + 145800000.0 + ], + "native": 128400000.0, + "native_iqr": [ + 105900000.0, + 152500000.0 + ], + "ratio": 1.124 + }, + "fast T1 f32 LUT, Philox": { + "portable": 105000000.0, + "portable_iqr": [ + 86780000.0, + 120400000.0 + ], + "native": 102500000.0, + "native_iqr": [ + 76790000.0, + 117000000.0 + ], + "ratio": 0.976 + }, + "fast T0 multispin, splitmix": { + "portable": 7559000000.0, + "portable_iqr": [ + 6396000000.0, + 9152000000.0 + ], + "native": 7586000000.0, + "native_iqr": [ + 5742000000.0, + 10070000000.0 + ], + "ratio": 1.004 + }, + "fast T0 multispin, Philox": { + "portable": 6658000000.0, + "portable_iqr": [ + 4590000000.0, + 7151000000.0 + ], + "native": 6163000000.0, + "native_iqr": [ + 5106000000.0, + 7366000000.0 + ], + "ratio": 0.926 + }, + "pbit stats IR ring, 1 thread": { + "portable": 6440000.0, + "portable_iqr": [ + 5513000.0, + 6779000.0 + ], + "native": 5104000.0, + "native_iqr": [ + 4959000.0, + 5856000.0 + ], + "ratio": 0.792 + }, + "pbit stats IR ring, 4 threads": { + "portable": 19300000.0, + "portable_iqr": [ + 14970000.0, + 20750000.0 + ], + "native": 16600000.0, + "native_iqr": [ + 14440000.0, + 22160000.0 + ], + "ratio": 0.86 + }, + "pbit decide 300 tasks, 4 thr (sampler)": { + "portable": 11970000.0, + "portable_iqr": [ + 9235000.0, + 13570000.0 + ], + "native": 9228000.0, + "native_iqr": [ + 8638000.0, + 13610000.0 + ], + "ratio": 0.771 + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB, rustc 1.98.1; 5 alternating runs; medians [IQR]; ratio = native / portable", + "T1 SoA f32 heat-bath LUT, splitmix: portable 7.319e+07 [6e+07-1.008e+08] | native 8.88e+07 [7.004e+07-9.463e+07] | ratio 1.213", + "T1 SoA f32 heat-bath LUT, Philox: portable 6.815e+07 [4.792e+07-9.116e+07] | native 7.775e+07 [5.738e+07-8.198e+07] | ratio 1.141", + "T0 multispin 64-lane, splitmix: portable 5.384e+09 [4.326e+09-6.368e+09] | native 5.95e+09 [4.162e+09-6.476e+09] | ratio 1.105", + "T0 multispin 64-lane, Philox: portable 4.283e+09 [3.87e+09-5.44e+09] | native 4.495e+09 [3.164e+09-4.682e+09] | ratio 1.049", + "T0 PT-in-a-word prefix-mask, splitm: portable 1.083e+09 [9.202e+08-1.245e+09] | native 1.173e+09 [1.002e+09-1.219e+09] | ratio 1.083", + "T0 multispin splitmix x 4 threads: portable 1.331e+10 [1.117e+10-1.705e+10] | native 1.327e+10 [1.088e+10-1.546e+10] | ratio 0.997", + "T0 multispin splitmix x10 threads: portable 1.289e+10 [1.123e+10-1.642e+10] | native 1.337e+10 [1.212e+10-1.633e+10] | ratio 1.037", + "fast multispin splitmix x 4 thr: portable 1.987e+10 [1.496e+10-2.3e+10] | native 1.696e+10 [1.266e+10-2.152e+10] | ratio 0.854", + "fast multispin splitmix x10 thr: portable 2.083e+10 [1.654e+10-2.522e+10] | native 1.786e+10 [1.552e+10-2.101e+10] | ratio 0.857", + "fast T1 f32 LUT, splitmix: portable 1.142e+08 [9.505e+07-1.458e+08] | native 1.284e+08 [1.059e+08-1.525e+08] | ratio 1.124", + "fast T1 f32 LUT, Philox: portable 1.05e+08 [8.678e+07-1.204e+08] | native 1.025e+08 [7.679e+07-1.17e+08] | ratio 0.976", + "fast T0 multispin, splitmix: portable 7.559e+09 [6.396e+09-9.152e+09] | native 7.586e+09 [5.742e+09-1.007e+10] | ratio 1.004", + "fast T0 multispin, Philox: portable 6.658e+09 [4.59e+09-7.151e+09] | native 6.163e+09 [5.106e+09-7.366e+09] | ratio 0.926", + "pbit stats IR ring, 1 thread: portable 6.44e+06 [5.513e+06-6.779e+06] | native 5.104e+06 [4.959e+06-5.856e+06] | ratio 0.792", + "pbit stats IR ring, 4 threads: portable 1.93e+07 [1.497e+07-2.075e+07] | native 1.66e+07 [1.444e+07-2.216e+07] | ratio 0.860", + "pbit decide 300 tasks, 4 thr (sampler): portable 1.197e+07 [9.235e+06-1.357e+07] | native 9.228e+06 [8.638e+06-1.361e+07] | ratio 0.771", + "kernels checksum identical across builds: True | decide answer (odds+plan+gate) identical across builds: True" + ], + "stderr_tail": null, + "note": "the script's first line is hard-coded to the M4; the machine is the one in `machine`" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 295.4, + "wall_iqr": [ + 272.9, + 298.7 + ], + "cpu_ms": 309.0, + "site_updates_per_s_M": 3.86, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 258.0, + "wall_iqr": [ + 225.4, + 310.7 + ], + "cpu_ms": 342.6, + "site_updates_per_s_M": 6.59, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 304.1, + "wall_iqr": [ + 249.8, + 363.4 + ], + "cpu_ms": 489.8, + "site_updates_per_s_M": 10.02, + "verdict": "refused" + }, + "default": { + "threads": 4, + "wall_ms": 551.4, + "wall_iqr": [ + 532.5, + 566.1 + ], + "sweeps": 11792, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 295.4 [272.9-298.7], CPU ms 309.0, site updates/s 3.86 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 258.0 [225.4-310.7], CPU ms 342.6, site updates/s 6.59 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 304.1 [249.8-363.4], CPU ms 489.8, site updates/s 10.02 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 4): wall ms 551.4 [532.5-566.1], sweeps 11792, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "attached": { + "checks": { + "build": "success", + "native_build": "success", + "pack_zip": "skipped", + "callers": "success", + "install_sh": "success", + "npm": "skipped", + "npm_windows": "skipped", + "install_ps1_pwsh": "skipped", + "install_ps1_ps51": "skipped", + "ps_callers": "skipped", + "rss_powershell": "skipped", + "cross_darwin": "skipped" + } + }, + "cells": { + "cpu_model": "Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz", + "logical_cores": 4, + "binary_size_bytes": 1270104, + "stats_1t": { + "median": 5663274, + "q1": 4270034.0, + "q3": 6624280.5, + "min": 4227391, + "max": 6855806, + "n": 5, + "values": [ + 6392755, + 6855806, + 5663274, + 4312677, + 4227391 + ] + }, + "stats_default": { + "median": 16654846, + "q1": 13430895.0, + "q3": 18558171.5, + "min": 12828345, + "max": 19632520, + "n": 5, + "values": [ + 19632520, + 17483823, + 14033445, + 16654846, + 12828345 + ] + }, + "stats_default_threads": [ + 4 + ], + "kernel_multispin_fast_4t": { + "median": 20540000000.0, + "q1": 18865000000.0, + "q3": 23575000000.0, + "min": 18540000000.0, + "max": 25570000000.0, + "n": 5, + "values": [ + 21580000000.0, + 19190000000.0, + 20540000000.0, + 18540000000.0, + 25570000000.0 + ] + }, + "kernel_multispin_fast_10t": { + "median": 23460000000.0, + "q1": 16620000000.0, + "q3": 25290000000.0, + "min": 13630000000.0, + "max": 25530000000.0, + "n": 5, + "values": [ + 19610000000.0, + 13630000000.0, + 25050000000.0, + 23460000000.0, + 25530000000.0 + ] + }, + "kernel_multispin_fast_1t": { + "median": 7829000000.0, + "q1": 3594500000.0, + "q3": 8973000000.0, + "min": 3367000000.0, + "max": 9724000000.0, + "n": 5, + "values": [ + 3367000000.0, + 7829000000.0, + 3822000000.0, + 8222000000.0, + 9724000000.0 + ] + }, + "kernel_heatbath_fast_1t": { + "median": 129800000.0, + "q1": 78315000.0, + "q3": 147500000.0, + "min": 70350000.0, + "max": 163400000.0, + "n": 5, + "values": [ + 70350000.0, + 86280000.0, + 131600000.0, + 129800000.0, + 163400000.0 + ] + }, + "decide_12_wall_ms": { + "median": 69.60024600004999, + "q1": 63.16287800001419, + "q3": 78.27322700001105, + "min": 61.85951199995543, + "max": 84.27907800000867, + "n": 5, + "values": [ + 84.27907800000867, + 72.26737600001343, + 69.60024600004999, + 64.46624400007295, + 61.85951199995543 + ] + }, + "decide_12_pbit_ms": { + "median": 47.83686, + "q1": 43.4171595, + "q3": 53.7968705, + "min": 41.723932, + "max": 56.069897, + "n": 5, + "values": [ + 56.069897, + 47.83686, + 51.523844, + 45.110387, + 41.723932 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 642.3366230000056, + "q1": 576.7561979999982, + "q3": 667.2402094999939, + "min": 574.0179809999972, + "max": 672.9500769999959, + "n": 5, + "values": [ + 672.9500769999959, + 661.5303419999918, + 579.4944149999992, + 574.0179809999972, + 642.3366230000056 + ] + }, + "decide_300_pbit_ms": { + "median": 616.291193, + "q1": 550.9512515, + "q3": 638.087972, + "min": 547.5859, + "max": 641.962333, + "n": 5, + "values": [ + 641.962333, + 634.213611, + 554.316603, + 547.5859, + 616.291193 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 10627692, + "q1": 9386599.0, + "q3": 11972655.5, + "min": 9067606, + "max": 12438879, + "n": 5, + "values": [ + 9067606, + 9705592, + 12438879, + 11506432, + 10627692 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 26288128, + "q1": 24580096.0, + "q3": 26697728.0, + "min": 24580096, + "max": 26697728, + "n": 3, + "values": [ + 24580096, + 26288128, + 26697728 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -l (maximum resident set size, bytes)", + "native_over_portable": { + "T1 SoA f32 heat-bath LUT, splitmix": 1.213, + "T1 SoA f32 heat-bath LUT, Philox": 1.141, + "T0 multispin 64-lane, splitmix": 1.105, + "T0 multispin 64-lane, Philox": 1.049, + "T0 PT-in-a-word prefix-mask, splitm": 1.083, + "T0 multispin splitmix x 4 threads": 0.997, + "T0 multispin splitmix x10 threads": 1.037, + "fast multispin splitmix x 4 thr": 0.854, + "fast multispin splitmix x10 thr": 0.857, + "fast T1 f32 LUT, splitmix": 1.124, + "fast T1 f32 LUT, Philox": 0.976, + "fast T0 multispin, splitmix": 1.004, + "fast T0 multispin, Philox": 0.926, + "pbit stats IR ring, 1 thread": 0.792, + "pbit stats IR ring, 4 threads": 0.86, + "pbit decide 300 tasks, 4 thr (sampler)": 0.771 + }, + "threads_rows": { + "threads_1": { + "wall_ms": 295.4, + "wall_iqr": [ + 272.9, + 298.7 + ], + "cpu_ms": 309.0, + "site_updates_per_s_M": 3.86, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 258.0, + "wall_iqr": [ + 225.4, + 310.7 + ], + "cpu_ms": 342.6, + "site_updates_per_s_M": 6.59, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 304.1, + "wall_iqr": [ + 249.8, + 363.4 + ], + "cpu_ms": 489.8, + "site_updates_per_s_M": 10.02, + "verdict": "refused" + }, + "default": { + "threads": 4, + "wall_ms": 551.4, + "wall_iqr": [ + 532.5, + 566.1 + ], + "sweeps": 11792, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 255.3 +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/bench-windows-x86_64.json b/scripts/bench/results/2026-10-01/run-36969399069/bench-windows-x86_64.json new file mode 100644 index 0000000..3ea03eb --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/bench-windows-x86_64.json @@ -0,0 +1,1241 @@ +{ + "schema": "pbit-bench-matrix/1", + "label": "windows-x86_64", + "date_utc": "2026-10-02T05:35:01Z", + "git_sha": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "runner": { + "RUNNER_OS": "Windows", + "RUNNER_ARCH": "X64", + "ImageOS": "win25-vs2026", + "ImageVersion": "20260925.250.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "bench", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "build_s=28", + "runner=windows-latest" + ], + "machine": { + "system": "Windows", + "os_release": "2025Server", + "arch": "AMD64", + "python": "3.12.10", + "logical_cores": 2, + "cpu_cmd": "Get-CimInstance Win32_Processor", + "cpu_model": "AMD EPYC 9V74 80-Core Processor", + "physical_cores": 1, + "cim_logical": 2, + "memory_bytes": 8584425472, + "os_version": "Microsoft Windows Server 2025 Datacenter (build 26100)" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-pc-windows-msvc", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": null, + "uptime": null, + "t": "05:35:03" + }, + "after_stats": { + "loadavg": null, + "uptime": null, + "t": "05:35:13" + }, + "after_decide": { + "loadavg": null, + "uptime": null, + "t": "05:35:15" + }, + "after_kernels": { + "loadavg": null, + "uptime": null, + "t": "05:36:15" + }, + "after_pvn": { + "loadavg": null, + "uptime": null, + "t": "05:38:19" + }, + "end": { + "loadavg": null, + "uptime": null, + "t": "05:38:20" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 966656, + "version": "pbit 0.2.0", + "linkage": { + "file": "PE32+ executable for MS Windows 6.00 (console), x86-64, 5 sections", + "dumpbin_found_via": "vswhere (not on PATH)", + "dll_imports": [ + "bcryptprimitives.dll", + "api-ms-win-core-synch-l1-2-0.dll", + "KERNEL32.dll", + "ntdll.dll", + "VCRUNTIME140.dll", + "api-ms-win-crt-math-l1-1-0.dll", + "api-ms-win-crt-string-l1-1-0.dll", + "api-ms-win-crt-runtime-l1-1-0.dll", + "api-ms-win-crt-stdio-l1-1-0.dll", + "api-ms-win-crt-locale-l1-1-0.dll", + "api-ms-win-crt-heap-l1-1-0.dll" + ], + "dumpbin_rc": 0 + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "windows", + "arch": "x86_64", + "logical_cpus": 2 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 2, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 12645217, + "q1": 12324133.5, + "q3": 13241031.0, + "min": 12245368, + "max": 13336756, + "n": 5, + "values": [ + 13336756, + 13145306, + 12402899, + 12245368, + 12645217 + ] + }, + "site_updates_per_s": { + "median": 17278842, + "q1": 15664056.5, + "q3": 17647574.0, + "min": 15298137, + "max": 17996981, + "n": 5, + "values": [ + 16029976, + 15298137, + 17278842, + 17298167, + 17996981 + ] + }, + "threads": [ + 2 + ], + "speedup": { + "median": 1.39, + "q1": 1.18, + "q3": 1.415, + "min": 1.16, + "max": 1.42, + "n": 5, + "values": [ + 1.2, + 1.16, + 1.39, + 1.41, + 1.42 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 21.600100000000566, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.8469, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 20.618300000023737, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.4243, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 20.410799999922347, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.4998, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 20.442300000013347, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.419, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 20.66669999999249, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.6842, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 20.618300000023737, + "q1": 20.426549999967847, + "q3": 21.133399999996527, + "min": 20.410799999922347, + "max": 21.600100000000566, + "n": 5, + "values": [ + 21.600100000000566, + 20.618300000023737, + 20.410799999922347, + 20.442300000013347, + 20.66669999999249 + ] + }, + "pbit_ms": { + "median": 12.4998, + "q1": 12.42165, + "q3": 12.765550000000001, + "min": 12.419, + "max": 12.8469, + "n": 5, + "values": [ + 12.8469, + 12.4243, + 12.4998, + 12.419, + 12.6842 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 477.7017999999771, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 464.9719, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 10776, + "site_updates_per_s": 10754273, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 469.82330000003003, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 456.0813, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 10888, + "site_updates_per_s": 10874870, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 455.75429999996686, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 442.7194, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 10656, + "site_updates_per_s": 10631296, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 457.8821000000062, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 444.7189, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 10808, + "site_updates_per_s": 10788774, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 456.29230000008647, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 443.1207, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 10496, + "site_updates_per_s": 10482711, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 457.8821000000062, + "q1": 456.02330000002667, + "q3": 473.76255000000356, + "min": 455.75429999996686, + "max": 477.7017999999771, + "n": 5, + "values": [ + 477.7017999999771, + 469.82330000003003, + 455.75429999996686, + 457.8821000000062, + 456.29230000008647 + ] + }, + "pbit_ms": { + "median": 444.7189, + "q1": 442.92005, + "q3": 460.52660000000003, + "min": 442.7194, + "max": 464.9719, + "n": 5, + "values": [ + 464.9719, + 456.0813, + 442.7194, + 444.7189, + 443.1207 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 10754273, + "q1": 10557003.5, + "q3": 10831822.0, + "min": 10482711, + "max": 10874870, + "n": 5, + "values": [ + 10754273, + 10874870, + 10631296, + 10788774, + 10482711 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "K32GetProcessMemoryInfo PeakWorkingSetSize (process handle, after exit)", + "bytes": { + "median": 21155840, + "q1": 21151744.0, + "q3": 21282816.0, + "min": 21151744, + "max": 21282816, + "n": 3, + "values": [ + 21282816, + 21155840, + 21151744 + ] + }, + "self_reported_peak_rss_mb": [ + null, + null, + null + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + { + "peak_pagefile_bytes": 1079914496 + }, + { + "peak_pagefile_bytes": 1079472128 + }, + { + "peak_pagefile_bytes": 1079771136 + } + ] + }, + "kernels": { + "cmd": "target/release/examples/kernels.exe", + "checksums": [ + "8520834" + ], + "lines": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "median": 217300000.0, + "q1": 201450000.0, + "q3": 220950000.0, + "min": 194900000.0, + "max": 221400000.0, + "n": 5, + "values": [ + 217300000.0, + 220500000.0, + 194900000.0, + 208000000.0, + 221400000.0 + ] + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "median": 151000000.0, + "q1": 146400000.0, + "q3": 154700000.0, + "min": 144900000.0, + "max": 157000000.0, + "n": 5, + "values": [ + 144900000.0, + 151000000.0, + 152400000.0, + 157000000.0, + 147900000.0 + ] + }, + "T0 multispin 64-lane, splitmix": { + "median": 10500000000.0, + "q1": 9734500000.0, + "q3": 11510000000.0, + "min": 9029000000.0, + "max": 11670000000.0, + "n": 5, + "values": [ + 10500000000.0, + 9029000000.0, + 10440000000.0, + 11350000000.0, + 11670000000.0 + ] + }, + "T0 multispin 64-lane, Philox": { + "median": 8252000000.0, + "q1": 7529500000.0, + "q3": 8674500000.0, + "min": 7474000000.0, + "max": 8830000000.0, + "n": 5, + "values": [ + 7474000000.0, + 7585000000.0, + 8252000000.0, + 8519000000.0, + 8830000000.0 + ] + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "median": 2075000000.0, + "q1": 2052000000.0, + "q3": 2120000000.0, + "min": 2050000000.0, + "max": 2127000000.0, + "n": 5, + "values": [ + 2054000000.0, + 2075000000.0, + 2113000000.0, + 2127000000.0, + 2050000000.0 + ] + }, + "T0 multispin splitmix x 4 threads": { + "median": 10880000000.0, + "q1": 10440000000.0, + "q3": 11020000000.0, + "min": 10110000000.0, + "max": 11050000000.0, + "n": 5, + "values": [ + 10110000000.0, + 10770000000.0, + 11050000000.0, + 10880000000.0, + 10990000000.0 + ] + }, + "T0 multispin splitmix x10 threads": { + "median": 10930000000.0, + "q1": 10765000000.0, + "q3": 11300000000.0, + "min": 10750000000.0, + "max": 11340000000.0, + "n": 5, + "values": [ + 10750000000.0, + 11260000000.0, + 10930000000.0, + 11340000000.0, + 10780000000.0 + ] + }, + "fast multispin splitmix x 4 thr": { + "median": 14740000000.0, + "q1": 14575000000.0, + "q3": 14960000000.0, + "min": 14450000000.0, + "max": 15180000000.0, + "n": 5, + "values": [ + 15180000000.0, + 14740000000.0, + 14450000000.0, + 14700000000.0, + 14740000000.0 + ] + }, + "fast multispin splitmix x10 thr": { + "median": 14450000000.0, + "q1": 13970000000.0, + "q3": 14690000000.0, + "min": 13750000000.0, + "max": 14820000000.0, + "n": 5, + "values": [ + 14560000000.0, + 13750000000.0, + 14190000000.0, + 14450000000.0, + 14820000000.0 + ] + }, + "fast T1 f32 LUT, splitmix": { + "median": 333700000.0, + "q1": 329500000.0, + "q3": 338800000.0, + "min": 326200000.0, + "max": 342700000.0, + "n": 5, + "values": [ + 333700000.0, + 332800000.0, + 326200000.0, + 334900000.0, + 342700000.0 + ] + }, + "fast T1 f32 LUT, Philox": { + "median": 187000000.0, + "q1": 182850000.0, + "q3": 193500000.0, + "min": 180900000.0, + "max": 199800000.0, + "n": 5, + "values": [ + 187200000.0, + 180900000.0, + 199800000.0, + 184800000.0, + 187000000.0 + ] + }, + "fast T0 multispin, splitmix": { + "median": 13670000000.0, + "q1": 12475000000.0, + "q3": 15205000000.0, + "min": 11470000000.0, + "max": 15240000000.0, + "n": 5, + "values": [ + 13670000000.0, + 11470000000.0, + 13480000000.0, + 15240000000.0, + 15170000000.0 + ] + }, + "fast T0 multispin, Philox": { + "median": 10110000000.0, + "q1": 8694000000.0, + "q3": 10370000000.0, + "min": 7969000000.0, + "max": 10410000000.0, + "n": 5, + "values": [ + 9419000000.0, + 7969000000.0, + 10110000000.0, + 10410000000.0, + 10330000000.0 + ] + } + } + }, + "windows_shim": [ + "target/release/pbit", + "target/release/examples/kernels", + "target/native/release/pbit", + "target/native/release/examples/kernels" + ], + "portable_vs_native": { + "cmd": "PBIT_DIR=target/release PBIT_NATIVE_DIR=target/native/release python3 bench/portable_vs_native.py", + "exit": 0, + "rows": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "portable": 214900000.0, + "portable_iqr": [ + 118600000.0, + 221400000.0 + ], + "native": 205100000.0, + "native_iqr": [ + 181900000.0, + 224200000.0 + ], + "ratio": 0.954 + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "portable": 145500000.0, + "portable_iqr": [ + 104800000.0, + 150400000.0 + ], + "native": 160300000.0, + "native_iqr": [ + 139800000.0, + 162800000.0 + ], + "ratio": 1.102 + }, + "T0 multispin 64-lane, splitmix": { + "portable": 11400000000.0, + "portable_iqr": [ + 7356000000.0, + 11650000000.0 + ], + "native": 10770000000.0, + "native_iqr": [ + 8510000000.0, + 11420000000.0 + ], + "ratio": 0.945 + }, + "T0 multispin 64-lane, Philox": { + "portable": 8483000000.0, + "portable_iqr": [ + 6170000000.0, + 9102000000.0 + ], + "native": 8023000000.0, + "native_iqr": [ + 7060000000.0, + 8704000000.0 + ], + "ratio": 0.946 + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "portable": 2110000000.0, + "portable_iqr": [ + 1756000000.0, + 2116000000.0 + ], + "native": 2902000000.0, + "native_iqr": [ + 2582000000.0, + 2918000000.0 + ], + "ratio": 1.375 + }, + "T0 multispin splitmix x 4 threads": { + "portable": 11230000000.0, + "portable_iqr": [ + 10980000000.0, + 11570000000.0 + ], + "native": 11200000000.0, + "native_iqr": [ + 10540000000.0, + 11820000000.0 + ], + "ratio": 0.997 + }, + "T0 multispin splitmix x10 threads": { + "portable": 11030000000.0, + "portable_iqr": [ + 10800000000.0, + 11230000000.0 + ], + "native": 11280000000.0, + "native_iqr": [ + 10980000000.0, + 11710000000.0 + ], + "ratio": 1.023 + }, + "fast multispin splitmix x 4 thr": { + "portable": 13940000000.0, + "portable_iqr": [ + 12960000000.0, + 14930000000.0 + ], + "native": 14710000000.0, + "native_iqr": [ + 14530000000.0, + 15010000000.0 + ], + "ratio": 1.055 + }, + "fast multispin splitmix x10 thr": { + "portable": 14500000000.0, + "portable_iqr": [ + 14080000000.0, + 14720000000.0 + ], + "native": 15400000000.0, + "native_iqr": [ + 14880000000.0, + 15600000000.0 + ], + "ratio": 1.062 + }, + "fast T1 f32 LUT, splitmix": { + "portable": 334100000.0, + "portable_iqr": [ + 168500000.0, + 340800000.0 + ], + "native": 330700000.0, + "native_iqr": [ + 228300000.0, + 383300000.0 + ], + "ratio": 0.99 + }, + "fast T1 f32 LUT, Philox": { + "portable": 179400000.0, + "portable_iqr": [ + 137700000.0, + 190200000.0 + ], + "native": 197700000.0, + "native_iqr": [ + 157200000.0, + 203700000.0 + ], + "ratio": 1.102 + }, + "fast T0 multispin, splitmix": { + "portable": 14940000000.0, + "portable_iqr": [ + 11710000000.0, + 16040000000.0 + ], + "native": 12790000000.0, + "native_iqr": [ + 10480000000.0, + 15330000000.0 + ], + "ratio": 0.856 + }, + "fast T0 multispin, Philox": { + "portable": 10260000000.0, + "portable_iqr": [ + 8446000000.0, + 10560000000.0 + ], + "native": 9554000000.0, + "native_iqr": [ + 7924000000.0, + 10480000000.0 + ], + "ratio": 0.931 + }, + "pbit stats IR ring, 1 thread": { + "portable": 12400000.0, + "portable_iqr": [ + 11760000.0, + 13400000.0 + ], + "native": 13400000.0, + "native_iqr": [ + 11690000.0, + 13700000.0 + ], + "ratio": 1.081 + }, + "pbit stats IR ring, 4 threads": { + "portable": 15320000.0, + "portable_iqr": [ + 13810000.0, + 16790000.0 + ], + "native": 15830000.0, + "native_iqr": [ + 14120000.0, + 18290000.0 + ], + "ratio": 1.033 + }, + "pbit decide 300 tasks, 4 thr (sampler)": { + "portable": 11410000.0, + "portable_iqr": [ + 8898000.0, + 11450000.0 + ], + "native": 11610000.0, + "native_iqr": [ + 11210000.0, + 11740000.0 + ], + "ratio": 1.017 + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB, rustc 1.98.1; 5 alternating runs; medians [IQR]; ratio = native / portable", + "T1 SoA f32 heat-bath LUT, splitmix: portable 2.149e+08 [1.186e+08-2.214e+08] | native 2.051e+08 [1.819e+08-2.242e+08] | ratio 0.954", + "T1 SoA f32 heat-bath LUT, Philox: portable 1.455e+08 [1.048e+08-1.504e+08] | native 1.603e+08 [1.398e+08-1.628e+08] | ratio 1.102", + "T0 multispin 64-lane, splitmix: portable 1.14e+10 [7.356e+09-1.165e+10] | native 1.077e+10 [8.51e+09-1.142e+10] | ratio 0.945", + "T0 multispin 64-lane, Philox: portable 8.483e+09 [6.17e+09-9.102e+09] | native 8.023e+09 [7.06e+09-8.704e+09] | ratio 0.946", + "T0 PT-in-a-word prefix-mask, splitm: portable 2.11e+09 [1.756e+09-2.116e+09] | native 2.902e+09 [2.582e+09-2.918e+09] | ratio 1.375", + "T0 multispin splitmix x 4 threads: portable 1.123e+10 [1.098e+10-1.157e+10] | native 1.12e+10 [1.054e+10-1.182e+10] | ratio 0.997", + "T0 multispin splitmix x10 threads: portable 1.103e+10 [1.08e+10-1.123e+10] | native 1.128e+10 [1.098e+10-1.171e+10] | ratio 1.023", + "fast multispin splitmix x 4 thr: portable 1.394e+10 [1.296e+10-1.493e+10] | native 1.471e+10 [1.453e+10-1.501e+10] | ratio 1.055", + "fast multispin splitmix x10 thr: portable 1.45e+10 [1.408e+10-1.472e+10] | native 1.54e+10 [1.488e+10-1.56e+10] | ratio 1.062", + "fast T1 f32 LUT, splitmix: portable 3.341e+08 [1.685e+08-3.408e+08] | native 3.307e+08 [2.283e+08-3.833e+08] | ratio 0.990", + "fast T1 f32 LUT, Philox: portable 1.794e+08 [1.377e+08-1.902e+08] | native 1.977e+08 [1.572e+08-2.037e+08] | ratio 1.102", + "fast T0 multispin, splitmix: portable 1.494e+10 [1.171e+10-1.604e+10] | native 1.279e+10 [1.048e+10-1.533e+10] | ratio 0.856", + "fast T0 multispin, Philox: portable 1.026e+10 [8.446e+09-1.056e+10] | native 9.554e+09 [7.924e+09-1.048e+10] | ratio 0.931", + "pbit stats IR ring, 1 thread: portable 1.24e+07 [1.176e+07-1.34e+07] | native 1.34e+07 [1.169e+07-1.37e+07] | ratio 1.081", + "pbit stats IR ring, 4 threads: portable 1.532e+07 [1.381e+07-1.679e+07] | native 1.583e+07 [1.412e+07-1.829e+07] | ratio 1.033", + "pbit decide 300 tasks, 4 thr (sampler): portable 1.141e+07 [8.898e+06-1.145e+07] | native 1.161e+07 [1.121e+07-1.174e+07] | ratio 1.017", + "kernels checksum identical across builds: True | decide answer (odds+plan+gate) identical across builds: True" + ], + "stderr_tail": null, + "note": "the script's first line is hard-coded to the M4; the machine is the one in `machine`" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 1, + "rows": {}, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]" + ], + "stderr_tail": "Traceback (most recent call last):\n File \".\\bench\\decide_threads.py\", line 15, in \n print(f\"--threads {t}: wall ms {statistics.median(ms):.1f} [{q[0]:.1f}-{q[2]:.1f}], CPU ms {statistics.median(cpu):.1f}, site updates/s {statistics.median(ups)/1e6:.2f} M; answer identical: yes; verdict {d['verdict']}\")\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\hostedtoolcache\\windows\\Python\\3.12.10\\x64\\Lib\\statistics.py\", line 614, in median\n data = sorted(data)\n ^^^^^^^^^^^^\nTypeError: '<' not supported between instances of 'NoneType' and 'NoneType'\n" + }, + "attached": { + "checks": { + "build": "success", + "native_build": "success", + "pack_zip": "success", + "callers": "success", + "install_sh": "skipped", + "npm": "skipped", + "npm_windows": "success", + "install_ps1_pwsh": "success", + "install_ps1_ps51": "failure", + "ps_callers": "failure", + "rss_powershell": "success", + "cross_darwin": "skipped" + }, + "peak_rss_powershell": { + "method": "Get-Process PeakWorkingSet64, polled every ~5 ms until exit (lower bound)", + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "powershell": "7.6.6", + "runs": [ + { + "peak_working_set_bytes": 20324352, + "samples": 17, + "verdict": "diagnostics_passed" + }, + { + "peak_working_set_bytes": 20955136, + "samples": 23, + "verdict": "partial" + }, + { + "peak_working_set_bytes": 21168128, + "samples": 25, + "verdict": "diagnostics_passed" + } + ] + } + }, + "cells": { + "cpu_model": "AMD EPYC 9V74 80-Core Processor", + "logical_cores": 2, + "binary_size_bytes": 966656, + "stats_1t": { + "median": 12645217, + "q1": 12324133.5, + "q3": 13241031.0, + "min": 12245368, + "max": 13336756, + "n": 5, + "values": [ + 13336756, + 13145306, + 12402899, + 12245368, + 12645217 + ] + }, + "stats_default": { + "median": 17278842, + "q1": 15664056.5, + "q3": 17647574.0, + "min": 15298137, + "max": 17996981, + "n": 5, + "values": [ + 16029976, + 15298137, + 17278842, + 17298167, + 17996981 + ] + }, + "stats_default_threads": [ + 2 + ], + "kernel_multispin_fast_4t": { + "median": 14740000000.0, + "q1": 14575000000.0, + "q3": 14960000000.0, + "min": 14450000000.0, + "max": 15180000000.0, + "n": 5, + "values": [ + 15180000000.0, + 14740000000.0, + 14450000000.0, + 14700000000.0, + 14740000000.0 + ] + }, + "kernel_multispin_fast_10t": { + "median": 14450000000.0, + "q1": 13970000000.0, + "q3": 14690000000.0, + "min": 13750000000.0, + "max": 14820000000.0, + "n": 5, + "values": [ + 14560000000.0, + 13750000000.0, + 14190000000.0, + 14450000000.0, + 14820000000.0 + ] + }, + "kernel_multispin_fast_1t": { + "median": 13670000000.0, + "q1": 12475000000.0, + "q3": 15205000000.0, + "min": 11470000000.0, + "max": 15240000000.0, + "n": 5, + "values": [ + 13670000000.0, + 11470000000.0, + 13480000000.0, + 15240000000.0, + 15170000000.0 + ] + }, + "kernel_heatbath_fast_1t": { + "median": 333700000.0, + "q1": 329500000.0, + "q3": 338800000.0, + "min": 326200000.0, + "max": 342700000.0, + "n": 5, + "values": [ + 333700000.0, + 332800000.0, + 326200000.0, + 334900000.0, + 342700000.0 + ] + }, + "decide_12_wall_ms": { + "median": 20.618300000023737, + "q1": 20.426549999967847, + "q3": 21.133399999996527, + "min": 20.410799999922347, + "max": 21.600100000000566, + "n": 5, + "values": [ + 21.600100000000566, + 20.618300000023737, + 20.410799999922347, + 20.442300000013347, + 20.66669999999249 + ] + }, + "decide_12_pbit_ms": { + "median": 12.4998, + "q1": 12.42165, + "q3": 12.765550000000001, + "min": 12.419, + "max": 12.8469, + "n": 5, + "values": [ + 12.8469, + 12.4243, + 12.4998, + 12.419, + 12.6842 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 457.8821000000062, + "q1": 456.02330000002667, + "q3": 473.76255000000356, + "min": 455.75429999996686, + "max": 477.7017999999771, + "n": 5, + "values": [ + 477.7017999999771, + 469.82330000003003, + 455.75429999996686, + 457.8821000000062, + 456.29230000008647 + ] + }, + "decide_300_pbit_ms": { + "median": 444.7189, + "q1": 442.92005, + "q3": 460.52660000000003, + "min": 442.7194, + "max": 464.9719, + "n": 5, + "values": [ + 464.9719, + 456.0813, + 442.7194, + 444.7189, + 443.1207 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 10754273, + "q1": 10557003.5, + "q3": 10831822.0, + "min": 10482711, + "max": 10874870, + "n": 5, + "values": [ + 10754273, + 10874870, + 10631296, + 10788774, + 10482711 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 21155840, + "q1": 21151744.0, + "q3": 21282816.0, + "min": 21151744, + "max": 21282816, + "n": 3, + "values": [ + 21282816, + 21155840, + 21151744 + ] + }, + "decide_300_peak_rss_method": "K32GetProcessMemoryInfo PeakWorkingSetSize (process handle, after exit)", + "native_over_portable": { + "T1 SoA f32 heat-bath LUT, splitmix": 0.954, + "T1 SoA f32 heat-bath LUT, Philox": 1.102, + "T0 multispin 64-lane, splitmix": 0.945, + "T0 multispin 64-lane, Philox": 0.946, + "T0 PT-in-a-word prefix-mask, splitm": 1.375, + "T0 multispin splitmix x 4 threads": 0.997, + "T0 multispin splitmix x10 threads": 1.023, + "fast multispin splitmix x 4 thr": 1.055, + "fast multispin splitmix x10 thr": 1.062, + "fast T1 f32 LUT, splitmix": 0.99, + "fast T1 f32 LUT, Philox": 1.102, + "fast T0 multispin, splitmix": 0.856, + "fast T0 multispin, Philox": 0.931, + "pbit stats IR ring, 1 thread": 1.081, + "pbit stats IR ring, 4 threads": 1.033, + "pbit decide 300 tasks, 4 thr (sampler)": 1.017 + }, + "threads_rows": "n/a: exit 1, TypeError: '<' not supported between instances of 'NoneType' and 'NoneType'" + }, + "elapsed_s": 199.1 +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/cross-linux-x86_64.json b/scripts/bench/results/2026-10-01/run-36969399069/cross-linux-x86_64.json new file mode 100644 index 0000000..1ab4957 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/cross-linux-x86_64.json @@ -0,0 +1,1753 @@ +{ + "schema": "pbit-cross/1", + "label": "cross-linux-x86_64", + "date_utc": "2026-10-02T05:35:29Z", + "outcomes": { + "musl_x86_64": "success", + "musl_aarch64": "success", + "wasm32_wasip1": "success", + "msrv": "success", + "measure": "success" + }, + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "cross", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "INTEL(R) XEON(R) PLATINUM 8573C", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "GenuineIntel", + "Model name": "INTEL(R) XEON(R) PLATINUM 8573C", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": true, + "avx512bw": true, + "avx512vl": true + }, + "memory_bytes": 8322932736, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "transcripts": { + "msrv.txt": [ + "rust 1.72: build FAILED, Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4): error[E0658]: use of unstable library feature 'int_roundings'", + "rust 1.73: build ok, Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4)", + "rust 1.77: build FAILED, --locked (Cargo.lock as committed, lockfile v4): error: failed to parse lock file at: /home/runner/work/pbit/pbit/Cargo.lock", + "rust 1.78: build ok, --locked (Cargo.lock as committed, lockfile v4)", + "Cargo.lock restored: 0 changes" + ], + "musl-aarch64.txt": [ + "$ sudo apt-get install -y gcc-aarch64-linux-gnu", + "", + "Running kernel seems to be up-to-date.", + "", + "No services need to be restarted.", + "", + "No containers need to be restarted.", + "", + "No user sessions are running outdated binaries.", + "", + "No VM guests are running outdated hypervisor (qemu) binaries on this host.", + "linker installed in 13 s", + "$ rustup target add aarch64-unknown-linux-musl", + "info: downloading component rust-std", + "$ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl", + " Finished `release` profile [optimized] target(s) in 19.31s", + "$ file target/aarch64-unknown-linux-musl/release/pbit", + "target/aarch64-unknown-linux-musl/release/pbit: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), statically linked, BuildID[sha1]=6b9608f1981e300fe1d149bd279ce6fa0ba3e383, not stripped", + "size 1477600 bytes", + "qemu-user-static installed in 4 s", + "$ qemu-aarch64-static target/aarch64-unknown-linux-musl/release/pbit version", + "pbit 0.2.0", + "$ qemu pbit demo --tasks 12 | qemu pbit decide: exit 0, verdict exact" + ], + "musl-x86_64.txt": [ + "$ rustup target add x86_64-unknown-linux-musl", + "info: downloading component rust-std", + "$ cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl", + " Finished `release` profile [optimized] target(s) in 18.87s", + "$ file target/x86_64-unknown-linux-musl/release/pbit", + "target/x86_64-unknown-linux-musl/release/pbit: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), static-pie linked, BuildID[sha1]=85ea1386df8c2738b288e74a66fd13f6a9490feb, not stripped", + "$ ldd target/x86_64-unknown-linux-musl/release/pbit", + "\tstatically linked", + "size 1545968 bytes", + "$ target/x86_64-unknown-linux-musl/release/pbit version", + "pbit 0.2.0", + "$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact" + ], + "wasm32-wasip1.txt": [ + "$ rustup target add wasm32-wasip1", + "info: downloading component rust-std", + "$ cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide", + "exit 0", + "first error: none", + " Checking pbit-ir v0.2.0 (/home/runner/work/pbit/pbit/pbit-ir)", + " Checking pbit-decide v0.2.0 (/home/runner/work/pbit/pbit/pbit-decide)", + " Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.69s" + ] + }, + "bench": { + "linux-x86_64-gnu-samevm": { + "schema": "pbit-bench-matrix/1", + "label": "linux-x86_64-gnu-samevm", + "date_utc": "2026-10-02T05:35:18Z", + "git_sha": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "cross", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "glibc build, same VM as the musl row" + ], + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "INTEL(R) XEON(R) PLATINUM 8573C", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "GenuineIntel", + "Model name": "INTEL(R) XEON(R) PLATINUM 8573C", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": true, + "avx512bw": true, + "avx512vl": true + }, + "memory_bytes": 8322932736, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 1.5732421875, + 0.88134765625, + 0.3720703125 + ], + "uptime": "05:35:18 up 4 min, 0 user, load average: 1.57, 0.88, 0.37", + "t": "05:35:18" + }, + "after_stats": { + "loadavg": [ + 1.52685546875, + 0.8837890625, + 0.37548828125 + ], + "uptime": "05:35:21 up 4 min, 0 user, load average: 1.53, 0.88, 0.38", + "t": "05:35:21" + }, + "after_decide": { + "loadavg": [ + 1.52685546875, + 0.8837890625, + 0.37548828125 + ], + "uptime": "05:35:23 up 4 min, 0 user, load average: 1.53, 0.88, 0.38", + "t": "05:35:23" + }, + "after_kernels": { + "loadavg": [ + 1.56494140625, + 0.90234375, + 0.38427734375 + ], + "uptime": "05:35:25 up 4 min, 0 user, load average: 1.56, 0.90, 0.38", + "t": "05:35:25" + }, + "after_pvn": { + "loadavg": [ + 1.56494140625, + 0.90234375, + 0.38427734375 + ], + "uptime": "05:35:25 up 4 min, 0 user, load average: 1.56, 0.90, 0.38", + "t": "05:35:25" + }, + "end": { + "loadavg": [ + 1.60009765625, + 0.9208984375, + 0.39306640625 + ], + "uptime": "05:35:29 up 4 min, 0 user, load average: 1.60, 0.92, 0.39", + "t": "05:35:29" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 1401248, + "version": "pbit 0.2.0", + "linkage": { + "file": "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 3.2.0, BuildID[sha1]=3a4791ff347b29ab20fc5b1ff998dd05660b1d15, not stripped", + "ldd": [ + "linux-vdso.so.1 (0x00007efc52923000)", + "\tlibgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007efc527b5000)", + "\tlibm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007efc526cc000)", + "\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007efc52400000)", + "\t/lib64/ld-linux-x86-64.so.2 (0x00007efc52925000)" + ], + "glibc_symbol_versions": [ + "2.2.5", + "2.3", + "2.3.4", + "2.14", + "2.16", + "2.17", + "2.18", + "2.25", + "2.28", + "2.29", + "2.30", + "2.32", + "2.33", + "2.34" + ], + "min_glibc": "2.34" + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "linux", + "arch": "x86_64", + "logical_cpus": 2 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 2, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 12050263, + "q1": 12034773.5, + "q3": 12066932.5, + "min": 12033304, + "max": 12070736, + "n": 5, + "values": [ + 12033304, + 12036243, + 12063129, + 12070736, + 12050263 + ] + }, + "site_updates_per_s": { + "median": 15575011, + "q1": 15445940.0, + "q3": 15599585.5, + "min": 15346475, + "max": 15616096, + "n": 5, + "values": [ + 15575011, + 15346475, + 15616096, + 15583075, + 15545405 + ] + }, + "threads": [ + 2 + ], + "speedup": { + "median": 1.29, + "q1": 1.2850000000000001, + "q3": 1.29, + "min": 1.28, + "max": 1.29, + "n": 5, + "values": [ + 1.29, + 1.28, + 1.29, + 1.29, + 1.29 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 16.310452999960035, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.852785, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 16.021156000022074, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.613127, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 15.986572000031174, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.621756, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 16.12302700004875, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.772905, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 15.932785999950738, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.607325, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 16.021156000022074, + "q1": 15.959678999990956, + "q3": 16.216740000004393, + "min": 15.932785999950738, + "max": 16.310452999960035, + "n": 5, + "values": [ + 16.310452999960035, + 16.021156000022074, + 15.986572000031174, + 16.12302700004875, + 15.932785999950738 + ] + }, + "pbit_ms": { + "median": 14.621756, + "q1": 14.610226, + "q3": 14.812845, + "min": 14.607325, + "max": 14.852785, + "n": 5, + "values": [ + 14.852785, + 14.613127, + 14.621756, + 14.772905, + 14.607325 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 454.34242500004984, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 449.546502, + "verdict": "partial", + "tier": null, + "released": 298, + "escalated": 2, + "violations": 0, + "sweeps": 8992, + "site_updates_per_s": 8974146, + "threads": 2, + "peak_rss_mb_self": 21.355469, + "stderr": null + }, + { + "wall_ms": 455.2757420000262, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 450.298547, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 8904, + "site_updates_per_s": 8893716, + "threads": 2, + "peak_rss_mb_self": 21.386719, + "stderr": null + }, + { + "wall_ms": 454.6698190000029, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 449.541653, + "verdict": "partial", + "tier": null, + "released": 299, + "escalated": 1, + "violations": 0, + "sweeps": 8872, + "site_updates_per_s": 8855982, + "threads": 2, + "peak_rss_mb_self": 21.207031, + "stderr": null + }, + { + "wall_ms": 455.9836380000206, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 450.782181, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 8904, + "site_updates_per_s": 8893969, + "threads": 2, + "peak_rss_mb_self": 21.390625, + "stderr": null + }, + { + "wall_ms": 454.67246999999134, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 449.679244, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 8968, + "site_updates_per_s": 8952163, + "threads": 2, + "peak_rss_mb_self": 21.277344, + "stderr": null + } + ], + "wall_ms": { + "median": 454.67246999999134, + "q1": 454.5061220000264, + "q3": 455.6296900000234, + "min": 454.34242500004984, + "max": 455.9836380000206, + "n": 5, + "values": [ + 454.34242500004984, + 455.2757420000262, + 454.6698190000029, + 455.9836380000206, + 454.67246999999134 + ] + }, + "pbit_ms": { + "median": 449.679244, + "q1": 449.54407749999996, + "q3": 450.54036399999995, + "min": 449.541653, + "max": 450.782181, + "n": 5, + "values": [ + 449.546502, + 450.298547, + 449.541653, + 450.782181, + 449.679244 + ] + }, + "released": { + "median": 300, + "q1": 298.5, + "q3": 300.0, + "min": 298, + "max": 300, + "n": 5, + "values": [ + 298, + 300, + 299, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed", + "partial" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 8893969, + "q1": 8874849.0, + "q3": 8963154.5, + "min": 8855982, + "max": 8974146, + "n": 5, + "values": [ + 8974146, + 8893716, + 8855982, + 8893969, + 8952163 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "bytes": { + "median": 22396928, + "q1": 22384640.0, + "q3": 22433792.0, + "min": 22384640, + "max": 22433792, + "n": 3, + "values": [ + 22384640, + 22433792, + 22396928 + ] + }, + "self_reported_peak_rss_mb": [ + 21.347656, + 21.394531, + 21.359375 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + {}, + {}, + {} + ] + }, + "kernels": { + "skipped": "no kernels example at target/release/examples/kernels (build it with -p pbit-core --example kernels)" + }, + "portable_vs_native": { + "skipped": "no --native-dir" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 165.3, + "wall_iqr": [ + 164.5, + 166.4 + ], + "cpu_ms": 168.2, + "site_updates_per_s_M": 6.12, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 141.5, + "wall_iqr": [ + 141.3, + 143.1 + ], + "cpu_ms": 199.6, + "site_updates_per_s_M": 8.8, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 142.4, + "wall_iqr": [ + 142.0, + 142.6 + ], + "cpu_ms": 199.4, + "site_updates_per_s_M": 8.71, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 449.8, + "wall_iqr": [ + 449.5, + 450.1 + ], + "sweeps": 8960, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 165.3 [164.5-166.4], CPU ms 168.2, site updates/s 6.12 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 141.5 [141.3-143.1], CPU ms 199.6, site updates/s 8.80 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 142.4 [142.0-142.6], CPU ms 199.4, site updates/s 8.71 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 2): wall ms 449.8 [449.5-450.1], sweeps 8960, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "cells": { + "cpu_model": "INTEL(R) XEON(R) PLATINUM 8573C", + "logical_cores": 2, + "binary_size_bytes": 1401248, + "stats_1t": { + "median": 12050263, + "q1": 12034773.5, + "q3": 12066932.5, + "min": 12033304, + "max": 12070736, + "n": 5, + "values": [ + 12033304, + 12036243, + 12063129, + 12070736, + 12050263 + ] + }, + "stats_default": { + "median": 15575011, + "q1": 15445940.0, + "q3": 15599585.5, + "min": 15346475, + "max": 15616096, + "n": 5, + "values": [ + 15575011, + 15346475, + 15616096, + 15583075, + 15545405 + ] + }, + "stats_default_threads": [ + 2 + ], + "kernel_multispin_fast_4t": null, + "kernel_multispin_fast_10t": null, + "kernel_multispin_fast_1t": null, + "kernel_heatbath_fast_1t": null, + "decide_12_wall_ms": { + "median": 16.021156000022074, + "q1": 15.959678999990956, + "q3": 16.216740000004393, + "min": 15.932785999950738, + "max": 16.310452999960035, + "n": 5, + "values": [ + 16.310452999960035, + 16.021156000022074, + 15.986572000031174, + 16.12302700004875, + 15.932785999950738 + ] + }, + "decide_12_pbit_ms": { + "median": 14.621756, + "q1": 14.610226, + "q3": 14.812845, + "min": 14.607325, + "max": 14.852785, + "n": 5, + "values": [ + 14.852785, + 14.613127, + 14.621756, + 14.772905, + 14.607325 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 454.67246999999134, + "q1": 454.5061220000264, + "q3": 455.6296900000234, + "min": 454.34242500004984, + "max": 455.9836380000206, + "n": 5, + "values": [ + 454.34242500004984, + 455.2757420000262, + 454.6698190000029, + 455.9836380000206, + 454.67246999999134 + ] + }, + "decide_300_pbit_ms": { + "median": 449.679244, + "q1": 449.54407749999996, + "q3": 450.54036399999995, + "min": 449.541653, + "max": 450.782181, + "n": 5, + "values": [ + 449.546502, + 450.298547, + 449.541653, + 450.782181, + 449.679244 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed", + "partial" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 298.5, + "q3": 300.0, + "min": 298, + "max": 300, + "n": 5, + "values": [ + 298, + 300, + 299, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 8893969, + "q1": 8874849.0, + "q3": 8963154.5, + "min": 8855982, + "max": 8974146, + "n": 5, + "values": [ + 8974146, + 8893716, + 8855982, + 8893969, + 8952163 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 22396928, + "q1": 22384640.0, + "q3": 22433792.0, + "min": 22384640, + "max": 22433792, + "n": 3, + "values": [ + 22384640, + 22433792, + 22396928 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "native_over_portable": "no --native-dir", + "threads_rows": { + "threads_1": { + "wall_ms": 165.3, + "wall_iqr": [ + 164.5, + 166.4 + ], + "cpu_ms": 168.2, + "site_updates_per_s_M": 6.12, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 141.5, + "wall_iqr": [ + 141.3, + 143.1 + ], + "cpu_ms": 199.6, + "site_updates_per_s_M": 8.8, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 142.4, + "wall_iqr": [ + 142.0, + 142.6 + ], + "cpu_ms": 199.4, + "site_updates_per_s_M": 8.71, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 449.8, + "wall_iqr": [ + 449.5, + 450.1 + ], + "sweeps": 8960, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 11.8 + }, + "linux-x86_64-musl": { + "schema": "pbit-bench-matrix/1", + "label": "linux-x86_64-musl", + "date_utc": "2026-10-02T05:35:05Z", + "git_sha": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "cross", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "static musl build, same VM as bench-linux-x86_64-gnu-samevm" + ], + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "INTEL(R) XEON(R) PLATINUM 8573C", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "GenuineIntel", + "Model name": "INTEL(R) XEON(R) PLATINUM 8573C", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": true, + "avx512bw": true, + "avx512vl": true + }, + "memory_bytes": 8322932736, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 1.67822265625, + 0.87646484375, + 0.365234375 + ], + "uptime": "05:35:05 up 4 min, 0 user, load average: 1.68, 0.88, 0.37", + "t": "05:35:05" + }, + "after_stats": { + "loadavg": [ + 1.62353515625, + 0.87890625, + 0.36865234375 + ], + "uptime": "05:35:09 up 4 min, 0 user, load average: 1.62, 0.88, 0.37", + "t": "05:35:09" + }, + "after_decide": { + "loadavg": [ + 1.62353515625, + 0.87890625, + 0.36865234375 + ], + "uptime": "05:35:11 up 4 min, 0 user, load average: 1.62, 0.88, 0.37", + "t": "05:35:11" + }, + "after_kernels": { + "loadavg": [ + 1.62353515625, + 0.87890625, + 0.36865234375 + ], + "uptime": "05:35:13 up 4 min, 0 user, load average: 1.62, 0.88, 0.37", + "t": "05:35:13" + }, + "after_pvn": { + "loadavg": [ + 1.62353515625, + 0.87890625, + 0.36865234375 + ], + "uptime": "05:35:13 up 4 min, 0 user, load average: 1.62, 0.88, 0.37", + "t": "05:35:13" + }, + "end": { + "loadavg": [ + 1.5732421875, + 0.88134765625, + 0.3720703125 + ], + "uptime": "05:35:18 up 4 min, 0 user, load average: 1.57, 0.88, 0.37", + "t": "05:35:18" + } + }, + "errors": [], + "binary": { + "path": "target/x86_64-unknown-linux-musl/release/pbit", + "size_bytes": 1545968, + "version": "pbit 0.2.0", + "linkage": { + "file": "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), static-pie linked, BuildID[sha1]=85ea1386df8c2738b288e74a66fd13f6a9490feb, not stripped", + "ldd": [ + "statically linked" + ], + "glibc_symbol_versions": [], + "min_glibc": null + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "linux", + "arch": "x86_64", + "logical_cpus": 2 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 2, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 12212185, + "q1": 11390408.5, + "q3": 12272753.0, + "min": 10825109, + "max": 12299880, + "n": 5, + "values": [ + 10825109, + 12299880, + 12212185, + 11955708, + 12245626 + ] + }, + "site_updates_per_s": { + "median": 14486702, + "q1": 13850414.5, + "q3": 14560172.0, + "min": 13368990, + "max": 14603511, + "n": 5, + "values": [ + 14486702, + 14331839, + 13368990, + 14516833, + 14603511 + ] + }, + "threads": [ + 2 + ], + "speedup": { + "median": 1.19, + "q1": 1.13, + "q3": 1.275, + "min": 1.09, + "max": 1.34, + "n": 5, + "values": [ + 1.34, + 1.17, + 1.09, + 1.21, + 1.19 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 16.301381999994646, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 15.011611, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 16.178147000005083, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.912711, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 15.907540999990033, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.736992, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 15.797739999982241, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.588939, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 15.718410000005179, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 14.633027, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 15.907540999990033, + "q1": 15.75807499999371, + "q3": 16.239764499999865, + "min": 15.718410000005179, + "max": 16.301381999994646, + "n": 5, + "values": [ + 16.301381999994646, + 16.178147000005083, + 15.907540999990033, + 15.797739999982241, + 15.718410000005179 + ] + }, + "pbit_ms": { + "median": 14.736992, + "q1": 14.610983000000001, + "q3": 14.962161, + "min": 14.588939, + "max": 15.011611, + "n": 5, + "values": [ + 15.011611, + 14.912711, + 14.736992, + 14.588939, + 14.633027 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 460.47870799998236, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 452.55244, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 8952, + "site_updates_per_s": 8936242, + "threads": 2, + "peak_rss_mb_self": 17.148438, + "stderr": null + }, + { + "wall_ms": 462.67710099999704, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 454.583414, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9504, + "site_updates_per_s": 9482343, + "threads": 2, + "peak_rss_mb_self": 17.351563, + "stderr": null + }, + { + "wall_ms": 462.6404050000019, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 454.493591, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9488, + "site_updates_per_s": 9472831, + "threads": 2, + "peak_rss_mb_self": 16.28125, + "stderr": null + }, + { + "wall_ms": 462.4570839999933, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 454.655133, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9480, + "site_updates_per_s": 9463056, + "threads": 2, + "peak_rss_mb_self": 18.203125, + "stderr": null + }, + { + "wall_ms": 461.15297099999, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 453.155324, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9000, + "site_updates_per_s": 8983317, + "threads": 2, + "peak_rss_mb_self": 16.835938, + "stderr": null + } + ], + "wall_ms": { + "median": 462.4570839999933, + "q1": 460.81583949998617, + "q3": 462.6587529999995, + "min": 460.47870799998236, + "max": 462.67710099999704, + "n": 5, + "values": [ + 460.47870799998236, + 462.67710099999704, + 462.6404050000019, + 462.4570839999933, + 461.15297099999 + ] + }, + "pbit_ms": { + "median": 454.493591, + "q1": 452.853882, + "q3": 454.61927349999996, + "min": 452.55244, + "max": 454.655133, + "n": 5, + "values": [ + 452.55244, + 454.583414, + 454.493591, + 454.655133, + 453.155324 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 9463056, + "q1": 8959779.5, + "q3": 9477587.0, + "min": 8936242, + "max": 9482343, + "n": 5, + "values": [ + 8936242, + 9482343, + 9472831, + 9463056, + 8983317 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "bytes": { + "median": 18997248, + "q1": 17096704.0, + "q3": 19005440.0, + "min": 17096704, + "max": 19005440, + "n": 3, + "values": [ + 17096704, + 19005440, + 18997248 + ] + }, + "self_reported_peak_rss_mb": [ + 16.304688, + 18.125, + 18.117188 + ], + "verdicts": [ + "partial", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + {}, + {}, + {} + ] + }, + "kernels": { + "skipped": "no kernels example at target/x86_64-unknown-linux-musl/release/examples/kernels (build it with -p pbit-core --example kernels)" + }, + "portable_vs_native": { + "skipped": "no --native-dir" + }, + "decide_threads": { + "cmd": "PBIT=target/x86_64-unknown-linux-musl/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 165.7, + "wall_iqr": [ + 165.2, + 166.4 + ], + "cpu_ms": 170.0, + "site_updates_per_s_M": 6.31, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 146.5, + "wall_iqr": [ + 145.3, + 149.3 + ], + "cpu_ms": 206.1, + "site_updates_per_s_M": 8.64, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 144.5, + "wall_iqr": [ + 143.5, + 146.2 + ], + "cpu_ms": 200.7, + "site_updates_per_s_M": 9.04, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 455.9, + "wall_iqr": [ + 453.9, + 456.9 + ], + "sweeps": 9080, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 165.7 [165.2-166.4], CPU ms 170.0, site updates/s 6.31 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 146.5 [145.3-149.3], CPU ms 206.1, site updates/s 8.64 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 144.5 [143.5-146.2], CPU ms 200.7, site updates/s 9.04 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 2): wall ms 455.9 [453.9-456.9], sweeps 9080, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "cells": { + "cpu_model": "INTEL(R) XEON(R) PLATINUM 8573C", + "logical_cores": 2, + "binary_size_bytes": 1545968, + "stats_1t": { + "median": 12212185, + "q1": 11390408.5, + "q3": 12272753.0, + "min": 10825109, + "max": 12299880, + "n": 5, + "values": [ + 10825109, + 12299880, + 12212185, + 11955708, + 12245626 + ] + }, + "stats_default": { + "median": 14486702, + "q1": 13850414.5, + "q3": 14560172.0, + "min": 13368990, + "max": 14603511, + "n": 5, + "values": [ + 14486702, + 14331839, + 13368990, + 14516833, + 14603511 + ] + }, + "stats_default_threads": [ + 2 + ], + "kernel_multispin_fast_4t": null, + "kernel_multispin_fast_10t": null, + "kernel_multispin_fast_1t": null, + "kernel_heatbath_fast_1t": null, + "decide_12_wall_ms": { + "median": 15.907540999990033, + "q1": 15.75807499999371, + "q3": 16.239764499999865, + "min": 15.718410000005179, + "max": 16.301381999994646, + "n": 5, + "values": [ + 16.301381999994646, + 16.178147000005083, + 15.907540999990033, + 15.797739999982241, + 15.718410000005179 + ] + }, + "decide_12_pbit_ms": { + "median": 14.736992, + "q1": 14.610983000000001, + "q3": 14.962161, + "min": 14.588939, + "max": 15.011611, + "n": 5, + "values": [ + 15.011611, + 14.912711, + 14.736992, + 14.588939, + 14.633027 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 462.4570839999933, + "q1": 460.81583949998617, + "q3": 462.6587529999995, + "min": 460.47870799998236, + "max": 462.67710099999704, + "n": 5, + "values": [ + 460.47870799998236, + 462.67710099999704, + 462.6404050000019, + 462.4570839999933, + 461.15297099999 + ] + }, + "decide_300_pbit_ms": { + "median": 454.493591, + "q1": 452.853882, + "q3": 454.61927349999996, + "min": 452.55244, + "max": 454.655133, + "n": 5, + "values": [ + 452.55244, + 454.583414, + 454.493591, + 454.655133, + 453.155324 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 9463056, + "q1": 8959779.5, + "q3": 9477587.0, + "min": 8936242, + "max": 9482343, + "n": 5, + "values": [ + 8936242, + 9482343, + 9472831, + 9463056, + 8983317 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 18997248, + "q1": 17096704.0, + "q3": 19005440.0, + "min": 17096704, + "max": 19005440, + "n": 3, + "values": [ + 17096704, + 19005440, + 18997248 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "native_over_portable": "no --native-dir", + "threads_rows": { + "threads_1": { + "wall_ms": 165.7, + "wall_iqr": [ + 165.2, + 166.4 + ], + "cpu_ms": 170.0, + "site_updates_per_s_M": 6.31, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 146.5, + "wall_iqr": [ + 145.3, + 149.3 + ], + "cpu_ms": 206.1, + "site_updates_per_s_M": 8.64, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 144.5, + "wall_iqr": [ + 143.5, + 146.2 + ], + "cpu_ms": 200.7, + "site_updates_per_s_M": 9.04, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 455.9, + "wall_iqr": [ + 453.9, + 456.9 + ], + "sweeps": 9080, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 12.3 + } + } +} \ No newline at end of file diff --git a/scripts/bench/results/2026-10-01/run-36969399069/test-linux-x86_64.json b/scripts/bench/results/2026-10-01/run-36969399069/test-linux-x86_64.json new file mode 100644 index 0000000..d7bae17 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/test-linux-x86_64.json @@ -0,0 +1,72 @@ +{ + "schema": "pbit-test-matrix/1", + "label": "linux-x86_64", + "date_utc": "2026-10-02T05:38:17Z", + "cmd": "cargo test --release --locked --workspace", + "exit": 0, + "passed": true, + "seconds": 326, + "test_binaries": 10, + "totals": { + "passed": 127, + "failed": 0, + "ignored": 1, + "measured": 0, + "filtered_out": 0 + }, + "failures": [], + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "test", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "AuthenticAMD", + "Model name": "AMD EPYC 7763 64-Core Processor", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": false, + "avx512bw": false, + "avx512vl": false + }, + "memory_bytes": 8323969024, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + } +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/test-macos-arm64.json b/scripts/bench/results/2026-10-01/run-36969399069/test-macos-arm64.json new file mode 100644 index 0000000..a4bc255 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/test-macos-arm64.json @@ -0,0 +1,55 @@ +{ + "schema": "pbit-test-matrix/1", + "label": "macos-arm64", + "date_utc": "2026-10-02T05:37:10Z", + "cmd": "cargo test --release --locked --workspace", + "exit": 0, + "passed": true, + "seconds": 250, + "test_binaries": 10, + "totals": { + "passed": 127, + "failed": 0, + "ignored": 1, + "measured": 0, + "filtered_out": 0 + }, + "failures": [], + "runner": { + "RUNNER_OS": "macOS", + "RUNNER_ARCH": "ARM64", + "ImageOS": "macos26", + "ImageVersion": "20260907.0351.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "test", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Darwin", + "os_release": "25.6.0", + "arch": "arm64", + "python": "3.14.7", + "logical_cores": 3, + "cpu_cmd": "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize", + "cpu_model": "Apple M1 (Virtual)", + "hw_ncpu": 3, + "memory_bytes": 7516192768, + "perf_cores": 3, + "efficiency_cores": 0, + "rosetta_translated": false, + "os_version": "26.6.2" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "aarch64-apple-darwin", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + } +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/test-macos-x86_64.json b/scripts/bench/results/2026-10-01/run-36969399069/test-macos-x86_64.json new file mode 100644 index 0000000..82095c5 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/test-macos-x86_64.json @@ -0,0 +1,65 @@ +{ + "schema": "pbit-test-matrix/1", + "label": "macos-x86_64", + "date_utc": "2026-10-02T05:38:16Z", + "cmd": "cargo test --release --locked --workspace", + "exit": 101, + "passed": false, + "seconds": 309, + "test_binaries": 2, + "totals": { + "passed": 47, + "failed": 1, + "ignored": 0, + "measured": 0, + "filtered_out": 0 + }, + "failures": [ + "run_deadline_ms_bounds_the_whole_call" + ], + "runner": { + "RUNNER_OS": "macOS", + "RUNNER_ARCH": "X64", + "ImageOS": "macos15", + "ImageVersion": "20260824.0482.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "test", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Darwin", + "os_release": "24.6.0", + "arch": "x86_64", + "python": "3.14.7", + "logical_cores": 4, + "cpu_cmd": "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize", + "cpu_model": "Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz", + "hw_ncpu": 4, + "memory_bytes": 15032385536, + "perf_cores": 4, + "efficiency_cores": 0, + "cpu_flags": { + "sse4.2": true, + "avx1.0": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": false + }, + "rosetta_translated": null, + "os_version": "15.7.9" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-apple-darwin", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + } +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/test-windows-x86_64.json b/scripts/bench/results/2026-10-01/run-36969399069/test-windows-x86_64.json new file mode 100644 index 0000000..2ca1575 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/test-windows-x86_64.json @@ -0,0 +1,55 @@ +{ + "schema": "pbit-test-matrix/1", + "label": "windows-x86_64", + "date_utc": "2026-10-02T05:42:16Z", + "cmd": "cargo test --release --locked --workspace", + "exit": 101, + "passed": false, + "seconds": 548, + "test_binaries": 6, + "totals": { + "passed": 79, + "failed": 1, + "ignored": 1, + "measured": 0, + "filtered_out": 0 + }, + "failures": [ + "acc1_tv_le_001_within_10ms" + ], + "runner": { + "RUNNER_OS": "Windows", + "RUNNER_ARCH": "X64", + "ImageOS": "win25-vs2026", + "ImageVersion": "20260925.250.1", + "GITHUB_RUN_ID": "36969399069", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "test", + "GITHUB_SHA": "ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Windows", + "os_release": "2025Server", + "arch": "AMD64", + "python": "3.12.10", + "logical_cores": 2, + "cpu_cmd": "Get-CimInstance Win32_Processor", + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "physical_cores": 1, + "cim_logical": 2, + "memory_bytes": 8584425472, + "os_version": "Microsoft Windows Server 2025 Datacenter (build 26100)" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-pc-windows-msvc", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + } +} diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-linux-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-linux-x86_64.txt new file mode 100644 index 0000000..9f3ef95 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-linux-x86_64.txt @@ -0,0 +1,33 @@ +$ pbit demo --tasks 12 | pbit decide --pretty (/home/runner/work/pbit/pbit/onpath/pbit) +exit 0 +{ + "engine": "pbit 0.2.0", + "tasks": 12, + "workers": 6, + "affinity": 0.8, + "verdict": "exact", + "tier": "enumerate", + "plan": { +$ pbit decide --budget-ms 200 < router.json > decision.json; case $? in ... +answer +exact +$ python3 python/test_pbit.py +......... +---------------------------------------------------------------------- +Ran 9 tests in 0.644s + +OK +$ python3 python/examples/route_agents.py +exact released 24 of 24 tasks + T000 -> human odds 0.895 + T001 -> human odds 0.883 + T002 -> codex odds 0.326 + T003 -> human odds 0.333 + T004 -> luna-pro odds 0.704 +$ node examples/node/decide.mjs (node v22.23.3) +pbit 0.2.0 (/home/runner/work/pbit/pbit/target/release/pbit) +exit 0 answer a 12-task demo document: verdict exact, 12 of 12 released, plan T000->human, T001->human, T002->codex, ... +exit 1 infeasible two tasks, one worker with room for one: verdict infeasible (no plan satisfies every rule (allowed sets + capacities + clamps)) +exit 2 bad_input a malformed document: schema at workers: must be an array +exit 3 refused exact enumeration of a 96-variable program, capped at 50 ms: verdict declined (enumeration stopped at --exact-ms; raise it, or use --op decide or sample) +all four exit codes mapped as expected diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-macos-arm64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-macos-arm64.txt new file mode 100644 index 0000000..f5647fe --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-macos-arm64.txt @@ -0,0 +1,33 @@ +$ pbit demo --tasks 12 | pbit decide --pretty (/Users/runner/work/pbit/pbit/onpath/pbit) +exit 0 +{ + "engine": "pbit 0.2.0", + "tasks": 12, + "workers": 6, + "affinity": 0.8, + "verdict": "exact", + "tier": "enumerate", + "plan": { +$ pbit decide --budget-ms 200 < router.json > decision.json; case $? in ... +answer +exact +$ python3 python/test_pbit.py +......... +---------------------------------------------------------------------- +Ran 9 tests in 0.672s + +OK +$ python3 python/examples/route_agents.py +exact released 24 of 24 tasks + T000 -> human odds 0.895 + T001 -> human odds 0.883 + T002 -> codex odds 0.326 + T003 -> human odds 0.333 + T004 -> luna-pro odds 0.704 +$ node examples/node/decide.mjs (node v24.20.0) +pbit 0.2.0 (/Users/runner/work/pbit/pbit/target/release/pbit) +exit 0 answer a 12-task demo document: verdict exact, 12 of 12 released, plan T000->human, T001->human, T002->codex, ... +exit 1 infeasible two tasks, one worker with room for one: verdict infeasible (no plan satisfies every rule (allowed sets + capacities + clamps)) +exit 2 bad_input a malformed document: schema at workers: must be an array +exit 3 refused exact enumeration of a 96-variable program, capped at 50 ms: verdict declined (enumeration stopped at --exact-ms; raise it, or use --op decide or sample) +all four exit codes mapped as expected diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-macos-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-macos-x86_64.txt new file mode 100644 index 0000000..a70f89b --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-macos-x86_64.txt @@ -0,0 +1,33 @@ +$ pbit demo --tasks 12 | pbit decide --pretty (/Users/runner/work/pbit/pbit/onpath/pbit) +exit 0 +{ + "engine": "pbit 0.2.0", + "tasks": 12, + "workers": 6, + "affinity": 0.8, + "verdict": "exact", + "tier": "enumerate", + "plan": { +$ pbit decide --budget-ms 200 < router.json > decision.json; case $? in ... +answer +exact +$ python3 python/test_pbit.py +......... +---------------------------------------------------------------------- +Ran 9 tests in 1.058s + +OK +$ python3 python/examples/route_agents.py +exact released 24 of 24 tasks + T000 -> human odds 0.895 + T001 -> human odds 0.883 + T002 -> codex odds 0.326 + T003 -> human odds 0.333 + T004 -> luna-pro odds 0.704 +$ node examples/node/decide.mjs (node v22.23.2) +pbit 0.2.0 (/Users/runner/work/pbit/pbit/target/release/pbit) +exit 0 answer a 12-task demo document: verdict exact, 12 of 12 released, plan T000->human, T001->human, T002->codex, ... +exit 1 infeasible two tasks, one worker with room for one: verdict infeasible (no plan satisfies every rule (allowed sets + capacities + clamps)) +exit 2 bad_input a malformed document: schema at workers: must be an array +exit 3 refused exact enumeration of a 96-variable program, capped at 50 ms: verdict declined (enumeration stopped at --exact-ms; raise it, or use --op decide or sample) +all four exit codes mapped as expected diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-powershell-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-powershell-windows-x86_64.txt new file mode 100644 index 0000000..1006e57 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-powershell-windows-x86_64.txt @@ -0,0 +1,33 @@ +PowerShell 7.6.6 (Core), pbit at D:\a\pbit\pbit\target\release\pbit.exe +> pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms + +verdict : exact +violations : 0 +ms : 19.688 + + +> Get-Content router.json -Raw | pbit decide --budget-ms 200 | ConvertFrom-Json; $LASTEXITCODE +verdict exact, released 24 of 24, violations 0; exit 0 +> one call per exit code (infeasible, bad input, declined) +pbit: bad problem: workers: must be an array +exit codes: infeasible 1 (want 1), bad input 2 (want 2), declined 3 (want 3) +PowerShell recipes: all passed +PowerShell 5.1.26100.33438 (Desktop), pbit at D:\a\pbit\pbit\target\release\pbit.exe +> pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms +pbit: bad JSON: unexpected character 'ï' at byte 0 + + +verdict : +violations : +ms : + + + + +demo pipeline: exit 2 +At D:\a\pbit\pbit\scripts\bench\callers.ps1:9 char:28 ++ ... f ($LASTEXITCODE -ne 0) { throw "demo pipeline: exit $LASTEXITCODE" } ++ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + CategoryInfo : OperationStopped: (demo pipeline: exit 2:String) [], RuntimeException + + FullyQualifiedErrorId : demo pipeline: exit 2 + diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-windows-x86_64.txt new file mode 100644 index 0000000..d2663c0 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/callers-windows-x86_64.txt @@ -0,0 +1,33 @@ +$ pbit demo --tasks 12 | pbit decide --pretty (/d/a/pbit/pbit/onpath/pbit) +exit 0 +{ + "engine": "pbit 0.2.0", + "tasks": 12, + "workers": 6, + "affinity": 0.8, + "verdict": "exact", + "tier": "enumerate", + "plan": { +$ pbit decide --budget-ms 200 < router.json > decision.json; case $? in ... +answer +exact +$ python python/test_pbit.py +......... +---------------------------------------------------------------------- +Ran 9 tests in 0.742s + +OK +$ python python/examples/route_agents.py +exact released 24 of 24 tasks + T000 -> human odds 0.895 + T001 -> human odds 0.883 + T002 -> codex odds 0.326 + T003 -> human odds 0.333 + T004 -> luna-pro odds 0.704 +$ node examples/node/decide.mjs (node v22.23.3) +pbit 0.2.0 (D:\a\pbit\pbit\target\release\pbit.exe) +exit 0 answer a 12-task demo document: verdict exact, 12 of 12 released, plan T000->human, T001->human, T002->codex, ... +exit 1 infeasible two tasks, one worker with room for one: verdict infeasible (no plan satisfies every rule (allowed sets + capacities + clamps)) +exit 2 bad_input a malformed document: schema at workers: must be an array +exit 3 refused exact enumeration of a 96-variable program, capped at 50 ms: verdict declined (enumeration stopped at --exact-ms; raise it, or use --op decide or sample) +all four exit codes mapped as expected diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/cross-x86_64-apple-darwin.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/cross-x86_64-apple-darwin.txt new file mode 100644 index 0000000..f49352c --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/cross-x86_64-apple-darwin.txt @@ -0,0 +1,18 @@ +$ rustup target add x86_64-apple-darwin +info: downloading component rust-std +$ cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin + Finished `release` profile [optimized] target(s) in 17.81s +$ file target/x86_64-apple-darwin/release/pbit +target/x86_64-apple-darwin/release/pbit: Mach-O 64-bit executable x86_64 +$ lipo -archs target/x86_64-apple-darwin/release/pbit +x86_64 +$ otool -L target/x86_64-apple-darwin/release/pbit +target/x86_64-apple-darwin/release/pbit: + /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0) +size 1270368 bytes +rosetta: present +$ arch -x86_64 target/x86_64-apple-darwin/release/pbit version +pbit 0.2.0 +$ arch -x86_64 pbit demo --tasks 12 | arch -x86_64 pbit decide: exit 0, verdict exact +$ arch -x86_64 pbit stats (self-test under Rosetta) +1 thread 10951089 | threads 3 19953406 diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-ps1-ps51-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-ps1-ps51-windows-x86_64.txt new file mode 100644 index 0000000..5a61437 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-ps1-ps51-windows-x86_64.txt @@ -0,0 +1,25 @@ +PowerShell 5.1.26100.33438 (Desktop) +== 1. & .\install.ps1 -DownloadBase http://127.0.0.1:50061/good -Version v0.2.0 -InstallDir \bin1 +pbit install: v0.2.0 for x86_64-pc-windows-msvc + fetching http://127.0.0.1:50061/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip + sha256 ok 2d92b06698b3c78f38a8d6e018313e67c2c8091c0dd8914de19b002bfe07ac0a + installed C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-216c1cbf1f214569872a604b13d639dd\bin1\pbit.exe (pbit 0.2.0) + +C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-216c1cbf1f214569872a604b13d639dd\bin1 is on this session's PATH only. To keep it, run (or re-run this installer with -AddToPath): + [Environment]::SetEnvironmentVariable('Path', 'C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-216c1cbf1f214569872a604b13d639dd\bin1;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User') + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +$ (Get-Command pbit).Source: C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-216c1cbf1f214569872a604b13d639dd\bin1\pbit.exe +$ pbit version: pbit 0.2.0 +pbit: bad JSON: unexpected character 'ï' at byte 0 +$ pbit demo --tasks 12 | pbit decide | ConvertFrom-Json: exit 2, verdict , released 1 of +FAIL: demo pipeline +At D:\a\pbit\pbit\scripts\bench\test_install_ps1.ps1:26 char:59 ++ ... ODE -ne 0 -or $d.verdict -ne 'exact') { throw 'FAIL: demo pipeline' } ++ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + CategoryInfo : OperationStopped: (FAIL: demo pipeline:String) [], RuntimeException + + FullyQualifiedErrorId : FAIL: demo pipeline + diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-ps1-pwsh-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-ps1-pwsh-windows-x86_64.txt new file mode 100644 index 0000000..c51b460 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-ps1-pwsh-windows-x86_64.txt @@ -0,0 +1,37 @@ +PowerShell 7.6.6 (Core) +== 1. & .\install.ps1 -DownloadBase http://127.0.0.1:50051/good -Version v0.2.0 -InstallDir \bin1 -AddToPath +pbit install: v0.2.0 for x86_64-pc-windows-msvc + fetching http://127.0.0.1:50051/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip + sha256 ok 2d92b06698b3c78f38a8d6e018313e67c2c8091c0dd8914de19b002bfe07ac0a + installed C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-44171e0b07df4a84a1b827d360f11481\bin1\pbit.exe (pbit 0.2.0) + added C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-44171e0b07df4a84a1b827d360f11481\bin1 to your user PATH (new terminals see it) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +$ (Get-Command pbit).Source: C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-44171e0b07df4a84a1b827d360f11481\bin1\pbit.exe +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide | ConvertFrom-Json: exit 0, verdict exact, released 12 of 12 +ok: user PATH now starts with C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-44171e0b07df4a84a1b827d360f11481\bin1 +== 2. irm | iex form: PBIT_* environment variables, script text through Invoke-Expression +pbit install: v0.2.0 for x86_64-pc-windows-msvc + fetching http://127.0.0.1:50051/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip + sha256 ok 2d92b06698b3c78f38a8d6e018313e67c2c8091c0dd8914de19b002bfe07ac0a + installed C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-44171e0b07df4a84a1b827d360f11481\bin2\pbit.exe (pbit 0.2.0) + +C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-44171e0b07df4a84a1b827d360f11481\bin2 is on this session's PATH only. To keep it, run (or re-run this installer with -AddToPath): + [Environment]::SetEnvironmentVariable('Path', 'C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-44171e0b07df4a84a1b827d360f11481\bin2;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User') + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: pbit 0.2.0 in C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-44171e0b07df4a84a1b827d360f11481\bin2 +== 3. a wrong .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-pc-windows-msvc + fetching http://127.0.0.1:50051/bad/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip +rejected: pbit install: checksum mismatch for pbit-v0.2.0-x86_64-pc-windows-msvc.zip: expected 0000000000000000000000000000000000000000000000000000000000000000, got 2d92b06698b3c78f38a8d6e018313e67c2c8091c0dd8914de19b002bfe07ac0a (nothing was installed) +== 4. -DownloadBase without -Version must fail +rejected: pbit install: set -Version (or PBIT_VERSION), e.g. v0.2.0, together with -DownloadBase +install.ps1: all checks passed (PowerShell 7.6.6) diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-linux-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-linux-x86_64.txt new file mode 100644 index 0000000..01983e2 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-linux-x86_64.txt @@ -0,0 +1,114 @@ +shell under test: sh (/usr/bin/sh); archive: pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz.sha256 +== 1. piped to sh, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47945/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok 37c918a62f2d0f930df5487beec13a4076ab647f5ed0f5a3e818ec32129b57dc + installed /tmp/tmp.Xso7PrR0aM/bin/pbit (pbit 0.2.0) + +/tmp/tmp.Xso7PrR0aM/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/tmp/tmp.Xso7PrR0aM/bin:$PATH" + +Next: + /tmp/tmp.Xso7PrR0aM/bin/pbit version + /tmp/tmp.Xso7PrR0aM/bin/pbit demo --tasks 12 | /tmp/tmp.Xso7PrR0aM/bin/pbit decide --pretty + /tmp/tmp.Xso7PrR0aM/bin/pbit stats --pretty +$ command -v pbit: /tmp/tmp.Xso7PrR0aM/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47945/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok 37c918a62f2d0f930df5487beec13a4076ab647f5ed0f5a3e818ec32129b57dc + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47945/bad/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 37c918a62f2d0f930df5487beec13a4076ab647f5ed0f5a3e818ec32129b57dc (nothing was installed) +ok: rejected, /tmp/tmp.Xso7PrR0aM/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (sh, x86_64-unknown-linux-gnu) +shell under test: bash (/usr/bin/bash); archive: pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz.sha256 +== 1. piped to bash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47791/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok e8af8a4b598c3b72de2d5cf6f75e610a768bb59c74a97333413e395626ecae20 + installed /tmp/tmp.o3x4RvtLf6/bin/pbit (pbit 0.2.0) + +/tmp/tmp.o3x4RvtLf6/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/tmp/tmp.o3x4RvtLf6/bin:$PATH" + +Next: + /tmp/tmp.o3x4RvtLf6/bin/pbit version + /tmp/tmp.o3x4RvtLf6/bin/pbit demo --tasks 12 | /tmp/tmp.o3x4RvtLf6/bin/pbit decide --pretty + /tmp/tmp.o3x4RvtLf6/bin/pbit stats --pretty +$ command -v pbit: /tmp/tmp.o3x4RvtLf6/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47791/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok e8af8a4b598c3b72de2d5cf6f75e610a768bb59c74a97333413e395626ecae20 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47791/bad/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got e8af8a4b598c3b72de2d5cf6f75e610a768bb59c74a97333413e395626ecae20 (nothing was installed) +ok: rejected, /tmp/tmp.o3x4RvtLf6/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (bash, x86_64-unknown-linux-gnu) +shell under test: dash (/usr/bin/dash); archive: pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz.sha256 +== 1. piped to dash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:43227/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok e8af8a4b598c3b72de2d5cf6f75e610a768bb59c74a97333413e395626ecae20 + installed /tmp/tmp.qoTkSAx0zg/bin/pbit (pbit 0.2.0) + +/tmp/tmp.qoTkSAx0zg/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/tmp/tmp.qoTkSAx0zg/bin:$PATH" + +Next: + /tmp/tmp.qoTkSAx0zg/bin/pbit version + /tmp/tmp.qoTkSAx0zg/bin/pbit demo --tasks 12 | /tmp/tmp.qoTkSAx0zg/bin/pbit decide --pretty + /tmp/tmp.qoTkSAx0zg/bin/pbit stats --pretty +$ command -v pbit: /tmp/tmp.qoTkSAx0zg/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:43227/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok e8af8a4b598c3b72de2d5cf6f75e610a768bb59c74a97333413e395626ecae20 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:43227/bad/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got e8af8a4b598c3b72de2d5cf6f75e610a768bb59c74a97333413e395626ecae20 (nothing was installed) +ok: rejected, /tmp/tmp.qoTkSAx0zg/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (dash, x86_64-unknown-linux-gnu) diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-macos-arm64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-macos-arm64.txt new file mode 100644 index 0000000..6d66f27 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-macos-arm64.txt @@ -0,0 +1,117 @@ +shell under test: sh (/bin/sh); archive: pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit-v0.2.0-aarch64-apple-darwin.tar.gz.sha256 +== 1. piped to sh, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49173/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok bbbca16f0eba20c351e7759a386d0b176a6d2c0b937a274d10556bf3a2d48099 + installed /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/bin/pbit (pbit 0.2.0) + +/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/bin:$PATH" + +Next: + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/bin/pbit version + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/bin/pbit demo --tasks 12 | /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/bin/pbit decide --pretty + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/bin/pbit stats --pretty +$ command -v pbit: /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49173/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok bbbca16f0eba20c351e7759a386d0b176a6d2c0b937a274d10556bf3a2d48099 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49173/bad/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-aarch64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got bbbca16f0eba20c351e7759a386d0b176a6d2c0b937a274d10556bf3a2d48099 (nothing was installed) +ok: rejected, /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.onitFH8jy0/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (sh, aarch64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 1824 Terminated: 15 "$PY" -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/srv" > "$WORK/http.log" 2>&1 +shell under test: bash (/bin/bash); archive: pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit-v0.2.0-aarch64-apple-darwin.tar.gz.sha256 +== 1. piped to bash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49182/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok dcdbc1bdeeb88e3d47e955e98c385062924f4791fe9807de5302c753db6c7428 + installed /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/bin/pbit (pbit 0.2.0) + +/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/bin:$PATH" + +Next: + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/bin/pbit version + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/bin/pbit demo --tasks 12 | /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/bin/pbit decide --pretty + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/bin/pbit stats --pretty +$ command -v pbit: /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49182/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok dcdbc1bdeeb88e3d47e955e98c385062924f4791fe9807de5302c753db6c7428 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49182/bad/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-aarch64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got dcdbc1bdeeb88e3d47e955e98c385062924f4791fe9807de5302c753db6c7428 (nothing was installed) +ok: rejected, /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CNbEhUFru1/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (bash, aarch64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 2372 Terminated: 15 "$PY" -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/srv" > "$WORK/http.log" 2>&1 +shell under test: dash (/bin/dash); archive: pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit-v0.2.0-aarch64-apple-darwin.tar.gz.sha256 +== 1. piped to dash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49192/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok 64e596800cb664ecca4343328d610d8372c0803ce4e4e973ff6a3b0e3ab18761 + installed /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/bin/pbit (pbit 0.2.0) + +/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/bin:$PATH" + +Next: + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/bin/pbit version + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/bin/pbit demo --tasks 12 | /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/bin/pbit decide --pretty + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/bin/pbit stats --pretty +$ command -v pbit: /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49192/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok 64e596800cb664ecca4343328d610d8372c0803ce4e4e973ff6a3b0e3ab18761 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49192/bad/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-aarch64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 64e596800cb664ecca4343328d610d8372c0803ce4e4e973ff6a3b0e3ab18761 (nothing was installed) +ok: rejected, /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.JRasdoqGV0/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (dash, aarch64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 3139 Terminated: 15 "$PY" -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/srv" > "$WORK/http.log" 2>&1 diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-macos-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-macos-x86_64.txt new file mode 100644 index 0000000..dbbe397 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/install-sh-macos-x86_64.txt @@ -0,0 +1,117 @@ +shell under test: sh (/bin/sh); archive: pbit-v0.2.0-x86_64-apple-darwin.tar.gz +pbit-v0.2.0-x86_64-apple-darwin.tar.gz.sha256 +== 1. piped to sh, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49183/good/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz + sha256 ok 3dca157a4263b49550f6c5ad2645ab3ec3b936c6ad77f6ef5632f4ae25517c20 + installed /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/bin/pbit (pbit 0.2.0) + +/var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/bin:$PATH" + +Next: + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/bin/pbit version + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/bin/pbit demo --tasks 12 | /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/bin/pbit decide --pretty + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/bin/pbit stats --pretty +$ command -v pbit: /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49183/good/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz + sha256 ok 3dca157a4263b49550f6c5ad2645ab3ec3b936c6ad77f6ef5632f4ae25517c20 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49183/bad/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 3dca157a4263b49550f6c5ad2645ab3ec3b936c6ad77f6ef5632f4ae25517c20 (nothing was installed) +ok: rejected, /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.4rNvxZMgTF/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (sh, x86_64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 5636 Terminated: 15 "$PY" -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/srv" > "$WORK/http.log" 2>&1 +shell under test: bash (/bin/bash); archive: pbit-v0.2.0-x86_64-apple-darwin.tar.gz +pbit-v0.2.0-x86_64-apple-darwin.tar.gz.sha256 +== 1. piped to bash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49193/good/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz + sha256 ok 943efddc395688b40c853fbc03dbcb10b56adc461263f6b482b0423f95af78dd + installed /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/bin/pbit (pbit 0.2.0) + +/var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/bin:$PATH" + +Next: + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/bin/pbit version + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/bin/pbit demo --tasks 12 | /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/bin/pbit decide --pretty + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/bin/pbit stats --pretty +$ command -v pbit: /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49193/good/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz + sha256 ok 943efddc395688b40c853fbc03dbcb10b56adc461263f6b482b0423f95af78dd + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49193/bad/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 943efddc395688b40c853fbc03dbcb10b56adc461263f6b482b0423f95af78dd (nothing was installed) +ok: rejected, /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.dIll06f370/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (bash, x86_64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 6374 Terminated: 15 "$PY" -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/srv" > "$WORK/http.log" 2>&1 +shell under test: dash (/bin/dash); archive: pbit-v0.2.0-x86_64-apple-darwin.tar.gz +pbit-v0.2.0-x86_64-apple-darwin.tar.gz.sha256 +== 1. piped to dash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49202/good/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz + sha256 ok 48a030141c020e7c35ff90212ab1bb1f8749476221b210358b5057ce3207690d + installed /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/bin/pbit (pbit 0.2.0) + +/var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/bin:$PATH" + +Next: + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/bin/pbit version + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/bin/pbit demo --tasks 12 | /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/bin/pbit decide --pretty + /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/bin/pbit stats --pretty +$ command -v pbit: /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49202/good/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz + sha256 ok 48a030141c020e7c35ff90212ab1bb1f8749476221b210358b5057ce3207690d + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-apple-darwin + fetching http://127.0.0.1:49202/bad/v0.2.0/pbit-v0.2.0-x86_64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 48a030141c020e7c35ff90212ab1bb1f8749476221b210358b5057ce3207690d (nothing was installed) +ok: rejected, /var/folders/20/jp1_0n3n7kndh6rnbqb5344m0000gn/T/tmp.sf6Rcnq3w0/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (dash, x86_64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 6888 Terminated: 15 "$PY" -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/srv" > "$WORK/http.log" 2>&1 diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/msrv.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/msrv.txt new file mode 100644 index 0000000..5baf58b --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/msrv.txt @@ -0,0 +1,5 @@ +rust 1.72: build FAILED, Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4): error[E0658]: use of unstable library feature 'int_roundings' +rust 1.73: build ok, Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4) +rust 1.77: build FAILED, --locked (Cargo.lock as committed, lockfile v4): error: failed to parse lock file at: /home/runner/work/pbit/pbit/Cargo.lock +rust 1.78: build ok, --locked (Cargo.lock as committed, lockfile v4) +Cargo.lock restored: 0 changes diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/musl-aarch64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/musl-aarch64.txt new file mode 100644 index 0000000..1bd8996 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/musl-aarch64.txt @@ -0,0 +1,23 @@ +$ sudo apt-get install -y gcc-aarch64-linux-gnu + +Running kernel seems to be up-to-date. + +No services need to be restarted. + +No containers need to be restarted. + +No user sessions are running outdated binaries. + +No VM guests are running outdated hypervisor (qemu) binaries on this host. +linker installed in 13 s +$ rustup target add aarch64-unknown-linux-musl +info: downloading component rust-std +$ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl + Finished `release` profile [optimized] target(s) in 19.31s +$ file target/aarch64-unknown-linux-musl/release/pbit +target/aarch64-unknown-linux-musl/release/pbit: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), statically linked, BuildID[sha1]=6b9608f1981e300fe1d149bd279ce6fa0ba3e383, not stripped +size 1477600 bytes +qemu-user-static installed in 4 s +$ qemu-aarch64-static target/aarch64-unknown-linux-musl/release/pbit version +pbit 0.2.0 +$ qemu pbit demo --tasks 12 | qemu pbit decide: exit 0, verdict exact diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/musl-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/musl-x86_64.txt new file mode 100644 index 0000000..b1f925b --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/musl-x86_64.txt @@ -0,0 +1,12 @@ +$ rustup target add x86_64-unknown-linux-musl +info: downloading component rust-std +$ cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl + Finished `release` profile [optimized] target(s) in 18.87s +$ file target/x86_64-unknown-linux-musl/release/pbit +target/x86_64-unknown-linux-musl/release/pbit: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), static-pie linked, BuildID[sha1]=85ea1386df8c2738b288e74a66fd13f6a9490feb, not stripped +$ ldd target/x86_64-unknown-linux-musl/release/pbit + statically linked +size 1545968 bytes +$ target/x86_64-unknown-linux-musl/release/pbit version +pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/npm-linux-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/npm-linux-x86_64.txt new file mode 100644 index 0000000..a4b0e1c --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/npm-linux-x86_64.txt @@ -0,0 +1,72 @@ +node v22.23.3, npm 10.9.9, target x86_64-unknown-linux-gnu +npm notice +npm notice 📦 pbit@0.2.0 +npm notice Tarball Contents +npm notice 11.4kB LICENSE +npm notice 1.6kB README.md +npm notice 1.7kB bin/pbit.js +npm notice 5.0kB install.js +npm notice 736B package.json +npm notice Tarball Details +npm notice name: pbit +npm notice version: 0.2.0 +npm notice filename: pbit-0.2.0.tgz +npm notice package size: 7.8 kB +npm notice unpacked size: 20.4 kB +npm notice shasum: 320a800c1bea6a3746f73441bf36f3c59c0c3499 +npm notice integrity: sha512-iuUFO3Oex1JpW[...]HF+pBF0+I8ChA== +npm notice total files: 5 +npm notice +== npm pack -> pbit-0.2.0.tgz: + package/LICENSE + package/install.js + package/bin/pbit.js + package/package.json + package/README.md +== 1. npm install -g pbit-0.2.0.tgz (PBIT_DOWNLOAD_BASE=http://127.0.0.1:51607/good) + +> pbit@0.2.0 postinstall +> node install.js + +pbit: fetching http://127.0.0.1:51607/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit: installed v0.2.0 (x86_64-unknown-linux-gnu), sha256 3aca12c306ed5b7105db53b477ea2581a3b9cb2dde708b30c72c7389a61defbf + +added 1 package in 440ms +$ which pbit: /tmp/tmp.AExae5SiEL/prefix1/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +exit codes through the wrapper: infeasible 1 (want 1), bad input 2 (want 2), declined 3 (want 3) + +removed 1 package in 132ms +ok: uninstalled, /tmp/tmp.AExae5SiEL/prefix1/bin/pbit absent +== 2. npm install -g --ignore-scripts: the binary is fetched on first run + +added 1 package in 141ms +pbit: the binary is not installed yet, fetching it now +pbit: fetching http://127.0.0.1:51607/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit: installed v0.2.0 (x86_64-unknown-linux-gnu), sha256 3aca12c306ed5b7105db53b477ea2581a3b9cb2dde708b30c72c7389a61defbf +pbit 0.2.0 +ok: first run fetched it; second run used it +== 3. PBIT_BINARY= (no download) + +> pbit@0.2.0 postinstall +> node install.js + +pbit: using /home/runner/work/pbit/pbit/target/release/pbit (PBIT_BINARY) + +added 1 package in 214ms +pbit 0.2.0 +== 4. tampered .sha256: npm install must fail + +> pbit@0.2.0 postinstall +> node install.js + +pbit: fetching http://127.0.0.1:51607/bad/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit: checksum mismatch for pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 3aca12c306ed5b7105db53b477ea2581a3b9cb2dde708b30c72c7389a61defbf; nothing was installed +npm error code 1 +npm error path /tmp/tmp.AExae5SiEL/prefix4/lib/node_modules/pbit +npm error command failed +npm error command sh -c node install.js +npm error A complete log of this run can be found in: /home/runner/.npm/_logs/2026-10-02T05_33_28_273Z-debug-0.log +ok: rejected +npm wrapper: all checks passed (x86_64-unknown-linux-gnu) diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/npm-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/npm-windows-x86_64.txt new file mode 100644 index 0000000..729e24e --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/npm-windows-x86_64.txt @@ -0,0 +1,72 @@ +node v22.23.3, npm 10.9.9, target x86_64-pc-windows-msvc +npm notice +npm notice package: pbit@0.2.0 +npm notice Tarball Contents +npm notice 11.6kB LICENSE +npm notice 1.6kB README.md +npm notice 1.8kB bin/pbit.js +npm notice 5.1kB install.js +npm notice 760B package.json +npm notice Tarball Details +npm notice name: pbit +npm notice version: 0.2.0 +npm notice filename: pbit-0.2.0.tgz +npm notice package size: 7.8 kB +npm notice unpacked size: 20.8 kB +npm notice shasum: e627337a1e165cdfb101bedbc6a178a24c3695d6 +npm notice integrity: sha512-H/JH23iMhSCfA[...]r7vAu3FiTe5YQ== +npm notice total files: 5 +npm notice +== npm pack -> pbit-0.2.0.tgz: + package/LICENSE + package/install.js + package/bin/pbit.js + package/package.json + package/README.md +== 1. npm install -g pbit-0.2.0.tgz (PBIT_DOWNLOAD_BASE=http://127.0.0.1:50031/good) + +> pbit@0.2.0 postinstall +> node install.js + +pbit: fetching http://127.0.0.1:50031/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip +pbit: installed v0.2.0 (x86_64-pc-windows-msvc), sha256 0bdba294ede614491cece19d55406c58549b3d73eadb284da3cab9533821979e + +added 1 package in 2s +$ which pbit: /tmp/tmp.rL0yBj3eyt/prefix1/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +exit codes through the wrapper: infeasible 1 (want 1), bad input 2 (want 2), declined 3 (want 3) + +removed 1 package in 353ms +ok: uninstalled, /tmp/tmp.rL0yBj3eyt/prefix1/pbit absent +== 2. npm install -g --ignore-scripts: the binary is fetched on first run + +added 1 package in 585ms +pbit: the binary is not installed yet, fetching it now +pbit: fetching http://127.0.0.1:50031/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip +pbit: installed v0.2.0 (x86_64-pc-windows-msvc), sha256 0bdba294ede614491cece19d55406c58549b3d73eadb284da3cab9533821979e +pbit 0.2.0 +ok: first run fetched it; second run used it +== 3. PBIT_BINARY= (no download) + +> pbit@0.2.0 postinstall +> node install.js + +pbit: using D:\a\pbit\pbit\target\release\pbit.exe (PBIT_BINARY) + +added 1 package in 651ms +pbit 0.2.0 +== 4. tampered .sha256: npm install must fail + +> pbit@0.2.0 postinstall +> node install.js + +pbit: fetching http://127.0.0.1:50031/bad/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip +pbit: checksum mismatch for pbit-v0.2.0-x86_64-pc-windows-msvc.zip: expected 0000000000000000000000000000000000000000000000000000000000000000, got 0bdba294ede614491cece19d55406c58549b3d73eadb284da3cab9533821979e; nothing was installed +npm error code 1 +npm error path C:\Users\RUNNER~1\AppData\Local\Temp\tmp.rL0yBj3eyt\prefix4\node_modules\pbit +npm error command failed +npm error command C:\Windows\system32\cmd.exe /d /s /c node install.js +npm error A complete log of this run can be found in: C:\Users\runneradmin\AppData\Local\npm-cache\_logs\2026-10-02T05_34_43_751Z-debug-0.log +ok: rejected +npm wrapper: all checks passed (x86_64-pc-windows-msvc) diff --git a/scripts/bench/results/2026-10-01/run-36969399069/transcripts/wasm32-wasip1.txt b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/wasm32-wasip1.txt new file mode 100644 index 0000000..69a962c --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36969399069/transcripts/wasm32-wasip1.txt @@ -0,0 +1,8 @@ +$ rustup target add wasm32-wasip1 +info: downloading component rust-std +$ cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide +exit 0 +first error: none + Checking pbit-ir v0.2.0 (/home/runner/work/pbit/pbit/pbit-ir) + Checking pbit-decide v0.2.0 (/home/runner/work/pbit/pbit/pbit-decide) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.69s diff --git a/scripts/bench/results/2026-10-01/run-36970631022/bench-linux-x86_64.json b/scripts/bench/results/2026-10-01/run-36970631022/bench-linux-x86_64.json new file mode 100644 index 0000000..ccf2502 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/bench-linux-x86_64.json @@ -0,0 +1,1345 @@ +{ + "schema": "pbit-bench-matrix/1", + "label": "linux-x86_64", + "date_utc": "2026-10-02T05:49:50Z", + "git_sha": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "bench", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "build_s=13", + "runner=ubuntu-latest" + ], + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "AMD EPYC 9V45 96-Core Processor", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "AuthenticAMD", + "Model name": "AMD EPYC 9V45 96-Core Processor", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": true, + "avx512bw": true, + "avx512vl": true + }, + "memory_bytes": 8323969024, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 0.9033203125, + 0.27734375, + 0.09521484375 + ], + "uptime": "05:49:50 up 4 min, 0 user, load average: 0.90, 0.28, 0.10", + "t": "05:49:50" + }, + "after_stats": { + "loadavg": [ + 0.9033203125, + 0.27734375, + 0.09521484375 + ], + "uptime": "05:49:52 up 4 min, 0 user, load average: 0.90, 0.28, 0.10", + "t": "05:49:52" + }, + "after_decide": { + "loadavg": [ + 0.9111328125, + 0.28955078125, + 0.10009765625 + ], + "uptime": "05:49:54 up 4 min, 0 user, load average: 0.91, 0.29, 0.10", + "t": "05:49:54" + }, + "after_kernels": { + "loadavg": [ + 3.9873046875, + 1.1376953125, + 0.39404296875 + ], + "uptime": "05:50:35 up 4 min, 0 user, load average: 3.99, 1.14, 0.39", + "t": "05:50:35" + }, + "after_pvn": { + "loadavg": [ + 6.87841796875, + 2.6171875, + 0.97119140625 + ], + "uptime": "05:51:58 up 6 min, 0 user, load average: 6.88, 2.62, 0.97", + "t": "05:51:58" + }, + "end": { + "loadavg": [ + 6.4072265625, + 2.59033203125, + 0.9716796875 + ], + "uptime": "05:52:02 up 6 min, 0 user, load average: 6.41, 2.59, 0.97", + "t": "05:52:02" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 1401248, + "version": "pbit 0.2.0", + "linkage": { + "file": "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 3.2.0, BuildID[sha1]=3a4791ff347b29ab20fc5b1ff998dd05660b1d15, not stripped", + "ldd": [ + "linux-vdso.so.1 (0x00007fe98a4e9000)", + "\tlibgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007fe98a37b000)", + "\tlibm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007fe98a292000)", + "\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fe98a000000)", + "\t/lib64/ld-linux-x86-64.so.2 (0x00007fe98a4eb000)" + ], + "glibc_symbol_versions": [ + "2.2.5", + "2.3", + "2.3.4", + "2.14", + "2.16", + "2.17", + "2.18", + "2.25", + "2.28", + "2.29", + "2.30", + "2.32", + "2.33", + "2.34" + ], + "min_glibc": "2.34" + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "linux", + "arch": "x86_64", + "logical_cpus": 2 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 2, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 17637485, + "q1": 17103334.0, + "q3": 17767678.5, + "min": 16952963, + "max": 17889808, + "n": 5, + "values": [ + 17253705, + 16952963, + 17889808, + 17645549, + 17637485 + ] + }, + "site_updates_per_s": { + "median": 23339333, + "q1": 23097044.0, + "q3": 23699545.5, + "min": 23011872, + "max": 24000627, + "n": 5, + "values": [ + 23011872, + 23182216, + 24000627, + 23339333, + 23398464 + ] + }, + "threads": [ + 2 + ], + "speedup": { + "median": 1.33, + "q1": 1.3250000000000002, + "q3": 1.355, + "min": 1.32, + "max": 1.37, + "n": 5, + "values": [ + 1.33, + 1.37, + 1.34, + 1.32, + 1.33 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 10.522428000001582, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 9.088063, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 10.505511000019396, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 9.103907, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 10.430859999985387, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 9.05987, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 10.539232000013499, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 9.165309, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 10.373764000007668, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 9.045869, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 10.505511000019396, + "q1": 10.402311999996527, + "q3": 10.53083000000754, + "min": 10.373764000007668, + "max": 10.539232000013499, + "n": 5, + "values": [ + 10.522428000001582, + 10.505511000019396, + 10.430859999985387, + 10.539232000013499, + 10.373764000007668 + ] + }, + "pbit_ms": { + "median": 9.088063, + "q1": 9.0528695, + "q3": 9.134608, + "min": 9.045869, + "max": 9.165309, + "n": 5, + "values": [ + 9.088063, + 9.103907, + 9.05987, + 9.165309, + 9.045869 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 423.65680800000405, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 419.000661, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 14520, + "site_updates_per_s": 14500705, + "threads": 2, + "peak_rss_mb_self": 21.429688, + "stderr": null + }, + { + "wall_ms": 422.52012199998035, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 417.940061, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 14904, + "site_updates_per_s": 14893883, + "threads": 2, + "peak_rss_mb_self": 21.316406, + "stderr": null + }, + { + "wall_ms": 424.36705899999083, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 419.914021, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15080, + "site_updates_per_s": 15067535, + "threads": 2, + "peak_rss_mb_self": 21.039063, + "stderr": null + }, + { + "wall_ms": 421.5770790000022, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 417.235029, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 15128, + "site_updates_per_s": 15108383, + "threads": 2, + "peak_rss_mb_self": 21.035156, + "stderr": null + }, + { + "wall_ms": 421.8094629999882, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 417.282844, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 14960, + "site_updates_per_s": 14940381, + "threads": 2, + "peak_rss_mb_self": 21.464844, + "stderr": null + } + ], + "wall_ms": { + "median": 422.52012199998035, + "q1": 421.6932709999952, + "q3": 424.01193349999744, + "min": 421.5770790000022, + "max": 424.36705899999083, + "n": 5, + "values": [ + 423.65680800000405, + 422.52012199998035, + 424.36705899999083, + 421.5770790000022, + 421.8094629999882 + ] + }, + "pbit_ms": { + "median": 417.940061, + "q1": 417.2589365, + "q3": 419.457341, + "min": 417.235029, + "max": 419.914021, + "n": 5, + "values": [ + 419.000661, + 417.940061, + 419.914021, + 417.235029, + 417.282844 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 14940381, + "q1": 14697294.0, + "q3": 15087959.0, + "min": 14500705, + "max": 15108383, + "n": 5, + "values": [ + 14500705, + 14893883, + 15067535, + 15108383, + 14940381 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "bytes": { + "median": 22052864, + "q1": 21991424.0, + "q3": 22351872.0, + "min": 21991424, + "max": 22351872, + "n": 3, + "values": [ + 22052864, + 22351872, + 21991424 + ] + }, + "self_reported_peak_rss_mb": [ + 21.03125, + 21.316406, + 20.972656 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + {}, + {}, + {} + ] + }, + "kernels": { + "cmd": "target/release/examples/kernels", + "checksums": [ + "8520834" + ], + "lines": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "median": 322900000.0, + "q1": 319050000.0, + "q3": 327600000.0, + "min": 318900000.0, + "max": 332300000.0, + "n": 5, + "values": [ + 332300000.0, + 322900000.0, + 319200000.0, + 318900000.0, + 322900000.0 + ] + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "median": 229500000.0, + "q1": 224250000.0, + "q3": 234650000.0, + "min": 222500000.0, + "max": 236400000.0, + "n": 5, + "values": [ + 236400000.0, + 232900000.0, + 226000000.0, + 222500000.0, + 229500000.0 + ] + }, + "T0 multispin 64-lane, splitmix": { + "median": 17310000000.0, + "q1": 16765000000.0, + "q3": 17540000000.0, + "min": 16230000000.0, + "max": 17640000000.0, + "n": 5, + "values": [ + 17640000000.0, + 17300000000.0, + 17310000000.0, + 16230000000.0, + 17440000000.0 + ] + }, + "T0 multispin 64-lane, Philox": { + "median": 13010000000.0, + "q1": 12525000000.0, + "q3": 13140000000.0, + "min": 12430000000.0, + "max": 13250000000.0, + "n": 5, + "values": [ + 12430000000.0, + 13030000000.0, + 13010000000.0, + 12620000000.0, + 13250000000.0 + ] + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "median": 2314000000.0, + "q1": 2288500000.0, + "q3": 2359000000.0, + "min": 2265000000.0, + "max": 2397000000.0, + "n": 5, + "values": [ + 2312000000.0, + 2397000000.0, + 2265000000.0, + 2321000000.0, + 2314000000.0 + ] + }, + "T0 multispin splitmix x 4 threads": { + "median": 16960000000.0, + "q1": 16735000000.0, + "q3": 17495000000.0, + "min": 16660000000.0, + "max": 17810000000.0, + "n": 5, + "values": [ + 16810000000.0, + 17810000000.0, + 16960000000.0, + 16660000000.0, + 17180000000.0 + ] + }, + "T0 multispin splitmix x10 threads": { + "median": 15850000000.0, + "q1": 15465000000.0, + "q3": 16030000000.0, + "min": 15440000000.0, + "max": 16070000000.0, + "n": 5, + "values": [ + 15990000000.0, + 16070000000.0, + 15490000000.0, + 15440000000.0, + 15850000000.0 + ] + }, + "fast multispin splitmix x 4 thr": { + "median": 22210000000.0, + "q1": 21165000000.0, + "q3": 23175000000.0, + "min": 20320000000.0, + "max": 23340000000.0, + "n": 5, + "values": [ + 23340000000.0, + 22210000000.0, + 20320000000.0, + 22010000000.0, + 23010000000.0 + ] + }, + "fast multispin splitmix x10 thr": { + "median": 22200000000.0, + "q1": 21810000000.0, + "q3": 22575000000.0, + "min": 21580000000.0, + "max": 22870000000.0, + "n": 5, + "values": [ + 22280000000.0, + 21580000000.0, + 22040000000.0, + 22200000000.0, + 22870000000.0 + ] + }, + "fast T1 f32 LUT, splitmix": { + "median": 502800000.0, + "q1": 500550000.0, + "q3": 529550000.0, + "min": 500000000.0, + "max": 537800000.0, + "n": 5, + "values": [ + 537800000.0, + 502800000.0, + 500000000.0, + 501100000.0, + 521300000.0 + ] + }, + "fast T1 f32 LUT, Philox": { + "median": 315100000.0, + "q1": 313400000.0, + "q3": 331150000.0, + "min": 311700000.0, + "max": 331900000.0, + "n": 5, + "values": [ + 330400000.0, + 311700000.0, + 315100000.0, + 315100000.0, + 331900000.0 + ] + }, + "fast T0 multispin, splitmix": { + "median": 24710000000.0, + "q1": 24300000000.0, + "q3": 25970000000.0, + "min": 24280000000.0, + "max": 26150000000.0, + "n": 5, + "values": [ + 25790000000.0, + 24280000000.0, + 24710000000.0, + 24320000000.0, + 26150000000.0 + ] + }, + "fast T0 multispin, Philox": { + "median": 16760000000.0, + "q1": 16480000000.0, + "q3": 17520000000.0, + "min": 16460000000.0, + "max": 17580000000.0, + "n": 5, + "values": [ + 17460000000.0, + 16500000000.0, + 16760000000.0, + 16460000000.0, + 17580000000.0 + ] + } + } + }, + "portable_vs_native": { + "cmd": "PBIT_DIR=target/release PBIT_NATIVE_DIR=target/native/release python3 bench/portable_vs_native.py", + "exit": 0, + "rows": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "portable": 313700000.0, + "portable_iqr": [ + 302400000.0, + 327600000.0 + ], + "native": 354900000.0, + "native_iqr": [ + 351200000.0, + 382200000.0 + ], + "ratio": 1.131 + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "portable": 226400000.0, + "portable_iqr": [ + 218400000.0, + 234900000.0 + ], + "native": 238100000.0, + "native_iqr": [ + 220800000.0, + 248500000.0 + ], + "ratio": 1.052 + }, + "T0 multispin 64-lane, splitmix": { + "portable": 17240000000.0, + "portable_iqr": [ + 16540000000.0, + 17860000000.0 + ], + "native": 17310000000.0, + "native_iqr": [ + 16820000000.0, + 18340000000.0 + ], + "ratio": 1.004 + }, + "T0 multispin 64-lane, Philox": { + "portable": 12910000000.0, + "portable_iqr": [ + 12470000000.0, + 13480000000.0 + ], + "native": 12570000000.0, + "native_iqr": [ + 12320000000.0, + 13460000000.0 + ], + "ratio": 0.974 + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "portable": 2336000000.0, + "portable_iqr": [ + 2246000000.0, + 2417000000.0 + ], + "native": 3719000000.0, + "native_iqr": [ + 3572000000.0, + 3914000000.0 + ], + "ratio": 1.592 + }, + "T0 multispin splitmix x 4 threads": { + "portable": 16900000000.0, + "portable_iqr": [ + 16400000000.0, + 17480000000.0 + ], + "native": 16830000000.0, + "native_iqr": [ + 16100000000.0, + 18040000000.0 + ], + "ratio": 0.996 + }, + "T0 multispin splitmix x10 threads": { + "portable": 15650000000.0, + "portable_iqr": [ + 15100000000.0, + 15920000000.0 + ], + "native": 15200000000.0, + "native_iqr": [ + 15020000000.0, + 16660000000.0 + ], + "ratio": 0.971 + }, + "fast multispin splitmix x 4 thr": { + "portable": 22570000000.0, + "portable_iqr": [ + 21660000000.0, + 23130000000.0 + ], + "native": 24170000000.0, + "native_iqr": [ + 23910000000.0, + 26220000000.0 + ], + "ratio": 1.071 + }, + "fast multispin splitmix x10 thr": { + "portable": 21250000000.0, + "portable_iqr": [ + 20300000000.0, + 22760000000.0 + ], + "native": 24070000000.0, + "native_iqr": [ + 23580000000.0, + 25770000000.0 + ], + "ratio": 1.133 + }, + "fast T1 f32 LUT, splitmix": { + "portable": 509800000.0, + "portable_iqr": [ + 494100000.0, + 529800000.0 + ], + "native": 537500000.0, + "native_iqr": [ + 527400000.0, + 575000000.0 + ], + "ratio": 1.054 + }, + "fast T1 f32 LUT, Philox": { + "portable": 323200000.0, + "portable_iqr": [ + 305300000.0, + 331100000.0 + ], + "native": 324900000.0, + "native_iqr": [ + 317800000.0, + 354400000.0 + ], + "ratio": 1.005 + }, + "fast T0 multispin, splitmix": { + "portable": 24240000000.0, + "portable_iqr": [ + 23600000000.0, + 25840000000.0 + ], + "native": 25800000000.0, + "native_iqr": [ + 25080000000.0, + 27680000000.0 + ], + "ratio": 1.064 + }, + "fast T0 multispin, Philox": { + "portable": 16230000000.0, + "portable_iqr": [ + 14790000000.0, + 17240000000.0 + ], + "native": 16470000000.0, + "native_iqr": [ + 16160000000.0, + 17810000000.0 + ], + "ratio": 1.015 + }, + "pbit stats IR ring, 1 thread": { + "portable": 18210000.0, + "portable_iqr": [ + 17250000.0, + 18850000.0 + ], + "native": 18200000.0, + "native_iqr": [ + 17500000.0, + 18930000.0 + ], + "ratio": 1.0 + }, + "pbit stats IR ring, 4 threads": { + "portable": 23900000.0, + "portable_iqr": [ + 23200000.0, + 24820000.0 + ], + "native": 24110000.0, + "native_iqr": [ + 23020000.0, + 25320000.0 + ], + "ratio": 1.009 + }, + "pbit decide 300 tasks, 4 thr (sampler)": { + "portable": 14730000.0, + "portable_iqr": [ + 14250000.0, + 15550000.0 + ], + "native": 14770000.0, + "native_iqr": [ + 14330000.0, + 15700000.0 + ], + "ratio": 1.002 + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB, rustc 1.98.1; 5 alternating runs; medians [IQR]; ratio = native / portable", + "T1 SoA f32 heat-bath LUT, splitmix: portable 3.137e+08 [3.024e+08-3.276e+08] | native 3.549e+08 [3.512e+08-3.822e+08] | ratio 1.131", + "T1 SoA f32 heat-bath LUT, Philox: portable 2.264e+08 [2.184e+08-2.349e+08] | native 2.381e+08 [2.208e+08-2.485e+08] | ratio 1.052", + "T0 multispin 64-lane, splitmix: portable 1.724e+10 [1.654e+10-1.786e+10] | native 1.731e+10 [1.682e+10-1.834e+10] | ratio 1.004", + "T0 multispin 64-lane, Philox: portable 1.291e+10 [1.247e+10-1.348e+10] | native 1.257e+10 [1.232e+10-1.346e+10] | ratio 0.974", + "T0 PT-in-a-word prefix-mask, splitm: portable 2.336e+09 [2.246e+09-2.417e+09] | native 3.719e+09 [3.572e+09-3.914e+09] | ratio 1.592", + "T0 multispin splitmix x 4 threads: portable 1.69e+10 [1.64e+10-1.748e+10] | native 1.683e+10 [1.61e+10-1.804e+10] | ratio 0.996", + "T0 multispin splitmix x10 threads: portable 1.565e+10 [1.51e+10-1.592e+10] | native 1.52e+10 [1.502e+10-1.666e+10] | ratio 0.971", + "fast multispin splitmix x 4 thr: portable 2.257e+10 [2.166e+10-2.313e+10] | native 2.417e+10 [2.391e+10-2.622e+10] | ratio 1.071", + "fast multispin splitmix x10 thr: portable 2.125e+10 [2.03e+10-2.276e+10] | native 2.407e+10 [2.358e+10-2.577e+10] | ratio 1.133", + "fast T1 f32 LUT, splitmix: portable 5.098e+08 [4.941e+08-5.298e+08] | native 5.375e+08 [5.274e+08-5.75e+08] | ratio 1.054", + "fast T1 f32 LUT, Philox: portable 3.232e+08 [3.053e+08-3.311e+08] | native 3.249e+08 [3.178e+08-3.544e+08] | ratio 1.005", + "fast T0 multispin, splitmix: portable 2.424e+10 [2.36e+10-2.584e+10] | native 2.58e+10 [2.508e+10-2.768e+10] | ratio 1.064", + "fast T0 multispin, Philox: portable 1.623e+10 [1.479e+10-1.724e+10] | native 1.647e+10 [1.616e+10-1.781e+10] | ratio 1.015", + "pbit stats IR ring, 1 thread: portable 1.821e+07 [1.725e+07-1.885e+07] | native 1.82e+07 [1.75e+07-1.893e+07] | ratio 1.000", + "pbit stats IR ring, 4 threads: portable 2.39e+07 [2.32e+07-2.482e+07] | native 2.411e+07 [2.302e+07-2.532e+07] | ratio 1.009", + "pbit decide 300 tasks, 4 thr (sampler): portable 1.473e+07 [1.425e+07-1.555e+07] | native 1.477e+07 [1.433e+07-1.57e+07] | ratio 1.002", + "kernels checksum identical across builds: True | decide answer (odds+plan+gate) identical across builds: True" + ], + "stderr_tail": null, + "note": "the script's first line is hard-coded to the M4; the machine is the one in `machine`" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 99.1, + "wall_iqr": [ + 99.1, + 99.8 + ], + "cpu_ms": 101.5, + "site_updates_per_s_M": 10.01, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 84.1, + "wall_iqr": [ + 82.4, + 85.1 + ], + "cpu_ms": 116.6, + "site_updates_per_s_M": 15.27, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 83.2, + "wall_iqr": [ + 83.1, + 83.6 + ], + "cpu_ms": 116.9, + "site_updates_per_s_M": 15.18, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 415.1, + "wall_iqr": [ + 414.5, + 416.5 + ], + "sweeps": 15712, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 99.1 [99.1-99.8], CPU ms 101.5, site updates/s 10.01 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 84.1 [82.4-85.1], CPU ms 116.6, site updates/s 15.27 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 83.2 [83.1-83.6], CPU ms 116.9, site updates/s 15.18 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 2): wall ms 415.1 [414.5-416.5], sweeps 15712, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "attached": { + "checks": { + "build": "success", + "native_build": "success", + "pack_zip": "skipped", + "callers": "success", + "install_sh": "success", + "npm": "success", + "npm_windows": "skipped", + "install_ps1_pwsh": "skipped", + "install_ps1_ps51": "skipped", + "ps_callers": "skipped", + "rss_powershell": "skipped", + "cross_darwin": "skipped" + } + }, + "cells": { + "cpu_model": "AMD EPYC 9V45 96-Core Processor", + "logical_cores": 2, + "binary_size_bytes": 1401248, + "stats_1t": { + "median": 17637485, + "q1": 17103334.0, + "q3": 17767678.5, + "min": 16952963, + "max": 17889808, + "n": 5, + "values": [ + 17253705, + 16952963, + 17889808, + 17645549, + 17637485 + ] + }, + "stats_default": { + "median": 23339333, + "q1": 23097044.0, + "q3": 23699545.5, + "min": 23011872, + "max": 24000627, + "n": 5, + "values": [ + 23011872, + 23182216, + 24000627, + 23339333, + 23398464 + ] + }, + "stats_default_threads": [ + 2 + ], + "kernel_multispin_fast_4t": { + "median": 22210000000.0, + "q1": 21165000000.0, + "q3": 23175000000.0, + "min": 20320000000.0, + "max": 23340000000.0, + "n": 5, + "values": [ + 23340000000.0, + 22210000000.0, + 20320000000.0, + 22010000000.0, + 23010000000.0 + ] + }, + "kernel_multispin_fast_10t": { + "median": 22200000000.0, + "q1": 21810000000.0, + "q3": 22575000000.0, + "min": 21580000000.0, + "max": 22870000000.0, + "n": 5, + "values": [ + 22280000000.0, + 21580000000.0, + 22040000000.0, + 22200000000.0, + 22870000000.0 + ] + }, + "kernel_multispin_fast_1t": { + "median": 24710000000.0, + "q1": 24300000000.0, + "q3": 25970000000.0, + "min": 24280000000.0, + "max": 26150000000.0, + "n": 5, + "values": [ + 25790000000.0, + 24280000000.0, + 24710000000.0, + 24320000000.0, + 26150000000.0 + ] + }, + "kernel_heatbath_fast_1t": { + "median": 502800000.0, + "q1": 500550000.0, + "q3": 529550000.0, + "min": 500000000.0, + "max": 537800000.0, + "n": 5, + "values": [ + 537800000.0, + 502800000.0, + 500000000.0, + 501100000.0, + 521300000.0 + ] + }, + "decide_12_wall_ms": { + "median": 10.505511000019396, + "q1": 10.402311999996527, + "q3": 10.53083000000754, + "min": 10.373764000007668, + "max": 10.539232000013499, + "n": 5, + "values": [ + 10.522428000001582, + 10.505511000019396, + 10.430859999985387, + 10.539232000013499, + 10.373764000007668 + ] + }, + "decide_12_pbit_ms": { + "median": 9.088063, + "q1": 9.0528695, + "q3": 9.134608, + "min": 9.045869, + "max": 9.165309, + "n": 5, + "values": [ + 9.088063, + 9.103907, + 9.05987, + 9.165309, + 9.045869 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 422.52012199998035, + "q1": 421.6932709999952, + "q3": 424.01193349999744, + "min": 421.5770790000022, + "max": 424.36705899999083, + "n": 5, + "values": [ + 423.65680800000405, + 422.52012199998035, + 424.36705899999083, + 421.5770790000022, + 421.8094629999882 + ] + }, + "decide_300_pbit_ms": { + "median": 417.940061, + "q1": 417.2589365, + "q3": 419.457341, + "min": 417.235029, + "max": 419.914021, + "n": 5, + "values": [ + 419.000661, + 417.940061, + 419.914021, + 417.235029, + 417.282844 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 14940381, + "q1": 14697294.0, + "q3": 15087959.0, + "min": 14500705, + "max": 15108383, + "n": 5, + "values": [ + 14500705, + 14893883, + 15067535, + 15108383, + 14940381 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 22052864, + "q1": 21991424.0, + "q3": 22351872.0, + "min": 21991424, + "max": 22351872, + "n": 3, + "values": [ + 22052864, + 22351872, + 21991424 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "native_over_portable": { + "T1 SoA f32 heat-bath LUT, splitmix": 1.131, + "T1 SoA f32 heat-bath LUT, Philox": 1.052, + "T0 multispin 64-lane, splitmix": 1.004, + "T0 multispin 64-lane, Philox": 0.974, + "T0 PT-in-a-word prefix-mask, splitm": 1.592, + "T0 multispin splitmix x 4 threads": 0.996, + "T0 multispin splitmix x10 threads": 0.971, + "fast multispin splitmix x 4 thr": 1.071, + "fast multispin splitmix x10 thr": 1.133, + "fast T1 f32 LUT, splitmix": 1.054, + "fast T1 f32 LUT, Philox": 1.005, + "fast T0 multispin, splitmix": 1.064, + "fast T0 multispin, Philox": 1.015, + "pbit stats IR ring, 1 thread": 1.0, + "pbit stats IR ring, 4 threads": 1.009, + "pbit decide 300 tasks, 4 thr (sampler)": 1.002 + }, + "threads_rows": { + "threads_1": { + "wall_ms": 99.1, + "wall_iqr": [ + 99.1, + 99.8 + ], + "cpu_ms": 101.5, + "site_updates_per_s_M": 10.01, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 84.1, + "wall_iqr": [ + 82.4, + 85.1 + ], + "cpu_ms": 116.6, + "site_updates_per_s_M": 15.27, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 83.2, + "wall_iqr": [ + 83.1, + 83.6 + ], + "cpu_ms": 116.9, + "site_updates_per_s_M": 15.18, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 415.1, + "wall_iqr": [ + 414.5, + 416.5 + ], + "sweeps": 15712, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 131.9 +} diff --git a/scripts/bench/results/2026-10-01/run-36970631022/bench-macos-arm64.json b/scripts/bench/results/2026-10-01/run-36970631022/bench-macos-arm64.json new file mode 100644 index 0000000..37afe35 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/bench-macos-arm64.json @@ -0,0 +1,1087 @@ +{ + "schema": "pbit-bench-matrix/1", + "label": "macos-arm64", + "date_utc": "2026-10-02T05:50:27Z", + "git_sha": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "runner": { + "RUNNER_OS": "macOS", + "RUNNER_ARCH": "ARM64", + "ImageOS": "macos26", + "ImageVersion": "20260907.0351.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "bench", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "build_s=27", + "runner=macos-latest" + ], + "machine": { + "system": "Darwin", + "os_release": "25.6.0", + "arch": "arm64", + "python": "3.14.7", + "logical_cores": 3, + "cpu_cmd": "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize", + "cpu_model": "Apple M1 (Virtual)", + "hw_ncpu": 3, + "memory_bytes": 7516192768, + "perf_cores": 3, + "efficiency_cores": 0, + "rosetta_translated": false, + "os_version": "26.6.2" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "aarch64-apple-darwin", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 10.0390625, + 16.64990234375, + 10.20751953125 + ], + "uptime": "5:50 up 5 mins, 1 user, load averages: 10.04 16.65 10.21", + "t": "05:50:27" + }, + "after_stats": { + "loadavg": [ + 9.63525390625, + 16.4560546875, + 10.1767578125 + ], + "uptime": "5:50 up 6 mins, 1 user, load averages: 9.64 16.46 10.18", + "t": "05:50:30" + }, + "after_decide": { + "loadavg": [ + 9.63525390625, + 16.4560546875, + 10.1767578125 + ], + "uptime": "5:50 up 6 mins, 1 user, load averages: 9.64 16.46 10.18", + "t": "05:50:32" + }, + "after_kernels": { + "loadavg": [ + 9.5654296875, + 15.705078125, + 10.1552734375 + ], + "uptime": "5:51 up 6 mins, 1 user, load averages: 9.57 15.71 10.16", + "t": "05:51:03" + }, + "after_pvn": { + "loadavg": [ + 9.5654296875, + 15.705078125, + 10.1552734375 + ], + "uptime": "5:51 up 6 mins, 1 user, load averages: 9.57 15.71 10.16", + "t": "05:51:03" + }, + "end": { + "loadavg": [ + 9.5654296875, + 15.705078125, + 10.1552734375 + ], + "uptime": "5:51 up 6 mins, 1 user, load averages: 9.57 15.71 10.16", + "t": "05:51:08" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 1146960, + "version": "pbit 0.2.0", + "linkage": { + "file": "Mach-O 64-bit executable arm64", + "otool_L": [ + "\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0)" + ], + "lipo_archs": "arm64" + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "macos", + "arch": "aarch64", + "logical_cpus": 3 + }, + "build": { + "target_features": [ + "neon", + "dotprod" + ], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 3, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 13181793, + "q1": 13144040.0, + "q3": 13570962.0, + "min": 13133773, + "max": 13697953, + "n": 5, + "values": [ + 13181793, + 13697953, + 13443971, + 13133773, + 13154307 + ] + }, + "site_updates_per_s": { + "median": 26321887, + "q1": 25687015.0, + "q3": 26375678.5, + "min": 25061594, + "max": 26387101, + "n": 5, + "values": [ + 26312436, + 26321887, + 26387101, + 26364256, + 25061594 + ] + }, + "threads": [ + 3 + ], + "speedup": { + "median": 1.96, + "q1": 1.915, + "q3": 2.005, + "min": 1.91, + "max": 2.01, + "n": 5, + "values": [ + 2, + 1.92, + 1.96, + 2.01, + 1.91 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 17.722832999993443, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 13.116958, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 17.848875000026965, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.8875, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 18.503374999966127, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.825, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 17.76937500000031, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 13.061542, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 17.71850000000086, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 12.993084, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 17.76937500000031, + "q1": 17.72066649999715, + "q3": 18.176124999996546, + "min": 17.71850000000086, + "max": 18.503374999966127, + "n": 5, + "values": [ + 17.722832999993443, + 17.848875000026965, + 18.503374999966127, + 17.76937500000031, + 17.71850000000086 + ] + }, + "pbit_ms": { + "median": 12.993084, + "q1": 12.85625, + "q3": 13.08925, + "min": 12.825, + "max": 13.116958, + "n": 5, + "values": [ + 13.116958, + 12.8875, + 12.825, + 13.061542, + 12.993084 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 439.2280840000353, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 431.368709, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 14816, + "site_updates_per_s": 14795806, + "threads": 3, + "peak_rss_mb_self": 30.421875, + "stderr": null + }, + { + "wall_ms": 437.1561669999551, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 428.79275, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 14280, + "site_updates_per_s": 14260742, + "threads": 3, + "peak_rss_mb_self": 30.09375, + "stderr": null + }, + { + "wall_ms": 439.7126249999701, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 430.681791, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 14680, + "site_updates_per_s": 14660912, + "threads": 3, + "peak_rss_mb_self": 30.359375, + "stderr": null + }, + { + "wall_ms": 438.44066700000894, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 431.212917, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 14768, + "site_updates_per_s": 14758929, + "threads": 3, + "peak_rss_mb_self": 30.40625, + "stderr": null + }, + { + "wall_ms": 449.6654999999805, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 440.538542, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 12560, + "site_updates_per_s": 12543775, + "threads": 3, + "peak_rss_mb_self": 29.0625, + "stderr": null + } + ], + "wall_ms": { + "median": 439.2280840000353, + "q1": 437.798416999982, + "q3": 444.6890624999753, + "min": 437.1561669999551, + "max": 449.6654999999805, + "n": 5, + "values": [ + 439.2280840000353, + 437.1561669999551, + 439.7126249999701, + 438.44066700000894, + 449.6654999999805 + ] + }, + "pbit_ms": { + "median": 431.212917, + "q1": 429.7372705, + "q3": 435.95362550000004, + "min": 428.79275, + "max": 440.538542, + "n": 5, + "values": [ + 431.368709, + 428.79275, + 430.681791, + 431.212917, + 440.538542 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 14660912, + "q1": 13402258.5, + "q3": 14777367.5, + "min": 12543775, + "max": 14795806, + "n": 5, + "values": [ + 14795806, + 14260742, + 14660912, + 14758929, + 12543775 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -l (maximum resident set size, bytes)", + "bytes": { + "median": 31211520, + "q1": 30162944.0, + "q3": 32112640.0, + "min": 30162944, + "max": 32112640, + "n": 3, + "values": [ + 30162944, + 31211520, + 32112640 + ] + }, + "self_reported_peak_rss_mb": [ + 28.71875, + 29.71875, + 30.5625 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + { + "peak_memory_footprint_bytes": 29164736 + }, + { + "peak_memory_footprint_bytes": 30213312 + }, + { + "peak_memory_footprint_bytes": 31114432 + } + ] + }, + "kernels": { + "cmd": "target/release/examples/kernels", + "checksums": [ + "8520834" + ], + "lines": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "median": 302500000.0, + "q1": 268750000.0, + "q3": 314650000.0, + "min": 257300000.0, + "max": 316100000.0, + "n": 5, + "values": [ + 316100000.0, + 313200000.0, + 280200000.0, + 302500000.0, + 257300000.0 + ] + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "median": 213700000.0, + "q1": 192400000.0, + "q3": 221250000.0, + "min": 176900000.0, + "max": 225100000.0, + "n": 5, + "values": [ + 217400000.0, + 225100000.0, + 207900000.0, + 213700000.0, + 176900000.0 + ] + }, + "T0 multispin 64-lane, splitmix": { + "median": 12290000000.0, + "q1": 11400000000.0, + "q3": 12775000000.0, + "min": 10540000000.0, + "max": 12800000000.0, + "n": 5, + "values": [ + 12800000000.0, + 12750000000.0, + 10540000000.0, + 12260000000.0, + 12290000000.0 + ] + }, + "T0 multispin 64-lane, Philox": { + "median": 10710000000.0, + "q1": 9150500000.0, + "q3": 11280000000.0, + "min": 8351000000.0, + "max": 11370000000.0, + "n": 5, + "values": [ + 11370000000.0, + 11190000000.0, + 8351000000.0, + 9950000000.0, + 10710000000.0 + ] + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "median": 1946000000.0, + "q1": 1646500000.0, + "q3": 1985500000.0, + "min": 1626000000.0, + "max": 1990000000.0, + "n": 5, + "values": [ + 1990000000.0, + 1667000000.0, + 1626000000.0, + 1946000000.0, + 1981000000.0 + ] + }, + "T0 multispin splitmix x 4 threads": { + "median": 32340000000.0, + "q1": 29975000000.0, + "q3": 36485000000.0, + "min": 27770000000.0, + "max": 37300000000.0, + "n": 5, + "values": [ + 35670000000.0, + 32180000000.0, + 32340000000.0, + 27770000000.0, + 37300000000.0 + ] + }, + "T0 multispin splitmix x10 threads": { + "median": 33150000000.0, + "q1": 28275000000.0, + "q3": 36945000000.0, + "min": 25950000000.0, + "max": 37330000000.0, + "n": 5, + "values": [ + 37330000000.0, + 25950000000.0, + 33150000000.0, + 30600000000.0, + 36560000000.0 + ] + }, + "fast multispin splitmix x 4 thr": { + "median": 46010000000.0, + "q1": 40950000000.0, + "q3": 54715000000.0, + "min": 37870000000.0, + "max": 55070000000.0, + "n": 5, + "values": [ + 55070000000.0, + 46010000000.0, + 37870000000.0, + 54360000000.0, + 44030000000.0 + ] + }, + "fast multispin splitmix x10 thr": { + "median": 57670000000.0, + "q1": 48330000000.0, + "q3": 58745000000.0, + "min": 45170000000.0, + "max": 59520000000.0, + "n": 5, + "values": [ + 57970000000.0, + 59520000000.0, + 45170000000.0, + 51490000000.0, + 57670000000.0 + ] + }, + "fast T1 f32 LUT, splitmix": { + "median": 495300000.0, + "q1": 439700000.0, + "q3": 512800000.0, + "min": 417500000.0, + "max": 526100000.0, + "n": 5, + "values": [ + 526100000.0, + 417500000.0, + 461900000.0, + 495300000.0, + 499500000.0 + ] + }, + "fast T1 f32 LUT, Philox": { + "median": 282200000.0, + "q1": 267850000.0, + "q3": 303600000.0, + "min": 256500000.0, + "max": 316000000.0, + "n": 5, + "values": [ + 316000000.0, + 256500000.0, + 282200000.0, + 291200000.0, + 279200000.0 + ] + }, + "fast T0 multispin, splitmix": { + "median": 20040000000.0, + "q1": 16185000000.0, + "q3": 20960000000.0, + "min": 15820000000.0, + "max": 21230000000.0, + "n": 5, + "values": [ + 21230000000.0, + 16550000000.0, + 20690000000.0, + 20040000000.0, + 15820000000.0 + ] + }, + "fast T0 multispin, Philox": { + "median": 13790000000.0, + "q1": 13385000000.0, + "q3": 14025000000.0, + "min": 13060000000.0, + "max": 14090000000.0, + "n": 5, + "values": [ + 14090000000.0, + 13960000000.0, + 13060000000.0, + 13790000000.0, + 13710000000.0 + ] + } + } + }, + "portable_vs_native": { + "skipped": "no --native-dir" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 135.6, + "wall_iqr": [ + 134.7, + 139.9 + ], + "cpu_ms": 140.4, + "site_updates_per_s_M": 7.27, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 109.4, + "wall_iqr": [ + 106.1, + 133.6 + ], + "cpu_ms": 148.5, + "site_updates_per_s_M": 13.82, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 119.4, + "wall_iqr": [ + 105.3, + 131.7 + ], + "cpu_ms": 160.7, + "site_updates_per_s_M": 13.98, + "verdict": "refused" + }, + "default": { + "threads": 3, + "wall_ms": 447.5, + "wall_iqr": [ + 442.0, + 472.2 + ], + "sweeps": 14392, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 135.6 [134.7-139.9], CPU ms 140.4, site updates/s 7.27 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 109.4 [106.1-133.6], CPU ms 148.5, site updates/s 13.82 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 119.4 [105.3-131.7], CPU ms 160.7, site updates/s 13.98 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 3): wall ms 447.5 [442.0-472.2], sweeps 14392, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "attached": { + "checks": { + "build": "success", + "native_build": "skipped", + "pack_zip": "skipped", + "callers": "success", + "install_sh": "success", + "npm": "skipped", + "npm_windows": "skipped", + "install_ps1_pwsh": "skipped", + "install_ps1_ps51": "skipped", + "ps_callers": "skipped", + "rss_powershell": "skipped", + "cross_darwin": "success" + }, + "cross_x86_64_apple_darwin": [ + "$ rustup target add x86_64-apple-darwin", + "info: downloading component rust-std", + "$ cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin", + " Finished `release` profile [optimized] target(s) in 21.67s", + "$ file target/x86_64-apple-darwin/release/pbit", + "target/x86_64-apple-darwin/release/pbit: Mach-O 64-bit executable x86_64", + "$ lipo -archs target/x86_64-apple-darwin/release/pbit", + "x86_64", + "$ otool -L target/x86_64-apple-darwin/release/pbit", + "target/x86_64-apple-darwin/release/pbit:", + "\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0)", + "size 1270368 bytes", + "rosetta: present", + "$ arch -x86_64 target/x86_64-apple-darwin/release/pbit version", + "pbit 0.2.0", + "$ arch -x86_64 pbit demo --tasks 12 | arch -x86_64 pbit decide: exit 0, verdict exact", + "$ arch -x86_64 pbit stats (self-test under Rosetta)", + "1 thread 9088089 | threads 3 14179921" + ] + }, + "cells": { + "cpu_model": "Apple M1 (Virtual)", + "logical_cores": 3, + "binary_size_bytes": 1146960, + "stats_1t": { + "median": 13181793, + "q1": 13144040.0, + "q3": 13570962.0, + "min": 13133773, + "max": 13697953, + "n": 5, + "values": [ + 13181793, + 13697953, + 13443971, + 13133773, + 13154307 + ] + }, + "stats_default": { + "median": 26321887, + "q1": 25687015.0, + "q3": 26375678.5, + "min": 25061594, + "max": 26387101, + "n": 5, + "values": [ + 26312436, + 26321887, + 26387101, + 26364256, + 25061594 + ] + }, + "stats_default_threads": [ + 3 + ], + "kernel_multispin_fast_4t": { + "median": 46010000000.0, + "q1": 40950000000.0, + "q3": 54715000000.0, + "min": 37870000000.0, + "max": 55070000000.0, + "n": 5, + "values": [ + 55070000000.0, + 46010000000.0, + 37870000000.0, + 54360000000.0, + 44030000000.0 + ] + }, + "kernel_multispin_fast_10t": { + "median": 57670000000.0, + "q1": 48330000000.0, + "q3": 58745000000.0, + "min": 45170000000.0, + "max": 59520000000.0, + "n": 5, + "values": [ + 57970000000.0, + 59520000000.0, + 45170000000.0, + 51490000000.0, + 57670000000.0 + ] + }, + "kernel_multispin_fast_1t": { + "median": 20040000000.0, + "q1": 16185000000.0, + "q3": 20960000000.0, + "min": 15820000000.0, + "max": 21230000000.0, + "n": 5, + "values": [ + 21230000000.0, + 16550000000.0, + 20690000000.0, + 20040000000.0, + 15820000000.0 + ] + }, + "kernel_heatbath_fast_1t": { + "median": 495300000.0, + "q1": 439700000.0, + "q3": 512800000.0, + "min": 417500000.0, + "max": 526100000.0, + "n": 5, + "values": [ + 526100000.0, + 417500000.0, + 461900000.0, + 495300000.0, + 499500000.0 + ] + }, + "decide_12_wall_ms": { + "median": 17.76937500000031, + "q1": 17.72066649999715, + "q3": 18.176124999996546, + "min": 17.71850000000086, + "max": 18.503374999966127, + "n": 5, + "values": [ + 17.722832999993443, + 17.848875000026965, + 18.503374999966127, + 17.76937500000031, + 17.71850000000086 + ] + }, + "decide_12_pbit_ms": { + "median": 12.993084, + "q1": 12.85625, + "q3": 13.08925, + "min": 12.825, + "max": 13.116958, + "n": 5, + "values": [ + 13.116958, + 12.8875, + 12.825, + 13.061542, + 12.993084 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 439.2280840000353, + "q1": 437.798416999982, + "q3": 444.6890624999753, + "min": 437.1561669999551, + "max": 449.6654999999805, + "n": 5, + "values": [ + 439.2280840000353, + 437.1561669999551, + 439.7126249999701, + 438.44066700000894, + 449.6654999999805 + ] + }, + "decide_300_pbit_ms": { + "median": 431.212917, + "q1": 429.7372705, + "q3": 435.95362550000004, + "min": 428.79275, + "max": 440.538542, + "n": 5, + "values": [ + 431.368709, + 428.79275, + 430.681791, + 431.212917, + 440.538542 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 14660912, + "q1": 13402258.5, + "q3": 14777367.5, + "min": 12543775, + "max": 14795806, + "n": 5, + "values": [ + 14795806, + 14260742, + 14660912, + 14758929, + 12543775 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 31211520, + "q1": 30162944.0, + "q3": 32112640.0, + "min": 30162944, + "max": 32112640, + "n": 3, + "values": [ + 30162944, + 31211520, + 32112640 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -l (maximum resident set size, bytes)", + "native_over_portable": "no --native-dir", + "threads_rows": { + "threads_1": { + "wall_ms": 135.6, + "wall_iqr": [ + 134.7, + 139.9 + ], + "cpu_ms": 140.4, + "site_updates_per_s_M": 7.27, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 109.4, + "wall_iqr": [ + 106.1, + 133.6 + ], + "cpu_ms": 148.5, + "site_updates_per_s_M": 13.82, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 119.4, + "wall_iqr": [ + 105.3, + 131.7 + ], + "cpu_ms": 160.7, + "site_updates_per_s_M": 13.98, + "verdict": "refused" + }, + "default": { + "threads": 3, + "wall_ms": 447.5, + "wall_iqr": [ + 442.0, + 472.2 + ], + "sweeps": 14392, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 40.6 +} diff --git a/scripts/bench/results/2026-10-01/run-36970631022/bench-windows-x86_64.json b/scripts/bench/results/2026-10-01/run-36970631022/bench-windows-x86_64.json new file mode 100644 index 0000000..2f7f63d --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/bench-windows-x86_64.json @@ -0,0 +1,1243 @@ +{ + "schema": "pbit-bench-matrix/1", + "label": "windows-x86_64", + "date_utc": "2026-10-02T05:51:30Z", + "git_sha": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "runner": { + "RUNNER_OS": "Windows", + "RUNNER_ARCH": "X64", + "ImageOS": "win25-vs2026", + "ImageVersion": "20260925.250.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "bench", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "build_s=37", + "runner=windows-latest" + ], + "machine": { + "system": "Windows", + "os_release": "2025Server", + "arch": "AMD64", + "python": "3.12.10", + "logical_cores": 2, + "cpu_cmd": "Get-CimInstance Win32_Processor", + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "physical_cores": 1, + "cim_logical": 2, + "memory_bytes": 8584425472, + "os_version": "Microsoft Windows Server 2025 Datacenter (build 26100)" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-pc-windows-msvc", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": null, + "uptime": null, + "t": "05:51:32" + }, + "after_stats": { + "loadavg": null, + "uptime": null, + "t": "05:51:39" + }, + "after_decide": { + "loadavg": null, + "uptime": null, + "t": "05:51:41" + }, + "after_kernels": { + "loadavg": null, + "uptime": null, + "t": "05:52:56" + }, + "after_pvn": { + "loadavg": null, + "uptime": null, + "t": "05:55:19" + }, + "end": { + "loadavg": null, + "uptime": null, + "t": "05:55:20" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 966656, + "version": "pbit 0.2.0", + "linkage": { + "file": "PE32+ executable for MS Windows 6.00 (console), x86-64, 5 sections", + "dumpbin_found_via": "vswhere (not on PATH)", + "dll_imports": [ + "bcryptprimitives.dll", + "api-ms-win-core-synch-l1-2-0.dll", + "KERNEL32.dll", + "ntdll.dll", + "VCRUNTIME140.dll", + "api-ms-win-crt-math-l1-1-0.dll", + "api-ms-win-crt-string-l1-1-0.dll", + "api-ms-win-crt-runtime-l1-1-0.dll", + "api-ms-win-crt-stdio-l1-1-0.dll", + "api-ms-win-crt-locale-l1-1-0.dll", + "api-ms-win-crt-heap-l1-1-0.dll" + ], + "dumpbin_rc": 0 + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "windows", + "arch": "x86_64", + "logical_cpus": 2 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 2, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 10278961, + "q1": 9970100.5, + "q3": 10966905.5, + "min": 9954876, + "max": 11246250, + "n": 5, + "values": [ + 9954876, + 10278961, + 11246250, + 9985325, + 10687561 + ] + }, + "site_updates_per_s": { + "median": 14610444, + "q1": 12761670.0, + "q3": 14644030.5, + "min": 12614362, + "max": 14655215, + "n": 5, + "values": [ + 14655215, + 14610444, + 12614362, + 14632846, + 12908978 + ] + }, + "threads": [ + 2 + ], + "speedup": { + "median": 1.42, + "q1": 1.165, + "q3": 1.47, + "min": 1.12, + "max": 1.47, + "n": 5, + "values": [ + 1.47, + 1.42, + 1.12, + 1.47, + 1.21 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 30.53740000001426, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.9074, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 30.857700000012755, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 18.2775, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 52.840799999955834, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 24.7737, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 30.733199999986027, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.9625, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 30.80610000000661, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.922, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 30.80610000000661, + "q1": 30.635300000000143, + "q3": 41.849249999984295, + "min": 30.53740000001426, + "max": 52.840799999955834, + "n": 5, + "values": [ + 30.53740000001426, + 30.857700000012755, + 52.840799999955834, + 30.733199999986027, + 30.80610000000661 + ] + }, + "pbit_ms": { + "median": 17.9625, + "q1": 17.9147, + "q3": 21.5256, + "min": 17.9074, + "max": 24.7737, + "n": 5, + "values": [ + 17.9074, + 18.2775, + 24.7737, + 17.9625, + 17.922 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 491.1162999999874, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 467.768, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 8872, + "site_updates_per_s": 8850496, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 490.4213000000368, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 467.5572, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 8840, + "site_updates_per_s": 8830393, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 497.3655000000008, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 480.9308, + "verdict": "partial", + "tier": null, + "released": 299, + "escalated": 1, + "violations": 0, + "sweeps": 8720, + "site_updates_per_s": 8687269, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 518.2927999999833, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 499.7997, + "verdict": "partial", + "tier": null, + "released": 296, + "escalated": 4, + "violations": 0, + "sweeps": 8272, + "site_updates_per_s": 8254094, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 493.0233000000044, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 470.1004, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 8648, + "site_updates_per_s": 8625832, + "threads": 2, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 493.0233000000044, + "q1": 490.7688000000121, + "q3": 507.82914999999207, + "min": 490.4213000000368, + "max": 518.2927999999833, + "n": 5, + "values": [ + 491.1162999999874, + 490.4213000000368, + 497.3655000000008, + 518.2927999999833, + 493.0233000000044 + ] + }, + "pbit_ms": { + "median": 470.1004, + "q1": 467.6626, + "q3": 490.36524999999995, + "min": 467.5572, + "max": 499.7997, + "n": 5, + "values": [ + 467.768, + 467.5572, + 480.9308, + 499.7997, + 470.1004 + ] + }, + "released": { + "median": 300, + "q1": 297.5, + "q3": 300.0, + "min": 296, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 299, + 296, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed", + "partial" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 8687269, + "q1": 8439963.0, + "q3": 8840444.5, + "min": 8254094, + "max": 8850496, + "n": 5, + "values": [ + 8850496, + 8830393, + 8687269, + 8254094, + 8625832 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "K32GetProcessMemoryInfo PeakWorkingSetSize (process handle, after exit)", + "bytes": { + "median": 20135936, + "q1": 20119552.0, + "q3": 21323776.0, + "min": 20119552, + "max": 21323776, + "n": 3, + "values": [ + 20135936, + 21323776, + 20119552 + ] + }, + "self_reported_peak_rss_mb": [ + null, + null, + null + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + { + "peak_pagefile_bytes": 1078857728 + }, + { + "peak_pagefile_bytes": 1079992320 + }, + { + "peak_pagefile_bytes": 1079435264 + } + ] + }, + "kernels": { + "cmd": "target/release/examples/kernels.exe", + "checksums": [ + "8520834" + ], + "lines": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "median": 157000000.0, + "q1": 128600000.0, + "q3": 167750000.0, + "min": 118900000.0, + "max": 169000000.0, + "n": 5, + "values": [ + 169000000.0, + 166500000.0, + 118900000.0, + 157000000.0, + 138300000.0 + ] + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "median": 102100000.0, + "q1": 101350000.0, + "q3": 107500000.0, + "min": 101200000.0, + "max": 107700000.0, + "n": 5, + "values": [ + 107300000.0, + 102100000.0, + 101500000.0, + 107700000.0, + 101200000.0 + ] + }, + "T0 multispin 64-lane, splitmix": { + "median": 8943000000.0, + "q1": 8171500000.0, + "q3": 9070500000.0, + "min": 8136000000.0, + "max": 9185000000.0, + "n": 5, + "values": [ + 9185000000.0, + 8956000000.0, + 8136000000.0, + 8207000000.0, + 8943000000.0 + ] + }, + "T0 multispin 64-lane, Philox": { + "median": 6908000000.0, + "q1": 6099500000.0, + "q3": 6997000000.0, + "min": 5637000000.0, + "max": 7069000000.0, + "n": 5, + "values": [ + 6925000000.0, + 6562000000.0, + 5637000000.0, + 7069000000.0, + 6908000000.0 + ] + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "median": 1785000000.0, + "q1": 1709500000.0, + "q3": 1850500000.0, + "min": 1654000000.0, + "max": 1856000000.0, + "n": 5, + "values": [ + 1845000000.0, + 1785000000.0, + 1654000000.0, + 1765000000.0, + 1856000000.0 + ] + }, + "T0 multispin splitmix x 4 threads": { + "median": 8502000000.0, + "q1": 7883500000.0, + "q3": 9326500000.0, + "min": 7327000000.0, + "max": 9412000000.0, + "n": 5, + "values": [ + 9412000000.0, + 8502000000.0, + 7327000000.0, + 8440000000.0, + 9241000000.0 + ] + }, + "T0 multispin splitmix x10 threads": { + "median": 8829000000.0, + "q1": 7115500000.0, + "q3": 9280500000.0, + "min": 5843000000.0, + "max": 9382000000.0, + "n": 5, + "values": [ + 8388000000.0, + 8829000000.0, + 5843000000.0, + 9179000000.0, + 9382000000.0 + ] + }, + "fast multispin splitmix x 4 thr": { + "median": 12030000000.0, + "q1": 10831500000.0, + "q3": 12630000000.0, + "min": 9683000000.0, + "max": 13130000000.0, + "n": 5, + "values": [ + 9683000000.0, + 11980000000.0, + 12130000000.0, + 12030000000.0, + 13130000000.0 + ] + }, + "fast multispin splitmix x10 thr": { + "median": 12110000000.0, + "q1": 11675000000.0, + "q3": 12515000000.0, + "min": 11600000000.0, + "max": 12580000000.0, + "n": 5, + "values": [ + 12580000000.0, + 11600000000.0, + 11750000000.0, + 12110000000.0, + 12450000000.0 + ] + }, + "fast T1 f32 LUT, splitmix": { + "median": 236600000.0, + "q1": 143875000.0, + "q3": 242650000.0, + "min": 68050000.0, + "max": 243500000.0, + "n": 5, + "values": [ + 68050000.0, + 243500000.0, + 241800000.0, + 236600000.0, + 219700000.0 + ] + }, + "fast T1 f32 LUT, Philox": { + "median": 141100000.0, + "q1": 123700000.0, + "q3": 163850000.0, + "min": 122400000.0, + "max": 164600000.0, + "n": 5, + "values": [ + 125000000.0, + 141100000.0, + 122400000.0, + 163100000.0, + 164600000.0 + ] + }, + "fast T0 multispin, splitmix": { + "median": 13160000000.0, + "q1": 11725000000.0, + "q3": 13300000000.0, + "min": 11560000000.0, + "max": 13420000000.0, + "n": 5, + "values": [ + 13420000000.0, + 13160000000.0, + 11560000000.0, + 11890000000.0, + 13180000000.0 + ] + }, + "fast T0 multispin, Philox": { + "median": 8983000000.0, + "q1": 8259000000.0, + "q3": 9161500000.0, + "min": 7874000000.0, + "max": 9270000000.0, + "n": 5, + "values": [ + 7874000000.0, + 9270000000.0, + 8644000000.0, + 9053000000.0, + 8983000000.0 + ] + } + } + }, + "windows_shim": [ + "target/release/pbit", + "target/release/examples/kernels", + "target/native/release/pbit", + "target/native/release/examples/kernels" + ], + "portable_vs_native": { + "cmd": "PBIT_DIR=target/release PBIT_NATIVE_DIR=target/native/release python3 bench/portable_vs_native.py", + "exit": 0, + "rows": { + "T1 SoA f32 heat-bath LUT, splitmix": { + "portable": 150200000.0, + "portable_iqr": [ + 144800000.0, + 150700000.0 + ], + "native": 161900000.0, + "native_iqr": [ + 153100000.0, + 174000000.0 + ], + "ratio": 1.078 + }, + "T1 SoA f32 heat-bath LUT, Philox": { + "portable": 108500000.0, + "portable_iqr": [ + 105800000.0, + 109000000.0 + ], + "native": 124200000.0, + "native_iqr": [ + 122200000.0, + 130200000.0 + ], + "ratio": 1.145 + }, + "T0 multispin 64-lane, splitmix": { + "portable": 9008000000.0, + "portable_iqr": [ + 8432000000.0, + 9671000000.0 + ], + "native": 9223000000.0, + "native_iqr": [ + 8564000000.0, + 9758000000.0 + ], + "ratio": 1.024 + }, + "T0 multispin 64-lane, Philox": { + "portable": 6693000000.0, + "portable_iqr": [ + 6255000000.0, + 6862000000.0 + ], + "native": 7071000000.0, + "native_iqr": [ + 6564000000.0, + 7198000000.0 + ], + "ratio": 1.056 + }, + "T0 PT-in-a-word prefix-mask, splitm": { + "portable": 1840000000.0, + "portable_iqr": [ + 1816000000.0, + 1850000000.0 + ], + "native": 2487000000.0, + "native_iqr": [ + 2466000000.0, + 2516000000.0 + ], + "ratio": 1.352 + }, + "T0 multispin splitmix x 4 threads": { + "portable": 9437000000.0, + "portable_iqr": [ + 9010000000.0, + 9462000000.0 + ], + "native": 9430000000.0, + "native_iqr": [ + 9352000000.0, + 9484000000.0 + ], + "ratio": 0.999 + }, + "T0 multispin splitmix x10 threads": { + "portable": 9263000000.0, + "portable_iqr": [ + 9233000000.0, + 9326000000.0 + ], + "native": 9512000000.0, + "native_iqr": [ + 9482000000.0, + 9542000000.0 + ], + "ratio": 1.027 + }, + "fast multispin splitmix x 4 thr": { + "portable": 12670000000.0, + "portable_iqr": [ + 12300000000.0, + 12920000000.0 + ], + "native": 13170000000.0, + "native_iqr": [ + 13100000000.0, + 13540000000.0 + ], + "ratio": 1.039 + }, + "fast multispin splitmix x10 thr": { + "portable": 12510000000.0, + "portable_iqr": [ + 12420000000.0, + 12950000000.0 + ], + "native": 13410000000.0, + "native_iqr": [ + 12980000000.0, + 13450000000.0 + ], + "ratio": 1.072 + }, + "fast T1 f32 LUT, splitmix": { + "portable": 255700000.0, + "portable_iqr": [ + 224200000.0, + 256600000.0 + ], + "native": 256700000.0, + "native_iqr": [ + 253800000.0, + 303800000.0 + ], + "ratio": 1.004 + }, + "fast T1 f32 LUT, Philox": { + "portable": 151500000.0, + "portable_iqr": [ + 150500000.0, + 158900000.0 + ], + "native": 184400000.0, + "native_iqr": [ + 175400000.0, + 187100000.0 + ], + "ratio": 1.217 + }, + "fast T0 multispin, splitmix": { + "portable": 13260000000.0, + "portable_iqr": [ + 11670000000.0, + 13580000000.0 + ], + "native": 12990000000.0, + "native_iqr": [ + 11640000000.0, + 14160000000.0 + ], + "ratio": 0.98 + }, + "fast T0 multispin, Philox": { + "portable": 8490000000.0, + "portable_iqr": [ + 8089000000.0, + 8980000000.0 + ], + "native": 9273000000.0, + "native_iqr": [ + 8404000000.0, + 9527000000.0 + ], + "ratio": 1.092 + }, + "pbit stats IR ring, 1 thread": { + "portable": 10090000.0, + "portable_iqr": [ + 9885000.0, + 10820000.0 + ], + "native": 10610000.0, + "native_iqr": [ + 10550000.0, + 11210000.0 + ], + "ratio": 1.052 + }, + "pbit stats IR ring, 4 threads": { + "portable": 14550000.0, + "portable_iqr": [ + 13020000.0, + 14610000.0 + ], + "native": 14430000.0, + "native_iqr": [ + 12350000.0, + 14480000.0 + ], + "ratio": 0.992 + }, + "pbit decide 300 tasks, 4 thr (sampler)": { + "portable": 8708000.0, + "portable_iqr": [ + 8372000.0, + 9288000.0 + ], + "native": 9626000.0, + "native_iqr": [ + 9282000.0, + 9655000.0 + ], + "ratio": 1.105 + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB, rustc 1.98.1; 5 alternating runs; medians [IQR]; ratio = native / portable", + "T1 SoA f32 heat-bath LUT, splitmix: portable 1.502e+08 [1.448e+08-1.507e+08] | native 1.619e+08 [1.531e+08-1.74e+08] | ratio 1.078", + "T1 SoA f32 heat-bath LUT, Philox: portable 1.085e+08 [1.058e+08-1.09e+08] | native 1.242e+08 [1.222e+08-1.302e+08] | ratio 1.145", + "T0 multispin 64-lane, splitmix: portable 9.008e+09 [8.432e+09-9.671e+09] | native 9.223e+09 [8.564e+09-9.758e+09] | ratio 1.024", + "T0 multispin 64-lane, Philox: portable 6.693e+09 [6.255e+09-6.862e+09] | native 7.071e+09 [6.564e+09-7.198e+09] | ratio 1.056", + "T0 PT-in-a-word prefix-mask, splitm: portable 1.84e+09 [1.816e+09-1.85e+09] | native 2.487e+09 [2.466e+09-2.516e+09] | ratio 1.352", + "T0 multispin splitmix x 4 threads: portable 9.437e+09 [9.01e+09-9.462e+09] | native 9.43e+09 [9.352e+09-9.484e+09] | ratio 0.999", + "T0 multispin splitmix x10 threads: portable 9.263e+09 [9.233e+09-9.326e+09] | native 9.512e+09 [9.482e+09-9.542e+09] | ratio 1.027", + "fast multispin splitmix x 4 thr: portable 1.267e+10 [1.23e+10-1.292e+10] | native 1.317e+10 [1.31e+10-1.354e+10] | ratio 1.039", + "fast multispin splitmix x10 thr: portable 1.251e+10 [1.242e+10-1.295e+10] | native 1.341e+10 [1.298e+10-1.345e+10] | ratio 1.072", + "fast T1 f32 LUT, splitmix: portable 2.557e+08 [2.242e+08-2.566e+08] | native 2.567e+08 [2.538e+08-3.038e+08] | ratio 1.004", + "fast T1 f32 LUT, Philox: portable 1.515e+08 [1.505e+08-1.589e+08] | native 1.844e+08 [1.754e+08-1.871e+08] | ratio 1.217", + "fast T0 multispin, splitmix: portable 1.326e+10 [1.167e+10-1.358e+10] | native 1.299e+10 [1.164e+10-1.416e+10] | ratio 0.980", + "fast T0 multispin, Philox: portable 8.49e+09 [8.089e+09-8.98e+09] | native 9.273e+09 [8.404e+09-9.527e+09] | ratio 1.092", + "pbit stats IR ring, 1 thread: portable 1.009e+07 [9.885e+06-1.082e+07] | native 1.061e+07 [1.055e+07-1.121e+07] | ratio 1.052", + "pbit stats IR ring, 4 threads: portable 1.455e+07 [1.302e+07-1.461e+07] | native 1.443e+07 [1.235e+07-1.448e+07] | ratio 0.992", + "pbit decide 300 tasks, 4 thr (sampler): portable 8.708e+06 [8.372e+06-9.288e+06] | native 9.626e+06 [9.282e+06-9.655e+06] | ratio 1.105", + "kernels checksum identical across builds: True | decide answer (odds+plan+gate) identical across builds: True" + ], + "stderr_tail": null, + "note": "the script's first line is hard-coded to the M4; the machine is the one in `machine`" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 1, + "rows": {}, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]" + ], + "stderr_tail": "Traceback (most recent call last):\n File \".\\bench\\decide_threads.py\", line 15, in \n print(f\"--threads {t}: wall ms {statistics.median(ms):.1f} [{q[0]:.1f}-{q[2]:.1f}], CPU ms {statistics.median(cpu):.1f}, site updates/s {statistics.median(ups)/1e6:.2f} M; answer identical: yes; verdict {d['verdict']}\")\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\hostedtoolcache\\windows\\Python\\3.12.10\\x64\\Lib\\statistics.py\", line 614, in median\n data = sorted(data)\n ^^^^^^^^^^^^\nTypeError: '<' not supported between instances of 'NoneType' and 'NoneType'\n" + }, + "attached": { + "checks": { + "build": "success", + "native_build": "success", + "pack_zip": "success", + "callers": "success", + "install_sh": "skipped", + "npm": "skipped", + "npm_windows": "success", + "install_ps1_pwsh": "success", + "install_ps1_ps51": "failure", + "ps_callers": "failure", + "rss_powershell": "success", + "cross_darwin": "skipped" + }, + "peak_rss_powershell": { + "method": "Get-Process PeakWorkingSet64, polled every ~5 ms until exit (lower bound)", + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "powershell": "7.6.6", + "runs": [ + { + "peak_working_set_bytes": 20234240, + "samples": 27, + "verdict": "partial" + }, + { + "peak_working_set_bytes": 20877312, + "samples": 29, + "verdict": "partial" + }, + { + "peak_working_set_bytes": 21143552, + "samples": 30, + "verdict": "partial" + } + ] + } + }, + "cells": { + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "logical_cores": 2, + "binary_size_bytes": 966656, + "stats_1t": { + "median": 10278961, + "q1": 9970100.5, + "q3": 10966905.5, + "min": 9954876, + "max": 11246250, + "n": 5, + "values": [ + 9954876, + 10278961, + 11246250, + 9985325, + 10687561 + ] + }, + "stats_default": { + "median": 14610444, + "q1": 12761670.0, + "q3": 14644030.5, + "min": 12614362, + "max": 14655215, + "n": 5, + "values": [ + 14655215, + 14610444, + 12614362, + 14632846, + 12908978 + ] + }, + "stats_default_threads": [ + 2 + ], + "kernel_multispin_fast_4t": { + "median": 12030000000.0, + "q1": 10831500000.0, + "q3": 12630000000.0, + "min": 9683000000.0, + "max": 13130000000.0, + "n": 5, + "values": [ + 9683000000.0, + 11980000000.0, + 12130000000.0, + 12030000000.0, + 13130000000.0 + ] + }, + "kernel_multispin_fast_10t": { + "median": 12110000000.0, + "q1": 11675000000.0, + "q3": 12515000000.0, + "min": 11600000000.0, + "max": 12580000000.0, + "n": 5, + "values": [ + 12580000000.0, + 11600000000.0, + 11750000000.0, + 12110000000.0, + 12450000000.0 + ] + }, + "kernel_multispin_fast_1t": { + "median": 13160000000.0, + "q1": 11725000000.0, + "q3": 13300000000.0, + "min": 11560000000.0, + "max": 13420000000.0, + "n": 5, + "values": [ + 13420000000.0, + 13160000000.0, + 11560000000.0, + 11890000000.0, + 13180000000.0 + ] + }, + "kernel_heatbath_fast_1t": { + "median": 236600000.0, + "q1": 143875000.0, + "q3": 242650000.0, + "min": 68050000.0, + "max": 243500000.0, + "n": 5, + "values": [ + 68050000.0, + 243500000.0, + 241800000.0, + 236600000.0, + 219700000.0 + ] + }, + "decide_12_wall_ms": { + "median": 30.80610000000661, + "q1": 30.635300000000143, + "q3": 41.849249999984295, + "min": 30.53740000001426, + "max": 52.840799999955834, + "n": 5, + "values": [ + 30.53740000001426, + 30.857700000012755, + 52.840799999955834, + 30.733199999986027, + 30.80610000000661 + ] + }, + "decide_12_pbit_ms": { + "median": 17.9625, + "q1": 17.9147, + "q3": 21.5256, + "min": 17.9074, + "max": 24.7737, + "n": 5, + "values": [ + 17.9074, + 18.2775, + 24.7737, + 17.9625, + 17.922 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 493.0233000000044, + "q1": 490.7688000000121, + "q3": 507.82914999999207, + "min": 490.4213000000368, + "max": 518.2927999999833, + "n": 5, + "values": [ + 491.1162999999874, + 490.4213000000368, + 497.3655000000008, + 518.2927999999833, + 493.0233000000044 + ] + }, + "decide_300_pbit_ms": { + "median": 470.1004, + "q1": 467.6626, + "q3": 490.36524999999995, + "min": 467.5572, + "max": 499.7997, + "n": 5, + "values": [ + 467.768, + 467.5572, + 480.9308, + 499.7997, + 470.1004 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed", + "partial" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 297.5, + "q3": 300.0, + "min": 296, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 299, + 296, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 8687269, + "q1": 8439963.0, + "q3": 8840444.5, + "min": 8254094, + "max": 8850496, + "n": 5, + "values": [ + 8850496, + 8830393, + 8687269, + 8254094, + 8625832 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 20135936, + "q1": 20119552.0, + "q3": 21323776.0, + "min": 20119552, + "max": 21323776, + "n": 3, + "values": [ + 20135936, + 21323776, + 20119552 + ] + }, + "decide_300_peak_rss_method": "K32GetProcessMemoryInfo PeakWorkingSetSize (process handle, after exit)", + "native_over_portable": { + "T1 SoA f32 heat-bath LUT, splitmix": 1.078, + "T1 SoA f32 heat-bath LUT, Philox": 1.145, + "T0 multispin 64-lane, splitmix": 1.024, + "T0 multispin 64-lane, Philox": 1.056, + "T0 PT-in-a-word prefix-mask, splitm": 1.352, + "T0 multispin splitmix x 4 threads": 0.999, + "T0 multispin splitmix x10 threads": 1.027, + "fast multispin splitmix x 4 thr": 1.039, + "fast multispin splitmix x10 thr": 1.072, + "fast T1 f32 LUT, splitmix": 1.004, + "fast T1 f32 LUT, Philox": 1.217, + "fast T0 multispin, splitmix": 0.98, + "fast T0 multispin, Philox": 1.092, + "pbit stats IR ring, 1 thread": 1.052, + "pbit stats IR ring, 4 threads": 0.992, + "pbit decide 300 tasks, 4 thr (sampler)": 1.105 + }, + "threads_rows": "n/a: exit 1, TypeError: '<' not supported between instances of 'NoneType' and 'NoneType'" + }, + "elapsed_s": 230.3 +} diff --git a/scripts/bench/results/2026-10-01/run-36970631022/cross-linux-x86_64.json b/scripts/bench/results/2026-10-01/run-36970631022/cross-linux-x86_64.json new file mode 100644 index 0000000..b83cee0 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/cross-linux-x86_64.json @@ -0,0 +1,1751 @@ +{ + "schema": "pbit-cross/1", + "label": "cross-linux-x86_64", + "date_utc": "2026-10-02T05:52:19Z", + "outcomes": { + "musl_x86_64": "success", + "musl_aarch64": "success", + "wasm32_wasip1": "success", + "msrv": "success", + "measure": "success" + }, + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "cross", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "AuthenticAMD", + "Model name": "AMD EPYC 7763 64-Core Processor", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": false, + "avx512bw": false, + "avx512vl": false + }, + "memory_bytes": 8323964928, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "transcripts": { + "msrv.txt": [ + "rust 1.72: build FAILED, Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4): error[E0658]: use of unstable library feature 'int_roundings'", + "rust 1.73: build ok, Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4)", + "rust 1.77: build FAILED, --locked (Cargo.lock as committed, lockfile v4): error: failed to parse lock file at: /home/runner/work/pbit/pbit/Cargo.lock", + "rust 1.78: build ok, --locked (Cargo.lock as committed, lockfile v4)", + "Cargo.lock restored: 0 changes" + ], + "musl-aarch64.txt": [ + "$ sudo apt-get install -y gcc-aarch64-linux-gnu", + "", + "Running kernel seems to be up-to-date.", + "", + "No services need to be restarted.", + "", + "No containers need to be restarted.", + "", + "No user sessions are running outdated binaries.", + "", + "No VM guests are running outdated hypervisor (qemu) binaries on this host.", + "linker installed in 14 s", + "$ rustup target add aarch64-unknown-linux-musl", + "info: downloading component rust-std", + "$ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl", + " Finished `release` profile [optimized] target(s) in 18.43s", + "$ file target/aarch64-unknown-linux-musl/release/pbit", + "target/aarch64-unknown-linux-musl/release/pbit: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), statically linked, BuildID[sha1]=6b9608f1981e300fe1d149bd279ce6fa0ba3e383, not stripped", + "size 1477600 bytes", + "qemu-user-static installed in 3 s", + "$ qemu-aarch64-static target/aarch64-unknown-linux-musl/release/pbit version", + "pbit 0.2.0", + "$ qemu pbit demo --tasks 12 | qemu pbit decide: exit 0, verdict exact" + ], + "musl-x86_64.txt": [ + "$ rustup target add x86_64-unknown-linux-musl", + "info: downloading component rust-std", + "$ cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl", + " Finished `release` profile [optimized] target(s) in 18.18s", + "$ file target/x86_64-unknown-linux-musl/release/pbit", + "target/x86_64-unknown-linux-musl/release/pbit: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), static-pie linked, BuildID[sha1]=85ea1386df8c2738b288e74a66fd13f6a9490feb, not stripped", + "$ ldd target/x86_64-unknown-linux-musl/release/pbit", + "\tstatically linked", + "size 1545968 bytes", + "$ target/x86_64-unknown-linux-musl/release/pbit version", + "pbit 0.2.0", + "$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact" + ], + "wasm32-wasip1.txt": [ + "$ rustup target add wasm32-wasip1", + "info: downloading component rust-std", + "$ cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide", + "exit 0", + "first error: none", + " Checking pbit-ir v0.2.0 (/home/runner/work/pbit/pbit/pbit-ir)", + " Checking pbit-decide v0.2.0 (/home/runner/work/pbit/pbit/pbit-decide)", + " Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.68s" + ] + }, + "bench": { + "linux-x86_64-gnu-samevm": { + "schema": "pbit-bench-matrix/1", + "label": "linux-x86_64-gnu-samevm", + "date_utc": "2026-10-02T05:52:06Z", + "git_sha": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "cross", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "glibc build, same VM as the musl row" + ], + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "AuthenticAMD", + "Model name": "AMD EPYC 7763 64-Core Processor", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": false, + "avx512bw": false, + "avx512vl": false + }, + "memory_bytes": 8323964928, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 1.404296875, + 0.69970703125, + 0.2978515625 + ], + "uptime": "05:52:06 up 4 min, 0 user, load average: 1.40, 0.70, 0.30", + "t": "05:52:06" + }, + "after_stats": { + "loadavg": [ + 1.404296875, + 0.69970703125, + 0.2978515625 + ], + "uptime": "05:52:10 up 4 min, 0 user, load average: 1.40, 0.70, 0.30", + "t": "05:52:10" + }, + "after_decide": { + "loadavg": [ + 1.4521484375, + 0.7216796875, + 0.30712890625 + ], + "uptime": "05:52:12 up 4 min, 0 user, load average: 1.45, 0.72, 0.31", + "t": "05:52:12" + }, + "after_kernels": { + "loadavg": [ + 1.4521484375, + 0.7216796875, + 0.30712890625 + ], + "uptime": "05:52:14 up 4 min, 0 user, load average: 1.45, 0.72, 0.31", + "t": "05:52:14" + }, + "after_pvn": { + "loadavg": [ + 1.4521484375, + 0.7216796875, + 0.30712890625 + ], + "uptime": "05:52:14 up 4 min, 0 user, load average: 1.45, 0.72, 0.31", + "t": "05:52:14" + }, + "end": { + "loadavg": [ + 1.576171875, + 0.759765625, + 0.32177734375 + ], + "uptime": "05:52:19 up 4 min, 0 user, load average: 1.58, 0.76, 0.32", + "t": "05:52:19" + } + }, + "errors": [], + "binary": { + "path": "target/release/pbit", + "size_bytes": 1401248, + "version": "pbit 0.2.0", + "linkage": { + "file": "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 3.2.0, BuildID[sha1]=3a4791ff347b29ab20fc5b1ff998dd05660b1d15, not stripped", + "ldd": [ + "linux-vdso.so.1 (0x00007f3abc9b8000)", + "\tlibgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f3abc84a000)", + "\tlibm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f3abc761000)", + "\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f3abc400000)", + "\t/lib64/ld-linux-x86-64.so.2 (0x00007f3abc9ba000)" + ], + "glibc_symbol_versions": [ + "2.2.5", + "2.3", + "2.3.4", + "2.14", + "2.16", + "2.17", + "2.18", + "2.25", + "2.28", + "2.29", + "2.30", + "2.32", + "2.33", + "2.34" + ], + "min_glibc": "2.34" + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "linux", + "arch": "x86_64", + "logical_cpus": 2 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 2, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 11007845, + "q1": 10861048.0, + "q3": 11025536.5, + "min": 10826321, + "max": 11027321, + "n": 5, + "values": [ + 10826321, + 11007845, + 11027321, + 10895775, + 11023752 + ] + }, + "site_updates_per_s": { + "median": 13977567, + "q1": 13928104.0, + "q3": 14030362.5, + "min": 13886695, + "max": 14046167, + "n": 5, + "values": [ + 13969513, + 14014558, + 13977567, + 13886695, + 14046167 + ] + }, + "threads": [ + 2 + ], + "speedup": { + "median": 1.27, + "q1": 1.27, + "q3": 1.28, + "min": 1.27, + "max": 1.29, + "n": 5, + "values": [ + 1.29, + 1.27, + 1.27, + 1.27, + 1.27 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 19.573209000043335, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.813563, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 19.660393999970438, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.915416, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 19.761556999981167, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 18.086557, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 19.372113000031277, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.707783, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 19.614365999984784, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.905076, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 19.614365999984784, + "q1": 19.472661000037306, + "q3": 19.710975499975802, + "min": 19.372113000031277, + "max": 19.761556999981167, + "n": 5, + "values": [ + 19.573209000043335, + 19.660393999970438, + 19.761556999981167, + 19.372113000031277, + 19.614365999984784 + ] + }, + "pbit_ms": { + "median": 17.905076, + "q1": 17.760672999999997, + "q3": 18.0009865, + "min": 17.707783, + "max": 18.086557, + "n": 5, + "values": [ + 17.813563, + 17.915416, + 18.086557, + 17.707783, + 17.905076 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 468.7279220000278, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 462.818239, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9376, + "site_updates_per_s": 9364888, + "threads": 2, + "peak_rss_mb_self": 20.003906, + "stderr": null + }, + { + "wall_ms": 476.43883099999584, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 470.172218, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9360, + "site_updates_per_s": 9355008, + "threads": 2, + "peak_rss_mb_self": 19.605469, + "stderr": null + }, + { + "wall_ms": 475.620326000012, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 469.52516, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9384, + "site_updates_per_s": 9366740, + "threads": 2, + "peak_rss_mb_self": 21.164063, + "stderr": null + }, + { + "wall_ms": 471.8144789999883, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 465.503453, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9392, + "site_updates_per_s": 9384858, + "threads": 2, + "peak_rss_mb_self": 19.558594, + "stderr": null + }, + { + "wall_ms": 470.5941310000412, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 464.308337, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9392, + "site_updates_per_s": 9383823, + "threads": 2, + "peak_rss_mb_self": 21.105469, + "stderr": null + } + ], + "wall_ms": { + "median": 471.8144789999883, + "q1": 469.6610265000345, + "q3": 476.02957850000394, + "min": 468.7279220000278, + "max": 476.43883099999584, + "n": 5, + "values": [ + 468.7279220000278, + 476.43883099999584, + 475.620326000012, + 471.8144789999883, + 470.5941310000412 + ] + }, + "pbit_ms": { + "median": 465.503453, + "q1": 463.563288, + "q3": 469.84868900000004, + "min": 462.818239, + "max": 470.172218, + "n": 5, + "values": [ + 462.818239, + 470.172218, + 469.52516, + 465.503453, + 464.308337 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 9366740, + "q1": 9359948.0, + "q3": 9384340.5, + "min": 9355008, + "max": 9384858, + "n": 5, + "values": [ + 9364888, + 9355008, + 9366740, + 9384858, + 9383823 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "bytes": { + "median": 21045248, + "q1": 20615168.0, + "q3": 22089728.0, + "min": 20615168, + "max": 22089728, + "n": 3, + "values": [ + 21045248, + 20615168, + 22089728 + ] + }, + "self_reported_peak_rss_mb": [ + 20.070313, + 19.660156, + 21.066406 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + {}, + {}, + {} + ] + }, + "kernels": { + "skipped": "no kernels example at target/release/examples/kernels (build it with -p pbit-core --example kernels)" + }, + "portable_vs_native": { + "skipped": "no --native-dir" + }, + "decide_threads": { + "cmd": "PBIT=target/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 172.9, + "wall_iqr": [ + 172.3, + 173.7 + ], + "cpu_ms": 176.7, + "site_updates_per_s_M": 6.46, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 153.6, + "wall_iqr": [ + 150.3, + 155.3 + ], + "cpu_ms": 205.9, + "site_updates_per_s_M": 9.16, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 152.0, + "wall_iqr": [ + 150.6, + 152.6 + ], + "cpu_ms": 206.1, + "site_updates_per_s_M": 9.09, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 464.2, + "wall_iqr": [ + 462.5, + 469.9 + ], + "sweeps": 9328, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 172.9 [172.3-173.7], CPU ms 176.7, site updates/s 6.46 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 153.6 [150.3-155.3], CPU ms 205.9, site updates/s 9.16 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 152.0 [150.6-152.6], CPU ms 206.1, site updates/s 9.09 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 2): wall ms 464.2 [462.5-469.9], sweeps 9328, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "cells": { + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "logical_cores": 2, + "binary_size_bytes": 1401248, + "stats_1t": { + "median": 11007845, + "q1": 10861048.0, + "q3": 11025536.5, + "min": 10826321, + "max": 11027321, + "n": 5, + "values": [ + 10826321, + 11007845, + 11027321, + 10895775, + 11023752 + ] + }, + "stats_default": { + "median": 13977567, + "q1": 13928104.0, + "q3": 14030362.5, + "min": 13886695, + "max": 14046167, + "n": 5, + "values": [ + 13969513, + 14014558, + 13977567, + 13886695, + 14046167 + ] + }, + "stats_default_threads": [ + 2 + ], + "kernel_multispin_fast_4t": null, + "kernel_multispin_fast_10t": null, + "kernel_multispin_fast_1t": null, + "kernel_heatbath_fast_1t": null, + "decide_12_wall_ms": { + "median": 19.614365999984784, + "q1": 19.472661000037306, + "q3": 19.710975499975802, + "min": 19.372113000031277, + "max": 19.761556999981167, + "n": 5, + "values": [ + 19.573209000043335, + 19.660393999970438, + 19.761556999981167, + 19.372113000031277, + 19.614365999984784 + ] + }, + "decide_12_pbit_ms": { + "median": 17.905076, + "q1": 17.760672999999997, + "q3": 18.0009865, + "min": 17.707783, + "max": 18.086557, + "n": 5, + "values": [ + 17.813563, + 17.915416, + 18.086557, + 17.707783, + 17.905076 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 471.8144789999883, + "q1": 469.6610265000345, + "q3": 476.02957850000394, + "min": 468.7279220000278, + "max": 476.43883099999584, + "n": 5, + "values": [ + 468.7279220000278, + 476.43883099999584, + 475.620326000012, + 471.8144789999883, + 470.5941310000412 + ] + }, + "decide_300_pbit_ms": { + "median": 465.503453, + "q1": 463.563288, + "q3": 469.84868900000004, + "min": 462.818239, + "max": 470.172218, + "n": 5, + "values": [ + 462.818239, + 470.172218, + 469.52516, + 465.503453, + 464.308337 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 9366740, + "q1": 9359948.0, + "q3": 9384340.5, + "min": 9355008, + "max": 9384858, + "n": 5, + "values": [ + 9364888, + 9355008, + 9366740, + 9384858, + 9383823 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 21045248, + "q1": 20615168.0, + "q3": 22089728.0, + "min": 20615168, + "max": 22089728, + "n": 3, + "values": [ + 21045248, + 20615168, + 22089728 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "native_over_portable": "no --native-dir", + "threads_rows": { + "threads_1": { + "wall_ms": 172.9, + "wall_iqr": [ + 172.3, + 173.7 + ], + "cpu_ms": 176.7, + "site_updates_per_s_M": 6.46, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 153.6, + "wall_iqr": [ + 150.3, + 155.3 + ], + "cpu_ms": 205.9, + "site_updates_per_s_M": 9.16, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 152.0, + "wall_iqr": [ + 150.6, + 152.6 + ], + "cpu_ms": 206.1, + "site_updates_per_s_M": 9.09, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 464.2, + "wall_iqr": [ + 462.5, + 469.9 + ], + "sweeps": 9328, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 12.6 + }, + "linux-x86_64-musl": { + "schema": "pbit-bench-matrix/1", + "label": "linux-x86_64-musl", + "date_utc": "2026-10-02T05:51:53Z", + "git_sha": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "cross", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "notes": [ + "static musl build, same VM as bench-linux-x86_64-gnu-samevm" + ], + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "AuthenticAMD", + "Model name": "AMD EPYC 7763 64-Core Processor", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": false, + "avx512bw": false, + "avx512vl": false + }, + "memory_bytes": 8323964928, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + }, + "load": { + "start": { + "loadavg": [ + 1.23388671875, + 0.63232421875, + 0.26953125 + ], + "uptime": "05:51:53 up 4 min, 0 user, load average: 1.23, 0.63, 0.27", + "t": "05:51:53" + }, + "after_stats": { + "loadavg": [ + 1.29541015625, + 0.6552734375, + 0.279296875 + ], + "uptime": "05:51:57 up 4 min, 0 user, load average: 1.30, 0.66, 0.28", + "t": "05:51:57" + }, + "after_decide": { + "loadavg": [ + 1.29541015625, + 0.6552734375, + 0.279296875 + ], + "uptime": "05:51:59 up 4 min, 0 user, load average: 1.30, 0.66, 0.28", + "t": "05:51:59" + }, + "after_kernels": { + "loadavg": [ + 1.35205078125, + 0.677734375, + 0.28857421875 + ], + "uptime": "05:52:01 up 4 min, 0 user, load average: 1.35, 0.68, 0.29", + "t": "05:52:01" + }, + "after_pvn": { + "loadavg": [ + 1.35205078125, + 0.677734375, + 0.28857421875 + ], + "uptime": "05:52:01 up 4 min, 0 user, load average: 1.35, 0.68, 0.29", + "t": "05:52:01" + }, + "end": { + "loadavg": [ + 1.404296875, + 0.69970703125, + 0.2978515625 + ], + "uptime": "05:52:06 up 4 min, 0 user, load average: 1.40, 0.70, 0.30", + "t": "05:52:06" + } + }, + "errors": [], + "binary": { + "path": "target/x86_64-unknown-linux-musl/release/pbit", + "size_bytes": 1545968, + "version": "pbit 0.2.0", + "linkage": { + "file": "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), static-pie linked, BuildID[sha1]=85ea1386df8c2738b288e74a66fd13f6a9490feb, not stripped", + "ldd": [ + "statically linked" + ], + "glibc_symbol_versions": [], + "min_glibc": null + } + }, + "stats_first": { + "engine": "pbit 0.2.0", + "machine": { + "os": "linux", + "arch": "x86_64", + "logical_cpus": 2 + }, + "build": { + "target_features": [], + "debug_assertions": false + }, + "controls": { + "chains": { + "value": 4, + "source": "default" + }, + "threads": { + "value": 2, + "source": "default" + }, + "cpu_limit_pct": { + "value": 100, + "source": "default" + }, + "priority": { + "value": "normal", + "source": "default" + }, + "mem_limit_mb": { + "value": 1024, + "source": "default" + }, + "config_file": null + } + }, + "stats": { + "site_updates_per_s_1_thread": { + "median": 11213958, + "q1": 10861141.5, + "q3": 11292951.5, + "min": 10557652, + "max": 11305367, + "n": 5, + "values": [ + 10557652, + 11305367, + 11213958, + 11280536, + 11164631 + ] + }, + "site_updates_per_s": { + "median": 14192316, + "q1": 13969448.0, + "q3": 14219086.5, + "min": 13947304, + "max": 14221332, + "n": 5, + "values": [ + 14192316, + 14221332, + 13991592, + 14216841, + 13947304 + ] + }, + "threads": [ + 2 + ], + "speedup": { + "median": 1.26, + "q1": 1.25, + "q3": 1.3, + "min": 1.25, + "max": 1.34, + "n": 5, + "values": [ + 1.34, + 1.26, + 1.25, + 1.26, + 1.25 + ] + } + }, + "decide_12": { + "cmd": "pbit demo --tasks 12 | pbit decide ", + "runs": [ + { + "wall_ms": 18.961402999991606, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.537871, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 19.188352000014675, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.476225, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 18.98067400003356, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.562758, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 19.080287999997836, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.712016, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + }, + { + "wall_ms": 18.986130000030244, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 17.631546, + "verdict": "exact", + "tier": "enumerate", + "released": 12, + "escalated": 0, + "violations": 0, + "sweeps": null, + "site_updates_per_s": null, + "threads": null, + "peak_rss_mb_self": null, + "stderr": null + } + ], + "wall_ms": { + "median": 18.986130000030244, + "q1": 18.971038500012583, + "q3": 19.134320000006255, + "min": 18.961402999991606, + "max": 19.188352000014675, + "n": 5, + "values": [ + 18.961402999991606, + 19.188352000014675, + 18.98067400003356, + 19.080287999997836, + 18.986130000030244 + ] + }, + "pbit_ms": { + "median": 17.562758, + "q1": 17.507047999999998, + "q3": 17.671781, + "min": 17.476225, + "max": 17.712016, + "n": 5, + "values": [ + 17.537871, + 17.476225, + 17.562758, + 17.712016, + 17.631546 + ] + }, + "released": { + "median": 12, + "q1": 12.0, + "q3": 12.0, + "min": 12, + "max": 12, + "n": 5, + "values": [ + 12, + 12, + 12, + 12, + 12 + ] + }, + "violations_max": 0, + "verdicts": [ + "exact" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": null + }, + "decide_300": { + "cmd": "pbit demo --tasks 300 | pbit decide --budget-ms 300", + "runs": [ + { + "wall_ms": 478.4774459999994, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 470.644527, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9688, + "site_updates_per_s": 9669312, + "threads": 2, + "peak_rss_mb_self": 16.328125, + "stderr": null + }, + { + "wall_ms": 477.71077400000195, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 469.844102, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9720, + "site_updates_per_s": 9702899, + "threads": 2, + "peak_rss_mb_self": 16.25, + "stderr": null + }, + { + "wall_ms": 478.6707449999881, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 470.315261, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9912, + "site_updates_per_s": 9894959, + "threads": 2, + "peak_rss_mb_self": 17.113281, + "stderr": null + }, + { + "wall_ms": 487.9514849999964, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 479.748635, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9720, + "site_updates_per_s": 9696817, + "threads": 2, + "peak_rss_mb_self": 17.847656, + "stderr": null + }, + { + "wall_ms": 476.8139489999612, + "demo_exit": 0, + "decide_exit": 0, + "pbit_ms": 468.59489, + "verdict": "diagnostics_passed", + "tier": null, + "released": 300, + "escalated": 0, + "violations": 0, + "sweeps": 9568, + "site_updates_per_s": 9547642, + "threads": 2, + "peak_rss_mb_self": 16.285156, + "stderr": null + } + ], + "wall_ms": { + "median": 478.4774459999994, + "q1": 477.2623614999816, + "q3": 483.31111499999224, + "min": 476.8139489999612, + "max": 487.9514849999964, + "n": 5, + "values": [ + 478.4774459999994, + 477.71077400000195, + 478.6707449999881, + 487.9514849999964, + 476.8139489999612 + ] + }, + "pbit_ms": { + "median": 470.315261, + "q1": 469.21949600000005, + "q3": 475.196581, + "min": 468.59489, + "max": 479.748635, + "n": 5, + "values": [ + 470.644527, + 469.844102, + 470.315261, + 479.748635, + 468.59489 + ] + }, + "released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "violations_max": 0, + "verdicts": [ + "diagnostics_passed" + ], + "exit_codes": [ + 0 + ], + "site_updates_per_s": { + "median": 9696817, + "q1": 9608477.0, + "q3": 9798929.0, + "min": 9547642, + "max": 9894959, + "n": 5, + "values": [ + 9669312, + 9702899, + 9894959, + 9696817, + 9547642 + ] + } + }, + "peak_rss_300": { + "cmd": "pbit decide --budget-ms 300 < demo300.json", + "method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "bytes": { + "median": 18399232, + "q1": 17866752.0, + "q3": 18784256.0, + "min": 17866752, + "max": 18784256, + "n": 3, + "values": [ + 18399232, + 18784256, + 17866752 + ] + }, + "self_reported_peak_rss_mb": [ + 17.546875, + 17.914063, + 17.039063 + ], + "verdicts": [ + "diagnostics_passed", + "diagnostics_passed", + "diagnostics_passed" + ], + "extra": [ + {}, + {}, + {} + ] + }, + "kernels": { + "skipped": "no kernels example at target/x86_64-unknown-linux-musl/release/examples/kernels (build it with -p pbit-core --example kernels)" + }, + "portable_vs_native": { + "skipped": "no --native-dir" + }, + "decide_threads": { + "cmd": "PBIT=target/x86_64-unknown-linux-musl/release/pbit python3 bench/decide_threads.py", + "exit": 0, + "rows": { + "threads_1": { + "wall_ms": 176.4, + "wall_iqr": [ + 176.2, + 186.1 + ], + "cpu_ms": 181.1, + "site_updates_per_s_M": 6.65, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 153.5, + "wall_iqr": [ + 152.4, + 154.6 + ], + "cpu_ms": 206.4, + "site_updates_per_s_M": 9.83, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 156.6, + "wall_iqr": [ + 155.2, + 159.9 + ], + "cpu_ms": 211.3, + "site_updates_per_s_M": 9.54, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 471.6, + "wall_iqr": [ + 470.6, + 472.7 + ], + "sweeps": 10184, + "verdict": "diagnostics_passed" + } + }, + "stdout": [ + "machine: Apple M4 (4P+6E), 16 GB; 300-task demo, decide --sweeps 400 --chains 4 --polish-ms 0 (fixed work), 5 runs each, medians [IQR]", + "--threads 1: wall ms 176.4 [176.2-186.1], CPU ms 181.1, site updates/s 6.65 M; answer identical: yes; verdict refused", + "--threads 2: wall ms 153.5 [152.4-154.6], CPU ms 206.4, site updates/s 9.83 M; answer identical: yes; verdict refused", + "--threads 4: wall ms 156.6 [155.2-159.9], CPU ms 211.3, site updates/s 9.54 M; answer identical: yes; verdict refused", + "default path (--budget-ms 300, 4 chains, threads 2): wall ms 471.6 [470.6-472.7], sweeps 10184, verdict diagnostics_passed" + ], + "stderr_tail": null + }, + "cells": { + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "logical_cores": 2, + "binary_size_bytes": 1545968, + "stats_1t": { + "median": 11213958, + "q1": 10861141.5, + "q3": 11292951.5, + "min": 10557652, + "max": 11305367, + "n": 5, + "values": [ + 10557652, + 11305367, + 11213958, + 11280536, + 11164631 + ] + }, + "stats_default": { + "median": 14192316, + "q1": 13969448.0, + "q3": 14219086.5, + "min": 13947304, + "max": 14221332, + "n": 5, + "values": [ + 14192316, + 14221332, + 13991592, + 14216841, + 13947304 + ] + }, + "stats_default_threads": [ + 2 + ], + "kernel_multispin_fast_4t": null, + "kernel_multispin_fast_10t": null, + "kernel_multispin_fast_1t": null, + "kernel_heatbath_fast_1t": null, + "decide_12_wall_ms": { + "median": 18.986130000030244, + "q1": 18.971038500012583, + "q3": 19.134320000006255, + "min": 18.961402999991606, + "max": 19.188352000014675, + "n": 5, + "values": [ + 18.961402999991606, + 19.188352000014675, + 18.98067400003356, + 19.080287999997836, + 18.986130000030244 + ] + }, + "decide_12_pbit_ms": { + "median": 17.562758, + "q1": 17.507047999999998, + "q3": 17.671781, + "min": 17.476225, + "max": 17.712016, + "n": 5, + "values": [ + 17.537871, + 17.476225, + 17.562758, + 17.712016, + 17.631546 + ] + }, + "decide_12_verdicts": [ + "exact" + ], + "decide_12_exit_codes": [ + 0 + ], + "decide_300_wall_ms": { + "median": 478.4774459999994, + "q1": 477.2623614999816, + "q3": 483.31111499999224, + "min": 476.8139489999612, + "max": 487.9514849999964, + "n": 5, + "values": [ + 478.4774459999994, + 477.71077400000195, + 478.6707449999881, + 487.9514849999964, + 476.8139489999612 + ] + }, + "decide_300_pbit_ms": { + "median": 470.315261, + "q1": 469.21949600000005, + "q3": 475.196581, + "min": 468.59489, + "max": 479.748635, + "n": 5, + "values": [ + 470.644527, + 469.844102, + 470.315261, + 479.748635, + 468.59489 + ] + }, + "decide_300_verdicts": [ + "diagnostics_passed" + ], + "decide_300_exit_codes": [ + 0 + ], + "decide_300_released": { + "median": 300, + "q1": 300.0, + "q3": 300.0, + "min": 300, + "max": 300, + "n": 5, + "values": [ + 300, + 300, + 300, + 300, + 300 + ] + }, + "decide_300_violations_max": 0, + "decide_300_site_updates_per_s": { + "median": 9696817, + "q1": 9608477.0, + "q3": 9798929.0, + "min": 9547642, + "max": 9894959, + "n": 5, + "values": [ + 9669312, + 9702899, + 9894959, + 9696817, + 9547642 + ] + }, + "decide_300_peak_rss_bytes": { + "median": 18399232, + "q1": 17866752.0, + "q3": 18784256.0, + "min": 17866752, + "max": 18784256, + "n": 3, + "values": [ + 18399232, + 18784256, + 17866752 + ] + }, + "decide_300_peak_rss_method": "/usr/bin/time -v (Maximum resident set size, KiB x 1024)", + "native_over_portable": "no --native-dir", + "threads_rows": { + "threads_1": { + "wall_ms": 176.4, + "wall_iqr": [ + 176.2, + 186.1 + ], + "cpu_ms": 181.1, + "site_updates_per_s_M": 6.65, + "verdict": "refused" + }, + "threads_2": { + "wall_ms": 153.5, + "wall_iqr": [ + 152.4, + 154.6 + ], + "cpu_ms": 206.4, + "site_updates_per_s_M": 9.83, + "verdict": "refused" + }, + "threads_4": { + "wall_ms": 156.6, + "wall_iqr": [ + 155.2, + 159.9 + ], + "cpu_ms": 211.3, + "site_updates_per_s_M": 9.54, + "verdict": "refused" + }, + "default": { + "threads": 2, + "wall_ms": 471.6, + "wall_iqr": [ + 470.6, + 472.7 + ], + "sweeps": 10184, + "verdict": "diagnostics_passed" + } + } + }, + "elapsed_s": 12.9 + } + } +} \ No newline at end of file diff --git a/scripts/bench/results/2026-10-01/run-36970631022/test-linux-x86_64.json b/scripts/bench/results/2026-10-01/run-36970631022/test-linux-x86_64.json new file mode 100644 index 0000000..a65d166 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/test-linux-x86_64.json @@ -0,0 +1,72 @@ +{ + "schema": "pbit-test-matrix/1", + "label": "linux-x86_64", + "date_utc": "2026-10-02T05:52:37Z", + "cmd": "cargo test --release --locked --workspace --no-fail-fast", + "exit": 0, + "passed": true, + "seconds": 201, + "test_binaries": 10, + "totals": { + "passed": 127, + "failed": 0, + "ignored": 1, + "measured": 0, + "filtered_out": 0 + }, + "failures": [], + "runner": { + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", + "ImageVersion": "20260927.320.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "test", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Linux", + "os_release": "6.17.0-1022-azure", + "arch": "x86_64", + "python": "3.12.3", + "logical_cores": 2, + "cpu_cmd": "lscpu", + "cpu_model": "AMD EPYC 9V45 96-Core Processor", + "lscpu": { + "Architecture": "x86_64", + "CPU(s)": "2", + "Thread(s) per core": "2", + "Core(s) per socket": "1", + "Socket(s)": "1", + "Vendor ID": "AuthenticAMD", + "Model name": "AMD EPYC 9V45 96-Core Processor", + "Hypervisor vendor": "Microsoft" + }, + "cpu_flags": { + "sse4_2": true, + "avx": true, + "avx2": true, + "fma": true, + "bmi2": true, + "avx512f": true, + "avx512bw": true, + "avx512vl": true + }, + "memory_bytes": 8323969024, + "os_version": "Ubuntu 24.04.5 LTS", + "libc": "glibc 2.39" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-unknown-linux-gnu", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + } +} diff --git a/scripts/bench/results/2026-10-01/run-36970631022/test-macos-arm64.json b/scripts/bench/results/2026-10-01/run-36970631022/test-macos-arm64.json new file mode 100644 index 0000000..9547a22 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/test-macos-arm64.json @@ -0,0 +1,55 @@ +{ + "schema": "pbit-test-matrix/1", + "label": "macos-arm64", + "date_utc": "2026-10-02T05:52:40Z", + "cmd": "cargo test --release --locked --workspace --no-fail-fast", + "exit": 0, + "passed": true, + "seconds": 195, + "test_binaries": 10, + "totals": { + "passed": 127, + "failed": 0, + "ignored": 1, + "measured": 0, + "filtered_out": 0 + }, + "failures": [], + "runner": { + "RUNNER_OS": "macOS", + "RUNNER_ARCH": "ARM64", + "ImageOS": "macos26", + "ImageVersion": "20260907.0351.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "test", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Darwin", + "os_release": "25.6.0", + "arch": "arm64", + "python": "3.14.7", + "logical_cores": 3, + "cpu_cmd": "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize", + "cpu_model": "Apple M1 (Virtual)", + "hw_ncpu": 3, + "memory_bytes": 7516192768, + "perf_cores": 3, + "efficiency_cores": 0, + "rosetta_translated": false, + "os_version": "26.6.2" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "aarch64-apple-darwin", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + } +} diff --git a/scripts/bench/results/2026-10-01/run-36970631022/test-windows-x86_64.json b/scripts/bench/results/2026-10-01/run-36970631022/test-windows-x86_64.json new file mode 100644 index 0000000..4318a54 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/test-windows-x86_64.json @@ -0,0 +1,53 @@ +{ + "schema": "pbit-test-matrix/1", + "label": "windows-x86_64", + "date_utc": "2026-10-02T05:59:04Z", + "cmd": "cargo test --release --locked --workspace --no-fail-fast", + "exit": 0, + "passed": true, + "seconds": 572, + "test_binaries": 10, + "totals": { + "passed": 127, + "failed": 0, + "ignored": 1, + "measured": 0, + "filtered_out": 0 + }, + "failures": [], + "runner": { + "RUNNER_OS": "Windows", + "RUNNER_ARCH": "X64", + "ImageOS": "win25-vs2026", + "ImageVersion": "20260925.250.1", + "GITHUB_RUN_ID": "36970631022", + "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_JOB": "test", + "GITHUB_SHA": "bef46ca8ea1a4970e7b95d5dd7bc047c00a30344", + "GITHUB_REF_NAME": "r22-portability-matrix", + "GITHUB_WORKFLOW": "bench" + }, + "machine": { + "system": "Windows", + "os_release": "2025Server", + "arch": "AMD64", + "python": "3.12.10", + "logical_cores": 2, + "cpu_cmd": "Get-CimInstance Win32_Processor", + "cpu_model": "AMD EPYC 7763 64-Core Processor", + "physical_cores": 1, + "cim_logical": 2, + "memory_bytes": 8584425472, + "os_version": "Microsoft Windows Server 2025 Datacenter (build 26100)" + }, + "toolchain": { + "binary": "rustc", + "commit-hash": "b940084d7eb6a299eb4bfeb8e34901bc051e7ac4", + "commit-date": "2026-09-28", + "host": "x86_64-pc-windows-msvc", + "release": "1.99.0", + "LLVM version": "23.1.1", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "cargo": "cargo 1.99.0 (5f94df478 2026-08-27)" + } +} diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-linux-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-linux-x86_64.txt new file mode 100644 index 0000000..0ca6078 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-linux-x86_64.txt @@ -0,0 +1,33 @@ +$ pbit demo --tasks 12 | pbit decide --pretty (/home/runner/work/pbit/pbit/onpath/pbit) +exit 0 +{ + "engine": "pbit 0.2.0", + "tasks": 12, + "workers": 6, + "affinity": 0.8, + "verdict": "exact", + "tier": "enumerate", + "plan": { +$ pbit decide --budget-ms 200 < router.json > decision.json; case $? in ... +answer +exact +$ python3 python/test_pbit.py +......... +---------------------------------------------------------------------- +Ran 9 tests in 0.631s + +OK +$ python3 python/examples/route_agents.py +exact released 24 of 24 tasks + T000 -> human odds 0.895 + T001 -> human odds 0.883 + T002 -> codex odds 0.326 + T003 -> human odds 0.333 + T004 -> luna-pro odds 0.704 +$ node examples/node/decide.mjs (node v22.23.3) +pbit 0.2.0 (/home/runner/work/pbit/pbit/target/release/pbit) +exit 0 answer a 12-task demo document: verdict exact, 12 of 12 released, plan T000->human, T001->human, T002->codex, ... +exit 1 infeasible two tasks, one worker with room for one: verdict infeasible (no plan satisfies every rule (allowed sets + capacities + clamps)) +exit 2 bad_input a malformed document: schema at workers: must be an array +exit 3 refused exact enumeration of a 96-variable program, capped at 50 ms: verdict declined (enumeration stopped at --exact-ms; raise it, or use --op decide or sample) +all four exit codes mapped as expected diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-macos-arm64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-macos-arm64.txt new file mode 100644 index 0000000..0b2c0ae --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-macos-arm64.txt @@ -0,0 +1,33 @@ +$ pbit demo --tasks 12 | pbit decide --pretty (/Users/runner/work/pbit/pbit/onpath/pbit) +exit 0 +{ + "engine": "pbit 0.2.0", + "tasks": 12, + "workers": 6, + "affinity": 0.8, + "verdict": "exact", + "tier": "enumerate", + "plan": { +$ pbit decide --budget-ms 200 < router.json > decision.json; case $? in ... +answer +exact +$ python3 python/test_pbit.py +......... +---------------------------------------------------------------------- +Ran 9 tests in 0.744s + +OK +$ python3 python/examples/route_agents.py +exact released 24 of 24 tasks + T000 -> human odds 0.895 + T001 -> human odds 0.883 + T002 -> codex odds 0.326 + T003 -> human odds 0.333 + T004 -> luna-pro odds 0.704 +$ node examples/node/decide.mjs (node v24.20.0) +pbit 0.2.0 (/Users/runner/work/pbit/pbit/target/release/pbit) +exit 0 answer a 12-task demo document: verdict exact, 12 of 12 released, plan T000->human, T001->human, T002->codex, ... +exit 1 infeasible two tasks, one worker with room for one: verdict infeasible (no plan satisfies every rule (allowed sets + capacities + clamps)) +exit 2 bad_input a malformed document: schema at workers: must be an array +exit 3 refused exact enumeration of a 96-variable program, capped at 50 ms: verdict declined (enumeration stopped at --exact-ms; raise it, or use --op decide or sample) +all four exit codes mapped as expected diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-powershell-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-powershell-windows-x86_64.txt new file mode 100644 index 0000000..268491b --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-powershell-windows-x86_64.txt @@ -0,0 +1,35 @@ +PowerShell 7.6.6 (Core), pbit at D:\a\pbit\pbit\target\release\pbit.exe +$OutputEncoding: utf-8, 0-byte preamble +> pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms + +verdict : exact +violations : 0 +ms : 28.4564 + + +> Get-Content router.json -Raw | pbit decide --budget-ms 200 | ConvertFrom-Json; $LASTEXITCODE +verdict exact, released 24 of 24, violations 0; exit 0 +> one call per exit code (infeasible, bad input, declined) +pbit: bad problem: workers: must be an array +exit codes: infeasible 1 (want 1), bad input 2 (want 2), declined 3 (want 3) +PowerShell recipes: all passed +PowerShell 5.1.26100.33438 (Desktop), pbit at D:\a\pbit\pbit\target\release\pbit.exe +$OutputEncoding: us-ascii, 0-byte preamble +> pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms +pbit: bad JSON: unexpected character 'ï' at byte 0 + + +verdict : +violations : +ms : + + + + +demo pipeline: exit 2 +At D:\a\pbit\pbit\scripts\bench\callers.ps1:18 char:28 ++ ... f ($LASTEXITCODE -ne 0) { throw "demo pipeline: exit $LASTEXITCODE" } ++ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + CategoryInfo : OperationStopped: (demo pipeline: exit 2:String) [], RuntimeException + + FullyQualifiedErrorId : demo pipeline: exit 2 + diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-windows-x86_64.txt new file mode 100644 index 0000000..493a22e --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/callers-windows-x86_64.txt @@ -0,0 +1,33 @@ +$ pbit demo --tasks 12 | pbit decide --pretty (/d/a/pbit/pbit/onpath/pbit) +exit 0 +{ + "engine": "pbit 0.2.0", + "tasks": 12, + "workers": 6, + "affinity": 0.8, + "verdict": "exact", + "tier": "enumerate", + "plan": { +$ pbit decide --budget-ms 200 < router.json > decision.json; case $? in ... +answer +exact +$ python python/test_pbit.py +......... +---------------------------------------------------------------------- +Ran 9 tests in 0.785s + +OK +$ python python/examples/route_agents.py +exact released 24 of 24 tasks + T000 -> human odds 0.895 + T001 -> human odds 0.883 + T002 -> codex odds 0.326 + T003 -> human odds 0.333 + T004 -> luna-pro odds 0.704 +$ node examples/node/decide.mjs (node v22.23.3) +pbit 0.2.0 (D:\a\pbit\pbit\target\release\pbit.exe) +exit 0 answer a 12-task demo document: verdict exact, 12 of 12 released, plan T000->human, T001->human, T002->codex, ... +exit 1 infeasible two tasks, one worker with room for one: verdict infeasible (no plan satisfies every rule (allowed sets + capacities + clamps)) +exit 2 bad_input a malformed document: schema at workers: must be an array +exit 3 refused exact enumeration of a 96-variable program, capped at 50 ms: verdict declined (enumeration stopped at --exact-ms; raise it, or use --op decide or sample) +all four exit codes mapped as expected diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/cross-x86_64-apple-darwin.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/cross-x86_64-apple-darwin.txt new file mode 100644 index 0000000..dea7d2a --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/cross-x86_64-apple-darwin.txt @@ -0,0 +1,18 @@ +$ rustup target add x86_64-apple-darwin +info: downloading component rust-std +$ cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin + Finished `release` profile [optimized] target(s) in 21.67s +$ file target/x86_64-apple-darwin/release/pbit +target/x86_64-apple-darwin/release/pbit: Mach-O 64-bit executable x86_64 +$ lipo -archs target/x86_64-apple-darwin/release/pbit +x86_64 +$ otool -L target/x86_64-apple-darwin/release/pbit +target/x86_64-apple-darwin/release/pbit: + /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0) +size 1270368 bytes +rosetta: present +$ arch -x86_64 target/x86_64-apple-darwin/release/pbit version +pbit 0.2.0 +$ arch -x86_64 pbit demo --tasks 12 | arch -x86_64 pbit decide: exit 0, verdict exact +$ arch -x86_64 pbit stats (self-test under Rosetta) +1 thread 9088089 | threads 3 14179921 diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-ps1-ps51-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-ps1-ps51-windows-x86_64.txt new file mode 100644 index 0000000..a0c84d2 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-ps1-ps51-windows-x86_64.txt @@ -0,0 +1,25 @@ +PowerShell 5.1.26100.33438 (Desktop), $OutputEncoding us-ascii with a 0-byte preamble +== 1. & .\install.ps1 -DownloadBase http://127.0.0.1:63146/good -Version v0.2.0 -InstallDir \bin1 +pbit install: v0.2.0 for x86_64-pc-windows-msvc + fetching http://127.0.0.1:63146/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip + sha256 ok cdc652e2b0bdf47505b8a51928446cac695ff506ad6549b71ddbde29762b315b + installed C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-0ffaa80b1b0f4e8582b8fe24452fe24b\bin1\pbit.exe (pbit 0.2.0) + +C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-0ffaa80b1b0f4e8582b8fe24452fe24b\bin1 is on this session's PATH only. To keep it, run (or re-run this installer with -AddToPath): + [Environment]::SetEnvironmentVariable('Path', 'C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-0ffaa80b1b0f4e8582b8fe24452fe24b\bin1;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User') + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +$ (Get-Command pbit).Source: C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-0ffaa80b1b0f4e8582b8fe24452fe24b\bin1\pbit.exe +$ pbit version: pbit 0.2.0 +pbit: bad JSON: unexpected character 'ï' at byte 0 +$ pbit demo --tasks 12 | pbit decide | ConvertFrom-Json: exit 2, verdict , released 1 of +FAIL: demo pipeline +At D:\a\pbit\pbit\scripts\bench\test_install_ps1.ps1:32 char:59 ++ ... ODE -ne 0 -or $d.verdict -ne 'exact') { throw 'FAIL: demo pipeline' } ++ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + CategoryInfo : OperationStopped: (FAIL: demo pipeline:String) [], RuntimeException + + FullyQualifiedErrorId : FAIL: demo pipeline + diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-ps1-pwsh-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-ps1-pwsh-windows-x86_64.txt new file mode 100644 index 0000000..526d042 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-ps1-pwsh-windows-x86_64.txt @@ -0,0 +1,37 @@ +PowerShell 7.6.6 (Core), $OutputEncoding utf-8 with a 0-byte preamble +== 1. & .\install.ps1 -DownloadBase http://127.0.0.1:63137/good -Version v0.2.0 -InstallDir \bin1 -AddToPath +pbit install: v0.2.0 for x86_64-pc-windows-msvc + fetching http://127.0.0.1:63137/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip + sha256 ok cdc652e2b0bdf47505b8a51928446cac695ff506ad6549b71ddbde29762b315b + installed C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-144c85b5f1b841c5a8a63b297d6b9a25\bin1\pbit.exe (pbit 0.2.0) + added C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-144c85b5f1b841c5a8a63b297d6b9a25\bin1 to your user PATH (new terminals see it) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +$ (Get-Command pbit).Source: C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-144c85b5f1b841c5a8a63b297d6b9a25\bin1\pbit.exe +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide | ConvertFrom-Json: exit 0, verdict exact, released 12 of 12 +ok: user PATH now starts with C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-144c85b5f1b841c5a8a63b297d6b9a25\bin1 +== 2. irm | iex form: PBIT_* environment variables, script text through Invoke-Expression +pbit install: v0.2.0 for x86_64-pc-windows-msvc + fetching http://127.0.0.1:63137/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip + sha256 ok cdc652e2b0bdf47505b8a51928446cac695ff506ad6549b71ddbde29762b315b + installed C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-144c85b5f1b841c5a8a63b297d6b9a25\bin2\pbit.exe (pbit 0.2.0) + +C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-144c85b5f1b841c5a8a63b297d6b9a25\bin2 is on this session's PATH only. To keep it, run (or re-run this installer with -AddToPath): + [Environment]::SetEnvironmentVariable('Path', 'C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-144c85b5f1b841c5a8a63b297d6b9a25\bin2;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User') + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: pbit 0.2.0 in C:\Users\runneradmin\AppData\Local\Temp\pbit-ps-test-144c85b5f1b841c5a8a63b297d6b9a25\bin2 +== 3. a wrong .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-pc-windows-msvc + fetching http://127.0.0.1:63137/bad/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip +rejected: pbit install: checksum mismatch for pbit-v0.2.0-x86_64-pc-windows-msvc.zip: expected 0000000000000000000000000000000000000000000000000000000000000000, got cdc652e2b0bdf47505b8a51928446cac695ff506ad6549b71ddbde29762b315b (nothing was installed) +== 4. -DownloadBase without -Version must fail +rejected: pbit install: set -Version (or PBIT_VERSION), e.g. v0.2.0, together with -DownloadBase +install.ps1: all checks passed (PowerShell 7.6.6) diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-sh-linux-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-sh-linux-x86_64.txt new file mode 100644 index 0000000..b84bf22 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-sh-linux-x86_64.txt @@ -0,0 +1,114 @@ +shell under test: sh (/usr/bin/sh); archive: pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz.sha256 +== 1. piped to sh, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:45897/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok bc90a727b4bab740c4cfd714311d281a469723e0305d3fcd644eb01768dec4ab + installed /tmp/tmp.LFHjgb2ZIM/bin/pbit (pbit 0.2.0) + +/tmp/tmp.LFHjgb2ZIM/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/tmp/tmp.LFHjgb2ZIM/bin:$PATH" + +Next: + /tmp/tmp.LFHjgb2ZIM/bin/pbit version + /tmp/tmp.LFHjgb2ZIM/bin/pbit demo --tasks 12 | /tmp/tmp.LFHjgb2ZIM/bin/pbit decide --pretty + /tmp/tmp.LFHjgb2ZIM/bin/pbit stats --pretty +$ command -v pbit: /tmp/tmp.LFHjgb2ZIM/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:45897/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok bc90a727b4bab740c4cfd714311d281a469723e0305d3fcd644eb01768dec4ab + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:45897/bad/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got bc90a727b4bab740c4cfd714311d281a469723e0305d3fcd644eb01768dec4ab (nothing was installed) +ok: rejected, /tmp/tmp.LFHjgb2ZIM/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (sh, x86_64-unknown-linux-gnu) +shell under test: bash (/usr/bin/bash); archive: pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz.sha256 +== 1. piped to bash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47027/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok 2df70526c40ee68758afba40d616c00f6b96eb096361cc1576fefb08c3c93de8 + installed /tmp/tmp.Pg5lEXpSKe/bin/pbit (pbit 0.2.0) + +/tmp/tmp.Pg5lEXpSKe/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/tmp/tmp.Pg5lEXpSKe/bin:$PATH" + +Next: + /tmp/tmp.Pg5lEXpSKe/bin/pbit version + /tmp/tmp.Pg5lEXpSKe/bin/pbit demo --tasks 12 | /tmp/tmp.Pg5lEXpSKe/bin/pbit decide --pretty + /tmp/tmp.Pg5lEXpSKe/bin/pbit stats --pretty +$ command -v pbit: /tmp/tmp.Pg5lEXpSKe/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47027/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok 2df70526c40ee68758afba40d616c00f6b96eb096361cc1576fefb08c3c93de8 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:47027/bad/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 2df70526c40ee68758afba40d616c00f6b96eb096361cc1576fefb08c3c93de8 (nothing was installed) +ok: rejected, /tmp/tmp.Pg5lEXpSKe/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (bash, x86_64-unknown-linux-gnu) +shell under test: dash (/usr/bin/dash); archive: pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz.sha256 +== 1. piped to dash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:38833/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok 2df70526c40ee68758afba40d616c00f6b96eb096361cc1576fefb08c3c93de8 + installed /tmp/tmp.O7e91hAo0P/bin/pbit (pbit 0.2.0) + +/tmp/tmp.O7e91hAo0P/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/tmp/tmp.O7e91hAo0P/bin:$PATH" + +Next: + /tmp/tmp.O7e91hAo0P/bin/pbit version + /tmp/tmp.O7e91hAo0P/bin/pbit demo --tasks 12 | /tmp/tmp.O7e91hAo0P/bin/pbit decide --pretty + /tmp/tmp.O7e91hAo0P/bin/pbit stats --pretty +$ command -v pbit: /tmp/tmp.O7e91hAo0P/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:38833/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz + sha256 ok 2df70526c40ee68758afba40d616c00f6b96eb096361cc1576fefb08c3c93de8 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for x86_64-unknown-linux-gnu + fetching http://127.0.0.1:38833/bad/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 2df70526c40ee68758afba40d616c00f6b96eb096361cc1576fefb08c3c93de8 (nothing was installed) +ok: rejected, /tmp/tmp.O7e91hAo0P/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (dash, x86_64-unknown-linux-gnu) diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-sh-macos-arm64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-sh-macos-arm64.txt new file mode 100644 index 0000000..c3ea31c --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/install-sh-macos-arm64.txt @@ -0,0 +1,117 @@ +shell under test: sh (/bin/sh); archive: pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit-v0.2.0-aarch64-apple-darwin.tar.gz.sha256 +== 1. piped to sh, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49173/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok 11519e5a78073ff6a0c057ca3137a996aa9d5d5d84e7ce344b6d798470c1a6f0 + installed /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/bin/pbit (pbit 0.2.0) + +/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/bin:$PATH" + +Next: + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/bin/pbit version + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/bin/pbit demo --tasks 12 | /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/bin/pbit decide --pretty + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/bin/pbit stats --pretty +$ command -v pbit: /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49173/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok 11519e5a78073ff6a0c057ca3137a996aa9d5d5d84e7ce344b6d798470c1a6f0 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49173/bad/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-aarch64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 11519e5a78073ff6a0c057ca3137a996aa9d5d5d84e7ce344b6d798470c1a6f0 (nothing was installed) +ok: rejected, /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.v66HwO2bTX/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (sh, aarch64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 5025 Terminated: 15 "$PY" scripts/bench/serve.py "$WORK/srv" "$PORT" > "$WORK/http.log" 2>&1 +shell under test: bash (/bin/bash); archive: pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit-v0.2.0-aarch64-apple-darwin.tar.gz.sha256 +== 1. piped to bash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49182/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok 8d2c1bc5ab37df21912f74c863eb9bf2a49407933a0e35f4626601d29d85d8e5 + installed /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/bin/pbit (pbit 0.2.0) + +/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/bin:$PATH" + +Next: + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/bin/pbit version + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/bin/pbit demo --tasks 12 | /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/bin/pbit decide --pretty + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/bin/pbit stats --pretty +$ command -v pbit: /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49182/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok 8d2c1bc5ab37df21912f74c863eb9bf2a49407933a0e35f4626601d29d85d8e5 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49182/bad/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-aarch64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 8d2c1bc5ab37df21912f74c863eb9bf2a49407933a0e35f4626601d29d85d8e5 (nothing was installed) +ok: rejected, /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.CPSaRfj0vq/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (bash, aarch64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 5136 Terminated: 15 "$PY" scripts/bench/serve.py "$WORK/srv" "$PORT" > "$WORK/http.log" 2>&1 +shell under test: dash (/bin/dash); archive: pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit-v0.2.0-aarch64-apple-darwin.tar.gz.sha256 +== 1. piped to dash, PBIT_INSTALL_DIR set, version given without the v +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49191/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok 8d2c1bc5ab37df21912f74c863eb9bf2a49407933a0e35f4626601d29d85d8e5 + installed /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/bin/pbit (pbit 0.2.0) + +/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/bin is not on your PATH. Add it (e.g. in ~/.profile): + export PATH="/var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/bin:$PATH" + +Next: + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/bin/pbit version + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/bin/pbit demo --tasks 12 | /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/bin/pbit decide --pretty + /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/bin/pbit stats --pretty +$ command -v pbit: /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +== 2. default directory, scratch HOME +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49191/good/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz + sha256 ok 8d2c1bc5ab37df21912f74c863eb9bf2a49407933a0e35f4626601d29d85d8e5 + installed /usr/local/bin/pbit (pbit 0.2.0) + +Next: + pbit version + pbit demo --tasks 12 | pbit decide --pretty + pbit stats --pretty +ok: installed to /usr/local/bin/pbit (pbit 0.2.0) +== 3. tampered .sha256 must fail and install nothing +pbit install: v0.2.0 for aarch64-apple-darwin + fetching http://127.0.0.1:49191/bad/v0.2.0/pbit-v0.2.0-aarch64-apple-darwin.tar.gz +pbit install: error: checksum mismatch for pbit-v0.2.0-aarch64-apple-darwin.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got 8d2c1bc5ab37df21912f74c863eb9bf2a49407933a0e35f4626601d29d85d8e5 (nothing was installed) +ok: rejected, /var/folders/36/tjdph2t965j8snz9_vkdnw0r0000gn/T/tmp.nGOdULBLzq/badbin/pbit absent +== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail +pbit install: error: set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE +ok: rejected +install.sh: all checks passed (dash, aarch64-apple-darwin) +scripts/bench/test_install_sh.sh: line 12: 5248 Terminated: 15 "$PY" scripts/bench/serve.py "$WORK/srv" "$PORT" > "$WORK/http.log" 2>&1 diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/msrv.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/msrv.txt new file mode 100644 index 0000000..5baf58b --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/msrv.txt @@ -0,0 +1,5 @@ +rust 1.72: build FAILED, Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4): error[E0658]: use of unstable library feature 'int_roundings' +rust 1.73: build ok, Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4) +rust 1.77: build FAILED, --locked (Cargo.lock as committed, lockfile v4): error: failed to parse lock file at: /home/runner/work/pbit/pbit/Cargo.lock +rust 1.78: build ok, --locked (Cargo.lock as committed, lockfile v4) +Cargo.lock restored: 0 changes diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/musl-aarch64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/musl-aarch64.txt new file mode 100644 index 0000000..2fe774b --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/musl-aarch64.txt @@ -0,0 +1,23 @@ +$ sudo apt-get install -y gcc-aarch64-linux-gnu + +Running kernel seems to be up-to-date. + +No services need to be restarted. + +No containers need to be restarted. + +No user sessions are running outdated binaries. + +No VM guests are running outdated hypervisor (qemu) binaries on this host. +linker installed in 14 s +$ rustup target add aarch64-unknown-linux-musl +info: downloading component rust-std +$ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl + Finished `release` profile [optimized] target(s) in 18.43s +$ file target/aarch64-unknown-linux-musl/release/pbit +target/aarch64-unknown-linux-musl/release/pbit: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), statically linked, BuildID[sha1]=6b9608f1981e300fe1d149bd279ce6fa0ba3e383, not stripped +size 1477600 bytes +qemu-user-static installed in 3 s +$ qemu-aarch64-static target/aarch64-unknown-linux-musl/release/pbit version +pbit 0.2.0 +$ qemu pbit demo --tasks 12 | qemu pbit decide: exit 0, verdict exact diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/musl-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/musl-x86_64.txt new file mode 100644 index 0000000..cb61ebe --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/musl-x86_64.txt @@ -0,0 +1,12 @@ +$ rustup target add x86_64-unknown-linux-musl +info: downloading component rust-std +$ cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl + Finished `release` profile [optimized] target(s) in 18.18s +$ file target/x86_64-unknown-linux-musl/release/pbit +target/x86_64-unknown-linux-musl/release/pbit: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), static-pie linked, BuildID[sha1]=85ea1386df8c2738b288e74a66fd13f6a9490feb, not stripped +$ ldd target/x86_64-unknown-linux-musl/release/pbit + statically linked +size 1545968 bytes +$ target/x86_64-unknown-linux-musl/release/pbit version +pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/npm-linux-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/npm-linux-x86_64.txt new file mode 100644 index 0000000..ceea000 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/npm-linux-x86_64.txt @@ -0,0 +1,72 @@ +node v22.23.3, npm 10.9.9, target x86_64-unknown-linux-gnu +npm notice +npm notice 📦 pbit@0.2.0 +npm notice Tarball Contents +npm notice 11.4kB LICENSE +npm notice 1.6kB README.md +npm notice 1.7kB bin/pbit.js +npm notice 5.0kB install.js +npm notice 736B package.json +npm notice Tarball Details +npm notice name: pbit +npm notice version: 0.2.0 +npm notice filename: pbit-0.2.0.tgz +npm notice package size: 7.8 kB +npm notice unpacked size: 20.4 kB +npm notice shasum: 320a800c1bea6a3746f73441bf36f3c59c0c3499 +npm notice integrity: sha512-iuUFO3Oex1JpW[...]HF+pBF0+I8ChA== +npm notice total files: 5 +npm notice +== npm pack -> pbit-0.2.0.tgz: + package/LICENSE + package/install.js + package/bin/pbit.js + package/package.json + package/README.md +== 1. npm install -g pbit-0.2.0.tgz (PBIT_DOWNLOAD_BASE=http://127.0.0.1:35951/good) + +> pbit@0.2.0 postinstall +> node install.js + +pbit: fetching http://127.0.0.1:35951/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit: installed v0.2.0 (x86_64-unknown-linux-gnu), sha256 b1064e26615a3f68433e0578f4e117faf2ebf5bf767008faa7953a8e3bcbf8cc + +added 1 package in 340ms +$ which pbit: /tmp/tmp.S8EG08YHen/prefix1/bin/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +exit codes through the wrapper: infeasible 1 (want 1), bad input 2 (want 2), declined 3 (want 3) + +removed 1 package in 139ms +ok: uninstalled, /tmp/tmp.S8EG08YHen/prefix1/bin/pbit absent +== 2. npm install -g --ignore-scripts: the binary is fetched on first run + +added 1 package in 152ms +pbit: the binary is not installed yet, fetching it now +pbit: fetching http://127.0.0.1:35951/good/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit: installed v0.2.0 (x86_64-unknown-linux-gnu), sha256 b1064e26615a3f68433e0578f4e117faf2ebf5bf767008faa7953a8e3bcbf8cc +pbit 0.2.0 +ok: first run fetched it; second run used it +== 3. PBIT_BINARY= (no download) + +> pbit@0.2.0 postinstall +> node install.js + +pbit: using /home/runner/work/pbit/pbit/target/release/pbit (PBIT_BINARY) + +added 1 package in 181ms +pbit 0.2.0 +== 4. tampered .sha256: npm install must fail + +> pbit@0.2.0 postinstall +> node install.js + +pbit: fetching http://127.0.0.1:35951/bad/v0.2.0/pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz +pbit: checksum mismatch for pbit-v0.2.0-x86_64-unknown-linux-gnu.tar.gz: expected 0000000000000000000000000000000000000000000000000000000000000000, got b1064e26615a3f68433e0578f4e117faf2ebf5bf767008faa7953a8e3bcbf8cc; nothing was installed +npm error code 1 +npm error path /tmp/tmp.S8EG08YHen/prefix4/lib/node_modules/pbit +npm error command failed +npm error command sh -c node install.js +npm error A complete log of this run can be found in: /home/runner/.npm/_logs/2026-10-02T05_49_49_925Z-debug-0.log +ok: rejected +npm wrapper: all checks passed (x86_64-unknown-linux-gnu) diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/npm-windows-x86_64.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/npm-windows-x86_64.txt new file mode 100644 index 0000000..b02083b --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/npm-windows-x86_64.txt @@ -0,0 +1,72 @@ +node v22.23.3, npm 10.9.9, target x86_64-pc-windows-msvc +npm notice +npm notice package: pbit@0.2.0 +npm notice Tarball Contents +npm notice 11.6kB LICENSE +npm notice 1.6kB README.md +npm notice 1.8kB bin/pbit.js +npm notice 5.1kB install.js +npm notice 760B package.json +npm notice Tarball Details +npm notice name: pbit +npm notice version: 0.2.0 +npm notice filename: pbit-0.2.0.tgz +npm notice package size: 7.8 kB +npm notice unpacked size: 20.8 kB +npm notice shasum: e627337a1e165cdfb101bedbc6a178a24c3695d6 +npm notice integrity: sha512-H/JH23iMhSCfA[...]r7vAu3FiTe5YQ== +npm notice total files: 5 +npm notice +== npm pack -> pbit-0.2.0.tgz: + package/LICENSE + package/install.js + package/bin/pbit.js + package/package.json + package/README.md +== 1. npm install -g pbit-0.2.0.tgz (PBIT_DOWNLOAD_BASE=http://127.0.0.1:63120/good) + +> pbit@0.2.0 postinstall +> node install.js + +pbit: fetching http://127.0.0.1:63120/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip +pbit: installed v0.2.0 (x86_64-pc-windows-msvc), sha256 fe695b5d46bbfa8fcadb0a1ce9200cafd296dfd3cc340d635149af5ce19e36e8 + +added 1 package in 1s +$ which pbit: /tmp/tmp.5rSfeA9JT6/prefix1/pbit +$ pbit version: pbit 0.2.0 +$ pbit demo --tasks 12 | pbit decide: exit 0, verdict exact | released 12 of 12 +exit codes through the wrapper: infeasible 1 (want 1), bad input 2 (want 2), declined 3 (want 3) + +removed 1 package in 474ms +ok: uninstalled, /tmp/tmp.5rSfeA9JT6/prefix1/pbit absent +== 2. npm install -g --ignore-scripts: the binary is fetched on first run + +added 1 package in 625ms +pbit: the binary is not installed yet, fetching it now +pbit: fetching http://127.0.0.1:63120/good/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip +pbit: installed v0.2.0 (x86_64-pc-windows-msvc), sha256 fe695b5d46bbfa8fcadb0a1ce9200cafd296dfd3cc340d635149af5ce19e36e8 +pbit 0.2.0 +ok: first run fetched it; second run used it +== 3. PBIT_BINARY= (no download) + +> pbit@0.2.0 postinstall +> node install.js + +pbit: using D:\a\pbit\pbit\target\release\pbit.exe (PBIT_BINARY) + +added 1 package in 752ms +pbit 0.2.0 +== 4. tampered .sha256: npm install must fail + +> pbit@0.2.0 postinstall +> node install.js + +pbit: fetching http://127.0.0.1:63120/bad/v0.2.0/pbit-v0.2.0-x86_64-pc-windows-msvc.zip +pbit: checksum mismatch for pbit-v0.2.0-x86_64-pc-windows-msvc.zip: expected 0000000000000000000000000000000000000000000000000000000000000000, got fe695b5d46bbfa8fcadb0a1ce9200cafd296dfd3cc340d635149af5ce19e36e8; nothing was installed +npm error code 1 +npm error path C:\Users\RUNNER~1\AppData\Local\Temp\tmp.5rSfeA9JT6\prefix4\node_modules\pbit +npm error command failed +npm error command C:\Windows\system32\cmd.exe /d /s /c node install.js +npm error A complete log of this run can be found in: C:\Users\runneradmin\AppData\Local\npm-cache\_logs\2026-10-02T05_51_17_370Z-debug-0.log +ok: rejected +npm wrapper: all checks passed (x86_64-pc-windows-msvc) diff --git a/scripts/bench/results/2026-10-01/run-36970631022/transcripts/wasm32-wasip1.txt b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/wasm32-wasip1.txt new file mode 100644 index 0000000..b4f14a6 --- /dev/null +++ b/scripts/bench/results/2026-10-01/run-36970631022/transcripts/wasm32-wasip1.txt @@ -0,0 +1,8 @@ +$ rustup target add wasm32-wasip1 +info: downloading component rust-std +$ cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide +exit 0 +first error: none + Checking pbit-ir v0.2.0 (/home/runner/work/pbit/pbit/pbit-ir) + Checking pbit-decide v0.2.0 (/home/runner/work/pbit/pbit/pbit-decide) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.68s diff --git a/scripts/bench/serve.py b/scripts/bench/serve.py new file mode 100644 index 0000000..33ea4dd --- /dev/null +++ b/scripts/bench/serve.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""Serve a directory on 127.0.0.1 for the installer tests (stdlib, Python >= 3.7). + + python3 scripts/bench/serve.py + +Same handler as `python3 -m http.server`, minus its startup reverse-DNS lookup: HTTPServer.server_bind calls +socket.getfqdn(), which stalled ~35 s per start on the GitHub macOS runners (bench run 36969399069). +""" +import functools, http.server, socketserver, sys + +directory, port = sys.argv[1], int(sys.argv[2]) +socketserver.TCPServer.allow_reuse_address = True +handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=directory) +with socketserver.TCPServer(("127.0.0.1", port), handler) as server: + server.serve_forever() diff --git a/scripts/bench/test_install_ps1.ps1 b/scripts/bench/test_install_ps1.ps1 new file mode 100644 index 0000000..b645173 --- /dev/null +++ b/scripts/bench/test_install_ps1.ps1 @@ -0,0 +1,59 @@ +# install.ps1 end to end with no GitHub release: serves a release-layout .zip on 127.0.0.1 (built beforehand with +# scripts/bench/package_like_release.sh into \good\\, plus a copy with a wrong .sha256 in \bad\\) and +# installs it with -DownloadBase, then the `irm | iex` form (environment variables), then the tampered copy (must fail). +# pwsh -File scripts/bench/test_install_ps1.ps1 -Srv [-Tag v0.2.0] [-AddToPath] +# powershell -ExecutionPolicy Bypass -File scripts/bench/test_install_ps1.ps1 -Srv +param([Parameter(Mandatory = $true)][string]$Srv, [string]$Tag = 'v0.2.0', [switch]$AddToPath) +$ErrorActionPreference = 'Stop' +"PowerShell $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" +# Windows PowerShell 5.1 adds a UTF-8 byte-order mark to text it pipes into a native program, which pbit 0.2.0 rejects +# (docs/agents.md, PowerShell): there the demo pipeline below goes through cmd.exe. +$desktop = $PSVersionTable.PSEdition -ne 'Core' +$l = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, 0); $l.Start(); $port = $l.LocalEndpoint.Port; $l.Stop() +$server = Start-Process -FilePath python -ArgumentList @('scripts/bench/serve.py', $Srv, "$port") -PassThru -WindowStyle Hidden +$base = "http://127.0.0.1:$port" +try { + for ($i = 0; $i -lt 50; $i++) { try { Invoke-WebRequest -UseBasicParsing -Uri "$base/" -TimeoutSec 2 | Out-Null; break } catch { Start-Sleep -Milliseconds 200 } } + $root = Join-Path ([IO.Path]::GetTempPath()) ('pbit-ps-test-' + [guid]::NewGuid().ToString('N')) + + "== 1. & .\install.ps1 -DownloadBase $base/good -Version $Tag -InstallDir \bin1$(if ($AddToPath) { ' -AddToPath' })" + $d1 = Join-Path $root 'bin1' + if ($AddToPath) { & .\install.ps1 -DownloadBase "$base/good" -Version $Tag -InstallDir $d1 -AddToPath } + else { & .\install.ps1 -DownloadBase "$base/good" -Version $Tag -InstallDir $d1 } + $cmd = Get-Command pbit -ErrorAction Stop + "`$ (Get-Command pbit).Source: $($cmd.Source)" + if ((Resolve-Path $cmd.Source).Path -ne (Resolve-Path (Join-Path $d1 'pbit.exe')).Path) { throw "FAIL: pbit resolves to $($cmd.Source)" } + "`$ pbit version: $(pbit version)" + if ($desktop) { $how = 'cmd /c "pbit demo --tasks 12 | pbit decide"'; $d = cmd /c "pbit demo --tasks 12 | pbit decide" | ConvertFrom-Json } + else { $how = 'pbit demo --tasks 12 | pbit decide'; $d = pbit demo --tasks 12 | pbit decide | ConvertFrom-Json } + "`$ $how | ConvertFrom-Json: exit $LASTEXITCODE, verdict $($d.verdict), released $(@($d.released).Count) of $($d.tasks)" + if ($LASTEXITCODE -ne 0 -or $d.verdict -ne 'exact') { throw 'FAIL: demo pipeline' } + if ($AddToPath) { + $u = [Environment]::GetEnvironmentVariable('Path', 'User') + if (-not ($u -split ';' -contains $d1)) { throw "FAIL: -AddToPath did not add $d1 to the user PATH" } + "ok: user PATH now starts with $(($u -split ';')[0])" + } + + '== 2. irm | iex form: PBIT_* environment variables, script text through Invoke-Expression' + $d2 = Join-Path $root 'bin2' + $env:PBIT_DOWNLOAD_BASE = "$base/good"; $env:PBIT_VERSION = $Tag.TrimStart('v'); $env:PBIT_INSTALL_DIR = $d2 + try { Get-Content -Raw .\install.ps1 | Invoke-Expression } + finally { Remove-Item Env:PBIT_DOWNLOAD_BASE, Env:PBIT_VERSION, Env:PBIT_INSTALL_DIR } + if (-not (Test-Path (Join-Path $d2 'pbit.exe'))) { throw "FAIL: $d2\pbit.exe missing" } + "ok: $(& (Join-Path $d2 'pbit.exe') version) in $d2" + + '== 3. a wrong .sha256 must fail and install nothing' + $d3 = Join-Path $root 'bin3' + $failed = $false + try { & .\install.ps1 -DownloadBase "$base/bad" -Version $Tag -InstallDir $d3 } catch { $failed = $true; "rejected: $($_.Exception.Message)" } + if (-not $failed) { throw 'FAIL: a wrong checksum was accepted' } + if (Test-Path (Join-Path $d3 'pbit.exe')) { throw 'FAIL: pbit.exe installed despite the checksum mismatch' } + + '== 4. -DownloadBase without -Version must fail' + $failed = $false + try { & .\install.ps1 -DownloadBase "$base/good" -InstallDir (Join-Path $root 'bin4') } catch { $failed = $true; "rejected: $($_.Exception.Message)" } + if (-not $failed) { throw 'FAIL: accepted' } + "install.ps1: all checks passed (PowerShell $($PSVersionTable.PSVersion))" +} finally { + Stop-Process -Id $server.Id -Force -ErrorAction SilentlyContinue +} diff --git a/scripts/bench/test_install_sh.sh b/scripts/bench/test_install_sh.sh new file mode 100755 index 0000000..d29e625 --- /dev/null +++ b/scripts/bench/test_install_sh.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# install.sh end to end with no GitHub release: pack a local binary as release.yml does, serve it on 127.0.0.1, then install +# it piped through sh (as `curl ... | sh` does), into the default directory under a scratch HOME, and from a server whose +# .sha256 is wrong (must fail and install nothing). +# scripts/bench/test_install_sh.sh [tag] SH=dash scripts/bench/... to pick the shell +set -eu +BIN=$1 TARGET=$2 TAG=${3:-v0.2.0} +SH=${SH:-sh} +PY=${PY:-$(command -v python3 || command -v python)} +WORK=$(mktemp -d 2> /dev/null || mktemp -d -t pbitinst) +SRV="" +cleanup() { if [ -n "$SRV" ]; then kill "$SRV" 2> /dev/null || true; fi; rm -rf "$WORK"; } +trap cleanup EXIT +sh scripts/bench/package_like_release.sh "$BIN" "$TARGET" "$TAG" "$WORK/srv/good/$TAG" > /dev/null +cp -R "$WORK/srv/good" "$WORK/srv/bad" +printf '%064d %s\n' 0 "pbit-$TAG-$TARGET.tar.gz" > "$WORK/srv/bad/$TAG/pbit-$TAG-$TARGET.tar.gz.sha256" +PORT=$("$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +"$PY" scripts/bench/serve.py "$WORK/srv" "$PORT" > "$WORK/http.log" 2>&1 & +SRV=$! +i=0 +until curl -fsS -o /dev/null "http://127.0.0.1:$PORT/" 2> /dev/null; do + i=$((i + 1)); [ "$i" -lt 50 ] || { echo "http server did not start"; exit 1; }; sleep 0.2 +done +BASE="http://127.0.0.1:$PORT" +echo "shell under test: $SH ($(command -v "$SH")); archive: $(ls "$WORK/srv/good/$TAG")" + +echo "== 1. piped to $SH, PBIT_INSTALL_DIR set, version given without the v" +PBIT_DOWNLOAD_BASE="$BASE/good" PBIT_VERSION="${TAG#v}" PBIT_INSTALL_DIR="$WORK/bin" "$SH" < install.sh +PATH="$WORK/bin:$PATH" +echo "\$ command -v pbit: $(command -v pbit)" +echo "\$ pbit version: $(pbit version)" +pbit demo --tasks 12 | pbit decide > "$WORK/d12.json" +echo "\$ pbit demo --tasks 12 | pbit decide: exit $?, $("$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); print("verdict", d["verdict"], "| released", len(d["released"]), "of", d["tasks"])' "$WORK/d12.json")" + +echo "== 2. default directory, scratch HOME" +HOME="$WORK/home" PBIT_DOWNLOAD_BASE="$BASE/good" PBIT_VERSION="$TAG" "$SH" install.sh +if [ -d /usr/local/bin ] && [ -w /usr/local/bin ]; then want=/usr/local/bin/pbit; else want="$WORK/home/.local/bin/pbit"; fi +[ -x "$want" ] || { echo "FAIL: expected $want"; exit 1; } +echo "ok: installed to $want ($("$want" version))" + +echo "== 3. tampered .sha256 must fail and install nothing" +if PBIT_DOWNLOAD_BASE="$BASE/bad" PBIT_VERSION="$TAG" PBIT_INSTALL_DIR="$WORK/badbin" "$SH" install.sh; then + echo "FAIL: a wrong checksum was accepted"; exit 1 +fi +[ ! -e "$WORK/badbin/pbit" ] || { echo "FAIL: pbit installed despite the checksum mismatch"; exit 1; } +echo "ok: rejected, $WORK/badbin/pbit absent" + +echo "== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail" +if PBIT_DOWNLOAD_BASE="$BASE/good" PBIT_INSTALL_DIR="$WORK/bin4" "$SH" install.sh; then echo "FAIL: accepted"; exit 1; fi +echo "ok: rejected" +echo "install.sh: all checks passed ($SH, $TARGET)" diff --git a/scripts/bench/test_npm.sh b/scripts/bench/test_npm.sh new file mode 100755 index 0000000..462f492 --- /dev/null +++ b/scripts/bench/test_npm.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# The npm wrapper end to end with no GitHub release and no registry: pack a local binary as release.yml does, serve it on +# 127.0.0.1, `npm pack` npm/, `npm install -g` the tarball (into a scratch prefix, so the machine's global packages are not +# touched): postinstall fetches + verifies the binary; run it (exit codes 0/1/2/3 must pass through); `npm uninstall -g`. +# Then an --ignore-scripts install (fetched on first run), PBIT_BINARY, and a tampered .sha256 (the install must fail). +# scripts/bench/test_npm.sh [tag] +set -eu +BIN=$1 TARGET=$2 TAG=${3:-v0.2.0} +PY=${PY:-$(command -v python3 || command -v python)} +WORK=$(mktemp -d 2> /dev/null || mktemp -d -t pbitnpm) +SRV="" +cleanup() { if [ -n "$SRV" ]; then kill "$SRV" 2> /dev/null || true; fi; rm -rf "$WORK"; } +trap cleanup EXIT +fail() { echo "FAIL: $*"; exit 1; } +BIN_ABS=$(cd "$(dirname "$BIN")" && pwd)/$(basename "$BIN") +if command -v cygpath > /dev/null; then BIN_ABS=$(cygpath -w "$BIN_ABS"); fi # Windows: node needs a native path +sh scripts/bench/package_like_release.sh "$BIN" "$TARGET" "$TAG" "$WORK/srv/good/$TAG" > /dev/null +cp -R "$WORK/srv/good" "$WORK/srv/bad" +case "$TARGET" in *windows*) EXT=zip ;; *) EXT=tar.gz ;; esac +printf '%064d %s\n' 0 "pbit-$TAG-$TARGET.$EXT" > "$WORK/srv/bad/$TAG/pbit-$TAG-$TARGET.$EXT.sha256" +PORT=$("$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +"$PY" scripts/bench/serve.py "$WORK/srv" "$PORT" > "$WORK/http.log" 2>&1 & +SRV=$! +i=0 +until curl -fsS -o /dev/null "http://127.0.0.1:$PORT/" 2> /dev/null; do + i=$((i + 1)); [ "$i" -lt 50 ] || fail "http server did not start"; sleep 0.2 +done +BASE="http://127.0.0.1:$PORT" +case "$(uname -s)" in MINGW* | MSYS* | CYGWIN*) gbin() { echo "$1"; } ;; *) gbin() { echo "$1/bin"; } ;; esac +NPM_FLAGS="--no-audit --no-fund --foreground-scripts" +echo "node $(node --version), npm $(npm --version), target $TARGET" + +(cd npm && npm pack --pack-destination "$WORK" > /dev/null) +TGZ=$(ls "$WORK"/pbit-*.tgz) +echo "== npm pack -> $(basename "$TGZ"):" +tar tzf "$TGZ" | sed 's/^/ /' + +echo "== 1. npm install -g $(basename "$TGZ") (PBIT_DOWNLOAD_BASE=$BASE/good)" +P1="$WORK/prefix1" +PBIT_DOWNLOAD_BASE="$BASE/good" npm install -g --prefix "$P1" $NPM_FLAGS "$TGZ" +OLDPATH=$PATH +PATH="$(gbin "$P1"):$OLDPATH" +echo "\$ which pbit: $(command -v pbit)" +echo "\$ pbit version: $(pbit version)" +pbit demo --tasks 12 | pbit decide > "$WORK/d12.json" +echo "\$ pbit demo --tasks 12 | pbit decide: exit $?, $("$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); print("verdict", d["verdict"], "| released", len(d["released"]), "of", d["tasks"])' "$WORK/d12.json")" +set +e +printf '%s' '{"workers":[{"id":"a","cap":1}],"tasks":[{"id":"t1","allowed":["a"],"scores":{"a":1}},{"id":"t2","allowed":["a"],"scores":{"a":1}}]}' | pbit decide > /dev/null; r1=$? +printf '%s' '{"workers": 1}' | pbit decide > /dev/null 2>&1; r2=$? +pbit run --op exact --exact-ms 50 < examples/denoise-8x12.json > /dev/null; r3=$? +set -e +echo "exit codes through the wrapper: infeasible $r1 (want 1), bad input $r2 (want 2), declined $r3 (want 3)" +[ "$r1" = 1 ] && [ "$r2" = 2 ] && [ "$r3" = 3 ] || fail "exit codes" +npm uninstall -g --prefix "$P1" $NPM_FLAGS pbit +PATH=$OLDPATH +[ ! -e "$(gbin "$P1")/pbit" ] || fail "npm uninstall left $(gbin "$P1")/pbit" +echo "ok: uninstalled, $(gbin "$P1")/pbit absent" + +echo "== 2. npm install -g --ignore-scripts: the binary is fetched on first run" +P2="$WORK/prefix2" +npm install -g --prefix "$P2" $NPM_FLAGS --ignore-scripts "$TGZ" +PBIT_DOWNLOAD_BASE="$BASE/good" "$(gbin "$P2")/pbit" version +"$(gbin "$P2")/pbit" demo --tasks 12 | "$(gbin "$P2")/pbit" decide > /dev/null +echo "ok: first run fetched it; second run used it" + +echo "== 3. PBIT_BINARY= (no download)" +P3="$WORK/prefix3" +PBIT_BINARY="$BIN_ABS" PBIT_DOWNLOAD_BASE="http://127.0.0.1:9/nowhere" npm install -g --prefix "$P3" $NPM_FLAGS "$TGZ" +"$(gbin "$P3")/pbit" version + +echo "== 4. tampered .sha256: npm install must fail" +if PBIT_DOWNLOAD_BASE="$BASE/bad" npm install -g --prefix "$WORK/prefix4" $NPM_FLAGS "$TGZ"; then fail "a wrong checksum was accepted"; fi +echo "ok: rejected" +echo "npm wrapper: all checks passed ($TARGET)" diff --git a/scripts/bench/test_summary.py b/scripts/bench/test_summary.py new file mode 100644 index 0000000..20f3ca7 --- /dev/null +++ b/scripts/bench/test_summary.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""One JSON for a `cargo test` run: pass/fail, wall seconds, summed counts of every "test result:" line, machine and toolchain. + + python3 scripts/bench/test_summary.py