From ab04b5ce1ef9c8367ddbbda5b8f4137b5e22dcd5 Mon Sep 17 00:00:00 2001 From: BitmapAsset Date: Thu, 1 Oct 2026 22:32:28 -0700 Subject: [PATCH 1/3] bench: portability matrix workflow, installers, npm wrapper (no publish) - .github/workflows/bench.yml: build, test and measure pbit on the ubuntu, macOS arm64, Windows and macOS Intel runners, plus static musl and cross builds, a wasm32-wasip1 check and minimum-Rust probes. Runs on pushes to this branch and by hand only. - scripts/bench/: matrix_bench.py (one JSON per platform, N = 5) and local-server tests for the installers and the npm wrapper. - install.sh and install.ps1: fetch a release archive, verify its SHA-256, install without sudo or administrator rights. - npm/: the `pbit` package (postinstall fetch with checksum, exit codes passed through, fetch on first run when install scripts are off). Not published. - docs/agents.md and examples/node/decide.mjs: calling pbit from a shell, Python, Node, PowerShell and agent harnesses. - release.yml: archives also carry examples/, python/ and docs/. - Crate manifests: readme, keywords, categories. --- .github/workflows/bench.yml | 348 ++++++++++++++++++++++ .github/workflows/release.yml | 1 + docs/agents.md | 97 ++++++ examples/node/decide.mjs | 87 ++++++ install.ps1 | 95 ++++++ install.sh | 144 +++++++++ npm/LICENSE | 202 +++++++++++++ npm/README.md | 29 ++ npm/bin/pbit.js | 39 +++ npm/install.js | 113 +++++++ npm/package.json | 24 ++ pbit-cli/Cargo.toml | 3 + pbit-core/Cargo.toml | 3 + pbit-decide/Cargo.toml | 3 + pbit-ir/Cargo.toml | 3 + scripts/bench/callers.ps1 | 27 ++ scripts/bench/matrix_bench.py | 406 ++++++++++++++++++++++++++ scripts/bench/package_like_release.sh | 23 ++ scripts/bench/peak_rss_windows.ps1 | 29 ++ scripts/bench/test_install_ps1.ps1 | 55 ++++ scripts/bench/test_install_sh.sh | 51 ++++ scripts/bench/test_npm.sh | 74 +++++ scripts/bench/test_summary.py | 19 ++ 23 files changed, 1875 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 docs/agents.md create mode 100644 examples/node/decide.mjs create mode 100644 install.ps1 create mode 100755 install.sh create mode 100644 npm/LICENSE create mode 100644 npm/README.md create mode 100755 npm/bin/pbit.js create mode 100644 npm/install.js create mode 100644 npm/package.json create mode 100644 scripts/bench/callers.ps1 create mode 100644 scripts/bench/matrix_bench.py create mode 100755 scripts/bench/package_like_release.sh create mode 100644 scripts/bench/peak_rss_windows.ps1 create mode 100644 scripts/bench/test_install_ps1.ps1 create mode 100755 scripts/bench/test_install_sh.sh create mode 100755 scripts/bench/test_npm.sh create mode 100644 scripts/bench/test_summary.py diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..000ddd4 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,348 @@ +name: bench +# Portability and benchmark matrix: pbit built, tested and measured on each standard GitHub runner OS; static and cross +# builds; the installers (install.sh, install.ps1), the npm wrapper and the docs/agents.md recipes, each against a local +# server (no release needed). Runs on pushes to its own branch and by hand only, never on pull requests or main: macOS +# minutes bill at 10x on a private repo. Standard runners only. Results: BENCHMARK-MATRIX.md, PORTABILITY.md. +on: + push: + branches: [r22-portability-matrix] + paths-ignore: ["**.md", "scripts/bench/results/**"] + workflow_dispatch: +permissions: + contents: read +env: + CARGO_TERM_COLOR: never + TAG: v0.2.0 + +jobs: + bench: + name: bench ${{ matrix.label }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, label: linux-x86_64, target: x86_64-unknown-linux-gnu, py: python3, native: true, timeout: 15 } + - { os: macos-latest, label: macos-arm64, target: aarch64-apple-darwin, py: python3, native: false, timeout: 15 } + - { os: windows-latest, label: windows-x86_64, target: x86_64-pc-windows-msvc, py: python, native: true, timeout: 15 } + - { os: macos-15-intel, label: macos-x86_64, target: x86_64-apple-darwin, py: python3, native: true, timeout: 12 } + runs-on: ${{ matrix.os }} + timeout-minutes: ${{ matrix.timeout }} + defaults: + run: + shell: bash + env: + PY: ${{ matrix.py }} + LABEL: ${{ matrix.label }} + TARGET: ${{ matrix.target }} + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: build (portable, as release.yml) + id: build + run: | + mkdir -p out + start=$(date +%s) + cargo build --release --locked --workspace + echo "build_s=$(( $(date +%s) - start ))" >> "$GITHUB_OUTPUT" + cargo build --release --locked -p pbit-core --example kernels + ls -l target/release/ + - name: build (-C target-cpu=native, for bench/portable_vs_native.py) + id: native + if: matrix.native + continue-on-error: true + env: + RUSTFLAGS: -C target-cpu=native + run: | + cargo build --release --locked -p pbit-cli --target-dir target/native + cargo build --release --locked -p pbit-core --example kernels --target-dir target/native + + - name: callers (shell pipe, python/pbit.py tests, examples/node/decide.mjs) + id: callers + continue-on-error: true + run: | + B="$PWD/target/release/pbit" + if [ "$RUNNER_OS" = Windows ]; then B="$(cygpath -w "$B.exe")"; fi + export PBIT_BIN="$B" + mkdir -p onpath && cp target/release/pbit* onpath/ 2> /dev/null || true + export PATH="$PWD/onpath:$PATH" + { + echo "\$ pbit demo --tasks 12 | pbit decide --pretty ($(command -v pbit))" + pbit demo --tasks 12 | pbit decide --pretty > d12.json + echo "exit $?"; sed -n 1,8p d12.json + pbit demo --tasks 24 > router.json + echo "\$ pbit decide --budget-ms 200 < router.json > decision.json; case \$? in ..." + pbit decide --budget-ms 200 < router.json > decision.json && rc=0 || rc=$? + case $rc in 0) echo answer ;; 1) echo infeasible ;; 2) echo "bad input" ;; 3) echo "refused: escalate" ;; *) echo "exit $rc"; exit 1 ;; esac + $PY -c 'import json,sys; print(json.load(sys.stdin)["verdict"])' < decision.json + echo "\$ $PY python/test_pbit.py" + $PY python/test_pbit.py + echo "\$ $PY python/examples/route_agents.py" + $PY python/examples/route_agents.py + echo "\$ node examples/node/decide.mjs (node $(node --version))" + node examples/node/decide.mjs + } 2>&1 | tee "out/callers-$LABEL.txt" + + - name: install.sh (release.yml layout, served on 127.0.0.1) + id: install_sh + if: runner.os != 'Windows' + continue-on-error: true + run: | + for s in sh bash dash; do + command -v "$s" > /dev/null || continue + SH=$s sh scripts/bench/test_install_sh.sh target/release/pbit "$TARGET" "$TAG" + done 2>&1 | tee "out/install-sh-$LABEL.txt" + + - name: npm wrapper (npm pack, npm install -g from 127.0.0.1, npm uninstall -g) + id: npm + if: runner.os == 'Linux' + continue-on-error: true + run: sh scripts/bench/test_npm.sh target/release/pbit "$TARGET" "$TAG" 2>&1 | tee "out/npm-$LABEL.txt" + - name: npm wrapper on Windows (informational) + id: npm_windows + if: runner.os == 'Windows' + continue-on-error: true + run: sh scripts/bench/test_npm.sh target/release/pbit.exe "$TARGET" "$TAG" 2>&1 | tee "out/npm-$LABEL.txt" + + # PowerShell scripts are started from bash (pwsh/powershell -File, exit code 1 on a throw): redirecting a native + # command's stderr inside a pwsh step can itself raise under the runner's ErrorActionPreference = stop. + - name: pack a release-layout .zip for install.ps1 + id: pack_zip + if: runner.os == 'Windows' + continue-on-error: true + run: | + sh scripts/bench/package_like_release.sh target/release/pbit.exe "$TARGET" "$TAG" "srv/good/$TAG" + cp -R srv/good srv/bad + printf '%064d %s\n' 0 "pbit-$TAG-$TARGET.zip" > "srv/bad/$TAG/pbit-$TAG-$TARGET.zip.sha256" + ls -l "srv/good/$TAG"; cat "srv/good/$TAG/pbit-$TAG-$TARGET.zip.sha256" + - name: install.ps1 under PowerShell 7 + id: install_ps1_pwsh + if: runner.os == 'Windows' + continue-on-error: true + run: pwsh -NoProfile -File scripts/bench/test_install_ps1.ps1 -Srv srv -Tag "$TAG" -AddToPath 2>&1 | tee "out/install-ps1-pwsh-$LABEL.txt" + - name: install.ps1 under Windows PowerShell 5.1 + id: install_ps1_ps51 + if: runner.os == 'Windows' + continue-on-error: true + run: powershell -NoProfile -ExecutionPolicy Bypass -File scripts/bench/test_install_ps1.ps1 -Srv srv -Tag "$TAG" 2>&1 | tee "out/install-ps1-ps51-$LABEL.txt" + - name: callers (PowerShell 7 and Windows PowerShell 5.1) + id: ps_callers + if: runner.os == 'Windows' + continue-on-error: true + run: | + export PATH="$PWD/target/release:$PATH" + { pwsh -NoProfile -File scripts/bench/callers.ps1 + powershell -NoProfile -ExecutionPolicy Bypass -File scripts/bench/callers.ps1; } 2>&1 | tee "out/callers-powershell-$LABEL.txt" + - name: peak RSS via Get-Process (cross-check of the harness value) + id: rss_ps + if: runner.os == 'Windows' + continue-on-error: true + run: pwsh -NoProfile -File scripts/bench/peak_rss_windows.ps1 -Exe target/release/pbit.exe -Out out/peak-rss-powershell.json + + - name: cross build x86_64-apple-darwin on the arm64 runner (Rosetta smoke if present) + id: cross_darwin + if: matrix.label == 'macos-arm64' + continue-on-error: true + run: | + { + echo "\$ rustup target add x86_64-apple-darwin"; rustup target add x86_64-apple-darwin + echo "\$ cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin" + cargo build --release --locked -p pbit-cli --target x86_64-apple-darwin 2>&1 | tail -1 + B=target/x86_64-apple-darwin/release/pbit + echo "\$ file $B"; file "$B" + echo "\$ lipo -archs $B"; lipo -archs "$B" + echo "\$ otool -L $B"; otool -L "$B" + echo "size $(stat -f %z "$B") bytes" + if /usr/bin/arch -x86_64 /usr/bin/true 2> /dev/null; then + echo "rosetta: present" + echo "\$ arch -x86_64 $B version"; /usr/bin/arch -x86_64 "$B" version + /usr/bin/arch -x86_64 "$B" demo --tasks 12 | /usr/bin/arch -x86_64 "$B" decide > d12-x86.json + echo "\$ arch -x86_64 pbit demo --tasks 12 | arch -x86_64 pbit decide: exit $?, verdict $(python3 -c 'import json; print(json.load(open("d12-x86.json"))["verdict"])')" + echo "\$ arch -x86_64 pbit stats (self-test under Rosetta)" + /usr/bin/arch -x86_64 "$B" stats | python3 -c 'import json,sys; s=json.load(sys.stdin)["self_test"]; print("1 thread", s["site_updates_per_s_1_thread"], "| threads", s["threads"], s["site_updates_per_s"])' + else + echo "rosetta: absent, build only" + fi + } 2>&1 | tee out/cross-x86_64-apple-darwin.txt + + - name: record check outcomes + if: always() + run: | + mkdir -p out + cat > "out/checks-$LABEL.json" << EOF + {"build": "${{ steps.build.outcome }}", "native_build": "${{ steps.native.outcome }}", "pack_zip": "${{ steps.pack_zip.outcome }}", "callers": "${{ steps.callers.outcome }}", "install_sh": "${{ steps.install_sh.outcome }}", + "npm": "${{ steps.npm.outcome }}", "npm_windows": "${{ steps.npm_windows.outcome }}", + "install_ps1_pwsh": "${{ steps.install_ps1_pwsh.outcome }}", "install_ps1_ps51": "${{ steps.install_ps1_ps51.outcome }}", + "ps_callers": "${{ steps.ps_callers.outcome }}", "rss_powershell": "${{ steps.rss_ps.outcome }}", + "cross_darwin": "${{ steps.cross_darwin.outcome }}"} + EOF + cat "out/checks-$LABEL.json" + - name: measure (scripts/bench/matrix_bench.py, N = 5) + id: measure + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: | + set -- --attach "checks=out/checks-$LABEL.json" --note "build_s=${{ steps.build.outputs.build_s }}" --note "runner=${{ matrix.os }}" + if [ "${{ steps.native.outcome }}" = success ]; then set -- "$@" --native-dir target/native/release; fi + if [ -f out/peak-rss-powershell.json ]; then set -- "$@" --attach peak_rss_powershell=out/peak-rss-powershell.json; fi + if [ -f out/cross-x86_64-apple-darwin.txt ]; then set -- "$@" --attach cross_x86_64_apple_darwin=out/cross-x86_64-apple-darwin.txt; fi + $PY scripts/bench/matrix_bench.py --bin target/release/pbit --label "$LABEL" --out "out/bench-$LABEL.json" "$@" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: bench-${{ matrix.label }} + path: out/ + if-no-files-found: warn + - name: fail the job if a required check failed + if: always() + run: | + fail=0 + for kv in callers=${{ steps.callers.outcome }} install_sh=${{ steps.install_sh.outcome }} npm=${{ steps.npm.outcome }} \ + install_ps1_pwsh=${{ steps.install_ps1_pwsh.outcome }} install_ps1_ps51=${{ steps.install_ps1_ps51.outcome }} \ + ps_callers=${{ steps.ps_callers.outcome }} measure=${{ steps.measure.outcome }}; do + case "$kv" in *=failure | *=cancelled) echo "required check failed: $kv"; fail=1 ;; esac + done + exit $fail + + test: + name: test ${{ matrix.label }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, label: linux-x86_64, py: python3, timeout: 15 } + - { os: macos-latest, label: macos-arm64, py: python3, timeout: 15 } + - { os: windows-latest, label: windows-x86_64, py: python, timeout: 15 } + - { os: macos-15-intel, label: macos-x86_64, py: python3, timeout: 12 } + runs-on: ${{ matrix.os }} + timeout-minutes: ${{ matrix.timeout }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: cargo test --release --locked --workspace + run: | + mkdir -p out + start=$(date +%s) + set +e + cargo test --release --locked --workspace 2>&1 | tee "out/test-${{ matrix.label }}.log" + rc=${PIPESTATUS[0]} + set -e + ${{ matrix.py }} scripts/bench/test_summary.py "out/test-${{ matrix.label }}.log" "$rc" "$(( $(date +%s) - start ))" \ + "${{ matrix.label }}" > "out/test-${{ matrix.label }}.json" || echo "test_summary.py failed" + grep -E '"(passed|seconds|exit)"' "out/test-${{ matrix.label }}.json" || true + exit "$rc" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: test-${{ matrix.label }} + path: out/ + if-no-files-found: warn + + cross: + name: static and cross builds (ubuntu-latest) + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: x86_64-unknown-linux-musl (static) + id: musl_x86 + continue-on-error: true + run: | + mkdir -p out + { + echo "\$ rustup target add x86_64-unknown-linux-musl"; rustup target add x86_64-unknown-linux-musl + echo "\$ cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl" + cargo build --release --locked -p pbit-cli --target x86_64-unknown-linux-musl 2>&1 | tail -1 + B=target/x86_64-unknown-linux-musl/release/pbit + echo "\$ file $B"; file "$B" + echo "\$ ldd $B"; ldd "$B" 2>&1 || true + echo "size $(stat -c %s "$B") bytes" + echo "\$ $B version"; "$B" version + "$B" demo --tasks 12 | "$B" decide > d12.json + echo "\$ pbit demo --tasks 12 | pbit decide: exit $?, verdict $(python3 -c 'import json; print(json.load(open("d12.json"))["verdict"])')" + } 2>&1 | tee out/musl-x86_64.txt + - name: glibc build in the same VM (for the musl vs glibc comparison) + run: cargo build --release --locked -p pbit-cli 2>&1 | tail -1 + - name: aarch64-unknown-linux-musl (cross-linked; qemu smoke if installable within 120 s) + id: musl_arm + continue-on-error: true + run: | + { + echo "\$ sudo apt-get install -y gcc-aarch64-linux-gnu" + t0=$(date +%s); sudo apt-get update -qq && sudo apt-get install -y -qq gcc-aarch64-linux-gnu > /dev/null + echo "linker installed in $(( $(date +%s) - t0 )) s" + echo "\$ rustup target add aarch64-unknown-linux-musl"; rustup target add aarch64-unknown-linux-musl + echo "\$ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl" + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --locked -p pbit-cli --target aarch64-unknown-linux-musl 2>&1 | tail -1 + B=target/aarch64-unknown-linux-musl/release/pbit + echo "\$ file $B"; file "$B" + echo "size $(stat -c %s "$B") bytes" + t0=$(date +%s) + if timeout 120 sudo apt-get install -y -qq qemu-user-static > /dev/null 2>&1; then + echo "qemu-user-static installed in $(( $(date +%s) - t0 )) s" + Q=$(command -v qemu-aarch64-static || command -v qemu-aarch64) + echo "\$ $(basename "$Q") $B version"; "$Q" "$B" version + "$Q" "$B" demo --tasks 12 | "$Q" "$B" decide > d12-arm.json + echo "\$ qemu pbit demo --tasks 12 | qemu pbit decide: exit $?, verdict $(python3 -c 'import json; print(json.load(open("d12-arm.json"))["verdict"])')" + else + echo "qemu-user-static: not installed within 120 s, build only" + fi + } 2>&1 | tee out/musl-aarch64.txt + - name: wasm32-wasip1 feasibility probe (cargo check, nothing fixed) + id: wasm + continue-on-error: true + run: | + { + echo "\$ rustup target add wasm32-wasip1"; rustup target add wasm32-wasip1 + echo "\$ cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide" + set +e; cargo check --target wasm32-wasip1 -p pbit-ir -p pbit-decide > wasm.log 2>&1; rc=$?; set -e + echo "exit $rc"; echo "first error: $(grep -m1 -E '^error' wasm.log || echo none)"; tail -3 wasm.log + } 2>&1 | tee out/wasm32-wasip1.txt + - name: minimum Rust version probes (debug builds of pbit-cli) + id: msrv + continue-on-error: true + run: | + { + for v in 1.72 1.73 1.77 1.78; do + rustup toolchain install "$v" --profile minimal --no-self-update > /dev/null 2>&1 || { echo "rust $v: toolchain install failed"; continue; } + flags=--locked; how="--locked (Cargo.lock as committed, lockfile v4)" + if [ "$v" = 1.72 ] || [ "$v" = 1.73 ]; then mv Cargo.lock Cargo.lock.v4; flags=""; how="Cargo.lock moved aside (cargo < 1.78 cannot read lockfile v4)"; fi + set +e; cargo "+$v" build $flags -p pbit-cli --target-dir "target/msrv-$v" > "msrv-$v.log" 2>&1; rc=$?; set -e + if [ -f Cargo.lock.v4 ]; then rm -f Cargo.lock; mv Cargo.lock.v4 Cargo.lock; fi + if [ "$rc" = 0 ]; then echo "rust $v: build ok, $how"; else echo "rust $v: build FAILED, $how: $(grep -m1 -E '^error' "msrv-$v.log")"; fi + done + echo "Cargo.lock restored: $(git status --short Cargo.lock | wc -l | tr -d ' ') changes" + } 2>&1 | tee out/msrv.txt + - name: measure the static (musl) and glibc binaries in this VM (N = 5) + id: measure + continue-on-error: true + run: | + python3 scripts/bench/matrix_bench.py --bin target/x86_64-unknown-linux-musl/release/pbit --label linux-x86_64-musl --out out/bench-linux-x86_64-musl.json --note "static musl build, same VM as bench-linux-x86_64-gnu-samevm" + python3 scripts/bench/matrix_bench.py --bin target/release/pbit --label linux-x86_64-gnu-samevm --out out/bench-linux-x86_64-gnu-samevm.json --note "glibc build, same VM as the musl row" + - name: one JSON for the job + if: always() + env: + OUTCOMES: '{"musl_x86_64": "${{ steps.musl_x86.outcome }}", "musl_aarch64": "${{ steps.musl_arm.outcome }}", "wasm32_wasip1": "${{ steps.wasm.outcome }}", "msrv": "${{ steps.msrv.outcome }}", "measure": "${{ steps.measure.outcome }}"}' + run: | + python3 - << 'EOF' + import glob, json, os, sys, time + sys.path.insert(0, "scripts/bench") + from matrix_bench import machine, runner, toolchain + out = {"schema": "pbit-cross/1", "label": "cross-linux-x86_64", "date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "outcomes": json.loads(os.environ["OUTCOMES"]), "runner": runner(), "machine": machine(), "toolchain": toolchain(), + "transcripts": {os.path.basename(p): open(p).read().splitlines() for p in sorted(glob.glob("out/*.txt"))}, + "bench": {os.path.basename(p)[6:-5]: json.load(open(p)) for p in sorted(glob.glob("out/bench-*.json"))}} + json.dump(out, open("out/cross-linux-x86_64.json", "w"), indent=1) + print(json.dumps(out["outcomes"])) + EOF + - uses: actions/upload-artifact@v7 + if: always() + with: + name: cross-linux-x86_64 + path: out/ + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68d1b89..6f61d7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,7 @@ jobs: cp "target/${{ matrix.target }}/release/pbit" "dist/$NAME/" fi cp README.md LICENSE CHANGELOG.md "dist/$NAME/" + cp -R examples python docs "dist/$NAME/" # JSON examples, the Python wrapper, the IR spec and docs/agents.md cd dist if [ "${{ matrix.archive }}" = "zip" ]; then ASSET="$NAME.zip"; 7z a "$ASSET" "$NAME" > /dev/null diff --git a/docs/agents.md b/docs/agents.md new file mode 100644 index 0000000..598965e --- /dev/null +++ b/docs/agents.md @@ -0,0 +1,97 @@ +# Calling pbit from anything + +`pbit` is one binary with a JSON contract: a document on stdin, one JSON document on stdout, and an exit code that says +what kind of answer it is. Anything that can start a process can use it: a shell, Python, Node, PowerShell, a CI job, or +an agent harness with a shell tool. No server, no bindings, no network. The document formats are in the README +("Use it from anything": the router document of `pbit decide`) and in [pbit-ir-json.md](pbit-ir-json.md) (programs for +`pbit run`). + +| exit | meaning | what to do | +|---|---|---| +| 0 | an answer: verdict `exact`, `diagnostics_passed` or `partial` | act on `released`; escalate `escalated` (empty unless `partial`) | +| 1 | `infeasible`: no plan satisfies the rules (a proof) | relax a rule or a cap; it is an answer, not a crash | +| 2 | bad input: one `{"error": {"code", "path", "message"}}` object on stdout; a bad flag prints one line on stderr instead | fix the document or the flag | +| 3 | `refused` / `declined` (the gate or a cap said no), or `{"error": {"code": "numeric"}}` | escalate the whole decision; the best-effort plan is still in the output | + +Every recipe below was run in the cross-platform matrix (`.github/workflows/bench.yml`; transcripts in +`scripts/bench/results/`), on the OSes named under each. + +## Shell (bash, zsh, sh) + +Run on Linux and macOS (and in Git Bash on Windows). + +```sh +pbit demo --tasks 12 | pbit decide --pretty # a synthetic 12-task routing document, decided +pbit decide --budget-ms 200 < router.json > decision.json # your document +case $? in 0) echo answer ;; 1) echo infeasible ;; 2) echo "bad input" ;; 3) echo "refused: escalate" ;; esac +``` + +Read fields with any JSON tool, e.g. `python3 -c 'import json,sys; print(json.load(sys.stdin)["verdict"])' < decision.json`. + +## Python + +Run on Linux, macOS and Windows: `python/pbit.py` is stdlib only (Python >= 3.9); `python3 python/test_pbit.py` is its test. + +```python +import pbit # python/pbit.py (copy it next to your code, or add python/ to sys.path) +decision = pbit.decide(pbit.demo(tasks=12), budget_ms=200) # exit 1 and 3 come back as answers +answer = pbit.run(program, deadline_ms=1000) # a pbit-ir program; bad input raises pbit.PbitInputError +``` + +The binary is found via `binary=`, `$PBIT_BIN`, `pbit` on `PATH`, or `../target/release/pbit` next to `pbit.py` +(on Windows set `PBIT_BIN` to the full path of `pbit.exe`). + +## Node + +Run on Linux, macOS and Windows: [examples/node/decide.mjs](../examples/node/decide.mjs), no dependencies (Node >= 18). +It exports `pbit(args, input)`, which spawns the binary, writes the JSON, and resolves to +`{kind: 'answer' | 'infeasible' | 'bad_input' | 'refused', exitCode, output, stderr}`. + +```sh +node examples/node/decide.mjs # a 12-task demo, then one call per exit code (0, 1, 2, 3) +node examples/node/decide.mjs router.json # decide your document +``` + +```js +import { pbit } from './decide.mjs'; +const r = await pbit(['decide', '--budget-ms', '200'], problem); // problem: an object or JSON text +if (r.kind === 'answer') act(r.output.plan, r.output.released); else escalate(r); +``` + +The npm package (`npm/`) installs the binary and a `pbit` command with exit codes passed through. On Windows, spawn +`pbit.exe` itself (set `PBIT_BIN`): Node cannot spawn npm's `pbit.cmd` shim without a shell. + +## PowerShell + +Run on Windows with PowerShell 7 (`pwsh`) and Windows PowerShell 5.1. + +```powershell +pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms +Get-Content router.json -Raw | pbit decide --budget-ms 200 | ConvertFrom-Json; $LASTEXITCODE # 0 / 1 / 2 / 3 +``` + +PowerShell 7.4 and later pipe bytes between native commands unchanged. Windows PowerShell 5.1 re-encodes text piped into +a native command with `$OutputEncoding` (ASCII by default), so a document with non-ASCII ids needs +`$OutputEncoding = [System.Text.UTF8Encoding]::new($false)` first. + +## For agents + +Coding agents with a shell tool (Claude Code, Codex, and other harnesses that can run commands) call pbit the same way a +script does: they run the command, read the JSON, and branch on the exit code. Nothing needs to be registered. A +paragraph like this one in the project's agent instructions (`CLAUDE.md`, `AGENTS.md`) is enough: + +> To route tasks to workers under hard rules (allowed workers, quotas) with odds, write a router document (README, "Use it +> from anything") and run `pbit decide --budget-ms 200 < doc.json`. Exit 0: act on `plan` for the ids in `released` and +> hand the ids in `escalated` to a human. Exit 1: no plan satisfies the rules. Exit 2: fix the document (the `error` +> object says where). Exit 3: escalate everything. Never read stderr as data. `pbit --help` lists every flag. + +Practical notes for agent use: + +- Bound the call: `--budget-ms` caps the sampler (the default answer takes ~300 ms on the 300-task demo), and + `pbit run --deadline-ms N` bounds a whole `run` call. Small documents (about 12 tasks by 6 workers) are answered exactly. +- Fixed work is reproducible: `--sweeps N --polish-ms 0` makes the answer a pure function of the document and `--seed`, + which is what a test or a replayed agent step wants. +- `pbit stats` prints the machine, the effective controls and a measured self-test; `--threads`, `--cpu-limit` and + `--priority low` keep it from crowding the agent's own process. +- An MCP server is planned for a later release and is not built yet; until then the shell command above is the + integration. diff --git a/examples/node/decide.mjs b/examples/node/decide.mjs new file mode 100644 index 0000000..1c979a2 --- /dev/null +++ b/examples/node/decide.mjs @@ -0,0 +1,87 @@ +// Call pbit from Node with no dependencies: one child process per call, JSON on stdin, JSON on stdout, the exit code mapped. +// node examples/node/decide.mjs # decides a 12-task demo document, then shows each exit code once +// node examples/node/decide.mjs router.json # decides your router document (README, "Use it from anything") +// The binary: $PBIT_BIN, else `pbit` on PATH. On Windows point PBIT_BIN at pbit.exe: Node cannot spawn the npm `pbit.cmd` +// shim without a shell. +import { execFileSync, spawn } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +const KIND = { 0: 'answer', 1: 'infeasible', 2: 'bad_input', 3: 'refused' }; + +/** + * Run `pbit ` with `input` (an object or JSON text; omit for none) on stdin. Resolves to {kind, exitCode, output, stderr}: + * 'answer' exit 0: verdict exact | diagnostics_passed | partial (act on output.released, escalate output.escalated) + * 'infeasible' exit 1: no plan satisfies the rules (a proof, not an error) + * 'bad_input' exit 2: output.error = {code, path, message} for a bad document; a bad flag leaves only a stderr line + * 'refused' exit 3: verdict refused / declined (escalate), or output.error.code === 'numeric' + * Rejects on a missing binary, a crash or signal, unparseable output, or after `timeoutMs`. + */ +export function pbit(args, input, { bin = process.env.PBIT_BIN || 'pbit', timeoutMs } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(bin, args, { stdio: ['pipe', 'pipe', 'pipe'] }); + const out = []; + const err = []; + const timer = timeoutMs && setTimeout(() => { child.kill(); reject(new Error(`pbit ${args[0]} passed ${timeoutMs} ms`)); }, timeoutMs); + child.stdout.on('data', (b) => out.push(b)); + child.stderr.on('data', (b) => err.push(b)); + child.on('error', (e) => { clearTimeout(timer); reject(e); }); // ENOENT: no binary + child.on('close', (code, signal) => { + clearTimeout(timer); + const stdout = Buffer.concat(out).toString('utf8'); + const stderr = Buffer.concat(err).toString('utf8').trim(); + if (!(code in KIND)) return reject(new Error(`pbit exited ${code ?? signal}: ${stderr.slice(0, 300)}`)); + let output = null; + try { + if (stdout.trim()) output = JSON.parse(stdout); + } catch { + return reject(new Error(`pbit ${args[0]}: unparseable output (exit ${code})`)); + } + resolve({ kind: KIND[code], exitCode: code, output, stderr }); + }); + child.stdin.on('error', () => {}); // EPIPE when pbit exits before reading all of it (a bad flag) + child.stdin.end(input === undefined ? undefined : typeof input === 'string' || Buffer.isBuffer(input) ? input : JSON.stringify(input)); + }); +} + +function describe(r) { + const o = r.output || {}; + if (r.kind === 'bad_input') return o.error ? `${o.error.code} at ${o.error.path}: ${o.error.message}` : r.stderr; + const plan = o.plan ? Object.entries(o.plan).slice(0, 3).map(([t, w]) => `${t}->${w}`).join(', ') : ''; + const released = Array.isArray(o.released) ? `, ${o.released.length} of ${o.tasks} released` : ''; + return `verdict ${o.verdict}${released}${plan ? `, plan ${plan}, ...` : ''}${o.reason ? ` (${o.reason})` : ''}`; +} + +async function main() { + const bin = process.env.PBIT_BIN || 'pbit'; + try { + console.log(`${execFileSync(bin, ['version'], { encoding: 'utf8' }).trim()} (${bin})`); + } catch (e) { + console.error(`cannot run ${bin} (${e.message}); install pbit or set PBIT_BIN`); + process.exit(127); + } + if (process.argv[2]) { + const r = await pbit(['decide', '--budget-ms', '200'], readFileSync(process.argv[2])); + console.log(`exit ${r.exitCode} ${r.kind}: ${describe(r)}`); + process.exit(r.exitCode); + } + const demo = (await pbit(['demo', '--tasks', '12'])).output; + const cases = [ + ['answer', 'a 12-task demo document', ['decide'], demo], + ['infeasible', 'two tasks, one worker with room for one', ['decide'], + { workers: [{ id: 'a', cap: 1 }], tasks: [{ id: 't1', allowed: ['a'], scores: { a: 1 } }, { id: 't2', allowed: ['a'], scores: { a: 1 } }] }], + ['bad_input', 'a malformed document', ['decide'], { workers: 1 }], + ['refused', 'exact enumeration of a 96-variable program, capped at 50 ms', ['run', '--op', 'exact', '--exact-ms', '50'], + readFileSync(new URL('../denoise-8x12.json', import.meta.url))], + ]; + let ok = true; + for (const [want, what, args, input] of cases) { + const r = await pbit(args, input); + ok &&= r.kind === want; + console.log(`exit ${r.exitCode} ${r.kind.padEnd(10)} ${what}: ${describe(r)}`); + } + console.log(ok ? 'all four exit codes mapped as expected' : 'UNEXPECTED exit-code mapping'); + process.exit(ok ? 0 : 1); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) main(); diff --git a/install.ps1 b/install.ps1 new file mode 100644 index 0000000..8ea3f57 --- /dev/null +++ b/install.ps1 @@ -0,0 +1,95 @@ +# pbit installer for Windows (Windows PowerShell 5.1 or PowerShell 7): fetches a prebuilt release archive, checks its SHA-256 +# and installs pbit.exe. No administrator rights needed. +# +# irm https://raw.githubusercontent.com/BitmapAsset/pbit/main/install.ps1 | iex +# & ([scriptblock]::Create((irm https://raw.githubusercontent.com/BitmapAsset/pbit/main/install.ps1))) -Version v0.2.0 +# +# Parameters (each also read from the environment variable named after it): +# -Version PBIT_VERSION release tag, e.g. v0.2.0 (default: the latest release) +# -InstallDir PBIT_INSTALL_DIR where pbit.exe goes (default: $HOME\.local\bin) +# -DownloadBase PBIT_DOWNLOAD_BASE the archive is fetched from //pbit--.zip +# (default: https://github.com/BitmapAsset/pbit/releases/download); needs -Version +# -Target PBIT_TARGET Rust target triple to fetch instead of the detected one +# -AddToPath PBIT_ADD_TO_PATH=1 also add InstallDir to your user PATH (otherwise only this session's PATH) +param( + [string]$Version = $env:PBIT_VERSION, + [string]$InstallDir = $env:PBIT_INSTALL_DIR, + [string]$DownloadBase = $env:PBIT_DOWNLOAD_BASE, + [string]$Target = $env:PBIT_TARGET, + [switch]$AddToPath = ($env:PBIT_ADD_TO_PATH -eq '1') +) +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' # the progress bar makes Invoke-WebRequest many times slower on 5.1 +$Repo = 'BitmapAsset/pbit' +if ($PSVersionTable.PSVersion.Major -lt 6) { + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 +} + +if (-not $Target) { + $arch = if ($env:PROCESSOR_ARCHITEW6432) { $env:PROCESSOR_ARCHITEW6432 } else { $env:PROCESSOR_ARCHITECTURE } + switch ($arch) { + 'AMD64' { $Target = 'x86_64-pc-windows-msvc' } + 'ARM64' { $Target = 'x86_64-pc-windows-msvc'; Write-Host 'pbit install: no native arm64 build yet; installing the x86_64 build (Windows runs it under emulation)' } + default { throw "pbit install: no prebuilt pbit for Windows on $arch" } + } +} +$customBase = [bool]$DownloadBase +if (-not $DownloadBase) { $DownloadBase = "https://github.com/$Repo/releases/download" } +$DownloadBase = $DownloadBase.TrimEnd('/') +if (-not $Version) { + if ($customBase) { throw 'pbit install: set -Version (or PBIT_VERSION), e.g. v0.2.0, together with -DownloadBase' } + $Version = (Invoke-RestMethod -UseBasicParsing -Uri "https://api.github.com/repos/$Repo/releases/latest").tag_name + if (-not $Version) { throw 'pbit install: no published release found (set -Version)' } +} +if (-not $Version.StartsWith('v')) { $Version = "v$Version" } +if (-not $InstallDir) { $InstallDir = Join-Path $HOME '.local\bin' } + +$name = "pbit-$Version-$Target" +$asset = "$name.zip" +$url = "$DownloadBase/$Version/$asset" +$tmp = Join-Path ([IO.Path]::GetTempPath()) ("pbit-install-" + [guid]::NewGuid().ToString('N')) +New-Item -ItemType Directory -Path $tmp | Out-Null +try { + Write-Host "pbit install: $Version for $Target" + Write-Host " fetching $url" + $zip = Join-Path $tmp $asset + Invoke-WebRequest -UseBasicParsing -Uri $url -OutFile $zip + Invoke-WebRequest -UseBasicParsing -Uri "$url.sha256" -OutFile "$zip.sha256" + $want = ((Get-Content -LiteralPath "$zip.sha256" -Raw).Trim() -split '\s+')[0].ToLowerInvariant() + $got = (Get-FileHash -Algorithm SHA256 -LiteralPath $zip).Hash.ToLowerInvariant() + if ($want -notmatch '^[0-9a-f]{64}$') { throw "pbit install: unreadable checksum file $asset.sha256" } + if ($want -ne $got) { throw "pbit install: checksum mismatch for ${asset}: expected $want, got $got (nothing was installed)" } + Write-Host " sha256 ok $got" + + Expand-Archive -LiteralPath $zip -DestinationPath (Join-Path $tmp 'x') -Force + $src = Join-Path $tmp "x\$name\pbit.exe" + if (-not (Test-Path -LiteralPath $src)) { throw "pbit install: $asset has no $name\pbit.exe" } + New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + $dest = Join-Path $InstallDir 'pbit.exe' + Copy-Item -LiteralPath $src -Destination $dest -Force + $ran = & $dest version + if ($LASTEXITCODE -ne 0) { throw "pbit install: installed $dest, but it does not run here: $ran" } + Write-Host " installed $dest ($ran)" +} finally { + Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue +} + +$full = (Resolve-Path -LiteralPath $InstallDir).Path.TrimEnd('\') +$onPath = { param($p) @($p -split ';' | Where-Object { $_ } | ForEach-Object { $_.TrimEnd('\') }) -contains $full } +if (-not (& $onPath $env:Path)) { $env:Path = "$full;$env:Path" } # this session +$userPath = [Environment]::GetEnvironmentVariable('Path', 'User') +if (-not (& $onPath $userPath)) { + if ($AddToPath) { + [Environment]::SetEnvironmentVariable('Path', ($(if ($userPath) { "$full;$userPath" } else { $full })), 'User') + Write-Host " added $full to your user PATH (new terminals see it)" + } else { + Write-Host '' + Write-Host "$full is on this session's PATH only. To keep it, run (or re-run this installer with -AddToPath):" + Write-Host " [Environment]::SetEnvironmentVariable('Path', '$full;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User')" + } +} +Write-Host '' +Write-Host 'Next:' +Write-Host ' pbit version' +Write-Host ' pbit demo --tasks 12 | pbit decide --pretty' +Write-Host ' pbit stats --pretty' diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..ebd2b70 --- /dev/null +++ b/install.sh @@ -0,0 +1,144 @@ +#!/bin/sh +# pbit installer: fetches a prebuilt release archive, checks its SHA-256 and installs the `pbit` binary. Never uses sudo. +# +# curl -fsSL https://raw.githubusercontent.com/BitmapAsset/pbit/main/install.sh | sh +# curl -fsSL https://raw.githubusercontent.com/BitmapAsset/pbit/main/install.sh | PBIT_VERSION=v0.2.0 sh +# +# Environment (all optional): +# PBIT_VERSION release tag, e.g. v0.2.0 (default: the latest release) +# PBIT_INSTALL_DIR where `pbit` goes (default: /usr/local/bin if you can write there, else ~/.local/bin) +# PBIT_DOWNLOAD_BASE the archive is fetched from $PBIT_DOWNLOAD_BASE//pbit--.tar.gz +# (default: https://github.com/BitmapAsset/pbit/releases/download); needs PBIT_VERSION +# PBIT_TARGET Rust target triple to fetch instead of the detected one +# Needs curl or wget, tar, and one of sha256sum, shasum or openssl. +set -eu + +REPO="BitmapAsset/pbit" + +say() { printf '%s\n' "$*"; } +die() { printf 'pbit install: error: %s\n' "$*" >&2; exit 1; } +has() { command -v "$1" > /dev/null 2>&1; } + +fetch() { # fetch URL FILE + if has curl; then + case "$1" in + https://*) curl --proto '=https' --proto-redir '=https' -fsSL "$1" -o "$2" ;; + *) curl -fsSL "$1" -o "$2" ;; + esac + elif has wget; then + wget -q -O "$2" "$1" + else + die "need curl or wget" + fi +} + +sha256() { + if has sha256sum; then sha256sum "$1" | cut -d ' ' -f 1 + elif has shasum; then shasum -a 256 "$1" | cut -d ' ' -f 1 + elif has openssl; then openssl dgst -sha256 "$1" | sed 's/^.*= *//' + else die "need sha256sum, shasum or openssl to check the download" + fi +} + +detect_target() { + os=$(uname -s) + arch=$(uname -m) + case "$os" in + Darwin) + # a shell running under Rosetta reports x86_64 on Apple silicon: take the native build + if [ "$arch" = x86_64 ] && [ "$(sysctl -n sysctl.proc_translated 2> /dev/null || echo 0)" = 1 ]; then arch=arm64; fi + case "$arch" in + arm64 | aarch64) echo aarch64-apple-darwin ;; + x86_64) echo x86_64-apple-darwin ;; + *) die "no prebuilt pbit for macOS on $arch" ;; + esac + ;; + Linux) + if (ldd --version 2>&1 || true) | grep -qi musl; then + die "this system uses musl libc (Alpine?) and the release ships a glibc build only; build from source: cargo install --git https://github.com/$REPO pbit-cli" + fi + case "$arch" in + x86_64 | amd64) echo x86_64-unknown-linux-gnu ;; + *) die "no prebuilt pbit for Linux on $arch yet; build from source: cargo install --git https://github.com/$REPO pbit-cli" ;; + esac + ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) die "on Windows use install.ps1 (PowerShell)" ;; + *) die "no prebuilt pbit for $os; build from source: cargo install --git https://github.com/$REPO pbit-cli" ;; + esac +} + +latest_tag() { + api="https://api.github.com/repos/$REPO/releases/latest" + fetch "$api" "$tmp/latest.json" || die "could not look up the latest release at $api (set PBIT_VERSION)" + sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p' "$tmp/latest.json" | head -n 1 +} + +main() { + base="${PBIT_DOWNLOAD_BASE:-https://github.com/$REPO/releases/download}" + base="${base%/}" + target="${PBIT_TARGET:-}" + if [ -z "$target" ]; then target=$(detect_target) || exit 1; fi + + tmp=$(mktemp -d 2> /dev/null || mktemp -d -t pbit) + trap 'rm -rf "$tmp"' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + version="${PBIT_VERSION:-}" + if [ -z "$version" ]; then + [ -z "${PBIT_DOWNLOAD_BASE:-}" ] || die "set PBIT_VERSION (e.g. v0.2.0) together with PBIT_DOWNLOAD_BASE" + version=$(latest_tag) || exit 1 + [ -n "$version" ] || die "no published release found (set PBIT_VERSION)" + fi + case "$version" in v*) ;; *) version="v$version" ;; esac + + name="pbit-$version-$target" + asset="$name.tar.gz" + url="$base/$version/$asset" + say "pbit install: $version for $target" + say " fetching $url" + fetch "$url" "$tmp/$asset" || die "download failed: $url" + fetch "$url.sha256" "$tmp/$asset.sha256" || die "download failed: $url.sha256" + + want=$(cut -d ' ' -f 1 < "$tmp/$asset.sha256" | tr 'ABCDEF' 'abcdef') + got=$(sha256 "$tmp/$asset" | tr 'ABCDEF' 'abcdef') + case "$want" in *[!0-9a-f]* | '') die "unreadable checksum file $asset.sha256" ;; esac + [ "${#want}" -eq 64 ] || die "unreadable checksum file $asset.sha256" + [ "$want" = "$got" ] || die "checksum mismatch for $asset: expected $want, got $got (nothing was installed)" + say " sha256 ok $got" + + tar -xzf "$tmp/$asset" -C "$tmp" || die "could not unpack $asset" + [ -f "$tmp/$name/pbit" ] || die "$asset has no $name/pbit" + + if [ -n "${PBIT_INSTALL_DIR:-}" ]; then + dir="$PBIT_INSTALL_DIR" + elif [ -d /usr/local/bin ] && [ -w /usr/local/bin ]; then + dir=/usr/local/bin + else + dir="${HOME:?HOME is not set; set PBIT_INSTALL_DIR}/.local/bin" + fi + mkdir -p "$dir" 2> /dev/null || die "cannot create $dir (set PBIT_INSTALL_DIR to a directory you can write)" + [ -w "$dir" ] || die "cannot write to $dir (set PBIT_INSTALL_DIR to a directory you can write; this script never uses sudo)" + cp "$tmp/$name/pbit" "$dir/.pbit.$$" && chmod 755 "$dir/.pbit.$$" && mv -f "$dir/.pbit.$$" "$dir/pbit" \ + || die "could not install into $dir" + ran=$("$dir/pbit" version 2>&1) || die "installed $dir/pbit, but it does not run here: $ran" + say " installed $dir/pbit ($ran)" + + cmd=pbit + case ":$PATH:" in + *":$dir:"*) ;; + *) + cmd="$dir/pbit" + say "" + say "$dir is not on your PATH. Add it (e.g. in ~/.profile):" + say " export PATH=\"$dir:\$PATH\"" + ;; + esac + say "" + say "Next:" + say " $cmd version" + say " $cmd demo --tasks 12 | $cmd decide --pretty" + say " $cmd stats --pretty" +} + +main "$@" diff --git a/npm/LICENSE b/npm/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/npm/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/npm/README.md b/npm/README.md new file mode 100644 index 0000000..c90b3e7 --- /dev/null +++ b/npm/README.md @@ -0,0 +1,29 @@ +# pbit (npm) + +Installs the prebuilt `pbit` command-line binary: a virtual p-bit processor that returns joint decisions under hard +rules, with gated odds. JSON in, JSON out. Source, documentation and every benchmark: +. + +```sh +npm install -g pbit +pbit demo --tasks 12 | pbit decide --pretty +pbit stats --pretty +``` + +The install step fetches the release archive for your platform from GitHub Releases (macOS arm64 and x86_64, Linux +x86_64 with glibc, Windows x86_64), checks it against the `.sha256` file published beside it and keeps the binary inside +this package. No dependencies; Node >= 18. If the install step could not run (`--ignore-scripts`, offline), the first +`pbit` call fetches the binary. A checksum mismatch fails the install and installs nothing. + +| variable | effect | +|---|---| +| `PBIT_VERSION` | release tag to fetch (default: `v` + this package's version) | +| `PBIT_DOWNLOAD_BASE` | fetch `$PBIT_DOWNLOAD_BASE//pbit--.tar.gz` (`.zip` on Windows) instead of GitHub | +| `PBIT_TARGET` | Rust target triple to fetch instead of the detected one | +| `PBIT_BINARY` | at install: copy this binary instead of downloading; at run time: run this binary | + +`pbit`'s exit code passes through unchanged: 0 answer (`exact`, `diagnostics_passed`, `partial`), 1 `infeasible` +(a proof), 2 bad input or flag, 3 `refused` / `declined`. Calling it from Node code (spawn, JSON on stdin, exit codes +mapped): `examples/node/decide.mjs` in the repository; from other languages and agent harnesses: `docs/agents.md`. + +License: Apache-2.0. diff --git a/npm/bin/pbit.js b/npm/bin/pbit.js new file mode 100755 index 0000000..6b98aec --- /dev/null +++ b/npm/bin/pbit.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node +'use strict'; +// `pbit` for npm installs: runs the native binary with this process's stdin/stdout/stderr and exits with its exit code +// (0 answer, 1 infeasible, 2 bad input, 3 refused / declined; see `pbit decide --help`). PBIT_BINARY=/path/to/pbit +// runs that binary instead. A missing binary (an install with --ignore-scripts) is fetched on first run. +const fs = require('fs'); +const os = require('os'); +const { spawn } = require('child_process'); +const { install, binaryPath } = require('../install.js'); + +async function main() { + let bin = process.env.PBIT_BINARY || binaryPath(); + if (!process.env.PBIT_BINARY && !fs.existsSync(bin)) { + process.stderr.write('pbit: the binary is not installed yet, fetching it now\n'); + bin = await install(); + } + const child = spawn(bin, process.argv.slice(2), { stdio: 'inherit' }); + const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'].filter((s) => process.platform !== 'win32' || s !== 'SIGHUP'); + const forward = (s) => { try { child.kill(s); } catch (e) { /* already gone */ } }; + for (const s of signals) process.on(s, forward); + child.on('error', (e) => { + process.stderr.write(`pbit: cannot run ${bin}: ${e.message}\n`); + process.exit(127); + }); + child.on('exit', (code, signal) => { + if (signal) { + for (const s of signals) process.removeListener(s, forward); + process.exitCode = 128 + (os.constants.signals[signal] || 0); // if the signal is ignored here (SIGPIPE) + process.kill(process.pid, signal); // die the same way, so the shell sees 128 + n + return; + } + process.exit(code === null ? 1 : code); + }); +} + +main().catch((e) => { + process.stderr.write(`pbit: ${e.message}\n`); + process.exit(127); +}); diff --git a/npm/install.js b/npm/install.js new file mode 100644 index 0000000..7ab460c --- /dev/null +++ b/npm/install.js @@ -0,0 +1,113 @@ +'use strict'; +// Fetches the prebuilt pbit binary for this platform from the GitHub release that matches this package's version, checks its +// SHA-256 against the .sha256 file beside it and puts it in vendor/. Runs as `postinstall`; bin/pbit.js calls it again on +// first use if the binary is missing (installs with --ignore-scripts). No dependencies: Node >= 18 (global fetch) and the +// system `tar` (Windows 10+ ships tar.exe, which also unpacks .zip). +// +// Environment (same names as install.sh): +// PBIT_VERSION release tag to fetch (default: v) +// PBIT_DOWNLOAD_BASE archive URL = $PBIT_DOWNLOAD_BASE//pbit--. +// (default: https://github.com/BitmapAsset/pbit/releases/download) +// PBIT_TARGET Rust target triple to fetch instead of the detected one +// PBIT_BINARY path to an existing pbit binary: copied into vendor/ instead of downloading +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const { execFileSync } = require('child_process'); +const pkg = require('./package.json'); + +const REPO = 'BitmapAsset/pbit'; +const TARGETS = { + 'darwin-arm64': 'aarch64-apple-darwin', + 'darwin-x64': 'x86_64-apple-darwin', + 'linux-x64': 'x86_64-unknown-linux-gnu', + 'win32-x64': 'x86_64-pc-windows-msvc', + 'win32-arm64': 'x86_64-pc-windows-msvc', // x64 emulation on Windows on Arm +}; +const EXE = process.platform === 'win32' ? 'pbit.exe' : 'pbit'; + +function binaryPath() { + return path.join(__dirname, 'vendor', EXE); +} + +class ChecksumError extends Error {} + +async function get(url) { + const res = await fetch(url, { redirect: 'follow' }); + if (!res.ok) throw new Error(`GET ${url}: HTTP ${res.status}`); + return Buffer.from(await res.arrayBuffer()); +} + +// copy then rename, so a concurrent first run never sees a half-written binary +function place(src, dest) { + fs.mkdirSync(path.dirname(dest), { recursive: true }); + const tmp = `${dest}.${process.pid}.tmp`; + fs.copyFileSync(src, tmp); + fs.chmodSync(tmp, 0o755); + try { + fs.renameSync(tmp, dest); + } catch (e) { + fs.rmSync(tmp, { force: true }); + if (!fs.existsSync(dest)) throw e; // another process put it there first + } +} + +async function install(log = (m) => process.stderr.write(`${m}\n`)) { + const env = process.env; + const dest = binaryPath(); + if (env.PBIT_BINARY) { + const src = path.resolve(env.PBIT_BINARY); + place(src, dest); + log(`pbit: using ${src} (PBIT_BINARY)`); + return dest; + } + const target = env.PBIT_TARGET || TARGETS[`${process.platform}-${process.arch}`]; + if (!target) { + throw new Error(`no prebuilt pbit for ${process.platform}-${process.arch}; build one (cargo build --release -p pbit-cli) ` + + 'and reinstall with PBIT_BINARY=/path/to/pbit'); + } + let tag = env.PBIT_VERSION || `v${pkg.version}`; + if (!tag.startsWith('v')) tag = `v${tag}`; + const base = (env.PBIT_DOWNLOAD_BASE || `https://github.com/${REPO}/releases/download`).replace(/\/+$/, ''); + const zip = target.includes('windows'); + const name = `pbit-${tag}-${target}`; + const asset = `${name}.${zip ? 'zip' : 'tar.gz'}`; + const url = `${base}/${tag}/${asset}`; + log(`pbit: fetching ${url}`); + const [archive, sums] = await Promise.all([get(url), get(`${url}.sha256`)]); + const want = sums.toString('utf8').trim().split(/\s+/)[0].toLowerCase(); + const got = crypto.createHash('sha256').update(archive).digest('hex'); + if (!/^[0-9a-f]{64}$/.test(want) || want !== got) { + throw new ChecksumError(`checksum mismatch for ${asset}: expected ${want}, got ${got}`); + } + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pbit-npm-')); + try { + const file = path.join(tmp, asset); + fs.writeFileSync(file, archive); + // Windows: call System32\tar.exe (bsdtar, reads .zip) by full path; a GNU tar earlier on PATH (Git) cannot + const tar = process.platform === 'win32' ? path.join(env.SystemRoot || 'C:\\Windows', 'System32', 'tar.exe') : 'tar'; + execFileSync(tar, [zip ? '-xf' : '-xzf', file, '-C', tmp], { stdio: ['ignore', 'ignore', 'inherit'] }); + const src = path.join(tmp, name, EXE); + if (!fs.existsSync(src)) throw new Error(`${asset} has no ${name}/${EXE}`); + place(src, dest); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + log(`pbit: installed ${tag} (${target}), sha256 ${got}`); + return dest; +} + +module.exports = { install, binaryPath, ChecksumError }; + +if (require.main === module) { + install().catch((e) => { + if (e instanceof ChecksumError) { + process.stderr.write(`pbit: ${e.message}; nothing was installed\n`); + process.exit(1); + } + // offline or no release yet: do not fail the npm install; bin/pbit.js retries on first run + process.stderr.write(`pbit: could not fetch the binary now (${e.message}); it will be fetched on first run, ` + + 'or reinstall with PBIT_BINARY=/path/to/pbit\n'); + }); +} diff --git a/npm/package.json b/npm/package.json new file mode 100644 index 0000000..d15e369 --- /dev/null +++ b/npm/package.json @@ -0,0 +1,24 @@ +{ + "name": "pbit", + "version": "0.2.0", + "description": "pbit, a virtual p-bit processor for joint decisions under hard rules: installs the prebuilt pbit CLI (JSON in, JSON out).", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/BitmapAsset/pbit.git", + "directory": "npm" + }, + "homepage": "https://github.com/BitmapAsset/pbit#readme", + "bugs": "https://github.com/BitmapAsset/pbit/issues", + "keywords": ["p-bit", "probabilistic", "decision", "constraints", "sampling", "cli"], + "bin": { + "pbit": "bin/pbit.js" + }, + "scripts": { + "postinstall": "node install.js" + }, + "files": ["bin/", "install.js", "README.md", "LICENSE"], + "engines": { + "node": ">=18" + } +} diff --git a/pbit-cli/Cargo.toml b/pbit-cli/Cargo.toml index d5f40b8..26025bf 100644 --- a/pbit-cli/Cargo.toml +++ b/pbit-cli/Cargo.toml @@ -5,6 +5,9 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +readme = "../README.md" +keywords = ["p-bit", "cli", "decision", "constraints", "probabilistic"] +categories = ["command-line-utilities", "algorithms", "science"] [[bin]] name = "pbit" diff --git a/pbit-core/Cargo.toml b/pbit-core/Cargo.toml index 0550fe6..da68e65 100644 --- a/pbit-core/Cargo.toml +++ b/pbit-core/Cargo.toml @@ -5,5 +5,8 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +readme = "../README.md" +keywords = ["p-bit", "ising", "monte-carlo", "philox", "probabilistic"] +categories = ["algorithms", "science", "simulation"] [dependencies] diff --git a/pbit-decide/Cargo.toml b/pbit-decide/Cargo.toml index 6ac47df..c732e42 100644 --- a/pbit-decide/Cargo.toml +++ b/pbit-decide/Cargo.toml @@ -5,6 +5,9 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +readme = "../README.md" +keywords = ["p-bit", "assignment", "routing", "constraints", "probabilistic"] +categories = ["algorithms", "science"] [dependencies] pbit-core = { path = "../pbit-core" } diff --git a/pbit-ir/Cargo.toml b/pbit-ir/Cargo.toml index 7dfe68f..52a4858 100644 --- a/pbit-ir/Cargo.toml +++ b/pbit-ir/Cargo.toml @@ -5,6 +5,9 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +readme = "../README.md" +keywords = ["p-bit", "constraints", "gibbs-sampling", "probabilistic", "ising"] +categories = ["algorithms", "science", "mathematics"] [dependencies] pbit-core = { path = "../pbit-core" } diff --git a/scripts/bench/callers.ps1 b/scripts/bench/callers.ps1 new file mode 100644 index 0000000..652b219 --- /dev/null +++ b/scripts/bench/callers.ps1 @@ -0,0 +1,27 @@ +# The PowerShell recipes of docs/agents.md, run as written, plus one call per exit code. `pbit` must be on PATH. +# pwsh -File scripts/bench/callers.ps1 powershell -ExecutionPolicy Bypass -File scripts/bench/callers.ps1 +# (No 2>$null on native calls: Windows PowerShell 5.1 turns redirected stderr into errors.) +$ErrorActionPreference = 'Continue' +"PowerShell $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition)), pbit at $((Get-Command pbit).Source)" + +'> pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms' +pbit demo --tasks 12 | pbit decide | ConvertFrom-Json | Select-Object verdict, violations, ms | Format-List | Out-String -Width 200 +if ($LASTEXITCODE -ne 0) { throw "demo pipeline: exit $LASTEXITCODE" } + +pbit demo --tasks 24 | Set-Content -Encoding ascii router.json +'> Get-Content router.json -Raw | pbit decide --budget-ms 200 | ConvertFrom-Json; $LASTEXITCODE' +$d = Get-Content router.json -Raw | pbit decide --budget-ms 200 | ConvertFrom-Json +$code = $LASTEXITCODE +"verdict $($d.verdict), released $(@($d.released).Count) of $($d.tasks), violations $($d.violations); exit $code" +if ($code -ne 0) { throw "router.json: exit $code" } + +'> one call per exit code (infeasible, bad input, declined)' +'{"workers":[{"id":"a","cap":1}],"tasks":[{"id":"t1","allowed":["a"],"scores":{"a":1}},{"id":"t2","allowed":["a"],"scores":{"a":1}}]}' | pbit decide | Out-Null +$r1 = $LASTEXITCODE +'{"workers": 1}' | pbit decide | Out-Null +$r2 = $LASTEXITCODE +Get-Content examples/denoise-8x12.json -Raw | pbit run --op exact --exact-ms 50 | Out-Null +$r3 = $LASTEXITCODE +"exit codes: infeasible $r1 (want 1), bad input $r2 (want 2), declined $r3 (want 3)" +if ($r1 -ne 1 -or $r2 -ne 2 -or $r3 -ne 3) { throw 'unexpected exit codes' } +'PowerShell recipes: all passed' diff --git a/scripts/bench/matrix_bench.py b/scripts/bench/matrix_bench.py new file mode 100644 index 0000000..0866d19 --- /dev/null +++ b/scripts/bench/matrix_bench.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""One row of BENCHMARK-MATRIX.md: machine facts, build facts and pbit timings for one OS/arch, written as one JSON file. + + python3 scripts/bench/matrix_bench.py --bin target/release/pbit --label linux-x86_64 --out bench-linux-x86_64.json \ + [--native-dir target-native/release] [--runs 5] [--attach KEY=FILE ...] + +Stdlib only, Python >= 3.8, Linux / macOS / Windows. Run from the repo root after +`cargo build --release --locked --workspace` and `cargo build --release --locked -p pbit-core --example kernels`. +`--native-dir` names a `-C target-cpu=native` build of the same two targets; with it, bench/portable_vs_native.py runs too. + +Every timing is N runs (default 5): median [IQR] (statistics.quantiles, n=4, as in bench/*.py) plus every raw value. +Wall clock is taken outside pbit (time.perf_counter around the processes); `pbit_ms` is what pbit reports itself. +Peak RSS of the 300-task decide: /usr/bin/time -l (macOS, "maximum resident set size", bytes), /usr/bin/time -v (Linux, +"Maximum resident set size", KiB), PeakWorkingSetSize from K32GetProcessMemoryInfo on the process handle (Windows; the +handle is still open after the wait). Absolute paths of this machine are scrubbed from the output. +""" +import argparse, json, os, platform, re, shutil, statistics, subprocess, sys, tempfile, time + +KLINE = re.compile(r"^(.+?)\s*:\s*([0-9.]+e[+-]?[0-9]+|[0-9.]+) updates/s") +NUM = r"([0-9.]+(?:e[+-]?[0-9]+)?)" +PVN = re.compile(r"^(.+?): portable " + NUM + r" \[" + NUM + "-" + NUM + r"\] \| native " + NUM + r" \[" + NUM + "-" + NUM + + r"\] \| ratio ([0-9.]+)$") +DT = re.compile(r"^--threads (\d+): wall ms ([0-9.]+) \[([0-9.]+)-([0-9.]+)\], CPU ms ([0-9.]+), site updates/s ([0-9.]+) M;" + r" answer identical: yes; verdict (\S+)$") +DD = re.compile(r"^default path \(--budget-ms 300, 4 chains, threads (\d+)\): wall ms ([0-9.]+) \[([0-9.]+)-([0-9.]+)\]," + r" sweeps (\d+), verdict (\S+)$") +# BENCHMARKS.md §1 rows, from pbit-core/examples/kernels.rs +KERNEL_CELLS = {"kernel_multispin_fast_4t": "fast multispin splitmix x 4 thr", "kernel_multispin_fast_10t": "fast multispin splitmix x10 thr", + "kernel_multispin_fast_1t": "fast T0 multispin, splitmix", "kernel_heatbath_fast_1t": "fast T1 f32 LUT, splitmix"} + + +def sh(args, input=None, env=None, timeout=900): + try: + p = subprocess.run(args, input=input, capture_output=True, text=True, env=env, timeout=timeout) + return p.returncode, p.stdout, p.stderr + except Exception as e: # tool missing, timeout + return None, "", "%s: %s" % (type(e).__name__, e) + + +def summary(xs): + xs = [x for x in xs if isinstance(x, (int, float))] + if not xs: + return None + q = statistics.quantiles(xs, n=4) if len(xs) > 1 else [xs[0]] * 3 + return {"median": statistics.median(xs), "q1": q[0], "q3": q[2], "min": min(xs), "max": max(xs), "n": len(xs), "values": xs} + + +def load(): + up = sh(["uptime"])[1].strip() if shutil.which("uptime") else None + return {"loadavg": list(os.getloadavg()) if hasattr(os, "getloadavg") else None, "uptime": up, "t": time.strftime("%H:%M:%S")} + + +def exe(path): + return path + ".exe" if platform.system() == "Windows" and not path.endswith(".exe") else path + + +def machine(): + m = {"system": platform.system(), "os_release": platform.release(), "arch": platform.machine(), + "python": platform.python_version(), "logical_cores": os.cpu_count()} + s = m["system"] + if s == "Darwin": + rc, out, _ = sh(["sysctl", "-n", "machdep.cpu.brand_string", "hw.ncpu", "hw.memsize"]) + v = out.strip().splitlines() + m["cpu_cmd"] = "sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize" + m["cpu_model"] = v[0] if v else None + m["hw_ncpu"] = int(v[1]) if len(v) > 1 else None + m["memory_bytes"] = int(v[2]) if len(v) > 2 else None + rc, out, _ = sh(["sysctl", "-n", "hw.perflevel0.logicalcpu", "hw.perflevel1.logicalcpu"]) + if rc == 0: + v = out.split(); m["perf_cores"], m["efficiency_cores"] = int(v[0]), (int(v[1]) if len(v) > 1 else 0) + rc, out, _ = sh(["sysctl", "-n", "machdep.cpu.features", "machdep.cpu.leaf7_features"]) + if rc == 0: + f = set(out.lower().split()); m["cpu_flags"] = {k: k in f for k in ["sse4.2", "avx1.0", "avx2", "fma", "bmi2", "avx512f"]} + rc, out, _ = sh(["sysctl", "-n", "sysctl.proc_translated"]) + m["rosetta_translated"] = out.strip() == "1" if rc == 0 else None + m["os_version"] = sh(["sw_vers", "-productVersion"])[1].strip() + elif s == "Linux": + rc, out, _ = sh(["lscpu"]) + kv = dict((a.strip(), b.strip()) for a, b in (l.split(":", 1) for l in out.splitlines() if ":" in l)) + m["cpu_cmd"] = "lscpu" + m["cpu_model"] = kv.get("Model name") + m["lscpu"] = {k: kv.get(k) for k in ["Architecture", "CPU(s)", "Thread(s) per core", "Core(s) per socket", "Socket(s)", + "Vendor ID", "Model name", "Hypervisor vendor", "L3"] if k in kv} + f = set(kv.get("Flags", "").split()) + m["cpu_flags"] = {k: k in f for k in ["sse4_2", "avx", "avx2", "fma", "bmi2", "avx512f", "avx512bw", "avx512vl"]} + try: + with open("/proc/meminfo") as fh: + m["memory_bytes"] = int(re.search(r"MemTotal:\s+(\d+) kB", fh.read()).group(1)) * 1024 + with open("/etc/os-release") as fh: + m["os_version"] = re.search(r'PRETTY_NAME="([^"]*)"', fh.read()).group(1) + except Exception as e: + m["meminfo_error"] = str(e) + m["libc"] = " ".join(platform.libc_ver()) + elif s == "Windows": + ps = ("$p = Get-CimInstance Win32_Processor | Select-Object -First 1; $c = Get-CimInstance Win32_ComputerSystem; " + "$o = Get-CimInstance Win32_OperatingSystem; [pscustomobject]@{Name = $p.Name; Cores = $p.NumberOfCores; " + "Logical = $p.NumberOfLogicalProcessors; Mem = $c.TotalPhysicalMemory; Os = $o.Caption; Build = $o.BuildNumber} | ConvertTo-Json") + rc, out, err = sh(["powershell", "-NoProfile", "-NonInteractive", "-Command", ps]) + m["cpu_cmd"] = "Get-CimInstance Win32_Processor" + try: + j = json.loads(out) + m.update(cpu_model=(j["Name"] or "").strip(), physical_cores=j["Cores"], cim_logical=j["Logical"], + memory_bytes=int(j["Mem"]), os_version="%s (build %s)" % (j["Os"], j["Build"])) + except Exception as e: + m["cim_error"] = "%s %s" % (e, err[:300]) + return m + + +def runner(): + keys = ["RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "GITHUB_JOB", "GITHUB_SHA", + "GITHUB_REF_NAME", "GITHUB_WORKFLOW"] + return {k: os.environ[k] for k in keys if k in os.environ} or None + + +def toolchain(): + rc, out, _ = sh(["rustc", "-vV"]) + t = dict((a.strip(), b.strip()) for a, b in (l.split(":", 1) for l in out.splitlines() if ":" in l)) + t["rustc"] = out.splitlines()[0] if out else None + t["cargo"] = sh(["cargo", "-V"])[1].strip() + return t + + +def linkage(bin_): + s = platform.system(); r = {} + if shutil.which("file"): + r["file"] = sh(["file", "-b", bin_])[1].strip() + if s == "Darwin": + r["otool_L"] = sh(["otool", "-L", bin_])[1].strip().splitlines()[1:] + r["lipo_archs"] = sh(["lipo", "-archs", bin_])[1].strip() + elif s == "Linux": + rc, out, err = sh(["ldd", bin_]); r["ldd"] = (out + err).strip().splitlines() + rc, out, _ = sh(["objdump", "-T", bin_]) + vers = sorted(set(re.findall(r"GLIBC_([0-9.]+)", out)), key=lambda v: [int(x) for x in v.split(".")]) + r["glibc_symbol_versions"] = vers + r["min_glibc"] = vers[-1] if vers else None + elif s == "Windows": + dump, how = shutil.which("dumpbin"), "PATH" + if not dump: + vsw = os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), "Microsoft Visual Studio", "Installer", "vswhere.exe") + if os.path.exists(vsw): + rc, out, _ = sh([vsw, "-latest", "-products", "*", "-find", r"VC\Tools\MSVC\**\bin\Hostx64\x64\dumpbin.exe"]) + cand = [l.strip() for l in out.splitlines() if l.strip().lower().endswith("dumpbin.exe")] + dump, how = (cand[0], "vswhere (not on PATH)") if cand else (None, None) + r["dumpbin_found_via"] = how if dump else "not found: dumpbin is not on PATH and vswhere found none" + if dump: + rc, out, err = sh([dump, "/nologo", "/dependents", bin_]) + r["dll_imports"] = [l.strip() for l in out.splitlines() if l.strip().lower().endswith(".dll")] + r["dumpbin_rc"] = rc + return r + + +def run_json(args, input=None): + rc, out, err = sh(args, input=input) + try: + return rc, json.loads(out), err + except Exception: + return rc, None, (err or "")[-500:] + " | stdout: " + (out or "")[:300] + + +def pipeline(bin_, demo_args, decide_args): + t0 = time.perf_counter() + p1 = subprocess.Popen([bin_, "demo"] + demo_args, stdout=subprocess.PIPE) + p2 = subprocess.Popen([bin_, "decide"] + decide_args, stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p1.stdout.close() + out, err = p2.communicate() + rc1 = p1.wait() + wall = (time.perf_counter() - t0) * 1e3 + try: + d = json.loads(out.decode("utf-8")) + except Exception: + d = None + return wall, rc1, p2.returncode, d, err.decode("utf-8", "replace")[-300:] + + +def win_peak(handle): + import ctypes + from ctypes import wintypes + + class PMC(ctypes.Structure): + _fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD)] + [(n, ctypes.c_size_t) for n in ( + "PeakWorkingSetSize", "WorkingSetSize", "QuotaPeakPagedPoolUsage", "QuotaPagedPoolUsage", "QuotaPeakNonPagedPoolUsage", + "QuotaNonPagedPoolUsage", "PagefileUsage", "PeakPagefileUsage")] + k32 = ctypes.WinDLL("kernel32", use_last_error=True) + k32.K32GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(PMC), wintypes.DWORD] + k32.K32GetProcessMemoryInfo.restype = wintypes.BOOL + pmc = PMC(); pmc.cb = ctypes.sizeof(PMC) + if not k32.K32GetProcessMemoryInfo(int(handle), ctypes.byref(pmc), pmc.cb): + raise OSError("K32GetProcessMemoryInfo failed, error %d" % ctypes.get_last_error()) + return pmc.PeakWorkingSetSize, pmc.PeakPagefileUsage + + +def peak_rss(bin_, doc, args): + """One run of `pbit decide args < doc` under the OS tool: (bytes, method, pbit's own peak_rss_mb, verdict, extra).""" + s = platform.system(); extra = {} + with open(doc, "rb") as f: + if s == "Windows": + p = subprocess.Popen([bin_, "decide"] + args, stdin=f, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + peak, peak_commit = win_peak(p._handle) + extra["peak_pagefile_bytes"] = peak_commit + method = "K32GetProcessMemoryInfo PeakWorkingSetSize (process handle, after exit)" + val = peak + elif s == "Darwin" or os.path.exists("/usr/bin/time"): + flag = "-l" if s == "Darwin" else "-v" + p = subprocess.run(["/usr/bin/time", flag, bin_, "decide"] + args, stdin=f, capture_output=True) + out, err = p.stdout, p.stderr + if s == "Darwin": + m = re.search(rb"(\d+)\s+maximum resident set size", err); val = int(m.group(1)) if m else None + fp = re.search(rb"(\d+)\s+peak memory footprint", err) + if fp: extra["peak_memory_footprint_bytes"] = int(fp.group(1)) + method = "/usr/bin/time -l (maximum resident set size, bytes)" + else: + m = re.search(rb"Maximum resident set size \(kbytes\): (\d+)", err); val = int(m.group(1)) * 1024 if m else None + method = "/usr/bin/time -v (Maximum resident set size, KiB x 1024)" + else: + p = subprocess.run([bin_, "decide"] + args, stdin=f, capture_output=True) + out, val, method = p.stdout, None, "none: /usr/bin/time missing" + try: + d = json.loads(out.decode("utf-8")) + except Exception: + d = {} + tel = d.get("telemetry") or {} + return val, method, tel.get("peak_rss_mb"), d.get("verdict"), extra + + +def parse_kernels(text): + r = {} + for line in text.splitlines(): + m = KLINE.match(line) + if m: + r[m.group(1).strip()] = float(m.group(2)) + if line.startswith("chk"): + r["chk"] = line.split()[1] + return r + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--bin", required=True); ap.add_argument("--label", required=True); ap.add_argument("--out", required=True) + ap.add_argument("--native-dir"); ap.add_argument("--runs", type=int, default=5) + ap.add_argument("--attach", action="append", default=[], help="KEY=FILE: embed FILE (JSON, else text) under attached.KEY") + ap.add_argument("--note", action="append", default=[]) + a = ap.parse_args() + bin_ = exe(a.bin); pdir = os.path.dirname(bin_) or "." + N = a.runs; t_start = time.time() + R = {"schema": "pbit-bench-matrix/1", "label": a.label, "date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "git_sha": sh(["git", "rev-parse", "HEAD"])[1].strip() or None, "runner": runner(), "notes": a.note, + "machine": machine(), "toolchain": toolchain(), "load": {"start": load()}, "errors": []} + print("machine:", R["machine"].get("cpu_model"), "| logical cores", R["machine"]["logical_cores"], "| load", R["load"]["start"]["loadavg"], flush=True) + + R["binary"] = {"path": a.bin, "size_bytes": os.path.getsize(bin_), "version": sh([bin_, "version"])[1].strip(), "linkage": linkage(bin_)} + + # pbit stats: the IR self-test (400-spin ring, fixed work), 1 thread and default threads + st = [] + for _ in range(N): + rc, j, err = run_json([bin_, "stats"]) + if j is None: + R["errors"].append("stats rc=%s %s" % (rc, err)); continue + st.append(j) + if st: + R["stats_first"] = {k: st[0].get(k) for k in ("engine", "machine", "build", "controls")} + R["stats"] = {"site_updates_per_s_1_thread": summary([j["self_test"]["site_updates_per_s_1_thread"] for j in st]), + "site_updates_per_s": summary([j["self_test"]["site_updates_per_s"] for j in st]), + "threads": sorted(set(j["self_test"]["threads"] for j in st)), + "speedup": summary([j["self_test"]["speedup"] for j in st])} + R["load"]["after_stats"] = load() + + # pbit demo --tasks 12 | pbit decide, and pbit demo --tasks 300 | pbit decide --budget-ms 300 + for key, demo, dec in [("decide_12", ["--tasks", "12"], []), ("decide_300", ["--tasks", "300"], ["--budget-ms", "300"])]: + runs = [] + for _ in range(N): + wall, rc1, rc2, d, err = pipeline(bin_, demo, dec) + d = d or {}; tel = d.get("telemetry") or {} + runs.append({"wall_ms": wall, "demo_exit": rc1, "decide_exit": rc2, "pbit_ms": d.get("ms"), "verdict": d.get("verdict"), + "tier": d.get("tier"), "released": len(d.get("released") or []), "escalated": len(d.get("escalated") or []), + "violations": d.get("violations"), "sweeps": tel.get("sweeps"), "site_updates_per_s": tel.get("site_updates_per_s"), + "threads": tel.get("threads"), "peak_rss_mb_self": tel.get("peak_rss_mb"), "stderr": err or None}) + R[key] = {"cmd": "pbit demo %s | pbit decide %s" % (" ".join(demo), " ".join(dec)), "runs": runs, + "wall_ms": summary([r["wall_ms"] for r in runs]), "pbit_ms": summary([r["pbit_ms"] for r in runs]), + "released": summary([r["released"] for r in runs]), "violations_max": max([r["violations"] or 0 for r in runs]), + "verdicts": sorted(set(str(r["verdict"]) for r in runs)), "exit_codes": sorted(set(r["decide_exit"] for r in runs)), + "site_updates_per_s": summary([r["site_updates_per_s"] for r in runs])} + R["load"]["after_decide"] = load() + + # peak RSS of the 300-task decide (3 runs; the document is generated once) + tmp = tempfile.mkdtemp(prefix="pbit-bench-") + doc = os.path.join(tmp, "demo300.json") + with open(doc, "w") as f: + f.write(sh([bin_, "demo", "--tasks", "300"])[1]) + rss = [] + for _ in range(3): + try: + rss.append(peak_rss(bin_, doc, ["--budget-ms", "300"])) + except Exception as e: + R["errors"].append("peak_rss: %s: %s" % (type(e).__name__, e)) + if rss: + R["peak_rss_300"] = {"cmd": "pbit decide --budget-ms 300 < demo300.json", "method": rss[0][1], + "bytes": summary([r[0] for r in rss]), "self_reported_peak_rss_mb": [r[2] for r in rss], + "verdicts": [r[3] for r in rss], "extra": [r[4] for r in rss]} + + # lattice kernels (pbit-core/examples/kernels.rs), portable build, N runs + kbin = exe(os.path.join(pdir, "examples", "kernels")) + if os.path.exists(kbin): + ks = [] + for _ in range(N): + rc, out, err = sh([kbin]) + if rc == 0: + ks.append(parse_kernels(out)) + else: + R["errors"].append("kernels rc=%s %s" % (rc, err[-300:])) + if ks: + R["kernels"] = {"cmd": os.path.relpath(kbin).replace(os.sep, "/"), "checksums": sorted(set(k.get("chk") for k in ks)), + "lines": {name: summary([k.get(name) for k in ks]) for name in ks[0] if name != "chk"}} + else: + R["kernels"] = {"skipped": "no kernels example at %s (build it with -p pbit-core --example kernels)" % os.path.relpath(kbin)} + R["load"]["after_kernels"] = load() + + # bench/portable_vs_native.py (unchanged; its header line names the M4 whatever the machine) + if a.native_dir: + if platform.system() == "Windows": + # the script runs /pbit and /examples/kernels; CreateProcess appends no .exe to a name with a path + R["windows_shim"] = [] + for d in (pdir, a.native_dir): + for name in ("pbit", os.path.join("examples", "kernels")): + src, dst = os.path.join(d, name + ".exe"), os.path.join(d, name) + if os.path.exists(src) and not os.path.exists(dst): + shutil.copyfile(src, dst); R["windows_shim"].append(os.path.relpath(dst).replace(os.sep, "/")) + env = dict(os.environ, PBIT_DIR=pdir, PBIT_NATIVE_DIR=a.native_dir) + rc, out, err = sh([sys.executable, "bench/portable_vs_native.py"], env=env, timeout=1800) + rows = {} + for line in out.splitlines(): + m = PVN.match(line) + if m: + g = m.groups() + rows[g[0]] = {"portable": float(g[1]), "portable_iqr": [float(g[2]), float(g[3])], "native": float(g[4]), + "native_iqr": [float(g[5]), float(g[6])], "ratio": float(g[7])} + R["portable_vs_native"] = {"cmd": "PBIT_DIR=%s PBIT_NATIVE_DIR=%s python3 bench/portable_vs_native.py" % ( + os.path.relpath(pdir).replace(os.sep, "/"), os.path.relpath(a.native_dir).replace(os.sep, "/")), "exit": rc, + "rows": rows, "stdout": out.splitlines(), "stderr_tail": err[-1500:] or None, + "note": "the script's first line is hard-coded to the M4; the machine is the one in `machine`"} + else: + R["portable_vs_native"] = {"skipped": "no --native-dir"} + R["load"]["after_pvn"] = load() + + # bench/decide_threads.py (unchanged; same hard-coded header) + rc, out, err = sh([sys.executable, "bench/decide_threads.py"], env=dict(os.environ, PBIT=bin_), timeout=1800) + th = {} + for line in out.splitlines(): + m = DT.match(line) + if m: + g = m.groups(); th["threads_%s" % g[0]] = {"wall_ms": float(g[1]), "wall_iqr": [float(g[2]), float(g[3])], "cpu_ms": float(g[4]), + "site_updates_per_s_M": float(g[5]), "verdict": g[6]} + m = DD.match(line) + if m: + g = m.groups(); th["default"] = {"threads": int(g[0]), "wall_ms": float(g[1]), "wall_iqr": [float(g[2]), float(g[3])], + "sweeps": int(g[4]), "verdict": g[5]} + R["decide_threads"] = {"cmd": "PBIT=%s python3 bench/decide_threads.py" % a.bin, "exit": rc, "rows": th, "stdout": out.splitlines(), + "stderr_tail": err[-1500:] or None} + R["load"]["end"] = load() + + for kv in a.attach: + k, _, path = kv.partition("=") + try: + with open(path) as f: + txt = f.read() + try: + R.setdefault("attached", {})[k] = json.loads(txt) + except ValueError: + R.setdefault("attached", {})[k] = txt.splitlines() + except OSError as e: + R.setdefault("attached", {})[k] = "missing: %s" % e + + # the cells BENCHMARK-MATRIX.md shows (each one also lives in full above) + C = {"cpu_model": R["machine"].get("cpu_model"), "logical_cores": R["machine"]["logical_cores"], + "binary_size_bytes": R["binary"]["size_bytes"]} + if "stats" in R: + C["stats_1t"] = R["stats"]["site_updates_per_s_1_thread"]; C["stats_default"] = R["stats"]["site_updates_per_s"] + C["stats_default_threads"] = R["stats"]["threads"] + for name, line in KERNEL_CELLS.items(): + C[name] = (R.get("kernels", {}).get("lines") or {}).get(line) + for key in ("decide_12", "decide_300"): + C[key + "_wall_ms"] = R[key]["wall_ms"]; C[key + "_pbit_ms"] = R[key]["pbit_ms"]; C[key + "_verdicts"] = R[key]["verdicts"] + C[key + "_exit_codes"] = R[key]["exit_codes"] + C["decide_300_released"] = R["decide_300"]["released"]; C["decide_300_violations_max"] = R["decide_300"]["violations_max"] + C["decide_300_site_updates_per_s"] = R["decide_300"]["site_updates_per_s"] + if "peak_rss_300" in R: + C["decide_300_peak_rss_bytes"] = R["peak_rss_300"]["bytes"]; C["decide_300_peak_rss_method"] = R["peak_rss_300"]["method"] + pv = R["portable_vs_native"].get("rows") or {} + C["native_over_portable"] = {k: v["ratio"] for k, v in pv.items()} or R["portable_vs_native"].get("skipped") or "n/a: exit %s" % R["portable_vs_native"].get("exit") + C["threads_rows"] = th or "n/a: exit %s, %s" % (rc, (err.strip().splitlines() or [""])[-1][:200]) + R["cells"] = C + R["elapsed_s"] = round(time.time() - t_start, 1) + + text = json.dumps(R, indent=1) + home = os.path.expanduser("~") + for p, rep in ((os.getcwd(), "."), (home, "~")): # no absolute paths of this machine in the output + if len(p) > 3: + text = text.replace(json.dumps(p)[1:-1], rep).replace(p, rep) + with open(a.out, "w") as f: + f.write(text + "\n") + print(json.dumps(C, indent=1)) + print("wrote", a.out, "in", R["elapsed_s"], "s;", len(R["errors"]), "errors", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/bench/package_like_release.sh b/scripts/bench/package_like_release.sh new file mode 100755 index 0000000..32a1047 --- /dev/null +++ b/scripts/bench/package_like_release.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Pack a local pbit binary the way .github/workflows/release.yml's "package" step does (same names, layout and .sha256), +# so install.sh, install.ps1 and the npm wrapper can be tested before any release exists. Keep in sync with release.yml. +# scripts/bench/package_like_release.sh +# Writes /pbit--.tar.gz (.zip for *-windows-*, with 7z as release.yml does) and the .sha256 beside it. +set -eu +BIN=$1 TARGET=$2 TAG=$3 OUT=$4 +NAME="pbit-$TAG-$TARGET" +STAGE=$(mktemp -d 2> /dev/null || mktemp -d -t pbitpkg) +mkdir -p "$STAGE/$NAME" "$OUT" +cp "$BIN" "$STAGE/$NAME/" +cp README.md LICENSE CHANGELOG.md "$STAGE/$NAME/" +cp -R examples python docs "$STAGE/$NAME/" +rm -rf "$STAGE/$NAME/python/__pycache__" +case "$TARGET" in + *windows*) ASSET="$NAME.zip"; (cd "$STAGE" && 7z a "$ASSET" "$NAME" > /dev/null) ;; + *) ASSET="$NAME.tar.gz"; (cd "$STAGE" && tar czf "$ASSET" "$NAME") ;; +esac +mv "$STAGE/$ASSET" "$OUT/" +rm -rf "$STAGE" +cd "$OUT" +if command -v sha256sum > /dev/null; then sha256sum "$ASSET" > "$ASSET.sha256"; else shasum -a 256 "$ASSET" > "$ASSET.sha256"; fi +echo "$OUT/$ASSET" diff --git a/scripts/bench/peak_rss_windows.ps1 b/scripts/bench/peak_rss_windows.ps1 new file mode 100644 index 0000000..9e151cd --- /dev/null +++ b/scripts/bench/peak_rss_windows.ps1 @@ -0,0 +1,29 @@ +# Peak working set of `pbit decide --budget-ms 300` on the 300-task demo, read with Get-Process every ~5 ms until the process +# exits: a lower bound (the last sample before exit). Cross-check of the exact value matrix_bench.py reads with +# K32GetProcessMemoryInfo on the process handle. +# pwsh -File scripts/bench/peak_rss_windows.ps1 [-Exe target\release\pbit.exe] [-Out peak-rss-powershell.json] [-Runs 3] +param([string]$Exe = 'target\release\pbit.exe', [string]$Out = 'peak-rss-powershell.json', [int]$Runs = 3) +$ErrorActionPreference = 'Stop' +$exePath = (Resolve-Path -LiteralPath $Exe).Path +$doc = Join-Path ([IO.Path]::GetTempPath()) 'pbit-demo300.json' +& $exePath demo --tasks 300 | Set-Content -Encoding ascii -LiteralPath $doc +$rows = foreach ($i in 1..$Runs) { + $o = Join-Path ([IO.Path]::GetTempPath()) "pbit-d300-$i.json" + $p = Start-Process -FilePath $exePath -ArgumentList 'decide', '--budget-ms', '300' -RedirectStandardInput $doc ` + -RedirectStandardOutput $o -NoNewWindow -PassThru + $peak = [long]0; $samples = 0 + while (-not $p.HasExited) { + $g = Get-Process -Id $p.Id -ErrorAction SilentlyContinue + if ($g) { $samples++; if ($g.PeakWorkingSet64 -gt $peak) { $peak = $g.PeakWorkingSet64 } } + Start-Sleep -Milliseconds 5 + } + $verdict = (Get-Content -Raw -LiteralPath $o | ConvertFrom-Json).verdict + [pscustomobject]@{ peak_working_set_bytes = $peak; samples = $samples; verdict = $verdict } +} +[pscustomobject]@{ + method = 'Get-Process PeakWorkingSet64, polled every ~5 ms until exit (lower bound)' + cmd = 'pbit decide --budget-ms 300 < demo300.json' + powershell = "$($PSVersionTable.PSVersion)" + runs = @($rows) +} | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $Out +Get-Content -LiteralPath $Out diff --git a/scripts/bench/test_install_ps1.ps1 b/scripts/bench/test_install_ps1.ps1 new file mode 100644 index 0000000..9c0b59a --- /dev/null +++ b/scripts/bench/test_install_ps1.ps1 @@ -0,0 +1,55 @@ +# install.ps1 end to end with no GitHub release: serves a release-layout .zip on 127.0.0.1 (built beforehand with +# scripts/bench/package_like_release.sh into \good\\, plus a copy with a wrong .sha256 in \bad\\) and +# installs it with -DownloadBase, then the `irm | iex` form (environment variables), then the tampered copy (must fail). +# pwsh -File scripts/bench/test_install_ps1.ps1 -Srv [-Tag v0.2.0] [-AddToPath] +# powershell -ExecutionPolicy Bypass -File scripts/bench/test_install_ps1.ps1 -Srv +param([Parameter(Mandatory = $true)][string]$Srv, [string]$Tag = 'v0.2.0', [switch]$AddToPath) +$ErrorActionPreference = 'Stop' +"PowerShell $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" +$l = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, 0); $l.Start(); $port = $l.LocalEndpoint.Port; $l.Stop() +$server = Start-Process -FilePath python -ArgumentList @('-m', 'http.server', "$port", '--bind', '127.0.0.1', '--directory', $Srv) -PassThru -WindowStyle Hidden +$base = "http://127.0.0.1:$port" +try { + for ($i = 0; $i -lt 50; $i++) { try { Invoke-WebRequest -UseBasicParsing -Uri "$base/" -TimeoutSec 2 | Out-Null; break } catch { Start-Sleep -Milliseconds 200 } } + $root = Join-Path ([IO.Path]::GetTempPath()) ('pbit-ps-test-' + [guid]::NewGuid().ToString('N')) + + "== 1. & .\install.ps1 -DownloadBase $base/good -Version $Tag -InstallDir \bin1$(if ($AddToPath) { ' -AddToPath' })" + $d1 = Join-Path $root 'bin1' + if ($AddToPath) { & .\install.ps1 -DownloadBase "$base/good" -Version $Tag -InstallDir $d1 -AddToPath } + else { & .\install.ps1 -DownloadBase "$base/good" -Version $Tag -InstallDir $d1 } + $cmd = Get-Command pbit -ErrorAction Stop + "`$ (Get-Command pbit).Source: $($cmd.Source)" + if ((Resolve-Path $cmd.Source).Path -ne (Resolve-Path (Join-Path $d1 'pbit.exe')).Path) { throw "FAIL: pbit resolves to $($cmd.Source)" } + "`$ pbit version: $(pbit version)" + $d = pbit demo --tasks 12 | pbit decide | ConvertFrom-Json + "`$ pbit demo --tasks 12 | pbit decide | ConvertFrom-Json: exit $LASTEXITCODE, verdict $($d.verdict), released $(@($d.released).Count) of $($d.tasks)" + if ($LASTEXITCODE -ne 0 -or $d.verdict -ne 'exact') { throw 'FAIL: demo pipeline' } + if ($AddToPath) { + $u = [Environment]::GetEnvironmentVariable('Path', 'User') + if (-not ($u -split ';' -contains $d1)) { throw "FAIL: -AddToPath did not add $d1 to the user PATH" } + "ok: user PATH now starts with $(($u -split ';')[0])" + } + + '== 2. irm | iex form: PBIT_* environment variables, script text through Invoke-Expression' + $d2 = Join-Path $root 'bin2' + $env:PBIT_DOWNLOAD_BASE = "$base/good"; $env:PBIT_VERSION = $Tag.TrimStart('v'); $env:PBIT_INSTALL_DIR = $d2 + try { Get-Content -Raw .\install.ps1 | Invoke-Expression } + finally { Remove-Item Env:PBIT_DOWNLOAD_BASE, Env:PBIT_VERSION, Env:PBIT_INSTALL_DIR } + if (-not (Test-Path (Join-Path $d2 'pbit.exe'))) { throw "FAIL: $d2\pbit.exe missing" } + "ok: $(& (Join-Path $d2 'pbit.exe') version) in $d2" + + '== 3. a wrong .sha256 must fail and install nothing' + $d3 = Join-Path $root 'bin3' + $failed = $false + try { & .\install.ps1 -DownloadBase "$base/bad" -Version $Tag -InstallDir $d3 } catch { $failed = $true; "rejected: $($_.Exception.Message)" } + if (-not $failed) { throw 'FAIL: a wrong checksum was accepted' } + if (Test-Path (Join-Path $d3 'pbit.exe')) { throw 'FAIL: pbit.exe installed despite the checksum mismatch' } + + '== 4. -DownloadBase without -Version must fail' + $failed = $false + try { & .\install.ps1 -DownloadBase "$base/good" -InstallDir (Join-Path $root 'bin4') } catch { $failed = $true; "rejected: $($_.Exception.Message)" } + if (-not $failed) { throw 'FAIL: accepted' } + "install.ps1: all checks passed (PowerShell $($PSVersionTable.PSVersion))" +} finally { + Stop-Process -Id $server.Id -Force -ErrorAction SilentlyContinue +} diff --git a/scripts/bench/test_install_sh.sh b/scripts/bench/test_install_sh.sh new file mode 100755 index 0000000..133950b --- /dev/null +++ b/scripts/bench/test_install_sh.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# install.sh end to end with no GitHub release: pack a local binary as release.yml does, serve it on 127.0.0.1, then install +# it piped through sh (as `curl ... | sh` does), into the default directory under a scratch HOME, and from a server whose +# .sha256 is wrong (must fail and install nothing). +# scripts/bench/test_install_sh.sh [tag] SH=dash scripts/bench/... to pick the shell +set -eu +BIN=$1 TARGET=$2 TAG=${3:-v0.2.0} +SH=${SH:-sh} +PY=${PY:-$(command -v python3 || command -v python)} +WORK=$(mktemp -d 2> /dev/null || mktemp -d -t pbitinst) +SRV="" +cleanup() { if [ -n "$SRV" ]; then kill "$SRV" 2> /dev/null || true; fi; rm -rf "$WORK"; } +trap cleanup EXIT +sh scripts/bench/package_like_release.sh "$BIN" "$TARGET" "$TAG" "$WORK/srv/good/$TAG" > /dev/null +cp -R "$WORK/srv/good" "$WORK/srv/bad" +printf '%064d %s\n' 0 "pbit-$TAG-$TARGET.tar.gz" > "$WORK/srv/bad/$TAG/pbit-$TAG-$TARGET.tar.gz.sha256" +PORT=$("$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +"$PY" -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/srv" > "$WORK/http.log" 2>&1 & +SRV=$! +i=0 +until curl -fsS -o /dev/null "http://127.0.0.1:$PORT/" 2> /dev/null; do + i=$((i + 1)); [ "$i" -lt 50 ] || { echo "http server did not start"; exit 1; }; sleep 0.2 +done +BASE="http://127.0.0.1:$PORT" +echo "shell under test: $SH ($(command -v "$SH")); archive: $(ls "$WORK/srv/good/$TAG")" + +echo "== 1. piped to $SH, PBIT_INSTALL_DIR set, version given without the v" +PBIT_DOWNLOAD_BASE="$BASE/good" PBIT_VERSION="${TAG#v}" PBIT_INSTALL_DIR="$WORK/bin" "$SH" < install.sh +PATH="$WORK/bin:$PATH" +echo "\$ command -v pbit: $(command -v pbit)" +echo "\$ pbit version: $(pbit version)" +pbit demo --tasks 12 | pbit decide > "$WORK/d12.json" +echo "\$ pbit demo --tasks 12 | pbit decide: exit $?, $("$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); print("verdict", d["verdict"], "| released", len(d["released"]), "of", d["tasks"])' "$WORK/d12.json")" + +echo "== 2. default directory, scratch HOME" +HOME="$WORK/home" PBIT_DOWNLOAD_BASE="$BASE/good" PBIT_VERSION="$TAG" "$SH" install.sh +if [ -d /usr/local/bin ] && [ -w /usr/local/bin ]; then want=/usr/local/bin/pbit; else want="$WORK/home/.local/bin/pbit"; fi +[ -x "$want" ] || { echo "FAIL: expected $want"; exit 1; } +echo "ok: installed to $want ($("$want" version))" + +echo "== 3. tampered .sha256 must fail and install nothing" +if PBIT_DOWNLOAD_BASE="$BASE/bad" PBIT_VERSION="$TAG" PBIT_INSTALL_DIR="$WORK/badbin" "$SH" install.sh; then + echo "FAIL: a wrong checksum was accepted"; exit 1 +fi +[ ! -e "$WORK/badbin/pbit" ] || { echo "FAIL: pbit installed despite the checksum mismatch"; exit 1; } +echo "ok: rejected, $WORK/badbin/pbit absent" + +echo "== 4. PBIT_DOWNLOAD_BASE without PBIT_VERSION must fail" +if PBIT_DOWNLOAD_BASE="$BASE/good" PBIT_INSTALL_DIR="$WORK/bin4" "$SH" install.sh; then echo "FAIL: accepted"; exit 1; fi +echo "ok: rejected" +echo "install.sh: all checks passed ($SH, $TARGET)" diff --git a/scripts/bench/test_npm.sh b/scripts/bench/test_npm.sh new file mode 100755 index 0000000..8fbd4db --- /dev/null +++ b/scripts/bench/test_npm.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# The npm wrapper end to end with no GitHub release and no registry: pack a local binary as release.yml does, serve it on +# 127.0.0.1, `npm pack` npm/, `npm install -g` the tarball (into a scratch prefix, so the machine's global packages are not +# touched): postinstall fetches + verifies the binary; run it (exit codes 0/1/2/3 must pass through); `npm uninstall -g`. +# Then an --ignore-scripts install (fetched on first run), PBIT_BINARY, and a tampered .sha256 (the install must fail). +# scripts/bench/test_npm.sh [tag] +set -eu +BIN=$1 TARGET=$2 TAG=${3:-v0.2.0} +PY=${PY:-$(command -v python3 || command -v python)} +WORK=$(mktemp -d 2> /dev/null || mktemp -d -t pbitnpm) +SRV="" +cleanup() { if [ -n "$SRV" ]; then kill "$SRV" 2> /dev/null || true; fi; rm -rf "$WORK"; } +trap cleanup EXIT +fail() { echo "FAIL: $*"; exit 1; } +BIN_ABS=$(cd "$(dirname "$BIN")" && pwd)/$(basename "$BIN") +if command -v cygpath > /dev/null; then BIN_ABS=$(cygpath -w "$BIN_ABS"); fi # Windows: node needs a native path +sh scripts/bench/package_like_release.sh "$BIN" "$TARGET" "$TAG" "$WORK/srv/good/$TAG" > /dev/null +cp -R "$WORK/srv/good" "$WORK/srv/bad" +case "$TARGET" in *windows*) EXT=zip ;; *) EXT=tar.gz ;; esac +printf '%064d %s\n' 0 "pbit-$TAG-$TARGET.$EXT" > "$WORK/srv/bad/$TAG/pbit-$TAG-$TARGET.$EXT.sha256" +PORT=$("$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') +"$PY" -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/srv" > "$WORK/http.log" 2>&1 & +SRV=$! +i=0 +until curl -fsS -o /dev/null "http://127.0.0.1:$PORT/" 2> /dev/null; do + i=$((i + 1)); [ "$i" -lt 50 ] || fail "http server did not start"; sleep 0.2 +done +BASE="http://127.0.0.1:$PORT" +case "$(uname -s)" in MINGW* | MSYS* | CYGWIN*) gbin() { echo "$1"; } ;; *) gbin() { echo "$1/bin"; } ;; esac +NPM_FLAGS="--no-audit --no-fund --foreground-scripts" +echo "node $(node --version), npm $(npm --version), target $TARGET" + +(cd npm && npm pack --pack-destination "$WORK" > /dev/null) +TGZ=$(ls "$WORK"/pbit-*.tgz) +echo "== npm pack -> $(basename "$TGZ"):" +tar tzf "$TGZ" | sed 's/^/ /' + +echo "== 1. npm install -g $(basename "$TGZ") (PBIT_DOWNLOAD_BASE=$BASE/good)" +P1="$WORK/prefix1" +PBIT_DOWNLOAD_BASE="$BASE/good" npm install -g --prefix "$P1" $NPM_FLAGS "$TGZ" +OLDPATH=$PATH +PATH="$(gbin "$P1"):$OLDPATH" +echo "\$ which pbit: $(command -v pbit)" +echo "\$ pbit version: $(pbit version)" +pbit demo --tasks 12 | pbit decide > "$WORK/d12.json" +echo "\$ pbit demo --tasks 12 | pbit decide: exit $?, $("$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); print("verdict", d["verdict"], "| released", len(d["released"]), "of", d["tasks"])' "$WORK/d12.json")" +set +e +printf '%s' '{"workers":[{"id":"a","cap":1}],"tasks":[{"id":"t1","allowed":["a"],"scores":{"a":1}},{"id":"t2","allowed":["a"],"scores":{"a":1}}]}' | pbit decide > /dev/null; r1=$? +printf '%s' '{"workers": 1}' | pbit decide > /dev/null 2>&1; r2=$? +pbit run --op exact --exact-ms 50 < examples/denoise-8x12.json > /dev/null; r3=$? +set -e +echo "exit codes through the wrapper: infeasible $r1 (want 1), bad input $r2 (want 2), declined $r3 (want 3)" +[ "$r1" = 1 ] && [ "$r2" = 2 ] && [ "$r3" = 3 ] || fail "exit codes" +npm uninstall -g --prefix "$P1" $NPM_FLAGS pbit +PATH=$OLDPATH +[ ! -e "$(gbin "$P1")/pbit" ] || fail "npm uninstall left $(gbin "$P1")/pbit" +echo "ok: uninstalled, $(gbin "$P1")/pbit absent" + +echo "== 2. npm install -g --ignore-scripts: the binary is fetched on first run" +P2="$WORK/prefix2" +npm install -g --prefix "$P2" $NPM_FLAGS --ignore-scripts "$TGZ" +PBIT_DOWNLOAD_BASE="$BASE/good" "$(gbin "$P2")/pbit" version +"$(gbin "$P2")/pbit" demo --tasks 12 | "$(gbin "$P2")/pbit" decide > /dev/null +echo "ok: first run fetched it; second run used it" + +echo "== 3. PBIT_BINARY= (no download)" +P3="$WORK/prefix3" +PBIT_BINARY="$BIN_ABS" PBIT_DOWNLOAD_BASE="http://127.0.0.1:9/nowhere" npm install -g --prefix "$P3" $NPM_FLAGS "$TGZ" +"$(gbin "$P3")/pbit" version + +echo "== 4. tampered .sha256: npm install must fail" +if PBIT_DOWNLOAD_BASE="$BASE/bad" npm install -g --prefix "$WORK/prefix4" $NPM_FLAGS "$TGZ"; then fail "a wrong checksum was accepted"; fi +echo "ok: rejected" +echo "npm wrapper: all checks passed ($TARGET)" diff --git a/scripts/bench/test_summary.py b/scripts/bench/test_summary.py new file mode 100644 index 0000000..ad82a78 --- /dev/null +++ b/scripts/bench/test_summary.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""One JSON for a `cargo test` run: pass/fail, wall seconds, summed counts of every "test result:" line, machine and toolchain. + + python3 scripts/bench/test_summary.py