From 1fc5ebc960fdb3dc5c24fd75655542bdc012052a Mon Sep 17 00:00:00 2001 From: ematipico Date: Wed, 14 Oct 2026 14:53:14 +0100 Subject: [PATCH 1/2] ci: better DX --- .github/workflows/publish-preview.yml | 40 ++++----- docker/release.Dockerfile | 35 ++++++++ justfile | 99 +++++++++++++++++++++ packages/forge/README.md | 65 ++++++++------ packages/forge/package.json | 4 +- packages/forge/scripts/archive.mjs | 14 +-- packages/forge/scripts/compile.mjs | 123 -------------------------- packages/forge/scripts/platforms.mjs | 18 ++-- packages/forge/scripts/publish.mjs | 50 ----------- 9 files changed, 209 insertions(+), 239 deletions(-) create mode 100644 docker/release.Dockerfile delete mode 100644 packages/forge/scripts/compile.mjs delete mode 100644 packages/forge/scripts/publish.mjs diff --git a/.github/workflows/publish-preview.yml b/.github/workflows/publish-preview.yml index 0d87f9880..c289cdfd5 100644 --- a/.github/workflows/publish-preview.yml +++ b/.github/workflows/publish-preview.yml @@ -35,6 +35,7 @@ on: - 'Cargo.toml' - 'Cargo.lock' - 'rust-toolchain.toml' + - 'justfile' - 'src/**' - 'packages/forge/**' - '!packages/forge/test/**' @@ -47,6 +48,7 @@ on: - 'Cargo.toml' - 'Cargo.lock' - 'rust-toolchain.toml' + - 'justfile' - 'src/**' - 'packages/forge/**' - '!packages/forge/test/**' @@ -70,10 +72,8 @@ jobs: # platforms.mjs; scripts/archive.mjs refuses to pack a package that misses # a platform. # - # Each job uploads its binary as forge-. upload-artifact keeps the - # path from the first wildcard on, so target/*/release/forge keeps - # /release/, where scripts/archive.mjs looks. A job builds only its - # own target, so the wildcard matches one file. + # Each job builds with `just build `, the recipe local builds use, + # and uploads target/binaries as forge-. build: name: Build (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} @@ -102,19 +102,16 @@ jobs: runner: ubuntu-24.04 exe: forge runs: true - # Link the C runtime in, or the binary needs VCRUNTIME140.dll from - # the Visual C++ Redistributable, which many machines don't have. + # `just build` links the C runtime into both Windows binaries. - platform: win32-arm64 target: aarch64-pc-windows-msvc runner: windows-latest exe: forge.exe - rustflags: -C target-feature=+crt-static runs: false - platform: win32-x64 target: x86_64-pc-windows-msvc runner: windows-latest exe: forge.exe - rustflags: -C target-feature=+crt-static runs: true defaults: run: @@ -133,6 +130,7 @@ jobs: uses: moonrepo/setup-rust@abb2d32350334249b178c401e5ec5836e0cd88d3 # v1.3.0 with: targets: ${{ matrix.target }} + bins: just@1.58.0 cache-target: ${{ matrix.target }}/release cache-base: main env: @@ -145,27 +143,28 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends musl-tools - # `wasm` gives `--target wasm://`, which runs plugins. + # The same recipe builds locally; it puts the binary in + # target/binaries//release/. - name: Build env: - RUSTFLAGS: ${{ matrix.rustflags }} TARGET: ${{ matrix.target }} - run: cargo build --release --locked --features wasm --target "$TARGET" + run: just build "$TARGET" # A binary the runner can execute gets a quick run, which catches a # broken link before the package is published. - name: Smoke test if: matrix.runs env: - BIN: target/${{ matrix.target }}/release/${{ matrix.exe }} + BIN: target/binaries/${{ matrix.target }}/release/${{ matrix.exe }} run: '"$BIN" --version' + # Uploading target/binaries keeps /release/ inside the artifact. # Artifacts drop the executable bit, which scripts/archive.mjs sets # again when it packs. - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: forge-${{ matrix.platform }} - path: target/*/release/${{ matrix.exe }} + path: target/binaries if-no-files-found: error retention-days: 7 @@ -182,19 +181,20 @@ jobs: - uses: ./.github/actions/setup-workspace - # Merges the build jobs' artifacts into binaries//release/, - # where scripts/archive.mjs picks them up. + - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 + with: + just-version: 1.58.0 + + # Merges the build jobs' artifacts into target/binaries//release/, + # where `just pack` picks them up, as it does locally. - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: forge-* merge-multiple: true - path: ${{ runner.temp }}/binaries + path: target/binaries - name: Pack - working-directory: packages/forge - env: - BINARIES: ${{ runner.temp }}/binaries - run: node scripts/archive.mjs --artifacts "$BINARIES" + run: just pack - name: Publish id: publish diff --git a/docker/release.Dockerfile b/docker/release.Dockerfile new file mode 100644 index 000000000..ed68ce1fc --- /dev/null +++ b/docker/release.Dockerfile @@ -0,0 +1,35 @@ +# Images for building forge's release binaries locally, for the targets a +# machine can't build natively. `just build-docker ` builds and runs +# them; see the justfile. CI builds on native runners instead. +# +# The justfile sets RUST_VERSION to the channel in rust-toolchain.toml, so the +# containers build with the same Rust as CI. The default, the latest 1.x, +# only keeps the image names valid for a plain `docker build`. +# +# The `extra-ca` build context holds extra CA certificates to trust, one per +# .crt file, for networks whose proxy re-signs HTTPS traffic, such as +# Cloudflare's Zero Trust gateway. Without them, cargo, cargo-xwin and the +# package managers reject every download. The justfile passes an empty +# directory when there are none. + +ARG RUST_VERSION=1 + +# Linux: musl, so the binary is static and runs on any distro. Alpine is +# musl-based, so its C compiler targets musl too, which the cc crate uses +# for wasmtime's C code. Run with --platform linux/arm64 or linux/amd64 to +# build for that architecture. +FROM rust:${RUST_VERSION}-alpine AS linux +COPY --from=extra-ca . /usr/local/share/ca-certificates/ +RUN update-ca-certificates && apk add --no-cache musl-dev + +# Windows: MSVC, cross-compiled with cargo-xwin, which downloads Microsoft's +# CRT and Windows SDK on first use. clang, lld and llvm are its toolchain. +FROM rust:${RUST_VERSION}-slim-bookworm AS windows +ARG CARGO_XWIN_VERSION=0.23.1 +COPY --from=extra-ca . /usr/local/share/ca-certificates/ +RUN update-ca-certificates +RUN apt-get update \ + && apt-get install -y --no-install-recommends clang lld llvm \ + && rm -rf /var/lib/apt/lists/* +RUN cargo install --locked cargo-xwin --version "${CARGO_XWIN_VERSION}" --root /usr/local \ + && rustup target add aarch64-pc-windows-msvc x86_64-pc-windows-msvc diff --git a/justfile b/justfile index 36399325e..5d23891de 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,7 @@ # Project commands. Run `just --list` to see them all. CI runs these same # recipes, so a recipe that passes here passes there. +_default: + just --list -u alias f := fmt alias l := lint @@ -24,3 +26,100 @@ fmt-check: lint: cargo clippy --locked --all-targets --all-features -- -D warnings cargo clippy --locked --all-targets --no-default-features -- -D warnings + +# Release builds +# +# Every way of getting release binaries puts them in target/binaries// +# release/, and `just pack` packs whatever is there. CI builds each target +# with `just build` on a native runner. Locally, `just build-all` builds them +# all: macOS targets natively, which needs a Mac, and the rest in Docker. +# `just fetch` downloads a CI run's binaries instead. + +# The Rust targets forge ships, as in packages/forge/scripts/platforms.mjs. +mac_targets := "aarch64-apple-darwin x86_64-apple-darwin" +docker_targets := "aarch64-unknown-linux-musl x86_64-unknown-linux-musl aarch64-pc-windows-msvc x86_64-pc-windows-msvc" +binaries := "target/binaries" + +# Link the C runtime into the Windows binaries, or they need VCRUNTIME140.dll +# from the Visual C++ Redistributable, which many machines don't have. Both +# `just build` (CI's Windows runners) and `just build-docker` (cargo-xwin, which +# keeps these flags) build with them. Other targets ignore them. +export CARGO_TARGET_AARCH64_PC_WINDOWS_MSVC_RUSTFLAGS := "-C target-feature=+crt-static" +export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS := "-C target-feature=+crt-static" + +# Build the release binary for on this machine +build target: + rustup target add {{ target }} + cargo build --release --locked --features wasm --target {{ target }} + mkdir -p {{ binaries }}/{{ target }}/release + cp target/{{ target }}/release/{{ if target =~ 'windows' { "forge.exe" } else { "forge" } }} {{ binaries }}/{{ target }}/release/ + +# Build the release binary for a Linux or Windows in Docker +build-docker target: + #!/usr/bin/env bash + set -euo pipefail + target='{{ target }}' + rust=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + case "$target" in + aarch64-unknown-linux-musl) stage=linux platform=linux/arm64 exe=forge cargo=(cargo build) ;; + x86_64-unknown-linux-musl) stage=linux platform=linux/amd64 exe=forge cargo=(cargo build) ;; + *-pc-windows-msvc) stage=windows platform= exe=forge.exe cargo=(cargo xwin build) ;; + *) echo "error: build-docker builds Linux and Windows targets; build $target with: just build $target" >&2; exit 1 ;; + esac + image="forge-release-$stage:$rust${platform:+-${platform#linux/}}" + # Extra CA certificates for the images to trust, behind a proxy that + # re-signs HTTPS traffic: FORGE_DOCKER_CA_CERT, or else the bundle Node + # trusts, NODE_EXTRA_CA_CERTS. They go one per file, because Alpine's + # update-ca-certificates skips a file that holds more than one. + ca_dir=target/docker/extra-ca + rm -rf "$ca_dir" && mkdir -p "$ca_dir" + ca="${FORGE_DOCKER_CA_CERT:-${NODE_EXTRA_CA_CERTS:-}}" + if [ -n "$ca" ]; then + [ -f "$ca" ] || { echo "error: no CA certificate file at $ca" >&2; exit 1; } + awk -v dir="$ca_dir" '/-----BEGIN CERTIFICATE-----/ { n++; out = sprintf("%s/extra-%d.crt", dir, n) } out { print > out } /-----END CERTIFICATE-----/ { close(out); out = "" }' "$ca" + echo "==> trusting $(ls "$ca_dir" | wc -l | tr -d ' ') extra CA certificate(s) from $ca" + fi + echo "==> $target in $image" + docker build --quiet ${platform:+--platform "$platform"} --target "$stage" --build-arg "RUST_VERSION=$rust" \ + --build-context "extra-ca=$ca_dir" \ + --tag "$image" --file docker/release.Dockerfile docker >/dev/null + # Runs as you, so everything it writes under target/ is yours. Each target + # builds into a directory of its own: the host's target/ holds build + # scripts compiled for the host, which can't run in the container. + # RUSTUP_TOOLCHAIN pins the image's Rust, so rustup doesn't try to install + # the components rust-toolchain.toml lists. + docker run --rm ${platform:+--platform "$platform"} --user "$(id -u):$(id -g)" \ + --volume "$PWD:/work" --workdir /work \ + --env HOME=/tmp \ + --env CARGO_HOME=/work/target/docker/cargo-home \ + --env CARGO_TARGET_DIR="/work/target/docker/$target" \ + --env XWIN_CACHE_DIR=/work/target/docker/xwin \ + --env RUSTUP_TOOLCHAIN="$rust" \ + --env CARGO_TARGET_AARCH64_PC_WINDOWS_MSVC_RUSTFLAGS \ + --env CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS \ + "$image" "${cargo[@]}" --release --locked --features wasm --target "$target" + mkdir -p "{{ binaries }}/$target/release" + cp "target/docker/$target/$target/release/$exe" "{{ binaries }}/$target/release/" + +# Build every release binary this machine can: macOS ones need a Mac +build-all: + #!/usr/bin/env bash + set -euo pipefail + if [ "$(uname -s)" = Darwin ]; then + for target in {{ mac_targets }}; do just build "$target"; done + else + echo "warning: skipping {{ mac_targets }}, which build only on a Mac; just fetch gets them from CI" >&2 + fi + for target in {{ docker_targets }}; do just build-docker "$target"; done + +# Download the release binaries a publish-preview run built, instead of building them +fetch run: + rm -rf target/fetch {{ binaries }} + gh run download {{ run }} --pattern 'forge-*' --dir target/fetch + mkdir -p {{ binaries }} + cp -R target/fetch/forge-*/. {{ binaries }}/ + rm -rf target/fetch + +# Pack the binaries in target/binaries into packages/forge/dist/; takes archive.mjs options +pack *args: + node packages/forge/scripts/archive.mjs --artifacts {{ binaries }} {{ args }} diff --git a/packages/forge/README.md b/packages/forge/README.md index 2f6757ec9..e8ff83a81 100644 --- a/packages/forge/README.md +++ b/packages/forge/README.md @@ -276,18 +276,20 @@ or list it in `package.json`. A path or URL to the tarball both work: ## Releasing -From `packages/forge` in the forge repository, on a Mac with Docker running: +From the root of the forge repository, with [just](https://just.systems) installed: ```sh -pnpm install # the dev dependencies, which build the plugin library -node scripts/version.mjs # after bumping the version in Cargo.toml -node scripts/wit.mjs # after changing the WIT in src/plugin/wasm.rs -node scripts/compile.mjs # builds all six binaries into target/ -node scripts/archive.mjs # packs them into dist/cloudflare-forge-.tgz -node scripts/publish.mjs # publishes that tarball; takes npm publish options +pnpm install # the dev dependencies, which build the plugin library +node packages/forge/scripts/version.mjs # after bumping the version in Cargo.toml +node packages/forge/scripts/wit.mjs # after changing the WIT in src/plugin/wasm.rs +just build-all # builds all six binaries into target/binaries/ (or: just fetch ) +just pack # packs them into packages/forge/dist/cloudflare-forge-.tgz ``` -The tarball that `archive.mjs` makes is the one that gets published, so a +To vendor the package in another repository, copy the tarball from +`packages/forge/dist/` and depend on it with a `file:` specifier. + +The tarball that `just pack` makes is the one that gets published, so a tarball you hand around and the package on the registry are the same. `npm pack` and `npm publish` refuse to run in this directory, which has no binaries of its own. @@ -315,35 +317,42 @@ runs its tests in Node: the tests of `forge::ts` in ### Building the binaries -`scripts/compile.mjs` builds each platform with the tool that can: +Every way of getting the binaries puts them in `target/binaries//release/`, +and `just pack` packs whatever is there: -| Platform | Tool | Needs | -| -------- | -------------------------------------------------------------- | --------------------------------------- | -| macOS | `cargo build` | a Mac, Xcode's command line tools | -| Linux | [`cross build`](https://github.com/cross-rs/cross) | Docker | -| Windows | [`cargo xwin build`](https://github.com/rust-cross/cargo-xwin) | nothing; it downloads the MSVC CRT once | +| Recipe | Builds | Needs | +| ---------------------------- | ------------------------------------------- | ------------------------ | +| `just build ` | one target, with this machine's `cargo` | a host that can build it | +| `just build-docker ` | a Linux (musl) or Windows (MSVC) target | Docker | +| `just build-all` | all six: macOS natively, the rest in Docker | a Mac and Docker | +| `just fetch ` | nothing; downloads a publish-preview run's | the GitHub CLI, `gh` | -One-time setup: +macOS binaries build only on a Mac. On Linux, `just build-all` builds the +other four, and `just fetch` gets all six from CI. The Docker images are in +`docker/release.Dockerfile` and build with the Rust in `rust-toolchain.toml`; +on an Apple silicon Mac, the x64 Linux build runs emulated, so it's slow. -```sh -rustup target add x86_64-apple-darwin aarch64-apple-darwin \ - aarch64-pc-windows-msvc x86_64-pc-windows-msvc -cargo install cross --git https://github.com/cross-rs/cross -cargo install --locked cargo-xwin -``` +Behind a proxy that re-signs HTTPS traffic, such as Cloudflare's Zero Trust +gateway, the containers need its CA certificate, or every download fails with +"self-signed certificate in certificate chain". `just build-docker` adds the +certificates in `FORGE_DOCKER_CA_CERT` to the images, or else those in +`NODE_EXTRA_CA_CERTS`, which Cloudflare machines usually set already. + +CI builds each target with `just build` on a runner that builds it natively, +in the publish-preview workflow, then packs them with `just pack`. -cross can't build macOS or Windows MSVC binaries, because it has no prebuilt -images for them. The Windows binaries link the C runtime statically. +The Windows binaries link the C runtime statically (see the `justfile`), so +they don't need the Visual C++ Redistributable. ### Trying it out locally -`compile.mjs` and `archive.mjs` take `--platform -`, repeated, to -build or pack only some platforms. For a package that runs on this machine: +`just pack` takes `--platform -`, repeated, to pack only some +platforms. For a package that runs on this machine: ```sh -cargo build --release --features wasm -node scripts/archive.mjs --platform darwin-arm64 +just build aarch64-apple-darwin +just pack --platform darwin-arm64 ``` A tarball that leaves platforms out lists the ones it has in its name, as in -`cloudflare-forge-0.1.0-darwin-arm64.tgz`, and `publish.mjs` won't publish it. +`cloudflare-forge-0.1.0-darwin-arm64.tgz`, so it can't pass for the full package. diff --git a/packages/forge/package.json b/packages/forge/package.json index 464260f19..1684ecc49 100644 --- a/packages/forge/package.json +++ b/packages/forge/package.json @@ -41,14 +41,12 @@ "access": "public" }, "scripts": { - "compile": "node scripts/compile.mjs", "wit": "node scripts/wit.mjs", "build": "node scripts/build.mjs", "test": "node scripts/build.mjs && tsc -p test && node --test \"test/*.test.js\"", "archive": "node scripts/archive.mjs", - "release": "node scripts/publish.mjs", "sync-version": "node scripts/version.mjs", - "prepack": "node -e \"console.error('error: this directory has no binaries; pack with npm run archive, publish with npm run release'); process.exit(1)\"" + "prepack": "node -e \"console.error('error: this directory has no binaries; pack the package with just pack from the repository root'); process.exit(1)\"" }, "devDependencies": { "@bytecodealliance/jco": "1.37.0", diff --git a/packages/forge/scripts/archive.mjs b/packages/forge/scripts/archive.mjs index 08f42daf0..4cd7dfc14 100644 --- a/packages/forge/scripts/archive.mjs +++ b/packages/forge/scripts/archive.mjs @@ -1,6 +1,6 @@ // Packs @cloudflare/forge, with the forge binary for every platform, into a -// tarball. That tarball is the package: scripts/publish.mjs publishes it, and -// another package can install it straight from the file: +// tarball. That tarball is the package: the publish-preview workflow publishes +// it, and another package can install it straight from the file: // // node scripts/archive.mjs [--artifacts ] [--platform ]... [--out ] // @@ -8,13 +8,14 @@ // # or, in package.json: "@cloudflare/forge": "file:../cloudflare-forge-0.1.0.tgz" // // The binaries come from , the crate's target directory unless you say -// otherwise (see findBinary in platforms.mjs); scripts/compile.mjs builds -// them all. Every platform needs one. +// otherwise (see findBinary in platforms.mjs). Every platform needs one. +// `just pack` runs this script on target/binaries, which `just build-all` +// fills, or `just fetch` from a CI run; see the justfile. // // --platform packs only the platforms it names, such as darwin-arm64, which // is enough to try the package out locally; `cargo build --release // --features wasm` builds the binary for this machine. The tarball's name -// then lists them, and scripts/publish.mjs won't publish it. +// then lists them, so it can't pass for the full package. // // It builds @cloudflare/forge/plugin into the package, as scripts/build.mjs // does, which needs this directory's dev dependencies: pnpm install here. @@ -77,7 +78,8 @@ function archive(artifacts, only, out) { if (missing.length > 0) { throw new Error( `no forge binary in ${artifacts} for ${missing.join(', ')}; ` + - `run node scripts/compile.mjs${only ? '' : ', or pack only some platforms with --platform'}`, + `build them with just build-all, or fetch a CI run's with just fetch ` + + (only ? '' : ', or pack only some platforms with --platform'), ); } diff --git a/packages/forge/scripts/compile.mjs b/packages/forge/scripts/compile.mjs deleted file mode 100644 index 43cf9844c..000000000 --- a/packages/forge/scripts/compile.mjs +++ /dev/null @@ -1,123 +0,0 @@ -// Builds the release forge binary for every platform, or for the ones you -// name, into the crate's target directory: -// -// node scripts/compile.mjs [--platform ]... -// -// Each platform builds with its own tool (see `builder` in platforms.mjs): -// -// macOS cargo build needs a Mac and Xcode's command line tools -// Linux cross build needs Docker running; see github.com/cross-rs/cross -// Windows cargo xwin build downloads Microsoft's CRT and SDK the first time -// -// Install the tools with: -// -// rustup target add ... # the targets in platforms.mjs -// cargo install cross --git https://github.com/cross-rs/cross -// cargo install --locked cargo-xwin -// -// The binaries are built with the `wasm` feature, so they run plugins. -// -// Then pack them into the package with scripts/archive.mjs. - -import { spawnSync } from 'node:child_process'; -import { existsSync } from 'node:fs'; -import { homedir } from 'node:os'; -import { delimiter, join } from 'node:path'; -import { parseArgs } from 'node:util'; - -import { fail, platforms, repoDir } from './platforms.mjs'; - -try { - const { values } = parseArgs({ - options: { platform: { type: 'string', multiple: true } }, - }); - compile(values.platform); -} catch (err) { - fail(err); -} - -/** Builds the binaries for the platforms in `only`, or for all of them. */ -function compile(only) { - const known = platforms.map((platform) => platform.id); - const unknown = only?.filter((id) => !known.includes(id)) ?? []; - if (unknown.length > 0) { - throw new Error(`unknown platform ${unknown.join(', ')}; forge supports ${known.join(', ')}`); - } - const chosen = platforms.filter((platform) => !only || only.includes(platform.id)); - if (process.platform !== 'darwin' && chosen.some((platform) => platform.os === 'darwin')) { - throw new Error('macOS binaries build only on a Mac; leave them out with --platform'); - } - - const installed = new Set(run('rustup', ['target', 'list', '--installed'], { capture: true }).split('\n')); - const missing = chosen - .filter((platform) => platform.builder !== 'cross' && !installed.has(platform.target)) - .map((platform) => platform.target); - if (missing.length > 0) { - // cross brings its own toolchain for its targets. - throw new Error(`run rustup target add ${missing.join(' ')}`); - } - - for (const platform of chosen) { - console.log(`\n==> ${platform.id} (${platform.target}) with ${platform.builder}`); - // `wasm` gives `--target wasm://`, which runs plugins. - const args = ['build', '--release', '--locked', '--features', 'wasm', '--target', platform.target]; - switch (platform.builder) { - case 'cargo': - run('cargo', args); - break; - case 'cross': - run(tool('cross', 'cargo install cross --git https://github.com/cross-rs/cross'), args); - break; - case 'xwin': { - tool('cargo-xwin', 'cargo install --locked cargo-xwin'); - // Link the C runtime in, or the binary needs VCRUNTIME140.dll from - // the Visual C++ Redistributable, which many machines don't have. - const flags = `CARGO_TARGET_${platform.target.toUpperCase().replaceAll('-', '_')}_RUSTFLAGS`; - const env = { [flags]: `${process.env[flags] ?? ''} -C target-feature=+crt-static`.trim() }; - run('cargo', ['xwin', ...args], { env }); - break; - } - default: - throw new Error(`${platform.id} has no builder named ${platform.builder}`); - } - } - console.log(`\nbuilt ${chosen.map((platform) => platform.id).join(', ')}`); -} - -/** - * Returns the path to the program `name`, looking on PATH and then in - * cargo's bin directory, which `cargo install` puts things in but which - * isn't always on PATH. Throws with `install` as the fix if there is none. - */ -function tool(name, install) { - const cargoBin = join(process.env.CARGO_HOME ?? join(homedir(), '.cargo'), 'bin'); - const exe = process.platform === 'win32' ? `${name}.exe` : name; - const found = [...(process.env.PATH ?? '').split(delimiter), cargoBin] - .map((dir) => join(dir, exe)) - .find((path) => existsSync(path)); - if (!found) { - throw new Error(`${name} is not installed; run ${install}`); - } - return found; -} - -/** - * Runs `cmd` with `args` in the crate's root, with `env` added to the - * environment, and throws if it fails. Returns its standard output if - * `capture` is set. - */ -function run(cmd, args, { capture = false, env = {} } = {}) { - const result = spawnSync(cmd, args, { - cwd: repoDir, - env: { ...process.env, ...env }, - encoding: 'utf8', - stdio: capture ? ['ignore', 'pipe', 'inherit'] : 'inherit', - }); - if (result.error) { - throw new Error(`cannot run ${cmd}: ${result.error.message}`); - } - if (result.status !== 0) { - throw new Error(`${cmd} ${args.join(' ')} failed`); - } - return result.stdout; -} diff --git a/packages/forge/scripts/platforms.mjs b/packages/forge/scripts/platforms.mjs index 0b32a20dd..53adb0e53 100644 --- a/packages/forge/scripts/platforms.mjs +++ b/packages/forge/scripts/platforms.mjs @@ -21,19 +21,19 @@ export const distDir = join(npmDir, 'dist'); * process.platform and process.arch, which the launcher looks the binary up * by, and `target` is the Rust target that builds it. * - * `builder` says how scripts/compile.mjs builds it: with cargo itself for - * macOS, which needs a Mac; with cross, in Docker, for Linux; and with - * cargo-xwin for Windows, since cross has no image for MSVC. + * `just build ` builds one (see the justfile); the publish-preview + * workflow's build matrix and the justfile's target lists must name the same + * platforms, and scripts/archive.mjs refuses to pack a package that misses one. * * Linux builds against musl so the binary is static and runs on any distro. */ export const platforms = [ - { os: 'darwin', cpu: 'arm64', target: 'aarch64-apple-darwin', builder: 'cargo' }, - { os: 'darwin', cpu: 'x64', target: 'x86_64-apple-darwin', builder: 'cargo' }, - { os: 'linux', cpu: 'arm64', target: 'aarch64-unknown-linux-musl', builder: 'cross' }, - { os: 'linux', cpu: 'x64', target: 'x86_64-unknown-linux-musl', builder: 'cross' }, - { os: 'win32', cpu: 'arm64', target: 'aarch64-pc-windows-msvc', builder: 'xwin' }, - { os: 'win32', cpu: 'x64', target: 'x86_64-pc-windows-msvc', builder: 'xwin' }, + { os: 'darwin', cpu: 'arm64', target: 'aarch64-apple-darwin' }, + { os: 'darwin', cpu: 'x64', target: 'x86_64-apple-darwin' }, + { os: 'linux', cpu: 'arm64', target: 'aarch64-unknown-linux-musl' }, + { os: 'linux', cpu: 'x64', target: 'x86_64-unknown-linux-musl' }, + { os: 'win32', cpu: 'arm64', target: 'aarch64-pc-windows-msvc' }, + { os: 'win32', cpu: 'x64', target: 'x86_64-pc-windows-msvc' }, ].map((platform) => ({ ...platform, id: `${platform.os}-${platform.cpu}`, diff --git a/packages/forge/scripts/publish.mjs b/packages/forge/scripts/publish.mjs deleted file mode 100644 index 5728caf17..000000000 --- a/packages/forge/scripts/publish.mjs +++ /dev/null @@ -1,50 +0,0 @@ -// Publishes the tarball scripts/archive.mjs made, which has the binary for -// every platform: -// -// node scripts/publish.mjs [npm publish options] -// -// Any options go to `npm publish`, such as --dry-run, --tag next, or -// --provenance. If the registry has this version already, it does nothing. - -import { execFileSync } from 'node:child_process'; -import { existsSync } from 'node:fs'; - -import { checkVersion, display, distDir, fail, tarball } from './platforms.mjs'; - -const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm'; -const name = '@cloudflare/forge'; - -try { - const version = checkVersion(); - const file = tarball(distDir, version); - if (!existsSync(file)) { - throw new Error(`${display(file)} does not exist; run node scripts/compile.mjs, then node scripts/archive.mjs`); - } - if (published(version)) { - console.log(`${name}@${version} is published already`); - } else { - // Run from dist/ so that npm ignores this directory's package.json. - execFileSync(npm, ['publish', file, ...process.argv.slice(2)], { - cwd: distDir, - stdio: 'inherit', - shell: npm.endsWith('.cmd'), - }); - } -} catch (err) { - fail(err); -} - -/** Reports whether the registry has @cloudflare/forge at `version`. */ -function published(version) { - try { - const out = execFileSync(npm, ['view', `${name}@${version}`, 'version'], { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'ignore'], - shell: npm.endsWith('.cmd'), - }); - return out.trim() === version; - } catch { - // npm view fails when the package doesn't exist at all. - return false; - } -} From ba0c424ad448eabb90a254e2c16d2a7c0b8265ba Mon Sep 17 00:00:00 2001 From: ematipico Date: Wed, 14 Oct 2026 15:00:31 +0100 Subject: [PATCH 2/2] chore: reduce certs stuff to a minimum --- docker/certs/.gitignore | 2 ++ docker/release.Dockerfile | 17 ++++++++--------- justfile | 13 ------------- packages/forge/README.md | 6 +----- 4 files changed, 11 insertions(+), 27 deletions(-) create mode 100644 docker/certs/.gitignore diff --git a/docker/certs/.gitignore b/docker/certs/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/docker/certs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docker/release.Dockerfile b/docker/release.Dockerfile index ed68ce1fc..6ae8ce7f5 100644 --- a/docker/release.Dockerfile +++ b/docker/release.Dockerfile @@ -6,11 +6,7 @@ # containers build with the same Rust as CI. The default, the latest 1.x, # only keeps the image names valid for a plain `docker build`. # -# The `extra-ca` build context holds extra CA certificates to trust, one per -# .crt file, for networks whose proxy re-signs HTTPS traffic, such as -# Cloudflare's Zero Trust gateway. Without them, cargo, cargo-xwin and the -# package managers reject every download. The justfile passes an empty -# directory when there are none. +# The images also trust any .crt files in docker/certs/, which is gitignored. ARG RUST_VERSION=1 @@ -19,15 +15,18 @@ ARG RUST_VERSION=1 # for wasmtime's C code. Run with --platform linux/arm64 or linux/amd64 to # build for that architecture. FROM rust:${RUST_VERSION}-alpine AS linux -COPY --from=extra-ca . /usr/local/share/ca-certificates/ -RUN update-ca-certificates && apk add --no-cache musl-dev +COPY certs/ /tmp/certs/ +RUN find /tmp/certs -name '*.crt' -exec cp {} /usr/local/share/ca-certificates/ \; \ + && update-ca-certificates \ + && apk add --no-cache musl-dev # Windows: MSVC, cross-compiled with cargo-xwin, which downloads Microsoft's # CRT and Windows SDK on first use. clang, lld and llvm are its toolchain. FROM rust:${RUST_VERSION}-slim-bookworm AS windows ARG CARGO_XWIN_VERSION=0.23.1 -COPY --from=extra-ca . /usr/local/share/ca-certificates/ -RUN update-ca-certificates +COPY certs/ /tmp/certs/ +RUN find /tmp/certs -name '*.crt' -exec cp {} /usr/local/share/ca-certificates/ \; \ + && update-ca-certificates RUN apt-get update \ && apt-get install -y --no-install-recommends clang lld llvm \ && rm -rf /var/lib/apt/lists/* diff --git a/justfile b/justfile index 5d23891de..77cc872cf 100644 --- a/justfile +++ b/justfile @@ -67,21 +67,8 @@ build-docker target: *) echo "error: build-docker builds Linux and Windows targets; build $target with: just build $target" >&2; exit 1 ;; esac image="forge-release-$stage:$rust${platform:+-${platform#linux/}}" - # Extra CA certificates for the images to trust, behind a proxy that - # re-signs HTTPS traffic: FORGE_DOCKER_CA_CERT, or else the bundle Node - # trusts, NODE_EXTRA_CA_CERTS. They go one per file, because Alpine's - # update-ca-certificates skips a file that holds more than one. - ca_dir=target/docker/extra-ca - rm -rf "$ca_dir" && mkdir -p "$ca_dir" - ca="${FORGE_DOCKER_CA_CERT:-${NODE_EXTRA_CA_CERTS:-}}" - if [ -n "$ca" ]; then - [ -f "$ca" ] || { echo "error: no CA certificate file at $ca" >&2; exit 1; } - awk -v dir="$ca_dir" '/-----BEGIN CERTIFICATE-----/ { n++; out = sprintf("%s/extra-%d.crt", dir, n) } out { print > out } /-----END CERTIFICATE-----/ { close(out); out = "" }' "$ca" - echo "==> trusting $(ls "$ca_dir" | wc -l | tr -d ' ') extra CA certificate(s) from $ca" - fi echo "==> $target in $image" docker build --quiet ${platform:+--platform "$platform"} --target "$stage" --build-arg "RUST_VERSION=$rust" \ - --build-context "extra-ca=$ca_dir" \ --tag "$image" --file docker/release.Dockerfile docker >/dev/null # Runs as you, so everything it writes under target/ is yours. Each target # builds into a directory of its own: the host's target/ holds build diff --git a/packages/forge/README.md b/packages/forge/README.md index e8ff83a81..239bcad10 100644 --- a/packages/forge/README.md +++ b/packages/forge/README.md @@ -332,11 +332,7 @@ other four, and `just fetch` gets all six from CI. The Docker images are in `docker/release.Dockerfile` and build with the Rust in `rust-toolchain.toml`; on an Apple silicon Mac, the x64 Linux build runs emulated, so it's slow. -Behind a proxy that re-signs HTTPS traffic, such as Cloudflare's Zero Trust -gateway, the containers need its CA certificate, or every download fails with -"self-signed certificate in certificate chain". `just build-docker` adds the -certificates in `FORGE_DOCKER_CA_CERT` to the images, or else those in -`NODE_EXTRA_CA_CERTS`, which Cloudflare machines usually set already. +The images also trust any `.crt` files in `docker/certs/`, which is gitignored. CI builds each target with `just build` on a runner that builds it natively, in the publish-preview workflow, then packs them with `just pack`.