diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03384ee..21bf4fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,34 +7,25 @@ on: env: CARGO_TERM_COLOR: always - RAYFORCE_SRC: ${{ github.workspace }}/rayforce-core - RAYFORCE_Q_SRC: ${{ github.workspace }}/rayforce-q jobs: test: runs-on: ubuntu-latest steps: + # The C core and the rayforce-q client are submodules under + # rayforce-sys/vendor/, so this one checkout brings the whole build. - name: Checkout bindings uses: actions/checkout@v4 - - - name: Checkout RayforceDB core - uses: actions/checkout@v4 with: - repository: RayforceDB/rayforce - path: rayforce-core + submodules: recursive - - name: Checkout rayforce-q client - uses: actions/checkout@v4 - with: - repository: RayforceDB/rayforce-q - path: rayforce-q + - name: Check vendored core pin + run: ./scripts/check-vendored-pin.sh - name: Install toolchain deps run: sudo apt-get update && sudo apt-get install -y clang libclang-dev build-essential - # The repo's .cargo/config.toml sets a macOS LIBCLANG_PATH for local dev. - # On Linux that path is invalid, so point bindgen at the apt libclang here - # — an env var set in the job takes precedence over the config default. + # bindgen needs to be told where libclang lives on the runner. - name: Locate libclang run: | LIB="$(find /usr/lib -name 'libclang*.so*' 2>/dev/null | head -1)" @@ -64,3 +55,17 @@ jobs: - name: Test run: cargo test --workspace + + # The vendored sources must actually land in the .crate — that is the + # whole reason docs.rs and other network-isolated builds work. Cheap to + # check here, and a broken package is invisible until someone consumes it. + - name: Check the packaged crate carries the vendored core + run: | + cargo package -p rayforce-sys --list --allow-dirty > /tmp/pkg.txt + for f in vendor/rayforce/include/rayforce.h vendor/rayforce/Makefile vendor/rayforce-q/q.c; do + grep -qx "$f" /tmp/pkg.txt || { echo "::error::$f missing from the packaged crate"; exit 1; } + done + if grep -qE '\.(o|d)$' /tmp/pkg.txt; then + echo "::error::build artifacts leaked into the packaged crate"; exit 1 + fi + echo "packaged $(wc -l < /tmp/pkg.txt) files" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c30d732..7c1b2a0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,10 +10,6 @@ on: env: CARGO_TERM_COLOR: always - # The publish verify build (and the downstream rayforce-sys build triggered - # when publishing `rayforce`) reads these to avoid re-cloning the C sources. - RAYFORCE_SRC: ${{ github.workspace }}/rayforce-core - RAYFORCE_Q_SRC: ${{ github.workspace }}/rayforce-q jobs: publish: @@ -24,30 +20,21 @@ jobs: id-token: write contents: read steps: + # The C sources are submodules under rayforce-sys/vendor/, so what is + # built and published here is exactly what a crates.io consumer gets — + # no separate checkout to keep in step with build.rs. - name: Checkout bindings uses: actions/checkout@v4 - - # Pin the same refs as rayforce-sys/build.rs so a source build here - # matches what a crates.io consumer gets. - - name: Checkout RayforceDB core (v2.5.1) - uses: actions/checkout@v4 with: - repository: RayforceDB/rayforce - ref: 'v2.5.1' - path: rayforce-core + submodules: recursive - - name: Checkout rayforce-q client (2.0.0) - uses: actions/checkout@v4 - with: - repository: RayforceDB/rayforce-q - ref: '2.0.0' - path: rayforce-q + - name: Check vendored core pin + run: ./scripts/check-vendored-pin.sh - name: Install toolchain deps run: sudo apt-get update && sudo apt-get install -y clang libclang-dev build-essential - # .cargo/config.toml sets a macOS LIBCLANG_PATH for local dev; on Linux - # that path is invalid, so point bindgen at the apt libclang here. + # bindgen needs to be told where libclang lives on the runner. - name: Locate libclang run: | LIB="$(find /usr/lib -name 'libclang*.so*' 2>/dev/null | head -1)" @@ -70,6 +57,19 @@ jobs: - name: Test workspace run: cargo test --workspace + # A .crate missing its vendored sources would build fine in this job (the + # submodule is on disk) and then fail for every consumer and on docs.rs. + # This is the last point at which that is catchable. + - name: Check the packaged crate carries the vendored core + run: | + cargo package -p rayforce-sys --list > /tmp/pkg.txt + for f in vendor/rayforce/include/rayforce.h vendor/rayforce/Makefile vendor/rayforce-q/q.c; do + grep -qx "$f" /tmp/pkg.txt || { echo "::error::$f missing from the packaged crate"; exit 1; } + done + if grep -qE '\.(o|d)$' /tmp/pkg.txt; then + echo "::error::build artifacts leaked into the packaged crate"; exit 1 + fi + # Trusted Publishing: exchanges the job's OIDC identity for a short-lived # crates.io token. No CARGO_REGISTRY_TOKEN secret needed — but both # `rayforce` and `rayforce-sys` must have this repo/workflow registered as diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..b8799e8 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "rayforce-sys/vendor/rayforce"] + path = rayforce-sys/vendor/rayforce + url = https://github.com/RayforceDB/rayforce.git +[submodule "rayforce-sys/vendor/rayforce-q"] + path = rayforce-sys/vendor/rayforce-q + url = https://github.com/RayforceDB/rayforce-q.git diff --git a/README.md b/README.md index 83720c9..966a120 100644 --- a/README.md +++ b/README.md @@ -88,8 +88,8 @@ println!("{result}"); ## Installation -The crate links two local checkouts: the RayforceDB **core** (`RAYFORCE_SRC`) and the -**rayforce-q** IPC client (`RAYFORCE_Q_SRC`). The build script compiles both and statically +The RayforceDB **core** and the **rayforce-q** IPC client are C. Both ship inside the +crate, so nothing is fetched at build time — the build script compiles them and statically links `librayforce.a`. ```toml @@ -98,18 +98,53 @@ links `librayforce.a`. rayforce = { git = "https://github.com/RayforceDB/rayforce-rs" } ``` -```sh -git clone https://github.com/RayforceDB/rayforce ~/rayforce # core -git clone https://github.com/RayforceDB/rayforce-q ~/rayforce-q # Q IPC client +Requirements: a C toolchain (`make`, `clang`) and `libclang` for `bindgen`. + +### Working on the bindings -export RAYFORCE_SRC=/path/to/rayforce # default: ~/rayforce -export RAYFORCE_Q_SRC=/path/to/rayforce-q # default: ~/rayforce-q +The C sources live in git submodules under `rayforce-sys/vendor/`, so a checkout needs +them initialized: + +```sh +git clone --recurse-submodules https://github.com/RayforceDB/rayforce-rs +# in an existing clone: +git submodule update --init --recursive cargo build cargo test ``` -Requirements: a C toolchain (`make`, `clang`) and `libclang` for `bindgen`. +### Choosing the core version + +Each release links one pinned core version. It lives in two places that must agree — the +`rayforce-sys/vendor/rayforce` submodule, and the `CORE_VERSION` / `CORE_COMMIT` constants +in `rayforce-sys/build.rs` that get stamped into `librayforce.a` (a crate unpacked from +crates.io has no git history for the core's Makefile to read a version from). + +To move the pin, move both: + +```sh +git -C rayforce-sys/vendor/rayforce fetch --tags +git -C rayforce-sys/vendor/rayforce checkout v2.6.0 +git add rayforce-sys/vendor/rayforce + +git -C rayforce-sys/vendor/rayforce rev-parse --short=7 HEAD # CORE_COMMIT +$EDITOR rayforce-sys/build.rs # CORE_VERSION, CORE_COMMIT + +./scripts/check-vendored-pin.sh # names the mismatch if they disagree +cargo test --workspace +``` + +`rayforce-sys/vendor/rayforce-q` works the same way, minus the constants — nothing is +stamped from it. + +To build against a core you are changing instead, point the build script at your own +checkout. These take precedence over the vendored copies: + +```sh +export RAYFORCE_SRC=/path/to/rayforce +export RAYFORCE_Q_SRC=/path/to/rayforce-q +``` `bindgen` locates `libclang` via `LIBCLANG_PATH`. This is deliberately **not** set in the repo's `.cargo/config.toml`. If bindgen can't auto-detect libclang, set it yourself: diff --git a/docs/docs/content/get-started/installation.md b/docs/docs/content/get-started/installation.md index 5a22888..b846d0f 100644 --- a/docs/docs/content/get-started/installation.md +++ b/docs/docs/content/get-started/installation.md @@ -1,8 +1,9 @@ # :octicons-package-16: Installation -`rayforce` builds against a local checkout of the RayforceDB core. The build -script compiles the core into a static library (`librayforce.a`) and statically -links it, so there is nothing to install at runtime. +The RayforceDB core is C, and it ships inside the `rayforce-sys` crate as a +pinned git submodule. The build script compiles it into a static library +(`librayforce.a`) and links it statically — so there is nothing to fetch while +building, and nothing to install at runtime. ## :material-clipboard-check-outline: Prerequisites @@ -11,7 +12,6 @@ links it, so there is nothing to install at runtime. - A **C toolchain** — `make` and `clang` — to build the RayforceDB core. - **`libclang`**, required by [`bindgen`](https://github.com/rust-lang/rust-bindgen) to generate the raw FFI bindings. -- A **RayforceDB core** checkout to link against (see below). !!! note "macOS: `LIBCLANG_PATH`" On macOS `bindgen` may not find `libclang` automatically. Point it at your @@ -24,23 +24,84 @@ links it, so there is nothing to install at runtime. The repository's `.cargo/config.toml` is the place to set this permanently for local builds. -## :material-source-branch: Building against a local core +## :material-tag-outline: Which core version gets linked -The build links a local RayforceDB core checkout. Point the `RAYFORCE_SRC` -environment variable at it; it defaults to `~/rayforce`. The build script runs -the core's `make lib` to produce `librayforce.a`, then links it. +Each release of `rayforce` links one specific core version. It is pinned in two +places that must agree: -```sh -git clone https://github.com/RayforceDB/rayforce-rs.git ~/rayforce +| What | Where | +| --- | --- | +| The core sources | the `rayforce-sys/vendor/rayforce` submodule | +| The version stamped into the library | `CORE_VERSION` / `CORE_COMMIT` in `rayforce-sys/build.rs` | + +The constants exist because the core's `Makefile` normally resolves its version +from `git describe`, and a crate unpacked from crates.io has no git history to +read. `scripts/check-vendored-pin.sh` asserts the two agree, and CI runs it on +every push. + +As a consumer you get the core that matches the `rayforce` version you depend +on — pick a different core by picking a different `rayforce` release. The two +sections below are for changing that pin yourself. -# Point the build at it (default is ~/rayforce, so this is optional there). -export RAYFORCE_SRC=~/rayforce +### :material-source-branch: Building against your own core checkout + +To develop against a core you are changing, point `RAYFORCE_SRC` at it. It takes +precedence over the vendored copy, and is built in place so your incremental +state and the version its git history reports are preserved. `RAYFORCE_Q_SRC` +does the same for the `rayforce-q` IPC client. + +```sh +export RAYFORCE_SRC=/path/to/rayforce +export RAYFORCE_Q_SRC=/path/to/rayforce-q -# Build and test the bindings. cargo build cargo test ``` +Unset them to go back to the vendored sources. + +### :material-arrow-up-bold-box-outline: Bumping the pinned version + +Moving the pin means moving the submodule and the constants together: + +```sh +# 1. Move the submodule to the new tag. +git -C rayforce-sys/vendor/rayforce fetch --tags +git -C rayforce-sys/vendor/rayforce checkout v2.6.0 +git add rayforce-sys/vendor/rayforce + +# 2. Read back the values build.rs must stamp. +git -C rayforce-sys/vendor/rayforce describe --tags --exact-match # -> v2.6.0 +git -C rayforce-sys/vendor/rayforce rev-parse --short=7 HEAD # -> e.g. 1a2b3c4 +``` + +Then edit `rayforce-sys/build.rs` to match — `CORE_VERSION` is the tag without +its leading `v`: + +```rust +const CORE_VERSION: &str = "2.6.0"; +const CORE_COMMIT: &str = "1a2b3c4"; +``` + +And check the result: + +```sh +./scripts/check-vendored-pin.sh # fails, with the mismatch named, if they disagree +cargo test --workspace +``` + +The same applies to `rayforce-sys/vendor/rayforce-q`, minus the constants — +nothing is stamped from it, so moving the submodule is the whole change. + +!!! warning "A new core may need the bindgen allowlist updated" + A few of the symbols the safe crate calls are not in the public + `rayforce.h` — they are read from the core's private headers instead. + `CORE_PRIVATE_HEADERS` and `INTERNAL_FNS` in `rayforce-sys/build.rs` name + those headers and symbols. Signatures need no maintenance, since bindgen + reads them from the core, but a bump that renames or relocates one will + fail the build: bindgen emits nothing for it and the safe crate stops + compiling against `rayforce_sys`. Fix it by updating those two lists. + !!! note "Tests run single-threaded" The engine runs on a single thread with one live runtime per process, so the test suite is serialized. Run it with `RUST_TEST_THREADS=1` (or via the @@ -58,7 +119,7 @@ or in `Cargo.toml`: ```toml [dependencies] -rayforce = "0.1" +rayforce = "1" ``` ### The `chrono` feature (default) @@ -72,7 +133,7 @@ To build without it, disable default features: ```toml [dependencies] -rayforce = { version = "0.1", default-features = false } +rayforce = { version = "1", default-features = false } ``` ## :material-arrow-right: Next steps diff --git a/rayforce-sys/Cargo.toml b/rayforce-sys/Cargo.toml index a32ca85..60355c4 100644 --- a/rayforce-sys/Cargo.toml +++ b/rayforce-sys/Cargo.toml @@ -15,6 +15,29 @@ rust-version.workspace = true links = "rayforce" build = "build.rs" +# The C sources ship inside the .crate so a build never needs the network +# (docs.rs and other sandboxes have none). Deny-by-default: the vendored repos +# carry test suites and websites that would otherwise bloat the package — +# `make lib` only needs src/*/*.c, include/ and the Makefile. +# +# The extension filters are load-bearing: an `include` list is matched against +# the filesystem, not against git, so a bare `src/**` would package the .o/.d +# artifacts of a local `make lib` despite the core's .gitignore. +include = [ + "src/**", + "tests/**", + "build.rs", + "Cargo.toml", + "vendor/rayforce/src/**/*.c", + "vendor/rayforce/src/**/*.h", + "vendor/rayforce/include/*.h", + "vendor/rayforce/Makefile", + "vendor/rayforce/LICENSE", + "vendor/rayforce-q/q.c", + "vendor/rayforce-q/q.h", + "vendor/rayforce-q/LICENSE", +] + [build-dependencies] bindgen = "0.70" cc = "1" diff --git a/rayforce-sys/build.rs b/rayforce-sys/build.rs index 6e2720c..a94a0d7 100644 --- a/rayforce-sys/build.rs +++ b/rayforce-sys/build.rs @@ -1,41 +1,121 @@ //! Build script for `rayforce-sys`. //! -//! 1. Locates the RayforceDB v2 core + `rayforce-q` source trees (see -//! [`core_src_dir`] / [`q_src_dir`] for the resolution order). When neither -//! an env override nor a local dev checkout is present — the common case for -//! a crate downloaded from crates.io — the pinned release tags are shallow -//! cloned from GitHub into `OUT_DIR`. -//! 2. Builds the static library `librayforce.a` via the core's `make lib` -//! (incremental — a no-op when objects are up to date). +//! 1. Locates the RayforceDB v2 core + `rayforce-q` source trees. Both ship +//! inside this crate as git submodules under `vendor/` (see [`core_src_dir`] +//! / [`q_src_dir`]), so a build never touches the network — docs.rs and +//! other sandboxes have none. +//! 2. Stages the core into `OUT_DIR` ([`stage_core`]) and builds the static +//! library `librayforce.a` there via the core's `make lib` (incremental — a +//! no-op when objects are up to date). //! 3. Emits the static link directives. -//! 4. Generates Rust bindings from `wrapper.h` with `bindgen`. +//! 4. Generates Rust bindings with `bindgen`, from the core's public header +//! plus the private ones declaring [`INTERNAL_FNS`]. +use std::collections::HashSet; use std::env; +use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -/// Pinned upstream sources cloned when no local checkout is available. Bump -/// these in lockstep with a `rayforce-sys` release. Override at build time with -/// the `RAYFORCE_REPO` / `RAYFORCE_REF` (and `_Q_` variants) env vars. -const RAYFORCE_REPO: &str = "https://github.com/RayforceDB/rayforce.git"; -const RAYFORCE_REF: &str = "v2.5.8"; -const RAYFORCE_Q_REPO: &str = "https://github.com/RayforceDB/rayforce-q.git"; -const RAYFORCE_Q_REF: &str = "2.0.0"; +/// Version stamped into the vendored core at compile time. The core's Makefile +/// normally resolves this from `git describe` (`Makefile:19`), but a crate +/// unpacked from crates.io has no git history — without this the core would +/// report itself as `0.0.0`. +/// +/// Must match the tag `vendor/rayforce` is pinned to. CI asserts the two agree; +/// see the "Check vendored core pin" step in `.github/workflows/ci.yml`. +const CORE_VERSION: &str = "2.5.8"; + +/// Commit the `vendor/rayforce` submodule is pinned to, stamped alongside +/// [`CORE_VERSION`]. Also checked by CI's "Check vendored core pin" step. +/// +/// This must be passed explicitly for the same reason as the version, and for +/// one more: `Makefile:27` resolves it with `git rev-parse --short HEAD`, and +/// git searches *upward* from the working directory. Since the core is built +/// under OUT_DIR, an unset value does not fall back to "unknown" — it silently +/// reports the HEAD of whatever unrelated repository happens to enclose the +/// build directory. +const CORE_COMMIT: &str = "f0d4bb4"; + +/// Warning flags for the vendored core build — the core's own `WARNS` +/// (`Makefile:30`) minus `-Werror`. Consumers compile this with whatever +/// toolchain they happen to have, and a new diagnostic from a future compiler +/// should not be a hard failure inside someone else's dependency tree. The +/// core's own CI is where `-Werror` belongs. +const CORE_WARNS: &str = "-Wall -Wextra -Wstrict-prototypes -Wno-unused-parameter"; + +/// Core headers outside `include/` that declare [`INTERNAL_FNS`]. Private to +/// the core, but they ship in this crate alongside the sources they belong to +/// (`Cargo.toml`'s `vendor/rayforce/src/**/*.h`), and each parses standalone +/// given `include/` and `src/` on the header search path — plus the +/// `-D_Atomic` workaround in [`main`]. +const CORE_PRIVATE_HEADERS: &[&str] = &[ + "lang/eval.h", + "lang/internal.h", + "ops/ops.h", + "store/serde.h", + "core/runtime.h", +]; + +/// Symbols the safe crate calls that `include/rayforce.h` does not declare. +/// They are exported by `librayforce.a` all the same, and are read from +/// [`CORE_PRIVATE_HEADERS`] rather than redeclared here — C linkage matches on +/// name alone, so a hand-copied signature that drifts from the core is +/// undefined behavior with no diagnostic anywhere in the build. +/// +/// Everything else bindgen generates comes from the public header; this list is +/// the entire deliberate exception to that, so a core bump that renames or +/// removes one of these fails the build rather than passing silently. +const INTERNAL_FNS: &[&str] = &[ + // src/lang/eval.h — evaluate an already-compiled AST object + "ray_eval", + // src/lang/internal.h — query builtins (variadic arg-array form) + "ray_update_fn", + "ray_insert_fn", + "ray_upsert_fn", + // src/lang/internal.h — CSV + on-disk table I/O + "ray_read_csv_fn", + "ray_write_csv_fn", + "ray_set_splayed_fn", + "ray_get_splayed_fn", + "ray_get_parted_fn", + // src/ops/ops.h — materialize a lazy DAG result + "ray_lazy_materialize", + // src/store/serde.h — serialize / deserialize a U8 vector with IPC header + "ray_ser", + "ray_de", + // src/core/runtime.h — last per-VM error message (set with a RAY_ERROR) + "ray_error_msg", +]; fn main() { - let core = core_src_dir(); - let include = core.join("include"); - let header = include.join("rayforce.h"); + // An explicit override means the caller brought their own core, with its + // own git history and its own version; only stamp CORE_VERSION on ours. + let core_is_vendored = env::var_os("RAYFORCE_SRC").is_none(); + let core_src = core_src_dir(); + let header_src = core_src.join("include/rayforce.h"); assert!( - header.exists(), + header_src.exists(), "rayforce core header not found at {}.\n\ - Set RAYFORCE_SRC to your rayforce checkout (default: ~/rayforce).", - header.display() + If this is a git checkout, the vendored core submodule is not \ + initialized — run `git submodule update --init --recursive`.\n\ + To build against a different core, point RAYFORCE_SRC at it.", + header_src.display() ); + // Our own vendored copy has to be staged into OUT_DIR before it is built; + // a checkout the caller pointed us at is theirs, and building it in place + // keeps their incremental state and the version its git history reports. + let core = if core_is_vendored { + stage_core(&core_src) + } else { + core_src.clone() + }; + let include = core.join("include"); + sanitize_libclang_path(); - build_core_lib(&core); + build_core_lib(&core, core_is_vendored); // --- Q IPC client (rayforce-q's q.c) --- // Linked BEFORE librayforce so its undefined `ray_*` symbols resolve from @@ -46,7 +126,9 @@ fn main() { assert!( q_c.exists(), "rayforce-q client not found at {}.\n\ - Set RAYFORCE_Q_SRC to your rayforce-q checkout (default: ~/rayforce-q).", + If this is a git checkout, the vendored submodule is not initialized \ + — run `git submodule update --init --recursive`.\n\ + To build against a different checkout, point RAYFORCE_Q_SRC at it.", q_c.display() ); cc::Build::new() @@ -58,7 +140,6 @@ fn main() { .compile("rayforce_q"); println!("cargo:rerun-if-changed={}", q_c.display()); println!("cargo:rerun-if-changed={}", q_src.join("q.h").display()); - println!("cargo:rerun-if-env-changed=RAYFORCE_Q_SRC"); // --- linking --- println!("cargo:rustc-link-search=native={}", core.display()); @@ -67,136 +148,216 @@ fn main() { if cfg!(target_os = "linux") { println!("cargo:rustc-link-lib=dylib=pthread"); } - // Expose the core dir to downstream crates (e.g. for symfile fixtures). + // Expose the staged core dir to downstream crates (e.g. for symfile + // fixtures) — this is where librayforce.a and the headers actually live. println!("cargo:root={}", core.display()); // --- bindgen --- - let bindings = bindgen::Builder::default() - .header("wrapper.h") + let mut builder = bindgen::Builder::default() + .header(include.join("rayforce.h").display().to_string()) .clang_arg(format!("-I{}", include.display())) - // Keep the surface tight and deterministic. - .allowlist_function("ray_.*") - .allowlist_type("ray_.*") - .allowlist_var("RAY_.*") - .allowlist_var("NULL_.*") - .allowlist_var("__ray_.*") - .allowlist_var("ray_type_sizes") + .clang_arg(format!("-I{}", core.join("src").display())) + // bindgen 0.70 cannot resolve C11 atomics and aborts the whole parse + // with "Couldn't resolve constant type" — reached here via + // `lang/internal.h` -> `mem/heap.h:442`, the only header declaring + // ray_{set,get}_splayed_fn / ray_get_parted_fn. Defining the keyword + // away costs nothing: the only two atomics in the parse are the file + // scope globals `ray_heap_pending_merge` (`mem/heap.h:442`) and + // `ray_parallel_flag` (`core/platform.h:179`), neither allowlisted, and + // no generated type contains one — `include/rayforce.h` never says + // `_Atomic`. So no layout bindgen emits can shift. This affects only + // bindgen's parse; the core itself is compiled by its own Makefile. + .clang_arg("-D_Atomic(T)=T") + // Everything the public header declares. This bound is load-bearing: + // the private headers added below declare ~550 functions and ~80 RAY_* + // constants between them, so a blanket `ray_.*` would drag in the whole + // internal surface. Anchored loosely because the staged path lives under + // OUT_DIR, which may itself contain regex metacharacters. + .allowlist_file(".*/include/rayforce\\.h") + // The public header leaves ray_runtime_s incomplete (`rayforce.h:656`) + // and `core/runtime.h:114` completes it. Left alone, bindgen would + // publish the runtime internals — ray_vm_t and friends, ~67 KB of + // private layout that would then churn on every core bump. Opaque + // keeps it a handle, which is all the public API ever passes around. + .opaque_type("ray_runtime_s") // ray_t is a union with a flexible array member + nested anon structs; // let bindgen represent it faithfully. .layout_tests(true) .derive_debug(false) .generate_comments(false) - .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) - .generate() - .expect("failed to generate rayforce bindings"); + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())); + for header in CORE_PRIVATE_HEADERS { + builder = builder.header(core.join("src").join(header).display().to_string()); + } + for func in INTERNAL_FNS { + builder = builder.allowlist_function(func); + } - let out = PathBuf::from(env::var("OUT_DIR").unwrap()); - bindings - .write_to_file(out.join("bindings.rs")) + builder + .generate() + .expect("failed to generate rayforce bindings") + .write_to_file(out_dir().join("bindings.rs")) .expect("failed to write bindings.rs"); - println!("cargo:rerun-if-changed=wrapper.h"); println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-env-changed=RAYFORCE_SRC"); - println!("cargo:rerun-if-changed={}", header.display()); - // Relink when the core archive changes (e.g. the C core was rebuilt). - // `make lib` is incremental, so this doesn't cause perpetual rebuilds. - // For a guaranteed pickup after editing core sources, touch build.rs or - // `cargo clean -p rayforce-sys`. - println!( - "cargo:rerun-if-changed={}", - core.join("librayforce.a").display() - ); } /// Resolve the RayforceDB core source tree, in order of precedence: -/// 1. `RAYFORCE_SRC` — an explicit checkout (used by CI and local overrides). -/// 2. `~/rayforce` — a developer's local clone, if it looks like the core. -/// 3. A shallow clone of [`RAYFORCE_REF`] into `OUT_DIR` (crates.io consumers). +/// 1. `RAYFORCE_SRC` — an explicit checkout, for building the bindings against +/// an unreleased core. +/// 2. `vendor/rayforce` — the submodule shipped inside this crate, pinned to +/// [`CORE_VERSION`]. Present both in a git checkout (once submodules are +/// initialized) and in the `.crate` published to crates.io. fn core_src_dir() -> PathBuf { println!("cargo:rerun-if-env-changed=RAYFORCE_SRC"); - if let Ok(p) = env::var("RAYFORCE_SRC") { - return PathBuf::from(p); - } - if let Ok(home) = env::var("HOME") { - let local = Path::new(&home).join("rayforce"); - if local.join("include/rayforce.h").exists() { - return local; - } - } - clone_pinned( - "rayforce", - "RAYFORCE_REPO", - RAYFORCE_REPO, - "RAYFORCE_REF", - RAYFORCE_REF, - ) + let src = match env::var("RAYFORCE_SRC") { + Ok(p) => PathBuf::from(p), + Err(_) => vendored("rayforce"), + }; + // Rebuild on a submodule bump, or on an edit to a RAYFORCE_SRC checkout. + println!("cargo:rerun-if-changed={}", src.join("Makefile").display()); + println!("cargo:rerun-if-changed={}", src.join("include").display()); + println!("cargo:rerun-if-changed={}", src.join("src").display()); + src } /// Resolve the `rayforce-q` source tree; same precedence as [`core_src_dir`], -/// keyed off `RAYFORCE_Q_SRC` / `~/rayforce-q` / a clone of [`RAYFORCE_Q_REF`]. +/// keyed off `RAYFORCE_Q_SRC` / the `vendor/rayforce-q` submodule. fn q_src_dir() -> PathBuf { println!("cargo:rerun-if-env-changed=RAYFORCE_Q_SRC"); - if let Ok(p) = env::var("RAYFORCE_Q_SRC") { - return PathBuf::from(p); + match env::var("RAYFORCE_Q_SRC") { + Ok(p) => PathBuf::from(p), + Err(_) => vendored("rayforce-q"), } - if let Ok(home) = env::var("HOME") { - let local = Path::new(&home).join("rayforce-q"); - if local.join("q.c").exists() { - return local; +} + +/// Path to a submodule under `vendor/`, resolved against the crate root so it +/// works from a git checkout and from an unpacked `.crate` alike. +fn vendored(name: &str) -> PathBuf { + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is always set")) + .join("vendor") + .join(name) +} + +fn out_dir() -> PathBuf { + PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR is always set")) +} + +/// Mirror the parts of the vendored core that `make lib` needs into +/// `OUT_DIR/core`, and return that path. +/// +/// The core's Makefile builds strictly in-tree — `Makefile:129` names objects +/// `src//.rel.o` and `Makefile:185` drops `librayforce.a` at the +/// root — so running it where the sources sit would write into the crate's own +/// directory. For a crates.io consumer that is the shared registry cache, and +/// it is what makes `cargo package`'s verify step fail with "files added". +/// Staging keeps the usual rule that a build script writes only under OUT_DIR. +/// +/// Copies are skipped when the destination is already current, so `make` stays +/// incremental across rebuilds (OUT_DIR persists). +fn stage_core(src: &Path) -> PathBuf { + let dst = out_dir().join("core"); + let mut staged = HashSet::new(); + copy_if_stale(&src.join("Makefile"), &dst.join("Makefile"), &mut staged); + mirror(&src.join("include"), &dst.join("include"), &mut staged); + mirror(&src.join("src"), &dst.join("src"), &mut staged); + prune_stale(&dst, &staged); + dst +} + +/// Recursively copy `.c` / `.h` files from `src` into `dst`, recording every +/// destination touched in `staged`. +fn mirror(src: &Path, dst: &Path, staged: &mut HashSet) { + let entries = + fs::read_dir(src).unwrap_or_else(|e| panic!("failed to read {}: {e}", src.display())); + for entry in entries.flatten() { + let from = entry.path(); + let to = dst.join(entry.file_name()); + if from.is_dir() { + mirror(&from, &to, staged); + } else if is_source(&from) { + copy_if_stale(&from, &to, staged); } } - clone_pinned( - "rayforce-q", - "RAYFORCE_Q_REPO", - RAYFORCE_Q_REPO, - "RAYFORCE_Q_REF", - RAYFORCE_Q_REF, - ) } -/// Shallow-clone `repo` at `git_ref` into `OUT_DIR/` and return the path. -/// Reuses an existing clone (`OUT_DIR` persists across incremental rebuilds) so -/// repeated builds don't re-hit the network. The repo URL and ref can be -/// overridden via the given env vars for testing against unreleased cores. -fn clone_pinned( - name: &str, - repo_env: &str, - repo_default: &str, - ref_env: &str, - ref_default: &str, -) -> PathBuf { - println!("cargo:rerun-if-env-changed={repo_env}"); - println!("cargo:rerun-if-env-changed={ref_env}"); - let repo = env::var(repo_env).unwrap_or_else(|_| repo_default.to_string()); - let git_ref = env::var(ref_env).unwrap_or_else(|_| ref_default.to_string()); - - let dst = PathBuf::from(env::var("OUT_DIR").unwrap()).join(name); - if dst.join(".git").exists() { - return dst; +fn is_source(p: &Path) -> bool { + matches!(p.extension().and_then(|e| e.to_str()), Some("c" | "h")) +} + +fn copy_if_stale(from: &Path, to: &Path, staged: &mut HashSet) { + staged.insert(to.to_path_buf()); + if is_current(from, to) { + return; } + let parent = to.parent().expect("staged paths always have a parent"); + fs::create_dir_all(parent) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", parent.display())); + fs::copy(from, to) + .unwrap_or_else(|e| panic!("failed to copy {} to {}: {e}", from.display(), to.display())); +} - eprintln!( - "rayforce-sys: cloning {name} {git_ref} from {repo} into {}", - dst.display() - ); - let status = Command::new("git") - .args(["clone", "--depth", "1", "--branch", &git_ref, &repo]) - .arg(&dst) - .status() - .unwrap_or_else(|e| panic!("failed to invoke `git clone` for {name}: {e}")); - assert!( - status.success(), - "`git clone --branch {git_ref} {repo}` failed (exit {:?}).\n\ - Provide a local checkout via {}_SRC to build offline.", - status.code(), - if name == "rayforce-q" { - "RAYFORCE_Q" - } else { - "RAYFORCE" - }, - ); - dst +/// `fs::copy` does not preserve mtime, so a freshly staged file is always newer +/// than its source; the size check guards against a same-instant edit. +fn is_current(from: &Path, to: &Path) -> bool { + let (Ok(f), Ok(t)) = (from.metadata(), to.metadata()) else { + return false; + }; + match (f.modified(), t.modified()) { + (Ok(fm), Ok(tm)) => tm >= fm && f.len() == t.len(), + _ => false, + } +} + +/// Delete staged sources that no longer exist upstream. Without this, a file +/// dropped by a core version bump would linger in OUT_DIR and still be compiled +/// in via the Makefile's `$(wildcard src/*/*.c)` (`Makefile:120`). Only `.c` / +/// `.h` are considered, so the objects and archive built here survive. +fn prune_stale(dst: &Path, staged: &HashSet) { + for root in [dst.join("src"), dst.join("include")] { + walk(&root, &mut |path| { + if is_source(path) && !staged.contains(path) { + let _ = fs::remove_file(path); + } + }); + } +} + +/// Visit every file under `root`. Missing directories are simply empty. +fn walk(root: &Path, visit: &mut dyn FnMut(&Path)) { + let mut dirs = vec![root.to_path_buf()]; + while let Some(dir) = dirs.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + dirs.push(path); + } else { + visit(&path); + } + } + } +} + +/// Drop the compiled objects when the flags stamped into them change. The +/// Makefile tracks header dependencies (`Makefile:143`) but not flag changes, +/// so editing [`CORE_VERSION`] on its own would otherwise leave the previous +/// string baked into objects that `make` still considers up to date. +fn invalidate_on_stamp_change(core: &Path, stamp: &str) { + let marker = core.join(".stamp"); + if fs::read_to_string(&marker).is_ok_and(|current| current == stamp) { + return; + } + walk(&core.join("src"), &mut |path| { + if path.extension().is_some_and(|e| e == "o") { + let _ = fs::remove_file(path); + } + }); + let _ = fs::remove_file(core.join("librayforce.a")); + fs::write(&marker, stamp) + .unwrap_or_else(|e| panic!("failed to write {}: {e}", marker.display())); } fn sanitize_libclang_path() { @@ -205,7 +366,7 @@ fn sanitize_libclang_path() { return; }; let dir = Path::new(&p); - let has_libclang = std::fs::read_dir(dir).is_ok_and(|entries| { + let has_libclang = fs::read_dir(dir).is_ok_and(|entries| { entries.flatten().any(|e| { let name = e.file_name(); let name = name.to_string_lossy(); @@ -223,9 +384,31 @@ fn sanitize_libclang_path() { } } -fn build_core_lib(core: &Path) { +/// Run the core's `make lib`. `stamp_version` is set when the core is our +/// pinned submodule staged under OUT_DIR, rather than a `RAYFORCE_SRC` +/// checkout building in place with its own git history. +fn build_core_lib(core: &Path, stamp_version: bool) { + // Cargo budgets build-script parallelism via NUM_JOBS. Without it make runs + // serially — minutes of wall clock for ~90 translation units at -O3, which + // matters inside docs.rs's capped build. + let jobs = env::var("NUM_JOBS").unwrap_or_else(|_| "1".to_string()); + + // Make command-line assignments override the Makefile's own definitions, + // including `?=` ones. + let mut defs = vec![format!("WARNS={CORE_WARNS}")]; + if stamp_version { + // Staged under OUT_DIR, with no git history of its own — and `git` + // searches upward, so leaving these unset would report the enclosing + // repository rather than falling back to "unknown". + defs.push(format!("RAY_VERSION={CORE_VERSION}")); + defs.push(format!("GIT_HASH={CORE_COMMIT}")); + invalidate_on_stamp_change(core, &defs.join(" ")); + } + let status = Command::new("make") .arg("lib") + .arg(format!("-j{jobs}")) + .args(&defs) .current_dir(core) .status() .expect("failed to invoke `make` to build librayforce.a"); diff --git a/rayforce-sys/src/lib.rs b/rayforce-sys/src/lib.rs index e6d6918..f7814cc 100644 --- a/rayforce-sys/src/lib.rs +++ b/rayforce-sys/src/lib.rs @@ -1,8 +1,10 @@ //! Raw FFI bindings to the RayforceDB v2 core (`librayforce`). //! -//! Generated by `bindgen` from `wrapper.h` (the public `rayforce.h` plus a few -//! hand-declared internal symbols). This crate is `unsafe` and 1:1 with the C -//! ABI; use the safe `rayforce` crate instead. +//! Generated by `bindgen` from the core's own headers: the public +//! `rayforce.h`, plus the private ones declaring the handful of internal +//! symbols the safe crate needs (`INTERNAL_FNS` in `build.rs` is the full +//! list). This crate is `unsafe` and 1:1 with the C ABI; use the safe +//! `rayforce` crate instead. //! //! The header's function-like macros (`ray_type`, `ray_len`, `ray_data`, //! `RAY_IS_ERR`, `RAY_IS_NULL`, …) are not emitted by bindgen — the safe crate diff --git a/rayforce-sys/vendor/rayforce b/rayforce-sys/vendor/rayforce new file mode 160000 index 0000000..f0d4bb4 --- /dev/null +++ b/rayforce-sys/vendor/rayforce @@ -0,0 +1 @@ +Subproject commit f0d4bb43a6a9b8e57b4afb0696017ee7070d89f7 diff --git a/rayforce-sys/vendor/rayforce-q b/rayforce-sys/vendor/rayforce-q new file mode 160000 index 0000000..ac5ab40 --- /dev/null +++ b/rayforce-sys/vendor/rayforce-q @@ -0,0 +1 @@ +Subproject commit ac5ab40fc2e365ac5b9ab411aa7257cede23bacd diff --git a/rayforce-sys/wrapper.h b/rayforce-sys/wrapper.h deleted file mode 100644 index cb379fc..0000000 --- a/rayforce-sys/wrapper.h +++ /dev/null @@ -1,49 +0,0 @@ -/* bindgen entry point for rayforce-sys. - * - * Pulls in the public v2 API, then hand-declares the handful of internal - * core symbols the safe crate needs (all present in librayforce.a but not in - * the public header — see PLAN.md). Signatures copied verbatim from the core - * sources: src/lang/eval.h, src/lang/internal.h, src/store/serde.h. */ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* src/lang/eval.h — evaluate an already-compiled AST object */ -ray_t* ray_eval(ray_t* obj); - -/* src/lang/internal.h — query builtins (variadic arg-array form) */ -ray_t* ray_update_fn(ray_t** args, int64_t n); -ray_t* ray_insert_fn(ray_t** args, int64_t n); -ray_t* ray_upsert_fn(ray_t** args, int64_t n); - -/* src/lang/internal.h — CSV + on-disk table I/O */ -ray_t* ray_read_csv_fn(ray_t** args, int64_t n); -ray_t* ray_write_csv_fn(ray_t** args, int64_t n); -ray_t* ray_set_splayed_fn(ray_t** args, int64_t n); -ray_t* ray_get_splayed_fn(ray_t** args, int64_t n); -ray_t* ray_get_parted_fn(ray_t** args, int64_t n); - -/* src/ops/ops.h — materialize a lazy DAG result (no-op for non-lazy inputs; - * consumes the lazy reference on success). RAY_LAZY (104) is the deferred type - * returned by ray_eval and graph-aware builtins. */ -ray_t* ray_lazy_materialize(ray_t* val); - -/* src/store/serde.h — serialize / deserialize to a U8 vector with IPC header */ -ray_t* ray_ser(ray_t* obj); -ray_t* ray_de(ray_t* bytes); - -/* src/core/runtime.c — last per-VM error message (set alongside a RAY_ERROR) */ -const char* ray_error_msg(void); - -/* The poll object the IPC client needs (ray_poll_create / ray_runtime_get_poll - * / ray_runtime_set_poll) is exported by the public header as of core v2.5.8, - * so it is no longer hand-declared here. ray_poll_create now returns the typed - * `ray_poll_t*` rather than the old `void*` — see the cast in - * rayforce/src/ipc.rs::ensure_poll. */ - -#ifdef __cplusplus -} -#endif diff --git a/scripts/check-vendored-pin.sh b/scripts/check-vendored-pin.sh new file mode 100755 index 0000000..1f57750 --- /dev/null +++ b/scripts/check-vendored-pin.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Assert the vendored core submodule matches the pin baked into +# rayforce-sys/build.rs. +# +# The build script stamps CORE_VERSION / CORE_COMMIT into librayforce.a because +# a crate unpacked from crates.io has no git history for the core's Makefile to +# read them from (it resolves them via `git describe` / `git rev-parse`, see +# Makefile:19 and Makefile:27 in the core). If the constants and the submodule +# disagree, published crates report a version they were not built from. +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd)" +build_rs="$root/rayforce-sys/build.rs" +core="$root/rayforce-sys/vendor/rayforce" + +field() { sed -n "s/^const $1: &str = \"\(.*\)\";\$/\1/p" "$build_rs"; } + +want_version="$(field CORE_VERSION)" +want_commit="$(field CORE_COMMIT)" +if [ -z "$want_version" ] || [ -z "$want_commit" ]; then + echo "could not read CORE_VERSION / CORE_COMMIT from $build_rs" >&2 + exit 1 +fi + +if [ ! -e "$core/include/rayforce.h" ]; then + echo "vendored core is missing from $core" >&2 + echo "run: git submodule update --init --recursive" >&2 + exit 1 +fi + +head_commit="$(git -C "$core" rev-parse HEAD)" +got_commit="$(git -C "$core" rev-parse --short="${#want_commit}" HEAD)" + +# Resolve the tag we expect and compare it to HEAD, rather than asking +# `git describe` what HEAD happens to be named. actions/checkout clones +# submodules with `git submodule update --depth=1`, and a shallow clone carries +# no tags at all, so `describe` on CI always answers "not on a tag" even when +# the pin is correct. Fetching the single tag we care about takes ~1s and makes +# the check behave the same on CI as in a full local clone. +want_tag="v$want_version" +resolve_tag() { git -C "$core" rev-parse -q --verify "refs/tags/$want_tag^{commit}" || true; } + +tag_commit="$(resolve_tag)" +if [ -z "$tag_commit" ]; then + # --depth=1 only where the clone is already shallow: on a full local checkout + # it would leave a .git/shallow behind and truncate history nobody asked to + # lose. Either way this fetches one ref, not the tag list. + if [ "$(git -C "$core" rev-parse --is-shallow-repository)" = true ]; then + git -C "$core" fetch --depth=1 --quiet origin "refs/tags/$want_tag:refs/tags/$want_tag" || true + else + git -C "$core" fetch --quiet origin "refs/tags/$want_tag:refs/tags/$want_tag" || true + fi + tag_commit="$(resolve_tag)" +fi + +status=0 +if [ -z "$tag_commit" ]; then + echo "core has no tag $want_tag, locally or on origin, but build.rs CORE_VERSION expects $want_version" >&2 + status=1 +elif [ "$tag_commit" != "$head_commit" ]; then + echo "core submodule is at $got_commit, but tag $want_tag is at $(git -C "$core" rev-parse --short="${#want_commit}" "$tag_commit")" >&2 + status=1 +fi +if [ "$got_commit" != "$want_commit" ]; then + echo "core submodule is at $got_commit, but build.rs CORE_COMMIT expects $want_commit" >&2 + status=1 +fi + +if [ "$status" -eq 0 ]; then + echo "vendored core pin OK: v$want_version ($want_commit)" +fi +exit "$status"