diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a014fa6..362080e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: steps: # The C core and the rayforce-q client are submodules under # rayforce-sys/vendor/, so this one checkout brings the whole build. + # Their URLs are SSH (git@github.com:); actions/checkout rewrites those to + # https with the job token, and persists that rewrite into each submodule + # so the pin check below can fetch its tag. Keep persist-credentials on. - name: Checkout bindings uses: actions/checkout@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c1b2a0..6ac4da0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,9 @@ jobs: steps: # The C sources are submodules under rayforce-sys/vendor/, so what is # built and published here is exactly what a crates.io consumer gets — - # no separate checkout to keep in step with build.rs. + # no separate checkout to keep in step with build.rs. The submodule URLs + # are SSH; actions/checkout rewrites them to https with the job token + # (see ci.yml). - name: Checkout bindings uses: actions/checkout@v4 with: diff --git a/.gitmodules b/.gitmodules index b8799e8..3be12a2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "rayforce-sys/vendor/rayforce"] path = rayforce-sys/vendor/rayforce - url = https://github.com/RayforceDB/rayforce.git + url = git@github.com:RayforceDB/rayforce.git [submodule "rayforce-sys/vendor/rayforce-q"] path = rayforce-sys/vendor/rayforce-q - url = https://github.com/RayforceDB/rayforce-q.git + url = git@github.com:RayforceDB/rayforce-q.git diff --git a/README.md b/README.md index 1b98ff8..73770f0 100644 --- a/README.md +++ b/README.md @@ -102,20 +102,40 @@ rayforce = { git = "https://github.com/RayforceDB/rayforce-rs" } Requirements: a C toolchain (`make`, `clang`) and `libclang` for `bindgen`. +The C sources are git submodules addressed over SSH (`git@github.com:`), and Cargo +fetches a git dependency's submodules itself. Without a GitHub SSH key, rewrite the +URLs to https and make Cargo fetch through `git`, which honours the rewrite: + +```sh +git config --global url."https://github.com/".insteadOf "git@github.com:" +``` + +```toml +# ~/.cargo/config.toml +[net] +git-fetch-with-cli = true +``` + +A crates.io dependency needs none of this — the sources ship inside the crate. + ### Working on the bindings -The C sources live in git submodules under `rayforce-sys/vendor/`, so a checkout needs -them initialized: +The C sources live in git submodules under `rayforce-sys/vendor/`, addressed over SSH, +so a checkout needs them initialized: ```sh -git clone --recurse-submodules https://github.com/RayforceDB/rayforce-rs +git clone --recurse-submodules git@github.com:RayforceDB/rayforce-rs.git # in an existing clone: +git submodule sync --recursive # picks up a URL change in .gitmodules git submodule update --init --recursive cargo build cargo test ``` +Without a GitHub SSH key, rewrite the submodule URLs to https once with +`git config --global url."https://github.com/".insteadOf "git@github.com:"`. + ### Choosing the core version Each release links one pinned core version. It lives in two places that must agree — the diff --git a/docs/docs/content/CHANGELOG.md b/docs/docs/content/CHANGELOG.md index a1dd126..fc72e41 100644 --- a/docs/docs/content/CHANGELOG.md +++ b/docs/docs/content/CHANGELOG.md @@ -29,6 +29,31 @@ All notable changes to `rayforce` are documented here. This project adheres to ### Changed +- **The vendored core is v2.6.0 and `rayforce-q` is 2.1.1** (from v2.5.8 and + 2.0.0). The core now recognises in-band nulls at construction, which changes + what a vector built from a raw buffer reports: `Value::vec(&[1i64, i64::MIN, 3])` + answers `is_null_at(1)` and `get(1)` returns the null singleton, where before + the sentinel was ordinary data until `set_null` marked it — the engine scans + the payload once and raises `HAS_NULLS`, so such values no longer aggregate as + data. The empty symbol and the empty string are now their types' nulls: + `is_null_at` reports them, but `get` returns the empty atom rather than the + null singleton, so `to_vec::()` keeps working and + `to_vec::>()` yields `None` for them. `set_null(idx, false)` is + a no-op in the core; overwrite the element with `set` instead. The docs no + longer describe a "null bitmap": nulls are sentinels behind a `HAS_NULLS` + fast-path hint. + +- **The submodules are addressed over SSH.** `.gitmodules` now points at + `git@github.com:RayforceDB/rayforce.git` and `rayforce-q.git`. An existing + clone picks the change up with `git submodule sync --recursive`; CI needs + nothing, since `actions/checkout` rewrites `git@github.com:` to https with the + job token. Without a GitHub SSH key, set + `git config --global url."https://github.com/".insteadOf "git@github.com:"` + before initializing the submodules — and, for a `git = "https://…"` Cargo + dependency, `net.git-fetch-with-cli = true` in `~/.cargo/config.toml` so Cargo + fetches through git and honours the rewrite. crates.io users are unaffected: + the C sources ship inside the crate. + - **A core-flavour switch rebuilds a `RAYFORCE_SRC` checkout from scratch.** Release and debug objects share every filename and `make` tracks headers but not flags, so a flavour flip would otherwise archive a mixed library. The diff --git a/docs/docs/content/documentation/data-types/integers.md b/docs/docs/content/documentation/data-types/integers.md index 4cbd107..34e6b98 100644 --- a/docs/docs/content/documentation/data-types/integers.md +++ b/docs/docs/content/documentation/data-types/integers.md @@ -76,4 +76,4 @@ assert!(v.as_slice::().is_err()); `Value::vec(&[1, 2, 3])` is an `I32` vector. For `I64` write `Value::vec(&[1i64, 2, 3])`; for `I16` write `Value::vec(&[1i16, 2, 3])`. -See [Vectors](vector.md) for indexing, mutation, slicing, and null bitmaps. +See [Vectors](vector.md) for indexing, mutation, slicing, and nulls. diff --git a/docs/docs/content/documentation/data-types/overview.md b/docs/docs/content/documentation/data-types/overview.md index d6dbc58..72f52cf 100644 --- a/docs/docs/content/documentation/data-types/overview.md +++ b/docs/docs/content/documentation/data-types/overview.md @@ -83,9 +83,11 @@ assert!(none.to_value().is_null()); assert_eq!(Value::i64(i64::MIN).extract::>()?, None); ``` -Vectors track nulls in a separate bitmap rather than by inspecting payload bytes. -A buffer that *happens* to hold a sentinel value is **not** null until it is -explicitly marked — see [Vectors](vector.md#null-bitmap) for the details. +Vector nulls are in-band too: an element is null when it holds its type's +sentinel, so a buffer handed to `Value::vec` that already contains one is null +from construction, and the empty symbol / empty string is the symbol / string +null. See [Vectors](vector.md#nulls) for how `is_null_at`, `get` and `set_null` +behave. Continue with [Values & Conversions](values.md) for how `Value` interoperates with native Rust types. diff --git a/docs/docs/content/documentation/data-types/vector.md b/docs/docs/content/documentation/data-types/vector.md index bffa4bc..42fac16 100644 --- a/docs/docs/content/documentation/data-types/vector.md +++ b/docs/docs/content/documentation/data-types/vector.md @@ -105,18 +105,25 @@ let c = a.concat(&b)?; assert_eq!(c.as_slice::()?, &[1, 2, 3, 4]); ``` -## Null bitmap { #null-bitmap } +## Nulls { #nulls } -Vectors track nulls in a **separate bitmap attribute**, not by scanning payload -bytes. A buffer that coincidentally contains a sentinel value (e.g. `i64::MIN`) -is *not* considered null until the element is explicitly marked. +Nulls live **in-band**: an element is null when it holds its type's sentinel — +`i16::MIN`, `i32::MIN`, `i64::MIN`, `NaN`, the all-zero GUID, the empty symbol, +the empty string. For the fixed-width types the engine also keeps a `HAS_NULLS` +attribute as a fast-path hint and checks it first; `Value::vec` raises it when +the buffer it is handed already contains a sentinel, and `set_null` raises it +when marking an element. Symbol and string vectors need no hint: the empty +value *is* the null. ```rust -// A coincidental sentinel is NOT null on its own: +// A buffer carrying a sentinel is null from construction: let raw = Value::vec(&[1i64, i64::MIN, 3]); -assert!(!raw.is_null_at(1)); +assert!(raw.is_null_at(1)); +assert!(raw.get(1)?.is_null()); +assert_eq!(raw.as_slice::()?, &[1, i64::MIN, 3]); // payload untouched +assert_eq!(raw.to_vec::>()?, vec![Some(1), None, Some(3)]); -// Explicitly marking an element is the supported path: +// Marking an element null writes the sentinel and raises the hint: let mut v = Value::vec(&[1i64, 2, 3]); v.set_null(1, true)?; assert!(v.is_null_at(1)); @@ -124,11 +131,29 @@ assert!(v.get(1)?.is_null()); assert_eq!(v.get(0)?.as_i64()?, 1); // neighbors untouched ``` -!!! note "Why the bitmap matters" - Decoupling nulls from payload bytes lets the engine store any in-band value - without ambiguity and lets a column be marked null in O(1) without touching - the data. Test for nulls with `is_null_at(idx)` rather than comparing against - a sentinel. +An empty symbol or string element is reported by `is_null_at`, but `get` hands +back the empty atom rather than the null singleton, so plain `String` extraction +keeps working and `Option` sees the null: + +```rust +let strs = Value::str_vec(&["hello", ""]); +assert!(strs.is_null_at(1)); +assert_eq!(strs.get(1)?.as_string()?, ""); +assert!(strs.get(1)?.is_atom_null()); +assert_eq!( + strs.to_vec::>()?, + vec![Some("hello".to_string()), None] +); +``` + +!!! note "Clearing a null, and the hint's blind spot" + `set_null(idx, false)` is a no-op — the engine cannot know the value the + sentinel replaced — so overwrite the element with `set` to un-null it. + Conversely, a numeric sentinel written through `set` or `push` does not + raise the hint, so `is_null_at` will not report it; the boxed atom still + answers `is_atom_null()` and extracts as `None`. Prefer `set_null` for + writing nulls, and `is_null_at(idx)` or `Option` extraction for reading + them, over hand-written sentinel comparisons. ## Constructed vectors match the engine diff --git a/docs/docs/content/get-started/installation.md b/docs/docs/content/get-started/installation.md index d78ed3b..c4be7f5 100644 --- a/docs/docs/content/get-started/installation.md +++ b/docs/docs/content/get-started/installation.md @@ -39,6 +39,21 @@ from `git describe`, and a crate unpacked from crates.io has no git history to read. `scripts/check-vendored-pin.sh` asserts the two agree, and CI runs it on every push. +!!! note "The submodules are fetched over SSH" + `.gitmodules` addresses both submodules as `git@github.com:RayforceDB/…`. An + existing clone picks that up with `git submodule sync --recursive` before + `git submodule update --init --recursive`. Without a GitHub SSH key, rewrite + the URLs to https once: + + ```sh + git config --global url."https://github.com/".insteadOf "git@github.com:" + ``` + + A `git = "https://…"` Cargo dependency fetches the submodules through Cargo, + which honours that rewrite only when it shells out to `git` — set + `net.git-fetch-with-cli = true` in `~/.cargo/config.toml`. A crates.io + dependency needs none of this; the sources ship inside the crate. + As a consumer you get the core that matches the `rayforce` version you depend on — pick a different core by picking a different `rayforce` release. The two sections below are for changing that pin yourself. diff --git a/rayforce-sys/build.rs b/rayforce-sys/build.rs index bcbebfd..639c05c 100644 --- a/rayforce-sys/build.rs +++ b/rayforce-sys/build.rs @@ -24,7 +24,7 @@ use std::process::Command; /// /// Must match the tag `vendor/rayforce` is pinned to. CI asserts the two agree; /// see the "Check vendored core pin" step in `.github/workflows/ci.yml`. -const CORE_VERSION: &str = "2.5.8"; +const CORE_VERSION: &str = "2.6.0"; /// Commit the `vendor/rayforce` submodule is pinned to, stamped alongside /// [`CORE_VERSION`]. Also checked by CI's "Check vendored core pin" step. @@ -35,7 +35,7 @@ const CORE_VERSION: &str = "2.5.8"; /// under OUT_DIR, an unset value does not fall back to "unknown" — it silently /// reports the HEAD of whatever unrelated repository happens to enclose the /// build directory. -const CORE_COMMIT: &str = "f0d4bb4"; +const CORE_COMMIT: &str = "b3e9aa1"; /// Warning flags for the vendored core build — the core's own `WARNS` /// (`Makefile:30`) minus `-Werror`. Consumers compile this with whatever diff --git a/rayforce-sys/vendor/rayforce b/rayforce-sys/vendor/rayforce index f0d4bb4..b3e9aa1 160000 --- a/rayforce-sys/vendor/rayforce +++ b/rayforce-sys/vendor/rayforce @@ -1 +1 @@ -Subproject commit f0d4bb43a6a9b8e57b4afb0696017ee7070d89f7 +Subproject commit b3e9aa187e93a97a0c04fd9f85f63b65e3c67905 diff --git a/rayforce-sys/vendor/rayforce-q b/rayforce-sys/vendor/rayforce-q index ac5ab40..c35ed3c 160000 --- a/rayforce-sys/vendor/rayforce-q +++ b/rayforce-sys/vendor/rayforce-q @@ -1 +1 @@ -Subproject commit ac5ab40fc2e365ac5b9ab411aa7257cede23bacd +Subproject commit c35ed3c574a05c233a674dd12a5dc7703d2a00b1 diff --git a/rayforce/src/vector.rs b/rayforce/src/vector.rs index c8a3261..1660e61 100644 --- a/rayforce/src/vector.rs +++ b/rayforce/src/vector.rs @@ -44,7 +44,9 @@ vec_elem!(f64, sys::RAY_F64); impl Value { // ---- construction ---- - /// Build a vector from a slice of fixed-width elements (single `memcpy`). + /// Build a vector from a slice of fixed-width elements: a single `memcpy`, + /// followed by one pass over the payload that raises `HAS_NULLS` if it + /// already holds the type's sentinel — see [`Value::is_null_at`]. pub fn vec(data: &[T]) -> Value { assert_on_runtime_thread("Value::vec"); unsafe { @@ -189,13 +191,34 @@ impl Value { } } - /// True if element `idx` is null (consults the vector's null bitmap). + /// True if element `idx` is null. + /// + /// Nulls are in-band. For the sentinel-encoded types (`i16`/`i32`/`i64`, + /// `f32`/`f64`, date/time/timestamp, GUID) the core first consults the + /// vector's `HAS_NULLS` attribute — raised by [`Value::vec`] when the raw + /// payload already holds a sentinel, and by [`Value::set_null`] — and only + /// then compares the element against its sentinel (`i64::MIN`, `NaN`, the + /// all-zero GUID, ...). Symbol and string vectors skip the gate: the empty + /// symbol and the empty string *are* their nulls. `bool`/`u8` vectors are + /// never null. + /// + /// Because of the gate, a numeric sentinel written later through + /// [`Value::set`] or [`Value::push`] is not reported here; the boxed atom + /// still answers [`Value::is_atom_null`], which is why + /// `to_vec::>()` maps it to `None` either way. Use `set_null` to + /// null an element. pub fn is_null_at(&self, idx: usize) -> bool { unsafe { sys::ray_vec_is_null(self.as_ptr(), idx as i64) } } - /// Box element `idx` as a [`Value`]. Returns the null singleton for null - /// elements. Bounds-checked. + /// Box element `idx` as a [`Value`]. Bounds-checked. + /// + /// Null elements of the sentinel-encoded types (integers, floats, + /// temporals, GUID) come back as the untyped null singleton + /// ([`Value::is_null`]). Symbol and string vectors carry their null + /// in-band, so an empty element comes back as the empty atom: + /// [`Value::is_atom_null`] is true for it and `Option` extraction + /// yields `None`, while plain `String` extraction still succeeds. pub fn get(&self, idx: usize) -> Result { let n = self.len(); if idx >= n { @@ -210,10 +233,15 @@ impl Value { return Ok(Value::from_borrowed(e)); } } - if self.is_null_at(idx) { + let t = self.abs_type() as u32; + // SYM/STR carry their null in-band (the empty symbol / empty string + // *is* the null): box it, so callers get an atom `is_atom_null` + // recognises. Every other nullable type has a sentinel that would + // otherwise box as an ordinary value, so those collapse to the + // untyped null singleton. + if !matches!(t, sys::RAY_SYM | sys::RAY_STR) && self.is_null_at(idx) { return Ok(Value::null()); } - let t = self.abs_type() as u32; unsafe { let base = raw::data(self.as_ptr()); let v = match t { @@ -319,8 +347,14 @@ impl Value { } } - /// Mark element `idx` as null (or clear it). Sets the vector's null sentinel - /// and `HAS_NULLS` attribute so [`Value::is_null_at`] reports it. + /// Mark element `idx` as null: writes the type's sentinel into the payload + /// (`i64::MIN`, `NaN`, symbol id 0, the empty string, the all-zero GUID) + /// and raises `HAS_NULLS` so [`Value::is_null_at`] reports it. Rejected for + /// `bool`/`u8` vectors and for slices. + /// + /// `is_null = false` is a no-op in the core — it cannot know the prior real + /// value — so the sentinel stays and the element remains null until the + /// caller overwrites it with [`Value::set`]. pub fn set_null(&mut self, idx: usize, is_null: bool) -> Result<()> { unsafe { let e = sys::ray_vec_set_null_checked(self.as_ptr(), idx as i64, is_null); diff --git a/rayforce/tests/containers.rs b/rayforce/tests/containers.rs index 5b03aa6..a38d51e 100644 --- a/rayforce/tests/containers.rs +++ b/rayforce/tests/containers.rs @@ -94,17 +94,36 @@ fn vector_slice_and_concat() { #[test] fn vector_nulls() { Runtime::scope(|_rt| { - // A raw buffer carrying a coincidental sentinel is NOT null until the - // HAS_NULLS attribute is set — matching the engine's design. + // A raw buffer carrying a sentinel is null from construction: the core + // scans the payload once in ray_vec_from_raw and raises HAS_NULLS. let raw = Value::vec(&[1i64, i64::MIN, 3]); - assert!(!raw.is_null_at(1)); + assert!(raw.is_null_at(1)); + assert!(!raw.is_null_at(0) && !raw.is_null_at(2)); + assert!(raw.get(1).unwrap().is_null()); + assert_eq!(raw.get(2).unwrap().as_i64().unwrap(), 3); + // The payload itself is untouched. + assert_eq!(raw.as_slice::().unwrap(), &[1, i64::MIN, 3]); + assert_eq!( + raw.to_vec::>().unwrap(), + vec![Some(1), None, Some(3)] + ); - // Explicitly marking an element null is the supported path. + // Explicitly marking an element null is still the supported path. let mut v = Value::vec(&[1i64, 2, 3]); + assert!(!v.is_null_at(1)); v.set_null(1, true).unwrap(); assert!(v.is_null_at(1)); assert!(v.get(1).unwrap().is_null()); + // set_null writes the sentinel; neighbours are untouched. + assert_eq!(v.as_slice::().unwrap(), &[1, i64::MIN, 3]); assert_eq!(v.get(0).unwrap().as_i64().unwrap(), 1); + + // Clearing is a no-op in the core; overwriting the element un-nulls it. + v.set_null(1, false).unwrap(); + assert!(v.is_null_at(1)); + v.set(1, 2i64).unwrap(); + assert!(!v.is_null_at(1)); + assert_eq!(v.get(1).unwrap().as_i64().unwrap(), 2); Ok(()) }) .unwrap(); @@ -124,6 +143,41 @@ fn symbol_and_string_vectors() { strs.get(1).unwrap().as_string().unwrap(), "a longer value here" ); + + // The empty symbol / empty string is the type's in-band null (core + // 2.6.0): is_null_at reports it, get() still hands back the empty atom + // so String extraction works and Option sees None. + let syms = Value::sym_vec(&["a", ""]); + assert!(!syms.is_null_at(0)); + assert!(syms.is_null_at(1)); + let e = syms.get(1).unwrap(); + assert!(!e.is_null()); + assert!(e.is_atom_null()); + assert_eq!(e.as_sym().unwrap(), ""); + assert_eq!( + syms.to_vec::().unwrap(), + vec!["a".to_string(), String::new()] + ); + assert_eq!( + syms.to_vec::>().unwrap(), + vec![Some("a".to_string()), None] + ); + + let strs = Value::str_vec(&["hello", ""]); + assert!(!strs.is_null_at(0)); + assert!(strs.is_null_at(1)); + let e = strs.get(1).unwrap(); + assert!(!e.is_null()); + assert!(e.is_atom_null()); + assert_eq!(e.as_string().unwrap(), ""); + assert_eq!( + strs.to_vec::().unwrap(), + vec!["hello".to_string(), String::new()] + ); + assert_eq!( + strs.to_vec::>().unwrap(), + vec![Some("hello".to_string()), None] + ); Ok(()) }) .unwrap();