From 62772cea938e2e20887d15f48e01d515c1a59701 Mon Sep 17 00:00:00 2001 From: Hiroshi Shinaoka Date: Thu, 29 Oct 2026 09:56:13 +0900 Subject: [PATCH 1/4] cost: store anchors and extrapolate linearly above the largest (#6 stage 1) `WidthCosts` no longer holds a table of widths 1..=6 with the width-6 value held flat above it. It stores the measured anchors (widths and nanoseconds, at most `MAX_ANCHORS` = 12) and evaluates the model per lookup: * below the first anchor: the first value, between two anchors: linear interpolation as before (i128, clamp at 0), above the largest anchor `w_m`: `ns[w_m] * w / w_m` in u128, saturating at `u64::MAX`. A wide section is never charged the cost of the largest anchor, so `width_for` no longer answers the full width just because the width-6 section pays. * `from_anchors(&[(usize, Duration)]) -> Option` adds measured anchors up to `u16::MAX` (const, allocation-free, no panic); `new` and `flat` keep their signatures and their 2/4/6 anchors. `min()` is the minimum over the anchors, which is the minimum over all widths. * `for_class(X86_64Avx512)` gains the measured width-8 team anchor 0.98 us from the post-fix run; in pool and outside keep 2/4/6 (their width-8 values are pre-fix data). Values at widths 1..=6 of every existing row are unchanged and pinned in a test; the generic row keeps 3/15/40 us at the anchors and now grows above them. Unit tests cover the flat-row law (1 us at 6/12/24/48), non-decreasing and at-least-linear extrapolation, anchors up to 64, `from_anchors` rejections and const use, equality ignoring unused capacity, a `Duration::MAX` row, and `width_for_by(24 us, 1, 64)` = 12 on a 1-us row. `benches/cost_lookup.rs` gains a budget-64 `width_for_by`, `min_cost`, a by-value `DispatchCosts` copy and the team-token `Exec` queries; the design records and the README state the new contract and keep the historical reason for the earlier flat model. --- README.md | 2 +- benches/cost_lookup.rs | 77 ++++++- docs/design/placement.md | 20 +- docs/design/thresholds.md | 61 ++++-- src/cost.rs | 440 +++++++++++++++++++++++++++++++++----- tests/dispatch_cost.rs | 91 ++++++-- 6 files changed, 585 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index 9f16cc7..0637c7d 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ the token to kernels by value. Kernels never store a token and never mint one. | `Pool::into_team(width)` -> `Team` (owns the pool; `IntoTeamError` gives it back), `Team::exec(budget)` / `try_exec`, `Team::pool`, `Team::into_pool` | persistent reservation for an engine that owns its pool (`Pool` is not `Clone`); minted tokens are ordinary `Exec`s whose `fork`, `for_ranges`, `broadcast`, `team` dispatch on the reserved members; one leader at a time (`exec` blocks, `try_exec` -> `Refused::Busy`); idle members park while reserved | | `PoolBuilder::qos(Qos)`, `raise_current_thread_qos(Qos)` | worker QoS on macOS (default `UserInitiated`); raise the leading thread without ever lowering it | | `PoolBuilder::idle_spin`, `idle_runs_rayon`, `acquire_timeout`; `Pool::release_idle` | waiting policy of idle team members | -| `DispatchCosts`, `PoolBuilder::dispatch_costs`, `Exec::dispatch_cost(width)`, `TeamExec::dispatch_cost(width)` | cost of one parallel section per kind of token (team, in pool, outside), steady state; one row per `HostClass` (`HostClass::detect()`, keyed by kernel dispatch: `x86_64-avx512`, `x86_64-avx2`, `x86_64-scalar`, `aarch64-apple`, `aarch64`, `generic`), `DispatchCosts::for_class(class)`; measured rows for `x86_64-avx512` and `aarch64-apple`, every other class takes the conservative `generic` row (3 / 15 / 40 us); `DispatchCosts::host_default()` is the detected class's row; `Duration::MAX` when serial; override per pool | +| `DispatchCosts`, `WidthCosts`, `PoolBuilder::dispatch_costs`, `Exec::dispatch_cost(width)`, `TeamExec::dispatch_cost(width)` | cost of one parallel section per kind of token (team, in pool, outside), steady state; one row per `HostClass` (`HostClass::detect()`, keyed by kernel dispatch: `x86_64-avx512`, `x86_64-avx2`, `x86_64-scalar`, `aarch64-apple`, `aarch64`, `generic`), `DispatchCosts::for_class(class)`; measured rows for `x86_64-avx512` (team anchors at widths 2, 4, 6 and 8) and `aarch64-apple`, every other class takes the conservative `generic` row (3 / 15 / 40 us at the anchors 2, 4, 6); a row holds measured anchors (`WidthCosts::new`, `WidthCosts::flat`, or `WidthCosts::from_anchors` for measured anchors up to 64 and beyond), interpolates between them and grows linearly above the largest one, so a width above the anchors is never charged the cost of the largest anchor; `DispatchCosts::host_default()` is the detected class's row; `Duration::MAX` when serial; override per pool | | `pays_at(est, sections, cost)`, `Exec::pays(est, sections, width)`, `Exec::width_for(est, sections)`, `width_for_by`, `PAYS` | the one rule: parallel iff `est >= PAYS * sections * D(width)`; `width_for` minimises `sections * D(k) + est / k` | | `Exec::external(f)` | enter the pool once and hand the budget as a plain width to an outside library (faer, a vendor BLAS); how it divides the width is that library's concern | diff --git a/benches/cost_lookup.rs b/benches/cost_lookup.rs index 7dad37b..a653407 100644 --- a/benches/cost_lookup.rs +++ b/benches/cost_lookup.rs @@ -1,5 +1,8 @@ -//! Cost of the dispatch-cost lookups themselves: `Exec::dispatch_cost`, `Exec::pays` and -//! `Exec::width_for` on a team token of a pool of 8 (what a kernel calls per GEMM and per gate). +//! Cost of the dispatch-cost lookups themselves: `Exec::dispatch_cost`, `Exec::pays`, +//! `Exec::width_for`, `DispatchCosts::min_cost` and a by-value copy of `DispatchCosts`, on a +//! team token of a pool of 8 (what a kernel calls per GEMM and per gate). The `width_for` +//! row takes a budget of 64 through `width_for_by`, so it reads the anchors up to 64 whatever +//! the pool size. //! //! `cargo bench --bench cost_lookup` @@ -26,10 +29,16 @@ fn ns_per_call(mut f: impl FnMut(u64) -> u64) -> f64 { fn main() { let costs = black_box(DispatchCosts::host_default()); let w8 = NonZeroUsize::new(8).unwrap(); - let budget = black_box(w8); + let budget8 = black_box(w8); + let budget64 = black_box(NonZeroUsize::new(64).unwrap()); // Varying input so nothing is hoisted: 5 us .. 1 ms. let est = |i: u64| Duration::from_nanos(5_000 + (i % 1024) * 1_000); let w = |i: u64| NonZeroUsize::new(2 + (i % 7) as usize).unwrap(); + let kind = |i: u64| match i % 3 { + 0 => DispatchKind::Team, + 1 => DispatchKind::InPool, + _ => DispatchKind::Outside, + }; println!( "at (width 2..8) {:>7.2} ns", ns_per_call(|i| costs.cost(DispatchKind::Team, w(i)).as_nanos() as u64) @@ -44,9 +53,67 @@ fn main() { println!( "width_for (budget 8) {:>7.2} ns", ns_per_call(|i| { - threadteam::width_for_by(est(i), 1, budget, |k| costs.cost(DispatchKind::Team, k)).get() - as u64 + threadteam::width_for_by(est(i), 1, budget8, |k| costs.cost(DispatchKind::Team, k)) + .get() as u64 }) ); + println!( + "width_for (budget 64) {:>7.2} ns", + ns_per_call(|i| { + threadteam::width_for_by(est(i), 1, budget64, |k| costs.cost(DispatchKind::Team, k)) + .get() as u64 + }) + ); + println!( + "min_cost {:>7.2} ns", + ns_per_call(|i| costs.min_cost(kind(i)).as_nanos() as u64) + ); + println!( + "copy DispatchCosts {:>7.2} ns", + ns_per_call(|i| { + // A by-value copy, as `TeamExec::dispatch_costs` and a consumer's + // shape hand it on; read a field so the copy cannot be elided. + let copy: DispatchCosts = black_box(costs); + copy.cost(DispatchKind::Team, w(i)).as_nanos() as u64 + }) + ); + + #[cfg(feature = "parallel")] + token_rows(costs); + let _ = WidthCosts::flat(Duration::ZERO); } + +/// The lookups on a real team token: the kind is read at call time, so these are +/// the paths a kernel takes. +#[cfg(feature = "parallel")] +fn token_rows(costs: DispatchCosts) { + use threadteam::{Pool, WaitOverride}; + + let pool = Pool::builder() + .threads(8) + .dispatch_costs(costs) + .acquire_timeout(Duration::from_secs(10)) + .build() + .expect("pool"); + // Park the reserved members at once: the timing loop must not compete with + // idle spinners on a host of fewer cores. + let team = pool + .into_team_with(8, WaitOverride::new().idle_spin(Duration::ZERO)) + .expect("idle pool"); + let exec = team.exec(8); + let w = |i: u64| NonZeroUsize::new(2 + (i % 7) as usize).unwrap(); + let est = |i: u64| Duration::from_nanos(5_000 + (i % 1024) * 1_000); + println!( + "Exec::dispatch_cost {:>7.2} ns", + ns_per_call(|i| exec.dispatch_cost(w(i)).as_nanos() as u64) + ); + println!( + "Exec::width_for (b 8) {:>7.2} ns", + ns_per_call(|i| exec.width_for(est(i), 1).get() as u64) + ); + println!( + "Exec::pays {:>7.2} ns", + ns_per_call(|i| exec.pays(est(i), 1, w(i)) as u64) + ); +} diff --git a/docs/design/placement.md b/docs/design/placement.md index 10c993b..b4a61aa 100644 --- a/docs/design/placement.md +++ b/docs/design/placement.md @@ -570,15 +570,25 @@ serial_estimate >= PAYS * sections * D(width) (PAYS = 3) `fork` children) is `DispatchKind::Team`; a plain token whose caller is a worker of its pool is `InPool`; a plain token called from outside is `Outside`. No thread-local state is involved and nothing allocates. -* `DispatchCosts` holds one `WidthCosts` per kind (values at widths 2, 4 and 6, - linear in between, flat above 6). +* `DispatchCosts` holds one `WidthCosts` per kind. A `WidthCosts` stores the + measured anchors of its row (widths 2, 4 and 6 from `new` / `flat`, or any + strictly increasing widths from 2 to `u16::MAX` from `from_anchors`, at most + `WidthCosts::MAX_ANCHORS` = 12 of them): the value at the first anchor below + it, linear interpolation between two anchors, and **linear extrapolation above + the largest anchor**, `cost(w_m) * w / w_m`. A wide section therefore never pays + the cost of the largest measured anchor; a row measured at width 8 keeps its + measured value there instead of the extrapolation. The model is evaluated on + every lookup by a scan over at most 12 anchors (no table, no allocation). `DispatchCosts::for_class(HostClass)` is the row of a host class, and `DispatchCosts::host_default()` the row of `HostClass::detect()`. Classes are keyed by the kernel dispatch the machine takes, not by the CPU model: `x86_64-avx512` (runtime `avx512f`), `x86_64-avx2` (`avx2` + `fma`), `x86_64-scalar`, `aarch64-apple` (aarch64 on macOS), `aarch64`, `generic`; detected once per process. - `x86_64-avx512` holds the measured Ryzen row (team 0.6/0.7/0.9, in pool 3.0/3.6/3.5, outside 9.7/10.1/8.7 us), - `aarch64-apple` the measured Apple M5 row, and every class without a measurement the conservative flat - `generic` row (3, 15, 40 microseconds). The rates of the consumer crates (strided-rs, tcontract-rs, faer) are + `x86_64-avx512` holds the measured Ryzen row (team 0.6/0.7/0.9 at widths 2/4/6 and the measured 0.98 at width 8, + in pool 3.0/3.6/3.5, outside 9.7/10.1/8.7 us), `aarch64-apple` the measured Apple M5 row at widths 2, 4 and 6, and + every class without a measurement the conservative generic row (3, 15, 40 microseconds at the anchors; the + extrapolation charges 32, 160 and 427 microseconds at width 64). A consumer that measured its own host at more + widths overrides the row with `WidthCosts::from_anchors` (up to `WidthCosts::MAX_ANCHORS` = 12 anchors). The rates + of the consumer crates (strided-rs, tcontract-rs, faer) are tables of their own, keyed by the same class names, and a class's rates and dispatch costs come from the same machine in the same clock state: boost on/off moves both by about 1.66x and leaves the ratio stable within a few percent (`faer-benchmark` `raw-exec3-boost`, `tables-*-exec3-boost.md`). Each crate carries an identical diff --git a/docs/design/thresholds.md b/docs/design/thresholds.md index 9792fcb..1827e28 100644 --- a/docs/design/thresholds.md +++ b/docs/design/thresholds.md @@ -282,7 +282,9 @@ checked (0.019 and 0.084). A scalar that is not native takes the `f64` or `c64` ```rust pub const PAYS: f64 = 3.0; pub enum DispatchKind { Team, InPool, Outside } -pub struct WidthCosts; // values at widths 2, 4, 6: const new([Duration; 3]), const flat(Duration), at(width), min() +pub struct WidthCosts; // measured anchors: const new([Duration; 3]) (widths 2, 4, 6), const flat(Duration), + // const from_anchors(&[(usize, Duration)]) -> Option, const MAX_ANCHORS = 12, + // at(width), min() pub struct DispatchCosts; // new(team, in_pool, outside), host_default(), for_class(HostClass), with(kind, costs), cost(kind, width), min_cost(kind) impl PoolBuilder { pub fn dispatch_costs(self, c: DispatchCosts) -> Self; } impl Pool { pub fn dispatch_costs(&self) -> DispatchCosts; } @@ -302,18 +304,30 @@ estimate / k` over `1..=budget`, counting only widths at which the region pays, authority (including `fork` children of a team token), else a worker of the pool (in pool), else outside. No thread-local state, no allocation (`tests/dispatch_cost.rs`). -Lookups are cheap: `WidthCosts::new` evaluates the model once into a table of nanoseconds for widths 1 to 6 (wider -tokens read the last entry, which is the flat model), so `at` is an index, and `pays_at` / `width_for_by` compare -integer nanoseconds (the objective of `width_for_by` stays `f64`). `cost_lookup` bench, cores 4-11, per call: -`at` 7.9 to 1.2 ns, `pays_at` after `at` 10.1 to 2.2 ns, `width_for` at budget 8 83 to 32 ns. `min_cost(kind)` / -`WidthCosts::min()` is the cheapest cost over all widths: a region below `PAYS * min_cost` pays at no width, a -rejection that stays right for a table whose cost rises with width. A test compares the table with the former -interpolation for widths 1 to 64. - -Width model: linear interpolation between the widths 2, 4 and 6, the width 2 value below, and above width 6 -the width 6 value held flat (the Linux width 8 data, section 3.6, is below the width 6 value, so flat is -conservative; it replaced growth in proportion to the width). A host that is not measured gets the flat generic table -(team 3, in pool 15, outside 40 microseconds), which errs towards staying serial. +Lookups are cheap: a `WidthCosts` stores its measured anchors (integers, widths and nanoseconds) and `at` is a +forward scan over at most `MAX_ANCHORS` = 12 of them, so the lookup allocates nothing and `pays_at` / +`width_for_by` compare integer nanoseconds (the objective of `width_for_by` stays `f64`). `cost_lookup` bench on +cores 4-11 (previous phase, flat table whose `at` was an index): `at` 7.9 to 1.2 ns, `pays_at` after `at` 10.1 to +2.2 ns, `width_for` at budget 8 83 to 32 ns. The anchor scan costs an integer division per interpolated width where +the table held the value: on the 4-core run of issue #6 (cores 56-59, best of 9 x 2M calls, same bench file on both +trees) `at` 0.89 -> 7.35 ns, `width_for` budget 8 25.8 -> 72.9 ns and budget 64 222 -> 553 ns, `min_cost` +6.0 -> 4.3 ns, a by-value `DispatchCosts` copy 4.5 -> 12.6 ns (144 -> 384 bytes), `Exec::dispatch_cost` on a team +3.3 -> 8.8 ns. That stays far below one section, which the rows above charge 0.6 to 427 us, but a consumer that +calls `width_for` on every tiny region should use `min_cost` first. `min_cost(kind)` / +`WidthCosts::min()` is the cheapest cost over all widths: the minimum over the anchors (neither interpolation nor +the extrapolation can undershoot the anchor it starts from), so a region below `PAYS * min_cost` pays at no width. +A test compares the rows with the former interpolation for widths 1 to 6 and pins the extrapolation above them. + +Width model: the value at the first anchor below it, linear interpolation between two anchors, and above the +**largest** anchor `w_m` linear extrapolation, `cost(w_m) * w / w_m` (issue #6). The largest anchor is 6 for the +rows built with `new` (widths 2, 4 and 6); the `x86_64-avx512` team row also carries the measured width-8 anchor, +and `from_anchors` accepts measured anchors up to `u16::MAX`, so a row measured on a many-core host replaces the +extrapolation where it has data. Above the last anchor the cost is never flat: a wide section is charged at least +in proportion to its width, so `width_for` no longer answers the full width just because the largest anchor pays (a +flat 1 us row gives 1, 2, 4 and 8 us at widths 6, 12, 24 and 48; a 24 us region on such a row selects width 12 of a +budget 64). Section 3.6 keeps the historical reason for the earlier flat model. A host that is not measured gets the +flat generic row (team 3, in pool 15, outside 40 microseconds at the anchors 2, 4 and 6, and the width proportion +above), which errs towards staying serial. faer: `ParShape::Team(n, DispatchCosts)` carries the costs of its token (copied by `Par::shape`); `ParShape::dispatch_cost(width)` and `pays(estimate, sections, width)` delegate to threadteam for team shapes. @@ -522,8 +536,10 @@ pool of 8, median of 3 processes, microseconds per section: After the per-member parker fix (see "Width dependence" below; same bench, pool of 8, 10 us share rows, median of 3 processes, all accepted; `tables-dispatch-exec2.md`), the team row is 0.6, 0.7, 0.9 us at widths 2, 4, 6 and 0.98 at w=8; in pool 3.0, 3.6, 3.5 and outside 9.7, 10.1, 8.7 were re-measured and are essentially unchanged. The table -above predates the parker fix; `for_class(HostClass::X86_64Avx512)` holds the post-fix row. The flat model above width 6 gives 0.9 us for a -team at w=8, 8 % optimistic against the measured 0.98; the model is left as it is. +above predates the parker fix; `for_class(HostClass::X86_64Avx512)` holds the post-fix row. The team row carries all +four measured widths as anchors, so width 8 keeps the measured 0.98 us; in pool and outside keep the widths of the +post-fix run only (their width-8 columns are from the pre-fix run and are not used), so the model extrapolates +there: width 8 is 4.67 us in the pool and 11.6 us from outside. | kind | w=2 | w=4 | w=6 | w=8 | | --- | --- | --- | --- | --- | @@ -535,7 +551,9 @@ The row is `DispatchCosts::for_class(HostClass::X86_64Avx512)` in `cost.rs`, and ```rust HostClass::X86_64Avx512 => DispatchCosts { - team: WidthCosts::new([us_f(0.6), us_f(0.7), us_f(0.9)]), + team: WidthCosts::from_anchors(&[ + (2, us_f(0.6)), (4, us_f(0.7)), (6, us_f(0.9)), (8, us_f(0.98)), + ]).unwrap(), in_pool: WidthCosts::new([us_f(3.0), us_f(3.6), us_f(3.5)]), outside: WidthCosts::new([us_f(9.7), us_f(10.1), us_f(8.7)]), }, @@ -545,10 +563,13 @@ What the Linux data says about the design: * The team cost is the M5's (1.1 to 2.8 us); a plain token costs about half the M5's. Parked members are dearer: a team after a 300 ms gap costs 82 to 84 us (M5 27), the first share starts 63 to 83 us after a park. -* The width model above the last anchor was 2 times pessimistic (it has since been changed to hold the width 6 - value flat, section 3.1): width 8 measures 1.11 us on a team, 3.0 in the - pool and 6.7 outside against 2.21, 4.69 and 11.7 from `cost(6) * 8 / 6`. With the generic table this refuses a - team GEMM of `64^3` at width 8 (estimate 9.7 us against `3 * 4.0`) that pays at width 4 (0.32 of serial). +* The width model above the last anchor was 2 times pessimistic when it grew in proportion to the width; the flat + model that replaced it (section 3.1) had the opposite error and was 8 % optimistic at width 8 on the post-fix team + row (0.9 against 0.98 us). Since issue #6 the model grows in proportion above the largest anchor again, with the + measured width-8 team anchor covering width 8 exactly. The historical numbers: width 8 measures 1.11 us on a team, + 3.0 in the pool and 6.7 outside (pre-fix run) against 2.21, 4.69 and 11.7 from `cost(6) * 8 / 6`. With the generic + row the extrapolation refuses a team GEMM of `64^3` at width 8 (estimate 9.7 us against `3 * 4.0`) that pays at + width 4 (0.32 of serial). * Every gate measured follows `estimate >= PAYS * sections * D` exactly; the deviations are in the inputs: the generic table on Linux, the M5 serial rates (the 2.0 GHz rates here are 1.4 to 3.7 times larger: GEMM 0.064 ns per MAC f64, 0.257 c64, 0.032 f32, 0.128 c32), and strided's class assignment (a contiguous `copy_into` and every diff --git a/src/cost.rs b/src/cost.rs index b38141a..a90f6c6 100644 --- a/src/cost.rs +++ b/src/cost.rs @@ -33,14 +33,24 @@ pub enum DispatchKind { Outside, } -/// Dispatch cost of one kind of token at the measured widths 2, 4 and 6. +/// Dispatch cost of one kind of token, stored as the measured anchors. /// -/// The model: linear interpolation between the anchors, the value at width 2 -/// below it, and above width 6 the width 6 value held flat (the Linux data at -/// width 8 is below the width 6 value, so flat is conservative and simple). -/// The model is evaluated once, when the value is built (`new` and `flat` are -/// `const`), into a table of nanoseconds, so a lookup is an index. Width 1 never -/// reaches the model: a token of one slot has no parallel section, see +/// The model at width `w`: the value at the first anchor below it, linear +/// interpolation between two anchors, and **linear extrapolation above the last +/// anchor** `w_m`, `ns[w_m] * w / w_m`, saturating at `u64::MAX`. Extrapolation +/// never falls below the last anchor and no width above it is charged a flat +/// cost: a wide section pays at least in proportion to its width, which is the +/// conservative reading of the measurements. +/// +/// The anchors are the widths a row was measured at. [`new`](Self::new) and +/// [`flat`](Self::flat) carry widths 2, 4 and 6; +/// [`from_anchors`](Self::from_anchors) takes any strictly increasing widths from +/// 2 to `u16::MAX`, at most [`MAX_ANCHORS`](Self::MAX_ANCHORS) of them, so a row +/// measured on a many-core host replaces the extrapolation where it has data. +/// The values are integer nanoseconds and a lookup scans at most +/// [`MAX_ANCHORS`](Self::MAX_ANCHORS) entries, so it allocates nothing and the +/// comparisons of the rule stay exact. Width 1 never reaches the model: a token +/// of one slot has no parallel section, see /// [`Exec::dispatch_cost`](crate::Exec::dispatch_cost). /// /// # Examples @@ -54,21 +64,26 @@ pub enum DispatchKind { /// let c = WidthCosts::new([us(3), us(2), us(1)]); /// assert_eq!(c.at(NonZeroUsize::new(2).unwrap()), us(3)); /// assert_eq!(c.at(NonZeroUsize::new(3).unwrap()), Duration::from_nanos(2500)); -/// assert_eq!(c.at(NonZeroUsize::new(12).unwrap()), us(1)); +/// assert_eq!(c.at(NonZeroUsize::new(6).unwrap()), us(1)); +/// // Above the last anchor (6) the cost grows with the width. +/// assert_eq!(c.at(NonZeroUsize::new(12).unwrap()), us(2)); +/// assert_eq!(c.at(NonZeroUsize::new(48).unwrap()), us(8)); /// assert_eq!(c.min(), us(1)); /// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct WidthCosts { - /// Nanoseconds at widths 1..=6 (index `w - 1`), the model evaluated once when - /// the table is built; every wider token reads the last entry. - ns: [u64; TABLE], + /// The measured widths, strictly increasing, in `widths[..len]`. + widths: [u16; CAPACITY], + /// The cost in nanoseconds at each anchor, in `ns[..len]`. + ns: [u64; CAPACITY], + /// The number of anchors. The slots past it are zeroed by construction, so + /// the derived `PartialEq` compares only the anchors. + len: u8, } -/// The last anchor: the model is flat from here on, so a table to this width -/// holds the whole model. -const TABLE: usize = 6; - -const ANCHORS: [u64; 3] = [2, 4, 6]; +/// The capacity of a row: enough for a measurement every power of two from 2 to +/// 4096 plus the widths a many-core campaign reports. +const CAPACITY: usize = 12; const fn to_ns(d: Duration) -> u64 { let n = d.as_nanos(); @@ -79,49 +94,123 @@ const fn to_ns(d: Duration) -> u64 { } } -/// The model at width `w` from the anchor values; the reference the table is -/// built from (linear between anchors, flat outside). -const fn model_ns(anchors: &[u64; 3], w: u64) -> u64 { - if w <= ANCHORS[0] { - anchors[0] - } else if w >= ANCHORS[2] { - // Flat above the last anchor. - anchors[2] +/// Linear interpolation between two anchors in `i128`, clamped at zero as the +/// model it replaces was. +#[inline] +fn interpolate(c0: u64, c1: u64, w0: u16, w1: u16, w: usize) -> u64 { + let (c0, c1) = (i128::from(c0), i128::from(c1)); + let v = c0 + (c1 - c0) * (w - usize::from(w0)) as i128 / i128::from(w1 - w0); + if v < 0 { + 0 } else { - let i = (w > ANCHORS[1]) as usize; - let (w0, w1) = (ANCHORS[i], ANCHORS[i + 1]); - let (c0, c1) = (anchors[i] as i128, anchors[i + 1] as i128); - let v = c0 + (c1 - c0) * (w - w0) as i128 / (w1 - w0) as i128; - if v < 0 { - 0 - } else { - v as u64 - } + v as u64 } } impl WidthCosts { + /// The largest number of anchors a row holds. + pub const MAX_ANCHORS: usize = CAPACITY; + /// The costs measured at widths 2, 4 and 6. pub const fn new(at_2_4_6: [Duration; 3]) -> Self { - let anchors = [to_ns(at_2_4_6[0]), to_ns(at_2_4_6[1]), to_ns(at_2_4_6[2])]; - let mut ns = [0u64; TABLE]; + Self::store(&[(2, at_2_4_6[0]), (4, at_2_4_6[1]), (6, at_2_4_6[2])]) + } + + /// The same cost at every anchor 2, 4 and 6 (the model of an unmeasured + /// host). Above width 6 the row grows with the width like every other row; a + /// host that measured equal costs out to a wide width says so with + /// [`from_anchors`](Self::from_anchors). + pub const fn flat(cost: Duration) -> Self { + WidthCosts::new([cost; 3]) + } + + /// A row from measured anchors `(width, cost)`: widths 2 to `u16::MAX`, + /// strictly increasing, at most [`MAX_ANCHORS`](Self::MAX_ANCHORS) of them. + /// + /// `None` for empty input, for more than [`MAX_ANCHORS`](Self::MAX_ANCHORS) + /// anchors, for a width below 2 or above `u16::MAX`, and for widths that are + /// not strictly increasing. No panic, no allocation, usable in a `const` + /// item. + /// + /// # Examples + /// + /// ``` + /// use std::num::NonZeroUsize; + /// use std::time::Duration; + /// use threadteam::WidthCosts; + /// + /// let us = Duration::from_micros; + /// let a = WidthCosts::from_anchors(&[(2, us(3)), (4, us(2)), (8, us(1))]).unwrap(); + /// assert_eq!(a.at(NonZeroUsize::new(6).unwrap()), Duration::from_nanos(1500)); + /// // Above the last anchor (8) the cost grows with the width. + /// assert_eq!(a.at(NonZeroUsize::new(16).unwrap()), us(2)); + /// assert!(WidthCosts::from_anchors(&[]).is_none()); + /// assert!(WidthCosts::from_anchors(&[(1, us(3))]).is_none()); + /// ``` + pub const fn from_anchors(anchors: &[(usize, Duration)]) -> Option { + let n = anchors.len(); + if n == 0 || n > CAPACITY { + return None; + } let mut k = 0; - while k < TABLE { - ns[k] = model_ns(&anchors, k as u64 + 1); + while k < n { + let w = anchors[k].0; + if w < 2 || w > u16::MAX as usize { + return None; + } + if k > 0 && w <= anchors[k - 1].0 { + return None; + } k += 1; } - WidthCosts { ns } + Some(Self::store(anchors)) } - /// The same cost at every anchor (the model of an unmeasured host). - pub const fn flat(cost: Duration) -> Self { - WidthCosts::new([cost; 3]) + /// Write the anchors into the fixed-capacity arrays; unused slots stay zero + /// so that the derived `PartialEq` sees only the anchors. The caller + /// validates. + const fn store(anchors: &[(usize, Duration)]) -> Self { + let mut widths = [0u16; CAPACITY]; + let mut ns = [0u64; CAPACITY]; + let mut k = 0; + while k < CAPACITY { + if k < anchors.len() { + widths[k] = anchors[k].0 as u16; + ns[k] = to_ns(anchors[k].1); + } + k += 1; + } + WidthCosts { + widths, + ns, + len: anchors.len() as u8, + } } - /// Nanoseconds of a section of `width` participants: one index. + /// Nanoseconds of a section of `width` participants: a forward scan over at + /// most `len` anchors. #[inline] pub(crate) fn ns_at(&self, width: NonZeroUsize) -> u64 { - self.ns[(width.get() - 1).min(TABLE - 1)] + let w = width.get(); + let len = usize::from(self.len); + if w <= usize::from(self.widths[0]) { + return self.ns[0]; + } + let mut i = 0; + while i + 1 < len { + let w1 = self.widths[i + 1]; + if w <= usize::from(w1) { + return interpolate(self.ns[i], self.ns[i + 1], self.widths[i], w1, w); + } + i += 1; + } + // Above the last anchor: at least linear in the width, saturating. + let v = u128::from(self.ns[len - 1]) * w as u128 / u128::from(self.widths[len - 1]); + if v > u64::MAX as u128 { + u64::MAX + } else { + v as u64 + } } /// The cost of a section of `width` participants, by the model above. @@ -130,15 +219,16 @@ impl WidthCosts { Duration::from_nanos(self.ns_at(width)) } - /// The cheapest cost over all widths (the model is evaluated at every width, - /// so this is right also for a table whose cost rises with width). A region - /// whose serial estimate is below `PAYS * min()` pays at no width, which is - /// the cheap rejection before a per-width search. + /// The cheapest cost over all widths: the minimum over the anchors (neither + /// interpolation nor the extrapolation can undershoot the anchor they start + /// from). A region whose serial estimate is below `PAYS * min()` pays at no + /// width, which is the cheap rejection before a per-width search. #[inline] pub fn min(&self) -> Duration { + let len = usize::from(self.len); let mut m = self.ns[0]; let mut k = 1; - while k < TABLE { + while k < len { if self.ns[k] < m { m = self.ns[k]; } @@ -280,19 +370,32 @@ impl DispatchCosts { /// `data/probes/20261026-awake-team-linux`, `docs/design/thresholds.md` /// section 3.6), in microseconds at widths 2, 4, 6: team 0.6, 0.7, 0.9; /// in pool 3.0, 3.6, 3.5; outside 9.7, 10.1, 8.7. The team row is the one - /// measured after the per-member parker fix. Width 8 measured 0.98 us on - /// a team, 8 % above the flat model (0.9). + /// measured after the per-member parker fix and carries the measured width + /// 8 anchor 0.98 us; in pool and outside keep the three widths of the same + /// run (their width 8 values are from the pre-fix run and are not used). /// * [`HostClass::Aarch64Apple`]: measured on an Apple M5 Max, 6 threads, /// QoS `UserInitiated`, from the back to back sections whose shares /// carry 10 us of work (`docs/design/thresholds-data/`), at widths 2, 4, /// 6: team 3.0, 2.3, 1.4; in pool 6.0, 11, 5.5; outside 14, 18, 19. /// * Every other class has no measurement and takes the conservative /// generic row: 3 us for a team, 15 us in the pool and 40 us from - /// outside, at every width. It errs towards staying serial. + /// outside at the anchors 2, 4 and 6, and above width 6 the width + /// proportion (32, 160 and 427 us at width 64). It errs towards staying + /// serial. + /// + /// Above the largest anchor of a row the cost grows with the width + /// ([`WidthCosts`]); [`WidthCosts::from_anchors`] replaces that + /// extrapolation where a host has measured anchors beyond width 6. pub fn for_class(class: HostClass) -> Self { match class { HostClass::X86_64Avx512 => DispatchCosts { - team: WidthCosts::new([us_f(0.6), us_f(0.7), us_f(0.9)]), + team: WidthCosts::from_anchors(&[ + (2, us_f(0.6)), + (4, us_f(0.7)), + (6, us_f(0.9)), + (8, us_f(0.98)), + ]) + .expect("the avx512 team row has four valid anchors"), in_pool: WidthCosts::new([us_f(3.0), us_f(3.6), us_f(3.5)]), outside: WidthCosts::new([us_f(9.7), us_f(10.1), us_f(8.7)]), }, @@ -340,6 +443,9 @@ impl DispatchCosts { } /// The cost of one section of `width` participants on a token of `kind`. + /// Measured anchors keep their values, widths between two anchors are + /// interpolated, and widths above the largest anchor of the row grow with + /// the width ([`WidthCosts`]). #[inline] pub fn cost(&self, kind: DispatchKind, width: NonZeroUsize) -> Duration { match kind { @@ -471,15 +577,157 @@ mod tests { assert_eq!(c.at(nz(1)), us(3)); assert_eq!(c.at(nz(3)), Duration::from_nanos(2500)); assert_eq!(c.at(nz(5)), Duration::from_nanos(1500)); - // Flat above the last anchor. - assert_eq!(c.at(nz(8)), us(1)); - assert_eq!(c.at(nz(12)), us(1)); + // Above the last anchor the cost grows with the width. + assert_eq!(c.at(nz(8)), Duration::from_nanos(1333)); + assert_eq!(c.at(nz(12)), us(2)); // Non-monotone anchors interpolate without underflow. let n = WidthCosts::new([us(1), us(5), us(0)]); assert_eq!(n.at(nz(5)), Duration::from_nanos(2500)); + assert_eq!(n.at(nz(12)), Duration::ZERO); + } + + #[test] + fn a_flat_row_extrapolates_in_proportion_to_the_width() { + let c = WidthCosts::flat(us(1)); + assert_eq!(c.at(nz(6)), us(1)); + assert_eq!(c.at(nz(12)), us(2)); + assert_eq!(c.at(nz(24)), us(4)); + assert_eq!(c.at(nz(48)), us(8)); + assert_eq!(c.at(nz(64)), Duration::from_nanos(10_666)); + assert_eq!(c.min(), us(1)); + } + + /// Above the last anchor `w_m` the cost never falls and is at least + /// `cost(w_m) * w / w_m`. + #[test] + fn above_the_last_anchor_is_non_decreasing_and_at_least_linear() { + let rows: [[Duration; 3]; 5] = [ + [us_f(0.6), us_f(0.7), us_f(0.9)], + [us_f(3.0), us_f(3.6), us_f(3.5)], + [us_f(9.7), us_f(10.1), us_f(8.7)], + [us(3), us(2), us(1)], + [us(1), us(5), us(0)], + ]; + for row in rows { + let c = WidthCosts::new(row); + let cm = u128::from(to_ns(c.at(nz(6)))); + let mut prev = c.at(nz(6)); + for w in 7..=64u64 { + let v = c.at(nz(w as usize)); + assert!(v >= prev, "{row:?}: width {w} fell"); + assert!( + u128::from(to_ns(v)) >= cm * u128::from(w) / 6, + "{row:?}: width {w} below the linear bound" + ); + prev = v; + } + } + } + + #[test] + fn anchors_beyond_six_interpolate_and_extrapolate() { + let c = WidthCosts::from_anchors(&[ + (2, Duration::from_nanos(8000)), + (4, Duration::from_nanos(4000)), + (8, Duration::from_nanos(2000)), + (16, Duration::from_nanos(1000)), + (32, Duration::from_nanos(500)), + (64, Duration::from_nanos(500)), + ]) + .unwrap(); + let ns = |w: usize| c.at(nz(w)).as_nanos() as u64; + assert_eq!(ns(2), 8000); + assert_eq!(ns(3), 6000); // between 2 and 4 + assert_eq!(ns(6), 3000); // between 4 and 8, past the old last anchor + assert_eq!(ns(12), 1500); // between 8 and 16 + assert_eq!(ns(24), 750); // between 16 and 32 + assert_eq!(ns(48), 500); // between 32 and 64 + assert_eq!(ns(64), 500); // the last anchor + assert_eq!(ns(96), 750); // above the last anchor + assert_eq!(ns(128), 1000); + assert_eq!(c.min(), Duration::from_nanos(500)); + } + + #[test] + fn from_anchors_rejects_invalid_rows() { + assert!(WidthCosts::from_anchors(&[]).is_none()); + assert!(WidthCosts::from_anchors(&[(0, us(1))]).is_none()); + assert!(WidthCosts::from_anchors(&[(1, us(1)), (2, us(1))]).is_none()); + assert!(WidthCosts::from_anchors(&[(2, us(1)), (2, us(1))]).is_none()); + assert!(WidthCosts::from_anchors(&[(4, us(1)), (2, us(1))]).is_none()); + let too_wide = [(usize::from(u16::MAX) + 1, us(1))]; + assert!(WidthCosts::from_anchors(&too_wide).is_none()); + assert!(WidthCosts::from_anchors(&[(usize::from(u16::MAX), us(1))]).is_some()); + // MAX_ANCHORS anchors fit, one more is rejected. + let mut many = [(0usize, Duration::ZERO); WidthCosts::MAX_ANCHORS + 1]; + for (k, a) in many.iter_mut().enumerate() { + *a = (k + 2, us(1)); + } + assert!(WidthCosts::from_anchors(&many[..WidthCosts::MAX_ANCHORS]).is_some()); + assert!(WidthCosts::from_anchors(&many).is_none()); } - /// The model as it was before the table: i128 interpolation on every call. + /// `from_anchors` is usable in a `const` item and the unused capacity does + /// not take part in equality. + #[test] + fn from_anchors_is_const_and_capacity_is_invisible() { + const ROW: WidthCosts = match WidthCosts::from_anchors(&[ + (2, Duration::from_micros(3)), + (4, Duration::from_micros(2)), + (8, Duration::from_micros(1)), + ]) { + Some(c) => c, + None => panic!("valid anchors"), + }; + assert_eq!(WidthCosts::MAX_ANCHORS, 12); + assert_eq!(ROW.at(nz(6)), Duration::from_nanos(1500)); + assert_eq!(ROW.at(nz(16)), us(2)); + assert_eq!(ROW.min(), us(1)); + + let three = WidthCosts::from_anchors(&[(2, us(3)), (4, us(2)), (6, us(1))]).unwrap(); + assert_eq!(three, WidthCosts::new([us(3), us(2), us(1)])); + let four = + WidthCosts::from_anchors(&[(2, us(3)), (4, us(2)), (6, us(1)), (8, us(1))]).unwrap(); + assert_ne!(three, four); + } + + #[test] + fn a_duration_max_row_never_pays() { + // `Duration::MAX` saturates to `u64::MAX` nanoseconds in a row. + let cap = Duration::from_nanos(u64::MAX); + let c = WidthCosts::flat(Duration::MAX); + assert_eq!(c.at(nz(6)), cap); + assert_eq!(c.at(nz(64)), cap); // the extrapolation saturates + assert_eq!(c.min(), cap); + assert!(!pays_at(cap, 1, c.at(nz(64)))); + assert!(!pays_at(cap, 2, c.at(nz(64)))); + let mixed = WidthCosts::from_anchors(&[(2, Duration::ZERO), (4, Duration::MAX)]).unwrap(); + assert_eq!(mixed.at(nz(64)), cap); + assert_eq!(mixed.min(), Duration::ZERO); + } + + /// A wide region selects a useful interior width above six, not always six + /// or the budget. + #[test] + fn a_wide_region_selects_an_interior_width_above_six() { + let row = WidthCosts::flat(us(1)); + let d = |k: NonZeroUsize| row.at(k); + let budget = nz(64); + // obj(k) = D(k) + 24 us / k with D(k) = 1 us * k / 6 above 6: obj(12) = + // 2 + 2 = 4 us against obj(6) = 1 + 4 = 5 us, obj(11) = 1.833 + 2.18 and + // obj(13) = 2.167 + 1.85. + assert_eq!(width_for_by(us(24), 1, budget, d).get(), 12); + // A large enough estimate pushes the optimum to the budget. + assert_eq!(width_for_by(us(1_000_000), 1, budget, d).get(), 64); + // A region that does not pay at width 6 does not pay at 64 either. + let below = Duration::from_nanos(3 * 1000 - 1); + assert!(!pays_at(below, 1, row.at(nz(6)))); + assert!(!pays_at(below, 1, row.at(nz(64)))); + assert_eq!(width_for_by(below, 1, budget, d).get(), 1); + } + + /// The model as it was before the anchors were stored: i128 interpolation on + /// every call, flat above width 6. fn old_at(ns: [u64; 3], w: u64) -> u64 { if w <= 2 { ns[0] @@ -494,7 +742,7 @@ mod tests { } #[test] - fn table_equals_the_interpolation_for_every_width() { + fn rows_keep_the_old_values_inside_the_anchors() { let tables = [ [3000, 2300, 1400], [2800, 2200, 1700], @@ -507,7 +755,7 @@ mod tests { ]; for ns in tables { let c = WidthCosts::new(ns.map(Duration::from_nanos)); - for w in 1..=64u64 { + for w in 1..=6u64 { assert_eq!( c.at(nz(w as usize)), Duration::from_nanos(old_at(ns, w)), @@ -515,6 +763,8 @@ mod tests { ); } } + // The minimum over the anchors is the minimum over all widths: neither + // interpolation nor the extrapolation can undershoot its endpoint. for d in [ DispatchCosts::for_class(HostClass::Aarch64Apple), DispatchCosts::for_class(HostClass::X86_64Avx512), @@ -531,6 +781,80 @@ mod tests { } } + /// The measured rows at widths 1 to 6 as the pre-anchor model gave them, so + /// this change moves no decision inside the anchors. + #[test] + fn measured_rows_are_pinned_at_widths_one_to_six() { + let avx512 = DispatchCosts::for_class(HostClass::X86_64Avx512); + let apple = DispatchCosts::for_class(HostClass::Aarch64Apple); + let generic = DispatchCosts::for_class(HostClass::Generic); + for (costs, kind, expected) in [ + ( + avx512, + DispatchKind::Team, + [600u64, 600, 650, 700, 800, 900], + ), + ( + avx512, + DispatchKind::InPool, + [3000, 3000, 3300, 3600, 3550, 3500], + ), + ( + avx512, + DispatchKind::Outside, + [9700, 9700, 9900, 10100, 9400, 8700], + ), + ( + apple, + DispatchKind::Team, + [3000, 3000, 2650, 2300, 1850, 1400], + ), + ( + apple, + DispatchKind::InPool, + [6000, 6000, 8500, 11000, 8250, 5500], + ), + ( + apple, + DispatchKind::Outside, + [14000, 14000, 16000, 18000, 18500, 19000], + ), + (generic, DispatchKind::Team, [3000; 6]), + (generic, DispatchKind::InPool, [15000; 6]), + (generic, DispatchKind::Outside, [40000; 6]), + ] { + for (k, want) in expected.into_iter().enumerate() { + assert_eq!( + costs.cost(kind, nz(k + 1)).as_nanos() as u64, + want, + "{kind:?} width {}", + k + 1 + ); + } + } + // The measured width-8 team anchor replaces the extrapolation there; + // width 7 interpolates between the width-6 and width-8 team anchors. + assert_eq!(avx512.cost(DispatchKind::Team, nz(8)), us_f(0.98)); + assert_eq!( + avx512.cost(DispatchKind::Team, nz(7)), + Duration::from_nanos(940) + ); + assert_eq!( + avx512.cost(DispatchKind::Team, nz(64)), + Duration::from_nanos(7_840) + ); + // The width-8 values in pool and outside come from the pre-fix run and + // are not anchors: the extrapolation applies to them. + assert_eq!( + avx512.cost(DispatchKind::InPool, nz(8)), + Duration::from_nanos(4666) + ); + assert_eq!( + avx512.cost(DispatchKind::Outside, nz(8)), + Duration::from_nanos(11_600) + ); + } + #[test] fn min_is_right_for_a_rising_table_and_a_falling_one() { let rising = WidthCosts::new([us(2), us(5), us(9)]); diff --git a/tests/dispatch_cost.rs b/tests/dispatch_cost.rs index abaf91f..a752bbb 100644 --- a/tests/dispatch_cost.rs +++ b/tests/dispatch_cost.rs @@ -6,7 +6,9 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::num::NonZeroUsize; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; -use threadteam::{pays_at, DispatchCosts, DispatchKind, Exec, HostClass, Pool, WidthCosts, PAYS}; +use threadteam::{ + pays_at, width_for_by, DispatchCosts, DispatchKind, Exec, HostClass, Pool, WidthCosts, PAYS, +}; struct Counting; static ALLOCS: AtomicUsize = AtomicUsize::new(0); @@ -148,20 +150,24 @@ fn decision_functions_do_not_allocate() { } #[test] -fn width_model_is_flat_above_the_last_anchor() { +fn width_model_extrapolates_above_the_last_anchor() { let c = WidthCosts::new([us(3), us(2), us(1)]); assert_eq!(c.at(nz(6)), us(1)); - assert_eq!(c.at(nz(8)), us(1)); - assert_eq!(c.at(nz(64)), us(1)); - // Linux row: width 8 measured below width 6, the model holds width 6. + // Above the last anchor the cost grows with the width, never flat. + assert_eq!(c.at(nz(8)), Duration::from_nanos(1333)); + assert_eq!(c.at(nz(64)), Duration::from_nanos(10_666)); + // Linux row: the team carries the measured width-8 anchor; in pool and + // outside have no anchor there and extrapolate from width 6. let l = DispatchCosts::for_class(HostClass::X86_64Avx512); - for (kind, w6) in [ - (DispatchKind::Team, 900), - (DispatchKind::InPool, 3_500), - (DispatchKind::Outside, 8_700), - ] { - assert_eq!(l.cost(kind, nz(8)), Duration::from_nanos(w6)); - } + assert_eq!(l.cost(DispatchKind::Team, nz(8)), Duration::from_nanos(980)); + assert_eq!( + l.cost(DispatchKind::InPool, nz(8)), + Duration::from_nanos(4666) + ); + assert_eq!( + l.cost(DispatchKind::Outside, nz(8)), + Duration::from_nanos(11_600) + ); // A team GEMM 64^3 (about 9.7 us) pays at width 8. assert!(pays_at( Duration::from_nanos(9_700), @@ -170,6 +176,36 @@ fn width_model_is_flat_above_the_last_anchor() { )); } +/// A row measured up to width 64 replaces the extrapolation where it has data. +#[test] +fn anchors_beyond_six_reach_the_decision_functions() { + let row = WidthCosts::from_anchors(&[ + (2, us(8)), + (4, us(4)), + (8, us(2)), + (16, us(1)), + (32, Duration::from_nanos(500)), + (64, Duration::from_nanos(500)), + ]) + .expect("valid anchors"); + let costs = DispatchCosts::for_class(HostClass::Generic).with(DispatchKind::Team, row); + assert_eq!(costs.cost(DispatchKind::Team, nz(6)), us(3)); + assert_eq!( + costs.cost(DispatchKind::Team, nz(64)), + Duration::from_nanos(500) + ); + // 24 us: obj(6) = 3 + 4 = 7 us against obj(64) = 0.5 + 0.375 us, and the + // anchors above six are what the search reads. + assert_eq!( + width_for_by(us(24), 1, nz(64), |k| costs.cost(DispatchKind::Team, k)).get(), + 64 + ); + assert_eq!( + costs.min_cost(DispatchKind::Team), + Duration::from_nanos(500) + ); +} + #[test] fn class_rows_and_host_default() { let ns = |x: f64| Duration::from_nanos((x * 1000.0) as u64); @@ -182,7 +218,8 @@ fn class_rows_and_host_default() { assert_eq!(m.cost(DispatchKind::Team, nz(2)), ns(3.0)); assert_eq!(m.cost(DispatchKind::InPool, nz(4)), ns(11.0)); assert_eq!(m.cost(DispatchKind::Outside, nz(6)), ns(19.0)); - // Classes without a measurement take the generic row: 3, 15, 40 us flat. + // Classes without a measurement take the generic row: 3, 15, 40 us at the + // anchors 2, 4, 6 and proportional to the width above them. for class in [ HostClass::X86_64Avx2, HostClass::X86_64Scalar, @@ -191,10 +228,30 @@ fn class_rows_and_host_default() { ] { let g = DispatchCosts::for_class(class); assert_eq!(g, DispatchCosts::for_class(HostClass::Generic), "{class:?}"); - for w in [2, 4, 6, 8, 64] { - assert_eq!(g.cost(DispatchKind::Team, nz(w)), ns(3.0)); - assert_eq!(g.cost(DispatchKind::InPool, nz(w)), ns(15.0)); - assert_eq!(g.cost(DispatchKind::Outside, nz(w)), ns(40.0)); + // The three anchors of the generic row are equal; above the last anchor + // the cost still grows with the width. + for (w, team, in_pool, outside) in [ + (2, 3_000, 15_000, 40_000), + (4, 3_000, 15_000, 40_000), + (6, 3_000, 15_000, 40_000), + (8, 4_000, 20_000, 53_333), + (64, 32_000, 160_000, 426_666), + ] { + assert_eq!( + g.cost(DispatchKind::Team, nz(w)), + Duration::from_nanos(team), + "{class:?} team width {w}" + ); + assert_eq!( + g.cost(DispatchKind::InPool, nz(w)), + Duration::from_nanos(in_pool), + "{class:?} in pool width {w}" + ); + assert_eq!( + g.cost(DispatchKind::Outside, nz(w)), + Duration::from_nanos(outside), + "{class:?} outside width {w}" + ); } } // The default is the row of the detected class, and detection is stable. From 125109603e554a8731e37771d6a1e2e52538d9ff Mon Sep 17 00:00:00 2001 From: Hiroshi Shinaoka Date: Thu, 29 Oct 2026 09:58:43 +0900 Subject: [PATCH 2/4] cost: 64-bit fast path for the anchor interpolation and extrapolation The 128-bit division is a library call; the 64-bit one gives the same truncated value whenever the product fits. Lookup at widths 2..8: 7.35 -> 4.12 ns (base table index 0.89 ns). Co-Authored-By: Claude Opus 5.5 --- src/cost.rs | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/cost.rs b/src/cost.rs index a90f6c6..b0e32d0 100644 --- a/src/cost.rs +++ b/src/cost.rs @@ -98,6 +98,15 @@ const fn to_ns(d: Duration) -> u64 { /// model it replaces was. #[inline] fn interpolate(c0: u64, c1: u64, w0: u16, w1: u16, w: usize) -> u64 { + // 64-bit fast path (a 128-bit division is a library call); the same + // truncating division, so the value is identical to the 128-bit one. + let dw = (w - usize::from(w0)) as i64; + if let (Ok(a), Ok(b)) = (i64::try_from(c0), i64::try_from(c1)) { + if let Some(t) = (b - a).checked_mul(dw) { + let v = a + t / i64::from(w1 - w0); + return if v < 0 { 0 } else { v as u64 }; + } + } let (c0, c1) = (i128::from(c0), i128::from(c1)); let v = c0 + (c1 - c0) * (w - usize::from(w0)) as i128 / i128::from(w1 - w0); if v < 0 { @@ -204,8 +213,13 @@ impl WidthCosts { } i += 1; } - // Above the last anchor: at least linear in the width, saturating. - let v = u128::from(self.ns[len - 1]) * w as u128 / u128::from(self.widths[len - 1]); + // Above the last anchor: at least linear in the width, saturating + // (64-bit when the product fits, as above). + let (last, w_m) = (self.ns[len - 1], u64::from(self.widths[len - 1])); + if let Some(p) = last.checked_mul(w as u64) { + return p / w_m; + } + let v = u128::from(last) * w as u128 / u128::from(w_m); if v > u64::MAX as u128 { u64::MAX } else { @@ -918,4 +932,22 @@ mod tests { ); assert_eq!(width_for_by(us(1_000_000), 1, nz(1), d).get(), 1); } + + #[test] + fn wide_values_take_the_128_bit_path_with_the_same_result() { + // Values whose products overflow 64 bits use the 128-bit fallback; it + // must give what the 128-bit formula gives. + let big = Duration::from_nanos(u64::MAX / 3); + let huge = Duration::from_nanos(u64::MAX / 2); + let c = WidthCosts::from_anchors(&[(2, big), (64, huge)]).unwrap(); + let (c0, c1) = (i128::from(u64::MAX / 3), i128::from(u64::MAX / 2)); + for w in [3usize, 17, 33, 63] { + let want = c0 + (c1 - c0) * (w as i128 - 2) / 62; + assert_eq!(u128::from(c.ns_at(nz(w))), want as u128, "width {w}"); + } + // Above the last anchor the product saturates instead of wrapping. + assert_eq!(c.ns_at(nz(200)), u64::MAX); + let want = u128::from(u64::MAX / 2) * 100 / 64; + assert_eq!(u128::from(c.ns_at(nz(100))), want.min(u128::from(u64::MAX))); + } } From e1310e659693b21c010614d09a9e189d8af33aea Mon Sep 17 00:00:00 2001 From: Hiroshi Shinaoka Date: Thu, 29 Oct 2026 10:02:19 +0900 Subject: [PATCH 3/4] cost: post-review fixes (final bench figures, min() wording, truncation test) Co-Authored-By: Claude Opus 5.5 --- benches/cost_lookup.rs | 4 +--- docs/design/thresholds.md | 11 ++++++----- src/cost.rs | 16 +++++++++++++--- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/benches/cost_lookup.rs b/benches/cost_lookup.rs index a653407..013d1c8 100644 --- a/benches/cost_lookup.rs +++ b/benches/cost_lookup.rs @@ -9,7 +9,7 @@ use std::hint::black_box; use std::num::NonZeroUsize; use std::time::{Duration, Instant}; -use threadteam::{DispatchCosts, DispatchKind, WidthCosts}; +use threadteam::{DispatchCosts, DispatchKind}; fn ns_per_call(mut f: impl FnMut(u64) -> u64) -> f64 { let iters = 2_000_000u64; @@ -80,8 +80,6 @@ fn main() { #[cfg(feature = "parallel")] token_rows(costs); - - let _ = WidthCosts::flat(Duration::ZERO); } /// The lookups on a real team token: the kind is read at call time, so these are diff --git a/docs/design/thresholds.md b/docs/design/thresholds.md index 1827e28..83ccd8f 100644 --- a/docs/design/thresholds.md +++ b/docs/design/thresholds.md @@ -310,12 +310,13 @@ forward scan over at most `MAX_ANCHORS` = 12 of them, so the lookup allocates no cores 4-11 (previous phase, flat table whose `at` was an index): `at` 7.9 to 1.2 ns, `pays_at` after `at` 10.1 to 2.2 ns, `width_for` at budget 8 83 to 32 ns. The anchor scan costs an integer division per interpolated width where the table held the value: on the 4-core run of issue #6 (cores 56-59, best of 9 x 2M calls, same bench file on both -trees) `at` 0.89 -> 7.35 ns, `width_for` budget 8 25.8 -> 72.9 ns and budget 64 222 -> 553 ns, `min_cost` -6.0 -> 4.3 ns, a by-value `DispatchCosts` copy 4.5 -> 12.6 ns (144 -> 384 bytes), `Exec::dispatch_cost` on a team -3.3 -> 8.8 ns. That stays far below one section, which the rows above charge 0.6 to 427 us, but a consumer that +trees, with the 64-bit fast path that falls back to 128 bits only when a product overflows) `at` 0.89 -> 4.12 ns, +`pays_at` after `at` 1.97 -> 7.84 ns, `width_for` budget 8 25.9 -> 61.8 ns and budget 64 222 -> 522 ns, `min_cost` +6.7 -> 4.8 ns, a by-value `DispatchCosts` copy 4.5 -> 11.0 ns (144 -> 384 bytes), `Exec::dispatch_cost` on a team +3.3 -> 6.3 ns. That stays far below one section, which the rows above charge 0.6 to 427 us, but a consumer that calls `width_for` on every tiny region should use `min_cost` first. `min_cost(kind)` / -`WidthCosts::min()` is the cheapest cost over all widths: the minimum over the anchors (neither interpolation nor -the extrapolation can undershoot the anchor it starts from), so a region below `PAYS * min_cost` pays at no width. +`WidthCosts::min()` is the cheapest cost over all widths: the minimum over the anchors (an interpolated value lies +between its two anchors, and the extrapolation never falls below the last anchor), so a region below `PAYS * min_cost` pays at no width. A test compares the rows with the former interpolation for widths 1 to 6 and pins the extrapolation above them. Width model: the value at the first anchor below it, linear interpolation between two anchors, and above the diff --git a/src/cost.rs b/src/cost.rs index b0e32d0..0b7ab23 100644 --- a/src/cost.rs +++ b/src/cost.rs @@ -233,9 +233,9 @@ impl WidthCosts { Duration::from_nanos(self.ns_at(width)) } - /// The cheapest cost over all widths: the minimum over the anchors (neither - /// interpolation nor the extrapolation can undershoot the anchor they start - /// from). A region whose serial estimate is below `PAYS * min()` pays at no + /// The cheapest cost over all widths: the minimum over the anchors (an + /// interpolated value lies between its two anchors, and the extrapolation + /// never falls below the last anchor). A region whose serial estimate is below `PAYS * min()` pays at no /// width, which is the cheap rejection before a per-width search. #[inline] pub fn min(&self) -> Duration { @@ -950,4 +950,14 @@ mod tests { let want = u128::from(u64::MAX / 2) * 100 / 64; assert_eq!(u128::from(c.ns_at(nz(100))), want.min(u128::from(u64::MAX))); } + + #[test] + fn falling_interpolation_truncates_toward_zero() { + // 4 + (0 - 4) * 1 / 3 = 4 - 1.33..: truncation toward zero gives 3, floor + // rounding would give 2. Pins the old model's integer semantics. + let ns = Duration::from_nanos; + let c = WidthCosts::from_anchors(&[(2, ns(4)), (5, ns(0))]).unwrap(); + assert_eq!(c.at(nz(3)), ns(3)); + assert_eq!(c.at(nz(4)), ns(2)); + } } From 220d3ba88f3be99bbf07003dea87d58da2bceb13 Mon Sep 17 00:00:00 2001 From: Hiroshi Shinaoka Date: Thu, 29 Oct 2026 09:58:49 +0900 Subject: [PATCH 4/4] release: threadteam 0.1.1 (WidthCosts extrapolates above the largest anchor; variable anchors) Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d15ffb1..dfe14f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -276,7 +276,7 @@ dependencies = [ [[package]] name = "threadteam" -version = "0.1.0" +version = "0.1.1" dependencies = [ "libc", "loom", diff --git a/Cargo.toml b/Cargo.toml index de48329..4e48328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "threadteam" -version = "0.1.0" +version = "0.1.1" edition = "2021" license = "MIT OR Apache-2.0" authors = ["Satoshi Terasaki", "Hiroshi Shinaoka"]