diff --git a/Cargo.lock b/Cargo.lock index 9c151f15..f65d19ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2831,9 +2831,9 @@ dependencies = [ [[package]] name = "threadteam" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d539fbcb550c65bbbb2b55350f279e0049bab0a3dbd6333274030839d4e4da70" +checksum = "d1bf24ef6ae1d37009be6b74c53027a756b9db97ee414e03c502362ca92b47cd" dependencies = [ "libc", "loom", diff --git a/PATCHSET.md b/PATCHSET.md index 7e5994f3..4921c5f8 100644 --- a/PATCHSET.md +++ b/PATCHSET.md @@ -123,6 +123,7 @@ gate has changed and two kinds of adoption are now in the ledger. | Team token runs the x86 kernel on its own members | `t4a-private-gemm-x86` 0.1.25 runs its two parallel stages (packing of the right-hand side, the dynamic job queue) on a caller-supplied `Fork` (`gemm_with`). A `Par::Team` is adapted (`utils::team::kernel::TeamFork`, over `TeamExec::run`, a sub-team of the gate's width) and used by the plain GEMM, `spicy_matmul` and the triangular product into a triangle (lower, upper, indexed destinations): gate `pays(estimate, 2 sections, width)`, width from `width_for`, below it the kernel on the calling thread. `gemm_tiled` stays as the non-x86 route; `outside_threads` and the triangular team recursion on x86 are gone (the recursion stays for non-x86 and non-native dtypes). A `Par::Team` of size 1 is `Par::Seq` in `par_pays`, `join_pays` and `region_pays`. Pins: threadteam `586738b` (Linux row), private-gemm-x86 git rev of 0.1.25. | EXPERIMENT (`exp/team-gemm-executor`, on top of `exp/par-team-lifetime`) | tensor4all-meta#51, tensor4all/t4a-private-gemm-x86#1 | No timing yet. Counter `gemm_team_kernel`. Tests: `utils::team::tests` (`the_kernel_forks_only_on_members_of_the_team`, `team_kernel_*`, `a_team_of_one_behaves_exactly_like_seq`). Record: [worklog](docs/worklogs/2026-10-26-team-gemm-executor.md). | | Serial path overhead of a team token | A QR 128 on a `Par::Team` that cannot pay was +11 % over `Par::Seq`: the 59 block applies asked every product and triangular step. The block apply asks one floor (`utils::cost::par_can_pay_time`, against `min_cost`, the cheapest width of the table) and runs the inner steps sequentially; `kernel_width` rejects against the same cheapest width instead of assuming a table non-increasing in width; gates read the token's table without a `ParShape` copy; the Rayon host table is cached. threadteam pinned at `7095893`. Fork-only (`Par::Team`). | KEEP (with the team executor) | | Threadteam is the only parallel route | `Par::Rayon`, `Par::rayon`, `ParShape::Rayon`, `ParShape::rayon` and `Par::__Lifetime` are removed, with the Rayon arms of `thread::join_raw`/`for_each`/`for_each_with_par`/`split`, of `ParShape::dispatch_cost`/`min_dispatch_cost`, of `par_pays_time`/`par_can_pay_time` and of the GEMM route; `parallelism_degree` folds into `Par::degree` and `is_team` goes. The Rayon cost model in `utils::cost` (`outside_cost_of`, `in_pool_cost_of`, `width_model`, `rayon_cost`, `min_rayon_cost`, `caller_in_pool`, the outside/in-pool anchors) and its tests are deleted; `ParShape::dispatch_cost` reads the costs of the team token the shape came from. faer's own serial `Rates` table and its `HostClass` key stay, and the `PAYS`/`pays_at` rule has one copy in `utils::cost` with a test against `threadteam`'s. `spindle` leaves `Cargo.toml` (no production code opened a group; the last users were tests and the two example harnesses). New: `faer::team::with_team(width, |par| ..)` owns a `threadteam::Pool` for a closure and hands out `Par::Team` (a width of one hands out `Par::Seq`), which is the caller-side answer #59 was waiting for and what the benchmark harness and the threshold probe use instead of `Par::rayon(n)`. `threadteam` becomes the feature that compiles the route (on by default, pulling `rayon` as its substrate); with it off there is no `Par::Team` and no team module, so a no_std build has `Par::Seq` only and nothing pulls std in through Rayon. 81 lib-test and 15 example sites converted to `with_team`; the tests that only asserted the removed route are deleted with the route. | KEEP | tensor4all/faer#85, tensor4all/faer#59 | Mechanism, not a speed-up: the acceptance measurement pairs the branch against `origin/main` with both rows on the team route, and the claim is only that deleting the second route did not regress the team route. Verification: `cargo test -p t4a-faer --lib`, `cargo test -p t4a-faer --doc`, `cargo check` for `--no-default-features`, `--no-default-features --features std`, `--no-default-features --features std,rayon`, default and `std,rayon,threadteam`, and `cargo +nightly-2025-01-08 run --profile nostd` in `faer-no-std-test` (the inherited nostd job). Record: [worklog](docs/worklogs/2026-10-29-threadteam-only.md). | +| Team dispatch cost read from threadteam | `utils::cost::team_section_cost` and `MEASURED_MAX_WIDTH` (faer's own linear extrapolation above width 8, #97) are deleted: the extrapolation lives in threadteam (`WidthCosts` 0.1.1, linear above a row's largest anchor; 0.1.2, a measured Zen2 team row for `x86_64-avx2` at widths 2 to 64). `ParShape::dispatch_cost` reads `DispatchCosts::cost(Team, w)`; `par_pays_time`, `team::kernel::kernel_width` and the tiled team product's floor ask `TeamExec::dispatch_cost(w)`. `kernel_width` searches from the full width down instead of stopping at the first narrow width that does not pay (measured rows fall with the width). Requirement `threadteam = "0.1.2"`. Decisions change at widths 7 and 8 for rows whose largest anchor is 6, at width 8 on `x86_64-avx512` (its 0.98 us anchor) and at every width on `x86_64-avx2`. | EXPERIMENT (pending the maintainer's acceptance decision) | tensor4all/threadteam-rs#6, tensor4all/faer#91 | No timing (gate inputs only; acceptance run left to the maintainer). Tests: `utils::cost::tests::the_team_cost_is_threadteams_table_at_every_width`, `utils::team::tests::kernel_width_finds_the_widest_paying_width_on_a_falling_row`. Record: [worklog](docs/worklogs/2026-10-29-team-cost-from-threadteam.md). | ## Ground rules diff --git a/docs/par-team-migration.md b/docs/par-team-migration.md index 1f81b7a4..b9a18dff 100644 --- a/docs/par-team-migration.md +++ b/docs/par-team-migration.md @@ -151,8 +151,9 @@ a region runs in parallel when `estimate >= PAYS * sections * dispatch_cost(widt measurement replaces its alias. **A row's rates and its dispatch costs must come from the same machine and clock state:** the boost moves both by about 1.66 times with a stable ratio (within a few percent, `tables-*-exec3-boost.md`), so mixing a boosted rate with an unboosted cost shifts every floor. The Rayon - table of a build without the feature is the model of threadteam (linear between the anchors at widths 2, 4, - 6, flat above) with the same values. Region constants of the `x86_64-avx512` row: the block apply streaming + table of a build without the feature was the model of threadteam at the time (linear between the anchors at + widths 2, 4, 6, flat above) with the same values; threadteam 0.1.1 grows the cost linearly above a row's largest + anchor, and faer reads the team cost from threadteam only (`docs/worklogs/2026-10-29-team-cost-from-threadteam.md`). Region constants of the `x86_64-avx512` row: the block apply streaming per multiply-add, the divide and conquer solves (median over `m` 64 to 512; per `m^2` 152.9 and 97.4 ns, size dependent, 145 to 178 and 88 to 102), the sparse product (median of the measured densities) and the triangular inverse (median over `m`; 0.047 per `m^3` at 64 falling to 0.0145 at 512, as on the M5 row). diff --git a/docs/worklogs/2026-10-29-team-cost-from-threadteam.md b/docs/worklogs/2026-10-29-team-cost-from-threadteam.md new file mode 100644 index 00000000..772061c6 --- /dev/null +++ b/docs/worklogs/2026-10-29-team-cost-from-threadteam.md @@ -0,0 +1,32 @@ +# Team dispatch cost read from threadteam (threadteam-rs#6) + +## Decisions +- The extrapolation of the team dispatch cost above the measured widths lives once, in threadteam + (`WidthCosts`, 0.1.1: linear above a row's largest anchor; 0.1.2: a measured Zen2 team row for + `x86_64-avx2` with anchors 2 to 64). faer's `utils::cost::team_section_cost` and + `MEASURED_MAX_WIDTH` (#97, provisional) are deleted; the four readers ask the table directly: + `ParShape::dispatch_cost` reads `DispatchCosts::cost(Team, w)`, `par_pays_time`, + `team::kernel::kernel_width` and the tiled team product's floor ask `TeamExec::dispatch_cost(w)`. + The requirement moves to `threadteam = "0.1.2"`, so a resolver cannot pick 0.1.0 (flat above 6). +- `kernel_width` asked the widths in increasing order and stopped at the first that did not pay, + assuming a cost that never falls with the width. Measured rows fall (`aarch64-apple` team 3.0, + 2.3, 1.4 us at widths 2, 4, 6), so a product that pays at 6 but not at 2 got no width. The widths + are now searched from the full width down and the first that pays is taken. + +## Verification conclusions and constraints +- Decisions change, by design, also at widths 8 and below, so the "bit-identical up to 8" argument of + #97 does not carry over: + - rows whose largest anchor is 6 (generic, `aarch64-apple`) grow from width 7 on (threadteam is + linear from its largest anchor; faer held the table flat to 8): generic width 8 is 4 us instead + of 3 us, so a 10 us one-section region at width 8 now stays serial; + - `x86_64-avx512` team reads its measured width-8 anchor (0.98 us instead of 0.9); + - `x86_64-avx2` (Zen2 row) is cheaper than what faer charged that class before at every width + (1.42 us against 3 us at width 2, 9.33 us against `3 * 64 / 8` = 24 us at width 64), and measured + up to 64 instead of extrapolated, so regions on such hosts go parallel more readily. +- Tests: `utils::cost::tests::the_team_cost_is_threadteams_table_at_every_width` (shape cost equals + threadteam's table at widths 2 to 128 for five rows; integer `cost(w_m) * w / w_m` above each + row's largest anchor, including width 128 on the Zen2 row; a region that does not pay at 8 does not + pay at 64), `utils::team::tests::kernel_width_finds_the_widest_paying_width_on_a_falling_row`. +- Not measured: the adoption protocol's blast-radius acceptance run (every family whose gate reads + the team cost). This change moves gate inputs, not kernels; per the maintainer's rule against a long + campaign per change it is left to the maintainer before merge. diff --git a/faer/Cargo.toml b/faer/Cargo.toml index 87d6f073..a9bf7c50 100644 --- a/faer/Cargo.toml +++ b/faer/Cargo.toml @@ -25,7 +25,7 @@ rayon = { version = "1.11.0", optional = true } # the parallel route: `Par::Team` runs its regions on a threadteam team, see src/utils/team.rs. # Optional, so that a no_std build (`--no-default-features`, `faer-no-std-test`) has no parallel # route at all: `Par::Seq` is the only token it carries, and nothing pulls std in through Rayon. -threadteam = { version = "0.1", default-features = false, optional = true } +threadteam = { version = "0.1.2", default-features = false, optional = true } log = { version = "0.4.29", optional = true } rand = { version = "0.9.2", optional = true } rand_distr = { version = "0.5.1", optional = true } diff --git a/faer/src/utils/cost.rs b/faer/src/utils/cost.rs index 0f085911..084f7ed4 100644 --- a/faer/src/utils/cost.rs +++ b/faer/src/utils/cost.rs @@ -582,8 +582,9 @@ pub(crate) fn par_pays_time(par: &crate::Par, estimate: Duration, sections: u32, crate::Par::Seq => false, #[cfg(feature = "threadteam")] crate::Par::Team(team) => { - // the same cost the shape reports, extrapolated beyond the measured widths - pays_at(estimate, sections, team_section_cost(team.dispatch_costs(), width)) + // the same cost the shape reports (threadteam's table, extrapolated by threadteam above + // its largest anchor) + pays_at(estimate, sections, team.dispatch_cost(width)) }, // a build without the route has no team token to ask #[cfg(not(feature = "threadteam"))] @@ -720,40 +721,15 @@ pub(crate) fn serial_time(count: usize, ns: f64) -> Duration { Duration::from_secs_f64(count as f64 * ns * 1e-9) } -/// The widest team the dispatch tables were measured at. `threadteam::DispatchCosts` holds the -/// anchors measured at widths 2, 4 and 6 and is flat above them (the Linux data at width 8 is below -/// the width-6 value, so flat is conservative that far); nothing was measured beyond 8. -pub(crate) const MEASURED_MAX_WIDTH: usize = 8; - -/// The dispatch cost of one section on a team token at `width`, beyond the measured range -/// *extrapolated*: from [`MEASURED_MAX_WIDTH`] on the cost grows linearly with the width, so a -/// region that does not pay at 8 members does not pay at 32 or 64 with the same work. Without it a -/// flat table lets a wide team open a region for a few microseconds of work and pick the full width -/// every time. -/// -/// Provisional until `tensor4all/faer#91`'s measurements (broccoli, widths 2 to 64) replace the -/// extrapolation with measured rows; `threadteam` itself is not changed. -#[cfg(feature = "threadteam")] -pub(crate) fn team_section_cost(costs: threadteam::DispatchCosts, width: NonZeroUsize) -> Duration { - let cost = costs.cost(threadteam::DispatchKind::Team, width); - let w = width.get(); - if w <= MEASURED_MAX_WIDTH { - cost - } else { - // at least linear in the width: never shallower than the table grew over the measured - // anchors, and a lower bound on what a wider team costs - cost.mul_f64(w as f64 / MEASURED_MAX_WIDTH as f64) - } -} - impl ParShape { /// The cost of one parallel section of `width` participants on this shape, in the steady /// state (sections back to back); [`Duration::MAX`] for a sequential shape or `width < 2`. /// `width` is clamped to the degree of the shape. /// /// `Team` is the cost of the team token the shape was taken from (threadteam's - /// `DispatchKind::Team`, with the costs of that token's pool), extrapolated beyond the widest - /// measured team ([`MEASURED_MAX_WIDTH`]). + /// `DispatchKind::Team`, with the costs of that token's pool). Above the largest measured + /// anchor of the row threadteam grows the cost linearly with the width (threadteam 0.1.1); + /// faer keeps no second model. pub fn dispatch_cost(self, width: NonZeroUsize) -> Duration { let width = width.min(NonZeroUsize::new(self.degree().max(1)).unwrap()); if width.get() < 2 { @@ -762,7 +738,7 @@ impl ParShape { match self { Self::Seq => Duration::MAX, #[cfg(feature = "threadteam")] - Self::Team(_, costs) => team_section_cost(costs, width), + Self::Team(_, costs) => costs.cost(threadteam::DispatchKind::Team, width), } } @@ -1004,27 +980,42 @@ mod tests { } } - /// Beyond the widest measured team the cost grows at least linearly with the width, so a region - /// that cannot pay at eight members cannot pay at 64 for the same work (#97 review, provisional - /// until #91 measures broccoli up to width 64). + /// The team cost is threadteam's table, nothing else: faer keeps no second extrapolation + /// (threadteam-rs#6). Above a row's largest anchor threadteam grows the cost in proportion to + /// the width, so a region that does not pay at eight members does not pay at 64. #[test] - fn widths_beyond_the_measured_range_cost_proportionally_more() { - use threadteam::{DispatchCosts, DispatchKind, WidthCosts}; + fn the_team_cost_is_threadteams_table_at_every_width() { + use threadteam::{DispatchCosts, DispatchKind, HostClass, WidthCosts}; let us = Duration::from_micros; - let costs = DispatchCosts::for_class(threadteam::HostClass::Generic) - .with(DispatchKind::Team, WidthCosts::flat(us(1))); - let s = ParShape::Team(nz(64), costs); - // the measured widths are unchanged - assert_eq!(s.dispatch_cost(nz(8)), us(1)); - assert_eq!(s.dispatch_cost(nz(4)), us(1)); - // beyond them: `cost(w_max) * w / w_max` - assert_eq!(s.dispatch_cost(nz(16)), us(2)); - assert_eq!(s.dispatch_cost(nz(32)), us(4)); - assert_eq!(s.dispatch_cost(nz(64)), us(8)); + let rows = [ + DispatchCosts::for_class(HostClass::Generic), + DispatchCosts::for_class(HostClass::X86_64Avx512), + DispatchCosts::for_class(HostClass::X86_64Avx2), + DispatchCosts::for_class(HostClass::Aarch64Apple), + DispatchCosts::for_class(HostClass::Generic).with(DispatchKind::Team, WidthCosts::flat(us(1))), + ]; + for costs in rows { + let s = ParShape::Team(nz(128), costs); + for w in 2..=128 { + assert_eq!(s.dispatch_cost(nz(w)), costs.cost(DispatchKind::Team, nz(w)), "{costs:?} width {w}"); + } + } + // above the largest anchor: `cost(w_m) * w / w_m` in integer nanoseconds. The flat row ends + // at 6, the avx512 team row at 8, the Zen2 avx2 row at 64. + let flat = ParShape::Team(nz(128), rows[4]); + for (w, ns) in [(8, 1333), (12, 2000), (16, 2666), (32, 5333), (64, 10666), (128, 21333)] { + assert_eq!(flat.dispatch_cost(nz(w)), Duration::from_nanos(ns), "flat width {w}"); + } + let avx512 = ParShape::Team(nz(128), rows[1]); + assert_eq!(avx512.dispatch_cost(nz(8)), Duration::from_nanos(980)); + assert_eq!(avx512.dispatch_cost(nz(64)), Duration::from_nanos(980 * 8)); + let zen2 = ParShape::Team(nz(128), rows[2]); + assert_eq!(zen2.dispatch_cost(nz(64)), Duration::from_nanos(9333)); + assert_eq!(zen2.dispatch_cost(nz(128)), Duration::from_nanos(18666)); // a region that does not pay at 8 members does not pay at 64 either - let at_8 = s.dispatch_cost(nz(8)) * 3; - assert!(s.pays(at_8, 1, nz(8))); - assert!(!s.pays(at_8, 1, nz(64))); + let at_8 = flat.dispatch_cost(nz(8)) * 3; + assert!(flat.pays(at_8, 1, nz(8))); + assert!(!flat.pays(at_8, 1, nz(64))); } #[test] diff --git a/faer/src/utils/team.rs b/faer/src/utils/team.rs index 6a93fd5b..5b74664b 100644 --- a/faer/src/utils/team.rs +++ b/faer/src/utils/team.rs @@ -385,20 +385,11 @@ pub(crate) mod kernel { stats::bump(&stats::GEMM_BELOW_FLOOR); return None; } - // the widest width that pays, on the cost the shapes read (`Exec::width_for` searches the - // raw table, which is flat beyond the measured widths and would always answer the full - // width); the cost never falls with the width, so the first width that does not pay ends - // the search + // the widest width that pays, on the cost the shapes read, searched from the full width + // down: a measured row can fall with the width (aarch64-apple's team row is 3.0, 2.3, + // 1.4 us at widths 2, 4, 6), so a narrow width that does not pay says nothing about a wide one let size = team.size().get(); - let mut best = None; - for w in 2..=size { - let width = core::num::NonZeroUsize::new(w).unwrap(); - if crate::utils::cost::pays_at(estimate, 2, crate::utils::cost::team_section_cost(team.dispatch_costs(), width)) { - best = Some(w); - } else { - break; - } - } + let best = (2..=size).rev().find(|&w| crate::utils::cost::pays_at(estimate, 2, team.dispatch_cost(core::num::NonZeroUsize::new(w).unwrap()))); match best { Some(width) => Some(width), None => { @@ -597,10 +588,7 @@ pub(crate) unsafe fn gemm_tiled( || !crate::utils::cost::pays_at( estimate, 1, - crate::utils::cost::team_section_cost( - team.dispatch_costs(), - core::num::NonZeroUsize::new(nt).unwrap(), - ), + team.dispatch_cost(core::num::NonZeroUsize::new(nt).unwrap()), ) { stats::bump(&stats::GEMM_BELOW_FLOOR); @@ -720,6 +708,7 @@ mod tests { /// for a table whose cost rises with the width: here 2 us at width 2 and 12 us at width 6, a /// 13 us product (two sections) pays at width 2 only, and a 11 us one nowhere. #[test] + #[cfg(all(target_arch = "x86_64", feature = "std"))] fn kernel_width_rejects_at_the_cheapest_width() { use threadteam::{DispatchCosts, DispatchKind, WidthCosts}; use core::time::Duration; @@ -746,6 +735,34 @@ mod tests { .expect("a team token"); } + /// A measured row can fall with the width (aarch64-apple's team row does): a 7 us product + /// (two sections) pays at width 6 (1 us) but not at width 2 (3 us), so the search must not stop + /// at the first width that does not pay. + #[test] + #[cfg(all(target_arch = "x86_64", feature = "std"))] + fn kernel_width_finds_the_widest_paying_width_on_a_falling_row() { + use threadteam::{DispatchCosts, DispatchKind, WidthCosts}; + use core::time::Duration; + let _turn = team_turn(); + let us = Duration::from_micros; + let costs = DispatchCosts::for_class(threadteam::HostClass::Generic).with(DispatchKind::Team, WidthCosts::new([us(3), us(2), us(1)])); + let team = Pool::builder() + .threads(6) + .acquire_timeout(std::time::Duration::from_secs(30)) + .dispatch_costs(costs) + .build() + .unwrap() + .into_team(6) + .expect("an idle pool"); + let mut exec = team.exec(6); + exec.team(|team| { + assert_eq!(kernel::kernel_width(&team, us(7)), Some(6)); + assert_eq!(kernel::kernel_width(&team, us(5)), None); + }) + .ok() + .expect("a team token"); + } + /// `par_pays_time` is `ParShape::pays` without the shape. #[test] fn par_pays_time_is_the_shape_gate() {