diff --git a/Cargo.lock b/Cargo.lock index dfe14f6..c325e89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -276,7 +276,7 @@ dependencies = [ [[package]] name = "threadteam" -version = "0.1.1" +version = "0.1.2" dependencies = [ "libc", "loom", diff --git a/Cargo.toml b/Cargo.toml index 4e48328..051ed78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "threadteam" -version = "0.1.1" +version = "0.1.2" edition = "2021" license = "MIT OR Apache-2.0" authors = ["Satoshi Terasaki", "Hiroshi Shinaoka"] diff --git a/README.md b/README.md index 0637c7d..d0e7dfe 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ the token to kernels by value. Kernels never store a token and never mint one. | `Pool::into_team(width)` -> `Team` (owns the pool; `IntoTeamError` gives it back), `Team::exec(budget)` / `try_exec`, `Team::pool`, `Team::into_pool` | persistent reservation for an engine that owns its pool (`Pool` is not `Clone`); minted tokens are ordinary `Exec`s whose `fork`, `for_ranges`, `broadcast`, `team` dispatch on the reserved members; one leader at a time (`exec` blocks, `try_exec` -> `Refused::Busy`); idle members park while reserved | | `PoolBuilder::qos(Qos)`, `raise_current_thread_qos(Qos)` | worker QoS on macOS (default `UserInitiated`); raise the leading thread without ever lowering it | | `PoolBuilder::idle_spin`, `idle_runs_rayon`, `acquire_timeout`; `Pool::release_idle` | waiting policy of idle team members | -| `DispatchCosts`, `WidthCosts`, `PoolBuilder::dispatch_costs`, `Exec::dispatch_cost(width)`, `TeamExec::dispatch_cost(width)` | cost of one parallel section per kind of token (team, in pool, outside), steady state; one row per `HostClass` (`HostClass::detect()`, keyed by kernel dispatch: `x86_64-avx512`, `x86_64-avx2`, `x86_64-scalar`, `aarch64-apple`, `aarch64`, `generic`), `DispatchCosts::for_class(class)`; measured rows for `x86_64-avx512` (team anchors at widths 2, 4, 6 and 8) and `aarch64-apple`, every other class takes the conservative `generic` row (3 / 15 / 40 us at the anchors 2, 4, 6); a row holds measured anchors (`WidthCosts::new`, `WidthCosts::flat`, or `WidthCosts::from_anchors` for measured anchors up to 64 and beyond), interpolates between them and grows linearly above the largest one, so a width above the anchors is never charged the cost of the largest anchor; `DispatchCosts::host_default()` is the detected class's row; `Duration::MAX` when serial; override per pool | +| `DispatchCosts`, `WidthCosts`, `PoolBuilder::dispatch_costs`, `Exec::dispatch_cost(width)`, `TeamExec::dispatch_cost(width)` | cost of one parallel section per kind of token (team, in pool, outside), steady state; one row per `HostClass` (`HostClass::detect()`, keyed by kernel dispatch: `x86_64-avx512`, `x86_64-avx2`, `x86_64-scalar`, `aarch64-apple`, `aarch64`, `generic`), `DispatchCosts::for_class(class)`; measured rows for `x86_64-avx512` (team anchors at widths 2, 4, 6 and 8) and `aarch64-apple`, a measured Zen2 team row for `x86_64-avx2` (anchors at ten widths from 2 to 64; its plain kinds stay generic), every other class takes the conservative `generic` row (3 / 15 / 40 us at the anchors 2, 4, 6); a row holds measured anchors (`WidthCosts::new`, `WidthCosts::flat`, or `WidthCosts::from_anchors` for measured anchors up to 64 and beyond), interpolates between them and grows linearly above the largest one, so a width above the anchors is never charged the cost of the largest anchor; `DispatchCosts::host_default()` is the detected class's row; `Duration::MAX` when serial; override per pool | | `pays_at(est, sections, cost)`, `Exec::pays(est, sections, width)`, `Exec::width_for(est, sections)`, `width_for_by`, `PAYS` | the one rule: parallel iff `est >= PAYS * sections * D(width)`; `width_for` minimises `sections * D(k) + est / k` | | `Exec::external(f)` | enter the pool once and hand the budget as a plain width to an outside library (faer, a vendor BLAS); how it divides the width is that library's concern | diff --git a/docs/design/placement.md b/docs/design/placement.md index b4a61aa..8194054 100644 --- a/docs/design/placement.md +++ b/docs/design/placement.md @@ -584,13 +584,15 @@ serial_estimate >= PAYS * sections * D(width) (PAYS = 3) machine takes, not by the CPU model: `x86_64-avx512` (runtime `avx512f`), `x86_64-avx2` (`avx2` + `fma`), `x86_64-scalar`, `aarch64-apple` (aarch64 on macOS), `aarch64`, `generic`; detected once per process. `x86_64-avx512` holds the measured Ryzen row (team 0.6/0.7/0.9 at widths 2/4/6 and the measured 0.98 at width 8, - in pool 3.0/3.6/3.5, outside 9.7/10.1/8.7 us), `aarch64-apple` the measured Apple M5 row at widths 2, 4 and 6, and - every class without a measurement the conservative generic row (3, 15, 40 microseconds at the anchors; the + in pool 3.0/3.6/3.5, outside 9.7/10.1/8.7 us), `aarch64-apple` the measured Apple M5 row at widths 2, 4 and 6, + `x86_64-avx2` a measured Zen2 team row at ten widths from 2 to 64 (1.42 to 9.33 us; in pool and outside generic; + thresholds section 3.7), and every class without a measurement the conservative generic row (3, 15, 40 microseconds at the anchors; the extrapolation charges 32, 160 and 427 microseconds at width 64). A consumer that measured its own host at more widths overrides the row with `WidthCosts::from_anchors` (up to `WidthCosts::MAX_ANCHORS` = 12 anchors). The rates of the consumer crates (strided-rs, tcontract-rs, faer) are - tables of their own, keyed by the same class names, and a class's rates and dispatch costs come from the same - machine in the same clock state: boost on/off moves both by about 1.66x and leaves the ratio stable within a + tables of their own, keyed by the same class names, and a measured class's rates and dispatch costs come from the + same machine in the same clock state (the exception is the `x86_64-avx2` team row, whose consumer rates are still + generic; thresholds section 3.7): boost on/off moves both by about 1.66x and leaves the ratio stable within a few percent (`faer-benchmark` `raw-exec3-boost`, `tables-*-exec3-boost.md`). Each crate carries an identical `HostClass` and `detect()`, kept in sync with `threadteam`'s by hand. `PoolBuilder::dispatch_costs` overrides it. diff --git a/docs/design/thresholds-data/raw/broccoli-dispatch-20261029.csv b/docs/design/thresholds-data/raw/broccoli-dispatch-20261029.csv new file mode 100644 index 0000000..bbeacb1 --- /dev/null +++ b/docs/design/thresholds-data/raw/broccoli-dispatch-20261029.csv @@ -0,0 +1,141 @@ +kind,width,ns +team/hot,2,1333.7 +team/gap1ms,2,1740.0 +team/gap10ms,2,2410.0 +team/gap300ms,2,10450.0 +team/load10us-minus-10us,2,1423.0 +team/fork-hot,2,1659.6 +team/hot,4,2111.0 +team/gap1ms,4,2490.0 +team/gap10ms,4,3050.0 +team/gap300ms,4,16680.0 +team/load10us-minus-10us,4,1991.6 +team/fork-hot,4,2258.6 +team/hot,6,2525.1 +team/gap1ms,6,2820.0 +team/gap10ms,6,3420.0 +team/gap300ms,6,21030.0 +team/load10us-minus-10us,6,2328.6 +team/fork-hot,6,2655.1 +team/hot,8,2950.1 +team/gap1ms,8,3250.0 +team/gap10ms,8,3890.0 +team/gap300ms,8,23771.0 +team/load10us-minus-10us,8,2737.7 +team/fork-hot,8,3277.8 +team/hot,12,3931.7 +team/gap1ms,12,3980.0 +team/gap10ms,12,4710.0 +team/gap300ms,12,31351.0 +team/load10us-minus-10us,12,3557.5 +team/fork-hot,12,4162.5 +team/hot,16,4581.1 +team/gap1ms,16,4710.0 +team/gap10ms,16,5460.0 +team/gap300ms,16,41570.0 +team/load10us-minus-10us,16,4076.0 +team/fork-hot,16,5161.3 +team/hot,24,4619.9 +team/gap1ms,24,4640.0 +team/gap10ms,24,5550.0 +team/gap300ms,24,58701.0 +team/load10us-minus-10us,24,4292.1 +team/fork-hot,24,5467.6 +team/hot,32,5044.2 +team/gap1ms,32,4750.0 +team/gap10ms,32,5700.0 +team/gap300ms,32,81231.0 +team/load10us-minus-10us,32,4837.8 +team/fork-hot,32,5679.8 +team/hot,48,6041.5 +team/gap1ms,48,5740.0 +team/gap10ms,48,6220.0 +team/gap300ms,48,107962.0 +team/load10us-minus-10us,48,6401.9 +team/fork-hot,48,6396.0 +team/hot,64,10334.5 +team/gap1ms,64,7670.0 +team/gap10ms,64,8620658.0 +team/gap300ms,64,143273.0 +team/load10us-minus-10us,64,9332.5 +team/fork-hot,64,8112.1 +plain/outside-hot,2,14398.6 +plain/outside-gap1ms,2,21711.0 +plain/outside-gap10ms,2,46121.0 +plain/outside-load10us-minus-10us,2,42975.1 +external/outside-hot,2,10932.7 +external/outside-gap1ms,2,20711.0 +plain/in-pool,2,816.3 +plain/in-pool-load10us-minus-10us,2,27529.1 +plain/outside-hot,4,21425.4 +plain/outside-gap1ms,4,24051.0 +plain/outside-gap10ms,4,49971.0 +plain/outside-load10us-minus-10us,4,70437.9 +external/outside-hot,4,10103.6 +external/outside-gap1ms,4,19851.0 +plain/in-pool,4,2761.1 +plain/in-pool-load10us-minus-10us,4,54263.4 +plain/outside-hot,6,28269.5 +plain/outside-gap1ms,6,26930.0 +plain/outside-gap10ms,6,54001.0 +plain/outside-load10us-minus-10us,6,83079.7 +external/outside-hot,6,12181.5 +external/outside-gap1ms,6,20200.0 +plain/in-pool,6,5834.0 +plain/in-pool-load10us-minus-10us,6,72908.3 +plain/outside-hot,8,32576.6 +plain/outside-gap1ms,8,71521.0 +plain/outside-gap10ms,8,59012.0 +plain/outside-load10us-minus-10us,8,76791.7 +external/outside-hot,8,9062.7 +external/outside-gap1ms,8,19560.0 +plain/in-pool,8,9396.2 +plain/in-pool-load10us-minus-10us,8,67186.3 +plain/outside-hot,12,39899.4 +plain/outside-gap1ms,12,74381.0 +plain/outside-gap10ms,12,275665.0 +plain/outside-load10us-minus-10us,12,96968.4 +external/outside-hot,12,9433.1 +external/outside-gap1ms,12,21260.0 +plain/in-pool,12,16428.7 +plain/in-pool-load10us-minus-10us,12,84907.8 +plain/outside-hot,16,45699.4 +plain/outside-gap1ms,16,146703.0 +plain/outside-gap10ms,16,75732.0 +plain/outside-load10us-minus-10us,16,91376.0 +external/outside-hot,16,12133.7 +external/outside-gap1ms,16,18720.0 +plain/in-pool,16,23073.7 +plain/in-pool-load10us-minus-10us,16,77622.0 +plain/outside-hot,24,56698.6 +plain/outside-gap1ms,24,146253.0 +plain/outside-gap10ms,24,395218.0 +plain/outside-load10us-minus-10us,24,100128.0 +external/outside-hot,24,11789.4 +external/outside-gap1ms,24,20230.0 +plain/in-pool,24,36303.3 +plain/in-pool-load10us-minus-10us,24,90913.5 +plain/outside-hot,32,63561.6 +plain/outside-gap1ms,32,266016.0 +plain/outside-gap10ms,32,386408.0 +plain/outside-load10us-minus-10us,32,83287.2 +external/outside-hot,32,8614.3 +external/outside-gap1ms,32,21001.0 +plain/in-pool,32,42314.6 +plain/in-pool-load10us-minus-10us,32,82865.7 +plain/outside-hot,48,75502.5 +plain/outside-gap1ms,48,268636.0 +plain/outside-gap10ms,48,396378.0 +plain/outside-load10us-minus-10us,48,95110.1 +external/outside-hot,48,10625.6 +external/outside-gap1ms,48,20660.0 +plain/in-pool,48,54374.3 +plain/in-pool-load10us-minus-10us,48,96198.8 +plain/outside-hot,64,76841.3 +plain/outside-gap1ms,64,323237.0 +plain/outside-gap10ms,64,390518.0 +plain/outside-load10us-minus-10us,64,99850.3 +external/outside-hot,64,9728.1 +external/outside-gap1ms,64,20520.0 +plain/in-pool,64,58670.7 +plain/in-pool-load10us-minus-10us,64,93638.2 diff --git a/docs/design/thresholds.md b/docs/design/thresholds.md index 83ccd8f..6d0c551 100644 --- a/docs/design/thresholds.md +++ b/docs/design/thresholds.md @@ -326,7 +326,8 @@ and `from_anchors` accepts measured anchors up to `u16::MAX`, so a row measured extrapolation where it has data. Above the last anchor the cost is never flat: a wide section is charged at least in proportion to its width, so `width_for` no longer answers the full width just because the largest anchor pays (a flat 1 us row gives 1, 2, 4 and 8 us at widths 6, 12, 24 and 48; a 24 us region on such a row selects width 12 of a -budget 64). Section 3.6 keeps the historical reason for the earlier flat model. A host that is not measured gets the +budget 64). Section 3.6 keeps the historical reason for the earlier flat model; section 3.7 is the `x86_64-avx2` team +row with anchors to width 64. A host that is not measured gets the flat generic row (team 3, in pool 15, outside 40 microseconds at the anchors 2, 4 and 6, and the width proportion above), which errs towards staying serial. @@ -548,7 +549,7 @@ there: width 8 is 4.67 us in the pool and 11.6 us from outside. | plain in pool | 3.0 | 3.6 | 3.5 | | | plain outside | 9.7 | 10.1 | 8.7 | | -The row is `DispatchCosts::for_class(HostClass::X86_64Avx512)` in `cost.rs`, and `host_default()` returns it on a host whose runtime detection finds `avx512f` (the other x86_64 classes, `x86_64-avx2` and `x86_64-scalar`, take the generic row until measured). The row and the rates of the consumer crates for the same class come from this machine at 2.0 GHz (boost off); boost on moves both by about 1.66x with a stable ratio (`raw-exec3-boost`, `tables-*-exec3-boost.md`): +The row is `DispatchCosts::for_class(HostClass::X86_64Avx512)` in `cost.rs`, and `host_default()` returns it on a host whose runtime detection finds `avx512f` (`x86_64-avx2` has its own team row since section 3.7; `x86_64-scalar` takes the generic row until measured). The row and the rates of the consumer crates for the same class come from this machine at 2.0 GHz (boost off); boost on moves both by about 1.66x with a stable ratio (`raw-exec3-boost`, `tables-*-exec3-boost.md`): ```rust HostClass::X86_64Avx512 => DispatchCosts { @@ -609,6 +610,43 @@ needs the ledger and `Participant::size` to exclude the empty shares to keep "a than k tasks of real work". the `x86_64-avx512` row now holds the post-fix row (0.6, 0.7, 0.9), re-measured by the Linux campaign. +### 3.7 Zen2 team row (x86_64-avx2, threadteam-rs#6) + +`for_class(HostClass::X86_64Avx2)` carries a measured team row with ten anchors. Host `broccoli`: AMD EPYC 7702P +(Zen2, 64 physical cores, SMT off, NUMA 1, 16 MiB L3 per 4-core CCX), `cpufreq/boost = 0`, governor `performance`, +nothing else running. The Linux port of this crate's `benches/dispatch_cost.rs` that takes the pool size and the +widths from the environment (`faer-benchmark` branch `bench/awake-team-linux`, +`data/probes/20261026-awake-team-linux/bench/tt/src/bin/dispatch_cost.rs`, unchanged; the checked-in bench here +does not read these variables) built against threadteam 0.1.0, `TT_POOL=64`, `TT_WIDTHS=2,4,6,8,12,16,24,32,48,64`, +process pinned to CPUs 0-63, 2026-10-29. The row is +`team/load10us-minus-10us`, the same quantity as the other rows (back to back sections whose shares carry 10 us of +work, minus the 10 us), rounded to the nearest nanosecond. Raw output: `thresholds-data/raw/broccoli-dispatch-20261029.csv`. + +| width | 2 | 4 | 6 | 8 | 12 | 16 | 24 | 32 | 48 | 64 | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| team, us | 1.42 | 1.99 | 2.33 | 2.74 | 3.56 | 4.08 | 4.29 | 4.84 | 6.40 | 9.33 | + +The cost grows more slowly than the width between 6 and 64 (2.33 to 9.33 us for a width ratio of 10.7). Widths +between the ten anchors interpolate, so a host of up to 64 cores never reaches the extrapolation; above 64 the +linear extrapolation is the chosen conservative policy, not a measured bound. + +What the row does not claim: + +* **Mixed provenance.** For `x86_64-avx512` and `aarch64-apple` the dispatch row and the consumer crates' rates of + the class come from one machine in one clock state. The consumer crates (faer, tcontract-rs, strided-rs) still + hold their generic rates for `x86_64-avx2`, so on this class the gate compares a measured dispatch cost with a + conservative rate. It is a heuristic exception, recorded here, until those rates are measured on the same host. +* **One class, many hosts.** `x86_64-avx2` covers every AVX2 + FMA host, small desktops included. The width 2 team + floor moves from the generic 9 us (`PAYS * 3 us`) to 4.3 us. The row is a representative default for the class, not + a guarantee for a given topology; a pool owner with its own measurement overrides it with + `PoolBuilder::dispatch_costs`. +* **Plain kinds unchanged.** In pool and outside deliberately keep the generic row (15 and 40 us at the anchors). The + run has plain-token numbers too (in the CSV), but they are a control the measurement was not designed for and are + not promoted to the row. +* **Fresh team entry.** tensor4all/faer#91 saw, on the same host, an intermittent section of about 0.62 ms when a + caller enters a fresh `Team::exec(w).team(...)` per call; the anchor instrument reuses one token and never shows it. + Its cause is not attributed. The row is the steady-state cost of a reused token and does not include it. + ## 4. Limits of this data * One host, rough, 6 threads only (widths 2 and 4 only for the dispatch cost), one process per mode diff --git a/src/cost.rs b/src/cost.rs index 0b7ab23..844e7de 100644 --- a/src/cost.rs +++ b/src/cost.rs @@ -328,7 +328,8 @@ impl HostClass { /// /// [`for_class`](DispatchCosts::for_class) holds measured rows for the host /// classes it knows (`x86_64-avx512`; `aarch64-apple`, QoS `UserInitiated`, the -/// pool default) and a conservative generic row for every other class; +/// pool default; the team kind of `x86_64-avx2`) and a conservative generic row +/// for everything else; /// [`host_default`](DispatchCosts::host_default) picks the row of the detected /// class. A pool owner that measured its own host overrides them with /// [`PoolBuilder::dispatch_costs`](crate::PoolBuilder::dispatch_costs). @@ -364,6 +365,39 @@ const fn us_f(x: f64) -> Duration { Duration::from_nanos((x * 1000.0) as u64) } +/// A measured row built at compile time: an invalid anchor list fails the +/// build, never a caller. +const fn anchored(anchors: &[(usize, Duration)]) -> WidthCosts { + match WidthCosts::from_anchors(anchors) { + Some(costs) => costs, + None => panic!("invalid measured anchors"), + } +} + +/// `x86_64-avx512` team row (post-fix run, with its width 8 anchor). +const AVX512_TEAM: WidthCosts = anchored(&[ + (2, us_f(0.6)), + (4, us_f(0.7)), + (6, us_f(0.9)), + (8, us_f(0.98)), +]); + +/// `x86_64-avx2` team row: broccoli (AMD EPYC 7702P, Zen2), pool 64, +/// `team/load10us-minus-10us`, nanoseconds rounded to the nearest +/// (`docs/design/thresholds-data/raw/broccoli-dispatch-20261029.csv`). +const AVX2_TEAM: WidthCosts = anchored(&[ + (2, Duration::from_nanos(1423)), + (4, Duration::from_nanos(1992)), + (6, Duration::from_nanos(2329)), + (8, Duration::from_nanos(2738)), + (12, Duration::from_nanos(3558)), + (16, Duration::from_nanos(4076)), + (24, Duration::from_nanos(4292)), + (32, Duration::from_nanos(4838)), + (48, Duration::from_nanos(6402)), + (64, Duration::from_nanos(9333)), +]); + impl DispatchCosts { /// Costs of the three kinds. pub fn new(team: WidthCosts, in_pool: WidthCosts, outside: WidthCosts) -> Self { @@ -374,9 +408,12 @@ impl DispatchCosts { } } - /// The row of a host class. Both the dispatch costs of a row and the rates - /// of the consumer crates' tables for the same class come from one machine - /// in one clock state, so a class is never mixed across crates' tables. + /// The row of a host class. For `x86_64-avx512` and `aarch64-apple` the + /// dispatch costs and the rates of the consumer crates' tables for the same + /// class come from one machine in one clock state. The `x86_64-avx2` team + /// row is the exception: it is measured (on a Zen2 host, below) while the + /// consumer crates' rates for that class are still their generic rows, so + /// the gate compares a measured dispatch cost with a conservative rate. /// /// * [`HostClass::X86_64Avx512`]: measured on an AMD Ryzen AI 9 HX 470 /// under Linux, 8 threads on cores 4-11, boost off (2.0 GHz), from the @@ -391,6 +428,18 @@ impl DispatchCosts { /// QoS `UserInitiated`, from the back to back sections whose shares /// carry 10 us of work (`docs/design/thresholds-data/`), at widths 2, 4, /// 6: team 3.0, 2.3, 1.4; in pool 6.0, 11, 5.5; outside 14, 18, 19. + /// * [`HostClass::X86_64Avx2`]: the team row measured on `broccoli`, an + /// AMD EPYC 7702P (Zen2, 64 cores, SMT off), boost off, governor + /// `performance`, pool 64 pinned to CPUs 0-63, with the Linux port of + /// `benches/dispatch_cost.rs` in `faer-benchmark` (back to back sections + /// whose shares carry 10 us of work, threadteam 0.1.0; + /// tensor4all/threadteam-rs#6, `docs/design/thresholds.md` section 3.7), in microseconds at widths 2, 4, 6, 8, + /// 12, 16, 24, 32, 48, 64: 1.42, 1.99, 2.33, 2.74, 3.56, 4.08, 4.29, + /// 4.84, 6.40, 9.33. In pool and outside deliberately keep the generic + /// row. The class covers every AVX2 host, also small desktops, so the + /// team row is a representative default for the class, not a guarantee + /// for a given topology; a pool owner overrides it with + /// [`PoolBuilder::dispatch_costs`](crate::PoolBuilder::dispatch_costs). /// * Every other class has no measurement and takes the conservative /// generic row: 3 us for a team, 15 us in the pool and 40 us from /// outside at the anchors 2, 4 and 6, and above width 6 the width @@ -403,13 +452,7 @@ impl DispatchCosts { pub fn for_class(class: HostClass) -> Self { match class { HostClass::X86_64Avx512 => DispatchCosts { - team: WidthCosts::from_anchors(&[ - (2, us_f(0.6)), - (4, us_f(0.7)), - (6, us_f(0.9)), - (8, us_f(0.98)), - ]) - .expect("the avx512 team row has four valid anchors"), + team: AVX512_TEAM, in_pool: WidthCosts::new([us_f(3.0), us_f(3.6), us_f(3.5)]), outside: WidthCosts::new([us_f(9.7), us_f(10.1), us_f(8.7)]), }, @@ -418,10 +461,12 @@ impl DispatchCosts { in_pool: WidthCosts::new([us_f(6.0), us_f(11.0), us_f(5.5)]), outside: WidthCosts::new([us_f(14.0), us_f(18.0), us_f(19.0)]), }, - HostClass::X86_64Avx2 - | HostClass::X86_64Scalar - | HostClass::Aarch64 - | HostClass::Generic => DispatchCosts { + HostClass::X86_64Avx2 => DispatchCosts { + team: AVX2_TEAM, + in_pool: WidthCosts::flat(us_f(15.0)), + outside: WidthCosts::flat(us_f(40.0)), + }, + HostClass::X86_64Scalar | HostClass::Aarch64 | HostClass::Generic => DispatchCosts { team: WidthCosts::flat(us_f(3.0)), in_pool: WidthCosts::flat(us_f(15.0)), outside: WidthCosts::flat(us_f(40.0)), diff --git a/tests/dispatch_cost.rs b/tests/dispatch_cost.rs index a752bbb..05d5a14 100644 --- a/tests/dispatch_cost.rs +++ b/tests/dispatch_cost.rs @@ -221,7 +221,6 @@ fn class_rows_and_host_default() { // Classes without a measurement take the generic row: 3, 15, 40 us at the // anchors 2, 4, 6 and proportional to the width above them. for class in [ - HostClass::X86_64Avx2, HostClass::X86_64Scalar, HostClass::Aarch64, HostClass::Generic, @@ -280,3 +279,58 @@ fn class_names() { assert_eq!(HostClass::Aarch64.name(), "aarch64"); assert_eq!(HostClass::Generic.name(), "generic"); } + +#[test] +fn avx2_class_has_the_measured_zen2_team_row() { + // threadteam-rs#6: broccoli (EPYC 7702P, Zen2), pool 64, + // team/load10us-minus-10us rounded to the nanosecond. + let a = DispatchCosts::for_class(HostClass::X86_64Avx2); + let g = DispatchCosts::for_class(HostClass::Generic); + assert_ne!(a, g); + for (w, ns) in [ + (2, 1423), + (4, 1992), + (6, 2329), + (8, 2738), + (12, 3558), + (16, 4076), + (24, 4292), + (32, 4838), + (48, 6402), + (64, 9333), + ] { + assert_eq!( + a.cost(DispatchKind::Team, nz(w)), + Duration::from_nanos(ns), + "team width {w}" + ); + } + // Between anchors: linear (width 10 is halfway between 8 and 12). + assert_eq!( + a.cost(DispatchKind::Team, nz(10)), + Duration::from_nanos(3148) + ); + // Above the last anchor: in proportion to the width. + assert_eq!( + a.cost(DispatchKind::Team, nz(128)), + Duration::from_nanos(18666) + ); + // The plain kinds stay the generic row, also above its anchors. + for w in [2, 4, 6, 8, 64] { + for kind in [DispatchKind::InPool, DispatchKind::Outside] { + assert_eq!( + a.cost(kind, nz(w)), + g.cost(kind, nz(w)), + "{kind:?} width {w}" + ); + } + } + assert_eq!( + a.cost(DispatchKind::InPool, nz(64)), + Duration::from_nanos(160_000) + ); + assert_eq!( + a.cost(DispatchKind::Outside, nz(64)), + Duration::from_nanos(426_666) + ); +}