diff --git a/Cargo.toml b/Cargo.toml index 1b21dc3..ac4359a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,13 +13,9 @@ categories = ["concurrency", "science"] [features] default = ["parallel"] -# Run entries on the pool's own worker threads (docs/design/own-threads.md). -# Without it the same types exist, every pool has width 1 and every entry runs -# inline on the calling thread. -parallel = [] -# Opt-in Rayon interop: `Exec::external_rayon` runs a callback inside a private -# Rayon pool of the token's width. Not needed by anything else. -rayon = ["parallel", "dep:rayon"] +# Run entries on a private Rayon pool. Without it the same types exist, every +# pool has width 1 and every entry runs inline on the calling thread. +parallel = ["dep:rayon"] # Test support: per-root live-task ledger and entry counters. Zero cost when off. ledger = [] # Test support: hooks that pause threads inside the park/wake handshake and @@ -49,8 +45,6 @@ libc = "0.2" [dev-dependencies] libc = "0.2" -# The Rayon reference rows of benches/wait_scenarios.rs. -rayon = "1.10" [[bench]] name = "team_dispatch" diff --git a/README.md b/README.md index 9069914..c732c9a 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,12 @@ type appears in any public signature. Used by strided-rs, tcontract-rs, cpueinsum-rs, tlinalg-rs and tenferro-rs. Design records: [`docs/design/placement.md`](docs/design/placement.md) (the token), -[`docs/design/own-threads.md`](docs/design/own-threads.md) (the worker threads), -[`docs/design/waiting.md`](docs/design/waiting.md) (how they wait). +[`docs/design/waiting.md`](docs/design/waiting.md) (how the workers wait). -The workers are threadteam's own `std::thread`s. A pool leases them from a -process-wide reserve and gives them back when dropped, so a pool built after another -one was dropped reuses running threads (faer's `with_team` builds one per call); -a pool with a thread name, a start hook, a stack size or a QoS of its own gets -dedicated workers that its drop joins. Rayon is not used unless the opt-in `rayon` -feature asks for `Exec::external_rayon`. +The workers belong to a private Rayon pool per `Pool`. A pool with the default +thread settings keeps its Rayon pool in a process-wide cache when it is +dropped, and the next pool of the same width takes it, so a pool built per call +(faer's `with_team`) reuses running threads instead of starting new ones. ## Usage @@ -67,9 +64,9 @@ the token to kernels by value. Kernels never store a token and never mint one. | `Exec::team(f)` / `team_with(over, f)` | reserve `budget - 1` awake pool members for a scope and run barrier-bearing sections through `TeamExec` (`run`, `run_with`, `split`, `fork`, `reborrow`), or `Err(Refused::Unavailable)`; see the design record | | `Pool::into_team(width)` -> `Team` (owns the pool; `IntoTeamError` gives it back), `Team::exec(budget)` / `try_exec`, `Team::pool`, `Team::into_pool` | persistent reservation for an engine that owns its pool (`Pool` is not `Clone`); minted tokens are ordinary `Exec`s whose `fork`, `for_ranges`, `broadcast`, `team` dispatch on the reserved members; one leader at a time (`exec` blocks, `try_exec` -> `Refused::Busy`); idle members park while reserved | | `PoolBuilder::qos(Qos)`, `raise_current_thread_qos(Qos)` | worker QoS on macOS (default `UserInitiated`); raise the leading thread without ever lowering it | -| `PoolBuilder::idle_spin`, `idle_runs_rayon`, `acquire_timeout`; `Pool::release_idle` | waiting policy of idle team members (an idle member spins for the spin budget unless `idle_spin` is zero, then sleeps; `idle_runs_rayon` has no effect and is kept for compatibility) | +| `PoolBuilder::idle_spin`, `idle_runs_rayon`, `acquire_timeout`; `Pool::release_idle` | waiting policy of idle team members (an idle member returns to Rayon after `idle_spin` or the spin budget, whichever comes first) | | `PoolBuilder::spin_polls`, `Pool::spin_polls` | spin budget of every team wait, in polls (the shape of libgomp's `GOMP_SPINCOUNT`): a reserved member waiting for work, the leader in a join and a participant in a barrier spin that many polls, yield a few times, then sleep until woken; throttled to 100 polls in an oversubscribed scope or after a waiter was descheduled ([`docs/design/waiting.md`](docs/design/waiting.md)) | -| `PoolBuilder::pin_participants(bool)`, `Pool::pin_participants()` | opt-in placement of the **members** of a root team scope (Linux; off by default, which is consistent with the rule it protects): member `k` (participant `k >= 1`) pins itself onto the `k`-th CPU of the calling thread's affinity set, `S[0]` is left to the leader, and a member restores its own mask before it serves the pool's queue, leaves the pool, or serves foreign work. **The leader is never pinned**, because a vendor library creates its thread team from the calling thread and a one-CPU caller would confine the whole provider team; a member that calls such a library directly inside a share does confine it, so use `Exec::external` (which restores the member's mask first) or leave pinning off. For tensor4all/faer#107; give each team its own CPU set (`taskset -c`), there is no CPU allocator | +| `PoolBuilder::pin_participants(bool)`, `Pool::pin_participants()` | opt-in placement of the **members** of a root team scope (Linux; off by default, which is consistent with the rule it protects): member `k` (participant `k >= 1`) pins itself onto the `k`-th CPU of the calling thread's affinity set, `S[0]` is left to the leader, and a member restores its own mask before it returns to Rayon or serves foreign work. **The leader is never pinned**, because a vendor library creates its thread team from the calling thread and a one-CPU caller would confine the whole provider team; a member that calls such a library directly inside a share does confine it, so use `Exec::external` (which restores the member's mask first) or leave pinning off. For tensor4all/faer#107; give each team its own CPU set (`taskset -c`), there is no CPU allocator | | `DispatchCosts`, `WidthCosts`, `PoolBuilder::dispatch_costs`, `Exec::dispatch_cost(width)`, `TeamExec::dispatch_cost(width)` | cost of one parallel section per kind of token (team, in pool, outside), steady state; one row per `HostClass` (`HostClass::detect()`, keyed by kernel dispatch: `x86_64-avx512`, `x86_64-avx2`, `x86_64-scalar`, `aarch64-apple`, `aarch64`, `generic`), `DispatchCosts::for_class(class)`; measured rows for `x86_64-avx512` (team anchors at widths 2, 4, 6 and 8) and `aarch64-apple`, a measured Zen2 team row for `x86_64-avx2` (anchors at ten widths from 2 to 64; its plain kinds stay generic), every other class takes the conservative `generic` row (3 / 15 / 40 us at the anchors 2, 4, 6); a row holds measured anchors (`WidthCosts::new`, `WidthCosts::flat`, or `WidthCosts::from_anchors` for measured anchors up to 64 and beyond), interpolates between them and grows linearly above the largest one, so a width above the anchors is never charged the cost of the largest anchor; `DispatchCosts::host_default()` is the detected class's row; `Duration::MAX` when serial; override per pool | | `pays_at(est, sections, cost)`, `Exec::pays(est, sections, width)`, `Exec::width_for(est, sections)`, `width_for_by`, `PAYS` | the one rule: parallel iff `est >= PAYS * sections * D(width)`; `width_for` minimises `sections * D(k) + est / k` | | `Exec::external(f)` | enter the pool once and hand the budget as a plain width to an outside library (faer, a vendor BLAS); how it divides the width is that library's concern | @@ -82,13 +79,11 @@ the token to kernels by value. Kernels never store a token and never mint one. once is at most its budget at every nesting depth. A parent waiting for its children keeps no slot. * **Lazy and inline.** A serial token, budget one, or one task runs inline on the - calling thread and makes no scheduler call; no worker starts before a pool is - built, and Rayon's process-global pool is never started. -* **Placement by caller.** A worker of the target pool runs its own batch's - tasks itself and serves the pool while it waits; a worker of another pool - publishes on the target pool and serves its own pool while it waits (so - chains between pools always make progress); a plain thread publishes and - sleeps until its tasks finished. No thread-local state exists in the crate. + calling thread and makes no scheduler call, so the process-global Rayon pool is + never started implicitly. +* **Placement by caller.** A worker of the target pool spawns into the current + scope without installing; a plain thread or a worker of another pool enters the + target pool once and waits. No thread-local state exists in the crate. * **`broadcast`** refuses with `Refused::ForkChild` for a token derived from `fork` (through `reborrow` or `limit` too), and with `Refused::CallerInPool` when the caller is a worker of the same pool. Roots, and their `reborrow`s and @@ -98,14 +93,19 @@ the token to kernels by value. Kernels never store a token and never mint one. blocking work of another token on those workers delays the broadcast. `can_broadcast` applies the same rule through one private helper, so the two cannot disagree. -* **Shutdown.** Dropping a `Pool` detaches every worker and waits until each - one acknowledged; after that no worker touches the pool. Dedicated workers (a - pool with a thread name, an `on_start` hook, a stack size or a QoS of its own) - are then joined: their thread-local destructors, including state created in - an `on_start` hook, have run when `drop` returns. Shared workers go back to the - process-wide reserve, asleep, for the next pool. A pool dropped from one of its - own workers does not wait for or join that worker, which acknowledges and - leaves after its task returns. Without `parallel` there is nothing to join. +* **Shutdown.** Dropping a `Pool` ends its team members and waits until they + left. A pool with a `thread_name`, an `on_start` hook, a `stack_size` or a + `qos` of its own has dedicated workers, and its drop joins them: when `drop` + returns, all of them have exited and their thread-local destructors + (including state created in an `on_start` hook) have run. A pool with the + default thread settings keeps its idle workers in a process-wide cache keyed + by width. The next default pool of that width takes them and gives each + worker the building thread's CPU set. A cached set belongs to one pool at a + time, and the cache holds at most as many threads as the larger of the + available parallelism and the widest pool built so far; it joins the oldest + entries beyond that. A pool dropped from one of its own workers cannot wait + for itself, so that drop only signals shutdown and returns without joining or + caching. Without `parallel` there is nothing to join. * **No scheduler type is public.** There is no constructor from, and no accessor returning, a scheduler pool. `scripts/check-no-public-rayon.sh` runs `cargo public-api -p threadteam` (when installed) and fails if `rayon::` @@ -115,39 +115,33 @@ the token to kernels by value. Kernels never store a token and never mint one. | Feature | Effect | | --- | --- | -| `parallel` (default) | run on the pool's own worker threads. Without it the same types exist, `PoolBuilder::build` returns a width-1 pool and every entry runs inline | -| `rayon` | opt-in Rayon interop: `Exec::external_rayon(f)` runs `f(width)` inside a private Rayon pool of the pool's width (separate threads, built on first use) | +| `parallel` (default) | run on a private pool. Without it the same types exist, `PoolBuilder::build` returns a width-1 pool and every entry runs inline | | `ledger` | test support: `ledger::max_live(root)`, `live`, `installs`, and `Exec::ledger_root`. Records live tasks per root token and pool installs; compiled out when off | Consumers assert `ledger::max_live(root) <= budget` in their own tests, and keep -a test binary that runs serial and pooled work and then checks -`ledger::global_pool_unstarted()` (with the `rayon` feature it checks that Rayon's -process-global pool was never started). +a test binary that runs serial and pooled work and then checks that +`rayon::ThreadPoolBuilder::new().build_global()` still succeeds. ## Upgrading from 0.1 -0.2 runs entries on threadteam's own worker threads instead of a private Rayon -pool (`docs/design/own-threads.md`). The signatures are those of 0.1 plus -`PoolBuilder::spin_polls` / `Pool::spin_polls`; the behaviour changes are: - -* **Pool drop.** A pool with default thread settings leases *shared* workers - from a process-wide reserve; dropping it waits until every worker detached - and returns them to the reserve, asleep, instead of joining them. A pool with - a `thread_name`, an `on_start` hook, a stack size or a QoS of its own still - gets dedicated workers that its drop joins (their thread-local destructors - have run when `drop` returns). -* **`external` on a team token** lends the members to the pool's own queue. It - no longer runs the callback inside a Rayon pool: Rayon calls in the callback - go to Rayon's global pool. Code that relied on the old behaviour enables the - `rayon` feature and calls `Exec::external_rayon`. -* **`parallel` pulls no Rayon.** Rayon is a dependency only with the `rayon` - feature. -* **`idle_runs_rayon`** has no effect: an idle worker always serves the pool's - queue. It is kept so that callers compile. +0.2 still runs entries on a private Rayon pool. The signatures are those of +0.1 plus `PoolBuilder::spin_polls` / `Pool::spin_polls`; the behaviour changes +are: + +* **Pool drop and reuse.** A pool with the default thread settings no longer + joins its workers on drop: after its members left, its Rayon pool goes to a + process-wide cache, and the next default pool of the same width reuses those + threads (each worker takes the building thread's CPU set). A pool with a + `thread_name`, an `on_start` hook, a `stack_size` or a `qos` of its own still + gets dedicated workers that its drop joins. * **Waits** follow libgomp (`docs/design/waiting.md`): a bounded spin of `spin_polls` polls (by default 300000 on x86-64, 2^21 on aarch64), 16 - yields, then sleep. `idle_spin`, `release_idle` and `resident_members` keep - their meaning on the native loop. + yields, then sleep. A reserved member that sleeps stays reserved; an idle + member returns to Rayon after `idle_spin` or the spin budget, whichever + comes first. +* **Unchanged:** `idle_runs_rayon` keeps its meaning (an idle member serves + Rayon jobs while it spins), `external` on a team token still lends the + members to the pool's Rayon jobs, and `parallel` still pulls Rayon. ## Wait benchmarks diff --git a/docs/design/own-threads.md b/docs/design/own-threads.md deleted file mode 100644 index 897c75c..0000000 --- a/docs/design/own-threads.md +++ /dev/null @@ -1,162 +0,0 @@ -# Own worker threads (threadteam-rs#15) - -threadteam stops running on a private Rayon `ThreadPool`. Its workers are `std::thread`s kept in a process-wide **reserve**; a `Pool` leases a fixed number of them and returns them when it is dropped. The waits stay libgomp's (`waiting.md`). Rayon becomes an opt-in interop layer. - -Design inputs: -- pi-council, plan mode (GPT-6-astra chair, DeepSeek critic, GLM minimalist). The brief and the synthesis are in the shinaoka-advisor run directory `work/runs/threadteam-own/`. -- The runtime sources read for it, without copying code: libgomp, LLVM libomp, oneTBB. -- The measurements of threadteam-rs#14, which stays a draft as their record. - -## Why: the one-call pool - -faer's `with_team` builds a `Pool`, and with it 64 fresh threads, per call. On a 64-CPU set under foreign load those threads start stacked: the scheduler places a new thread before it knows its load. The team's spinning waiters then never leave a CPU idle that could pull one over, and every 20 µs section of the call waits for a descheduled member (faer#107). - -The structural difference is the spawn, not the waiting. In one broccoli run (faer#107, 13:10Z, 0.1.3, 64 threads, `gemm128`), the same acquisition path at the same width gave: - -| arm | what happens per call | median call | -| --- | --- | --- | -| `into_team_per_call` | the pool is held; members are acquired and released per call | 0.03 ms | -| `fresh_pool` | a new pool, so 64 new threads, per call | 15.99 ms | - -On #14, no waiting policy fixed the fresh pool and kept held teams at libgomp's speed at the same time: -- the pool-wide throttle fixed (a) but cost (c); -- libgomp's waits met (b) and (c) but left the fresh pool at 11 ms; -- a wall-clock window could not tell the two apart. - -libgomp creates its threads once per process (per master thread) and reuses them for every region, docked on a futex between regions. TBB's RML server owns one process-wide set of workers that every arena reuses, each sleeping on its own monitor when idle. - -The answer to question (i) of #15 is therefore (ii): if no `Pool` spawns threads, there is no fresh-thread stacking to detect. - -**Caveat.** The first pool of a process still spawns its workers. The acceptance criterion (a) judges a process by its median call over 50, so one cold first call does not make it slow, and the benchmarks report the cold first call separately. - -## Decisions - -### Reserve and lease -- **One process-wide reserve: a list of idle workers.** It is created by the first explicit `Pool::build` that wants shared workers. Serial, budget-one and one-task entries never reach it, and no API submits work to it without a pool, so "no implicit global pool" holds. -- **Compatibility.** A worker of the reserve was spawned with the default stack (16 MiB), the platform's default QoS and no hooks. A pool with any `thread_name`, `on_start`, a different stack size or a QoS gets **dedicated** workers: spawned for it and joined when it is dropped, exactly as before. tcontract's C API names its workers and sets a thread-local in `on_start`; it keeps today's semantics. -- **`Pool::build`.** It takes `width` idle workers from the reserve and spawns the missing ones; a spawn failure is a build error. A worker learns its pool and its index when leased. -- **`Pool` drop (shared workers).** Every worker is told to detach. Each one finishes, restores its own affinity mask (a pinned member) and acknowledges. Only then is the worker back in the reserve. The drop waits for all acknowledgements (quiescence) and does not join. - - The reserve keeps at most `C` idle workers (pre-review resolution 6 sets `C`); a worker beyond the cap exits. - - A pool dropped from one of its own workers (a worker running a task that owns the pool) cannot wait for itself: it detaches the others, and that worker detaches when its task returns. -- **Contract change.** "Dropping a `Pool` joins every worker thread" now holds for dedicated workers only; a shared worker outlives the pool, parked in the reserve. - -### The worker loop -Each leased worker runs one native loop over its member slot, the slot of the existing team machinery (`awake.rs`). The slot's state word and its transitions are pre-review resolution 3 (`RESERVED`, `LENT`, `BUSY`, `DETACH`; free is `0`). - -Work, in the order the loop looks for it: -1. A **broadcast share** for this worker in the pool's broadcast list (pre-review resolution 2). -2. A **task** from the pool's queue. -3. Otherwise the staged wait: spin, yield, then park on its slot's `Parker`, until the queue generation, the broadcast generation, or the state changes. - -The `ABSENT` state, the claim jobs (`claim_cold`) and the `YIELD` state of the Rayon design disappear: every leased worker is always in its loop. - -### Plain tokens: `fork`, `for_ranges`, `external` -- **A call publishes a stack-allocated batch.** The batch holds the task function, the next unclaimed index, the task count and a completion counter (`Completion`). It is linked into the pool's queue: an intrusive list under a `std::sync::Mutex`, so nothing is allocated. -- **Claiming and freeing.** - - A worker claims an index **under the queue lock**, and the claim that takes the last index unlinks the batch. - - The batch's owner frees it only after the completion counter reaches zero. - - So no worker can reach a batch after it was unlinked, and an unlinked batch is gone only after all of its tasks finished. -- **The caller.** - - A caller that is a worker of the pool claims its own batch's remaining indices before it waits. - - A caller from outside the pool waits (staged; it sleeps on the pool's `sync` parker, woken by the last completion), as "a plain thread enters the pool once and waits" says today. -- **No deadlock.** Every unclaimed task is claimable by its owner, and an owner that waits only waits for tasks that are already running. A running task blocks only on its own children, which it can run itself. So no cycle of waits is possible; the worst case is a nested fork that runs its children on its own thread. -- **Budget conservation.** The structure is unchanged: a batch has at most its token's budget of tasks, and the shares of `fork` sum to the budget. -- **`external`.** - - A plain token: the callback runs on a pool worker, as a one-task batch, or inline when the caller is a worker of the pool. - - A team token: its members are lent to the pool's queue for the call, and serve tasks the callback's code publishes to the same pool. - -### Broadcast -- Broadcasts go into the pool's ordered broadcast list (pre-review resolution 2, which replaced a first draft's whole-broadcast lock and per-slot mailboxes). The caller is never a worker of the pool (`CallerInPool` refusal). Every worker runs the broadcasts that target it in publication order. -- The caller waits for all `budget` shares, as today; a reserved member serves no broadcast, again as today. - -### Identity without thread-locals -Each slot records its worker's `ThreadId` when leased. "Is the caller a worker of this pool, and which one" is a scan of the pool's slots (`width` comparisons), done at entry points only, never on the team dispatch path. - -### Rayon interop: opt-in -The Rayon lending of `external` has no production caller (downstream use: tests and benchmarks). It becomes a non-default feature, `rayon`: `Exec::external_rayon(f)` runs `f` inside `install` on a private Rayon pool of the token's width, built lazily per pool. Those are separate threads, which the feature documents. Without the feature, Rayon is not a dependency at all. - -### Unchanged -- **The team machinery:** sections, `Completion`, `Barrier`, `Parker`, pinning, root-scope oversubscription, `Team`/`TeamExec` and the spare worker. A pool of `w` workers gives a team of `w` participants with an outside leader, so one worker always stays free for the queue and for `external`. threadteam-rs#12 stays open. -- **The waits** (`waiting.md`): 300000 polls, 16 yields, futex; 100 polls in a statically oversubscribed scope. - -### Public semantics that change (for the maintainer) - -| item | before (Rayon) | now | -| --- | --- | --- | -| `Pool` drop | joins every worker | dedicated workers: joins. Shared workers: waits for every worker to detach, then returns them to the reserve | -| `idle_runs_rayon` | an idle member runs queued Rayon jobs | no effect: an idle worker always serves the pool's queue (kept for compatibility) | -| `idle_spin` | how long an unreserved member spins before returning to Rayon | how long an unreserved worker may spin before it sleeps; the spin budget bounds it as well | -| `release_idle` | idle members return to Rayon | idle workers stop spinning and sleep now | -| `resident_members` | members in a member loop (reserved or idle) | workers reserved by a team, or idle and awake | -| `external` on a team token | lends members to Rayon | lends members to the pool's queue; Rayon lending is `external_rayon` under the `rayon` feature | -| feature `parallel` | pulls Rayon | pulls nothing; `rayon` is a separate opt-in | - -## Pre-review resolutions - -The gpt-6.1-sol pre-review (`work/runs/threadteam-own/pre.out`) found one deadlock and seven gaps in the first draft. The design above is amended as follows. - -1. **Cross-pool liveness (A → B → A).** - - **The problem.** A worker of pool A that waits for a batch it published on pool B must not just sleep. If every worker of A does so, and B's tasks publish back on A, nobody runs A's queue. - - **The rule.** A waiting worker of any pool keeps serving **its own pool's queue** (and its broadcast sequence) until its batch completes. - - **The wake.** The batch records who to wake: the waiting worker's own slot parker, or the target pool's `sync` parker for a plain outside thread. That waiter sleeps on a predicate that covers both its batch and its pool's work. - - **Same pool.** A worker caller of the same pool still claims its own batch's indices first. -2. **Broadcast.** - - **Publication only.** The pool's broadcast lock covers publication, not completion. - - **Ordering.** Broadcasts are numbered in publication order and linked into the pool's broadcast list. Each worker runs the broadcasts that target it in sequence order, skipping the others. A broadcast stays linked, and alive on its caller's stack, until all of its targets have finished; its caller unlinks it. - - **No lock held while waiting**, so a cooperative waiter re-entering a broadcast cannot block on a lock held lower on its own stack. -3. **One state word per slot.** Bits `RESERVED`, `LENT`, `BUSY`, `DETACH`. - - A team claims a slot only from `0` (compare-and-swap). - - A free worker takes work only by winning `0 → BUSY`, and a lent member only by winning `RESERVED|LENT → RESERVED|LENT|BUSY`. Both happen before it removes a task from the queue; a failed compare-and-swap leaves the task unclaimed. - - Releasing a reservation clears `RESERVED|LENT` only. A worker still running a lent task stays `BUSY`, so it is not claimable until it finishes. - - Detach sets `DETACH`. Finishing a task clears `BUSY` only, so it cannot erase `DETACH`. - - Nested execution (a waiting worker helping its queue) keeps `BUSY`. - - These transitions are a small type (`SlotState`) that `loom` checks directly. -4. **Lease state and self-drop.** - - **Ownership.** Every leased worker holds an `Arc` of the pool's shared state through its epilogue. Its detach acknowledgement is its last access to that state. - - **Drop.** `Pool` drop marks every slot `DETACH`, wakes the workers and waits for the acknowledgements of all workers but itself. - - **Self-drop from a worker.** It does not wait for the dropping worker, which acknowledges after its task returns. The other workers cannot be running work of this pool, because any such work borrows the pool, so their acknowledgements come promptly. - - **Locks.** No reserve lock is held while waiting. - - **Workers above the cap.** They exit, and their handles are detached; only dedicated workers are joined. -5. **Affinity on lease.** `Pool::build` reads the calling thread's affinity mask, and every reused (shared) worker applies it to itself when it is leased (Linux), as a newly spawned thread would inherit it. A dedicated worker was spawned by that caller and inherited it already; it keeps whatever its `on_start` hook set (post-review 5). A worker whose pin restore failed is not returned to the reserve; it exits. -6. **Reserve cap.** The cap is the largest of the available parallelism and every width leased so far. So repeated wide pools (faer's per-call `with_team`) keep reusing their workers, whatever the first caller's affinity was. -7. **Panics.** - - Every claimed task and broadcast share runs under `catch_unwind` and always reaches its completion. The first payload is kept on the batch. - - The owner waits for completion (all claimed tasks, including tasks still unclaimed, which it runs itself) before it resumes the first panic. - - A broadcast share is unlinked by its caller only after completion; a worker never clears a share after signalling it. - - `Completion::finish`'s rule holds: nothing touches a batch after its final decrement. -8. **Verification.** - - The loom list grows to the transitions above: state word, queue claim against reserve, last completion against reclamation, broadcast sequence, detach against task end, and lease attach against first publication. - - The allocation test of the plain paths drops its Rayon-era minimum-window exemption and counts strictly after warm-up. - -Kept against the review's delete list: the opt-in Rayon layer, which the maintainer asked for ("Rayon との相互運用が要る機能は opt-in の層として残す"), and `idle_runs_rayon`, as a documented no-op, so the public API does not break. - -## Post-review resolutions - -The gpt-6.1-sol post-review of the implementation (`work/runs/threadteam-own/post.out`) found three deadlocks, an endless wake-up and an affinity regression. Each fix has a regression test that fails (or hangs) with the fix undone. - -1. **Attachment against the first section.** A worker marked its slot claimable before it read the slot's section generation; a leader that reserved it in between published a section the worker then took as already seen, and both waited forever. `serve` now reads the generation first and marks the slot attached after it. (The window is a few instructions wide; no test forces it.) -2. **Nested helping.** A wait that runs a task which waits in turn cleared the outer wait's `helping` flag on return, so `wake_for_work` skipped a worker still waiting, and A → B → A could hang. The flag is now saved and restored (`a_nested_wait_keeps_the_outer_wait_wakeable`). -3. **A later broadcast above an unfinished one.** A broadcast share that waits on another pool helps its own pool meanwhile; it could start a later broadcast's share, whose shares meet at a barrier, while the earlier broadcast's other share waited at its own barrier for this worker. A worker with a broadcast share on its stack now takes queued tasks but no further share (`a_waiting_broadcast_share_does_not_start_a_later_broadcast`). -4. **Cursor past unlinked broadcasts.** A worker outside a broadcast's budget that looked only after the broadcast was unlinked kept its cursor before it; `has_after` stayed true, so the worker never parked. The cursor now moves past every publication when nothing is left for the worker (`a_cursor_moves_past_a_broadcast_unlinked_before_it_looked`). -5. **Dedicated workers and `on_start` pinning.** The lease reset a dedicated worker's mask to the caller's, undoing a hook's pin. Only reused workers take the caller's mask now (`a_dedicated_worker_keeps_the_pin_of_its_start_hook`). -6. **Docs.** `Pool`'s and `Team`'s shutdown docs distinguish shared and dedicated workers, as the README does. The unused `shutdown` flag and `acquire`'s unused pool argument are gone. - -## Verification -- **loom, new paths, small models:** - - a batch published with one owner and two claimers, the owner asleep until completion; - - an idle worker's wait on the queue generation, the broadcast generation and its state, against a publish and a detach; - - the detach handshake of a pool drop. -- **The existing tests:** alloc-free (the plain paths and the team paths, warm), ledger, compile-fail, no-public-rayon, and the feature matrix. -- **New tests:** - - reuse: thread creation counted; a second pool of the same width spawns none; - - quiescence at drop; dedicated pools keep join-on-drop; - - nested forks; a worker caller running its own batch; - - concurrent broadcasts; a pinned member restoring its mask on detach. -- **Benchmarks:** `scripts/bench-ab.sh` against the base (`a3a9320`) and libgomp (`OMP=1`), and faer's `acquire_probe_107` (all arms), on broccoli and evo. The `fresh_pool` rows (`build`, `acquire`, `call`, `release`, `drop`) show the reuse, and the cold first call is reported separately. - -## Acceptance -As on #14: -- **(a)** faer#107 slow mode zero in every arm; -- **(b)** quiet hosts within 10 % of 0.1.3 (`held_team`, `held_pool`, gaps 0-1 ms); -- **(c)** under foreign load no worse than libgomp; -- **(d)** the 10 ms gap governed by the spin budget. diff --git a/docs/design/placement.md b/docs/design/placement.md index d07150d..ef37592 100644 --- a/docs/design/placement.md +++ b/docs/design/placement.md @@ -6,14 +6,6 @@ cpueinsum-rs, tlinalg-rs and tenferro-rs; the cross-repository rules are in tensor4all/tensor4all-meta#41 (`docs/design/core-boundary-and-execution-ownership.md`, "Where placement is decided"). - -> **Workers (2026-10-30).** threadteam no longer runs on a private Rayon pool. -> Its workers are its own threads, leased from a process-wide reserve; see -> [`own-threads.md`](own-threads.md). Where this record describes Rayon jobs, -> installs or claim jobs, read the native queue, the cooperative waits and the -> compare-and-swap reservation described there. The token contracts here are -> unchanged. - ## Problem The CPU stack places work with **four** mechanisms at once: @@ -153,12 +145,29 @@ deadlock it. `can_broadcast` reports, without consuming the token, whether refusal rule, so they cannot disagree. A kernel asks it before building a setup that only a broadcast needs. -Dropping a `Pool` is deterministic. The pool starts its workers through Rayon's -`spawn_handler` with `std::thread::Builder` (same name and stack size) and keeps -the join handles. `Drop` first drops the Rayon pool, which signals shutdown, and -then joins every handle, so when it returns all workers have exited and their -thread-local destructors have run. A pool dropped from one of its own workers -cannot join itself: that drop skips joining. Steady-state entries are unaffected. +The pool starts its workers through Rayon's `spawn_handler` with +`std::thread::Builder` (same name and stack size) and keeps the join handles. +Dropping a pool with dedicated workers (a `thread_name`, an `on_start` hook, a +`stack_size` or a `qos` of its own) is deterministic: `Drop` first drops the +Rayon pool, which signals shutdown, and then joins every handle, so when it +returns all workers have exited and their thread-local destructors have run. +A pool with the default thread settings waits until its team members left +their loops and then keeps the Rayon pool and its handles in a process-wide +cache keyed by width; the next default pool of that width takes it and each +worker applies the building thread's CPU set to itself (a Rayon `broadcast`). +Only idle entries are handed out, so two live pools never share workers. The +cache holds at most the larger of the available parallelism and the widest +pool built so far, in threads, and drops and joins its oldest entries beyond +that. A pool dropped from one of its own workers can neither join nor wait for +itself: that drop skips joining and caching. Steady-state entries are +unaffected. + +The cache is the Rayon-based answer to tensor4all/faer#107's per-call pools +(threadteam-rs#18, #19): the slow mode came from the fresh threads of every +pool, not from the waits. Own `std::thread` workers (#16) solved the same +problem with a much larger change; maintainer decision, 2026-10-30: 「しばらくは +Rayon版を出版し続けて，Rayonで何ができないか熟考してから判断する」. That +version is kept on the `develop` branch and would ship as 0.3.0. A parent waiting for its children keeps no slot; Rayon may let the waiting thread run one of its children, which is one of the tasks already counted. diff --git a/docs/design/waiting.md b/docs/design/waiting.md index d1d52d8..bf6de5b 100644 --- a/docs/design/waiting.md +++ b/docs/design/waiting.md @@ -90,11 +90,22 @@ The last member of a section therefore pays one extra uncontended read-modify-wr - 12288 polls (about 300 µs) made the first section after a 1 ms or 10 ms gap 2.3x and 10.8x slower than 0.1.3 on a quiet host (evo): the members had gone to sleep. 131072 closed the 1 ms gap; 300000 also closes the 10 ms gap (evo, `d430347` runs), and is accepted as the OpenMP default (criterion d). - libomp keeps 200 ms (`KMP_BLOCKTIME`); 0.1.3 kept 200 ms of wall time for members and never slept in joins. - **No calibration.** Converting a time to polls at pool build was considered and dropped (pre-review): a calibration taken while the CPU is contended yields too few polls, and a bare PAUSE loop is not the real poll anyway. The count is approximate in time, as libgomp's is. -- **Throttled budget.** 100 polls (libgomp's value) when the root scope is statically oversubscribed. +- **Throttled budget.** 100 polls (libgomp's value) when the root scope is statically oversubscribed, or while a new pool is under sustained pressure (below). -### Scheduling pressure: none (libgomp's waits) +### Scheduling pressure: only while a pool is new -Nothing detects preemption: every wait spins its budget, yields and sleeps, as libgomp's waits do. #14 tried several detectors on the Rayon base, and each traded one acceptance criterion for another (broccoli, threadteam-rs#14): +**Default: libgomp's waits.** Once a pool is past its warm-up, its waits detect nothing: every wait spins 300000 polls, yields 16 times and sleeps, as libgomp's do (`GOMP_SPINCOUNT`, futex). +- On broccoli (`e76036c`, 16:58Z) this was the only configuration that met criteria (b) and (c) together: on quiet hosts held teams within 10 % of 0.1.3, and under load `held_team` at 0.91x libgomp, `held_pool` +7.8 %, gaps on par with libgomp. +- Every detecting variant made held teams under load 3.5-4x slower than libgomp, because detection throttled waits that libgomp keeps spinning. +- The main path of real use is a held pool: tenferro's `CpuEngine` holds its pool, and libomp and libgomp hold hot teams. + +**Warm-up: detection while a pool is new (default 200 ms after it is built).** Without any detection, a pool built for one call (`fresh_pool`, faer's `with_team`) collapsed (87.6 ms): its freshly spawned threads start stacked, because the scheduler places new threads before it knows their load, and spinning members never leave a CPU idle to pull one over. So during the warm-up a new pool's waiters detect preemptions and the pool can throttle, as in `c0c024b`, which kept the faer#107 arms at zero: +- **Detection.** A spinning waiter reads the clock every 64 polls. A gap of more than 200 µs between two reads means it was descheduled while it spun: 64 polls take well under 10 µs, and a scheduler slice is 0.75 ms or more. +- **Response.** That waiter ends its spin (yields, then sleeps). The pool counts the observations, and once 4 come within 100 ms of each other, every wait of the pool adopts the throttled budget (100 polls) at its first clock read, for 100 ms (`fetch_max`). +- **Limits of the signal.** It is a pressure heuristic: foreign load, the team's own threads, interrupts and VM steal look alike, and it only shortens active waiting. +- **Cost after the warm-up.** None in the poll loop: a wait's first clock read finds detection off, and that wait stops reading the clock. + +**What was tried and measured on the way** (broccoli and evo, threadteam-rs#14): | detection | quiet held teams | held teams under load | fresh pool | | --- | --- | --- | --- | @@ -102,9 +113,14 @@ Nothing detects preemption: every wait spins its budget, yields and sleeps, as l | a pool-wide throttle on 4 preemptions within 100 ms (`c0c024b`) | within 10 % | 3.5-4x libgomp | fixed | | a per-thread 100 ms throttle (`2773c36`) | 1.2-1.8x slower | — | fixed | | none (`e76036c` variant) | within 10 % | about libgomp | collapsed (87.6 ms) | -| detection during a 200 ms warm-up (`1f2145b`) | `held_pool` 2.2x | 3.3x libgomp | collapsed (11 ms) | -The one configuration that met (b) and (c), no detection, failed only on the pool built per call. That pool's collapse comes from spawning fresh threads per call, not from waiting. With threadteam's own threads (`own-threads.md`) a pool reuses the reserve's running workers, so the waits stay libgomp's and the detection, the warm-up and their knobs are gone. +**Knobs and counters for the benchmarks** (`PoolBuilder::preempt_gap`, `warm_up`, `wait_tuning`, all hidden; bench flags `--preempt-gap-us`, `--warm-ms`, `--pressure-events`, `--pressure-hold-ms`, `--throttled-polls`, `--own-hold-ms`): +- detection after the warm-up (a nonzero gap); +- the warm-up length; +- the pool threshold, the hold and the throttled budget; +- a per-thread hold. + +`Pool::wait_counters` (hidden) counts join and barrier sleeps, member parks and observed preemptions, printed under every bench row. ### Start of a section: no arrival barrier diff --git a/scripts/bench-ab.sh b/scripts/bench-ab.sh index 095f63e..3ed5767 100755 --- a/scripts/bench-ab.sh +++ b/scripts/bench-ab.sh @@ -10,6 +10,9 @@ # taskset -c 0-63 scripts/bench-ab.sh a3a9320 feat/wait-openmp 20 \ # --scenario acquire --widths 64 --calls 40 --impl threadteam # +# HEAD2= adds a third side, labelled `head2`, built and run in every round; the order of the +# sides rotates by round. +# # HEAD_VARIANTS adds head runs with extra arguments in every round, `;`-separated, each named by # its arguments, e.g. HEAD_VARIANTS='--spin-polls 300000;--own-hold-ms 100;--pressure-events 0'. # OMP=1 also runs the libgomp driver (benches/omp/wait_omp.c) in every round, with the same @@ -21,7 +24,7 @@ csv=${CSV:-docs/design/waiting-data/$host-ab.csv} log=${LOG:-docs/design/waiting-data/$host-ab.log} mkdir -p "$(dirname "$csv")" work=$(mktemp -d) -trap 'git worktree remove --force "$work/base" 2>/dev/null; git worktree remove --force "$work/head" 2>/dev/null; rm -rf "$work"' EXIT +trap 'for d in base head head2; do git worktree remove --force "$work/$d" 2>/dev/null; done; rm -rf "$work"' EXIT build() { # $1 = rev, $2 = dir name; prints the bench binary git worktree add -q --detach "$work/$2" "$1" (cd "$work/$2" && cargo build -j "${JOBS:-4}" --release --bench wait_scenarios >/dev/null 2>&1) @@ -29,6 +32,8 @@ build() { # $1 = rev, $2 = dir name; prints the bench binary } bin_base=$(build "$base" base) bin_head=$(build "$head" head) +bin_head2="" +[[ -n ${HEAD2:-} ]] && bin_head2=$(build "$HEAD2" head2) omp="" if [[ ${OMP:-0} == 1 ]]; then gcc -O2 -fopenmp "$work/head/benches/omp/wait_omp.c" -o "$work/wait_omp" @@ -42,12 +47,20 @@ arg() { # value of bench option $1 in "$@" after it, or $2 omp_scenario=$(arg --scenario all "$@"); omp_widths=$(arg --widths "$(nproc)" "$@") omp_calls=$(arg --calls 200 "$@"); omp_load=$(arg --load "" "$@") IFS=';' read -r -a variants <<<"${HEAD_VARIANTS:-}" -echo "base $(git rev-parse --short "$base") head $(git rev-parse --short "$head") rounds $rounds args $*" | tee -a "$log" +echo "base $(git rev-parse --short "$base") head $(git rev-parse --short "$head")${HEAD2:+ head2 $(git rev-parse --short "$HEAD2")} rounds $rounds args $*" | tee -a "$log" for r in $(seq 1 "$rounds"); do - order=(base head); (( r % 2 == 0 )) && order=(head base) + if [[ -n $bin_head2 ]]; then + case $(( r % 3 )) in + 1) order=(base head head2) ;; 2) order=(head head2 base) ;; *) order=(head2 base head) ;; + esac + else + order=(base head); (( r % 2 == 0 )) && order=(head base) + fi for side in "${order[@]}"; do if [[ $side == base ]]; then "$bin_base" --label "base#$r" --csv "$csv" "$@" >>"$log" + elif [[ $side == head2 ]]; then + "$bin_head2" --label "head2#$r" --csv "$csv" "$@" >>"$log" else "$bin_head" --label "head#$r" --csv "$csv" "$@" >>"$log" if [[ -n $omp ]]; then diff --git a/src/awake.rs b/src/awake.rs index 1e626b5..25334a4 100644 --- a/src/awake.rs +++ b/src/awake.rs @@ -1,17 +1,23 @@ -//! The workers of a pool: their slots, the native loop, sections, acquisition. +//! Reserved awake members: per-member dispatch slots, sections, acquisition. //! -//! Every leased worker runs [`serve`] for its slot until the pool detaches it -//! (`docs/design/own-threads.md`). A slot's state word ([`crate::park::SlotState`]) -//! has the bits `RESERVED`, `LENT`, `BUSY`, `DETACH`: +//! A *member* is a worker of the pool that runs a long-lived loop as one +//! Rayon job. While the loop runs, the worker serves no other Rayon job. +//! Each worker index owns one cache-line padded [`Slot`]: //! -//! * `0`: free. The worker serves the pool's broadcast shares and queued tasks -//! (winning `BUSY` before it takes one) and otherwise waits (spin, yield, -//! park). A team claims it by a compare-and-swap from `0`. -//! * `RESERVED`: a team member. It waits for `gen` (spin, yield, park) and runs -//! the sections its leader publishes; it serves nothing else. -//! * `RESERVED|LENT`: a member lent to the pool's queue (`Exec::external`). -//! * `BUSY`: running a task or a share; not claimable. -//! * `DETACH`: the pool is being dropped; the worker acknowledges and leaves. +//! ```text +//! ABSENT --claim job--> RESERVED <--claim (CAS)-- IDLE +//! ^ | release (store) ^ | +//! +---- idle timeout ---+-----------------------+ +-- YIELD (inside a Rayon job) +//! ``` +//! +//! * `ABSENT`: no loop on this worker. +//! * `RESERVED`: owned by one leader; the member waits for `gen` (spin, yield, +//! then park) and runs the section the leader published. It neither runs +//! Rayon jobs nor leaves. +//! * `IDLE`: loop alive but unreserved; polls for the policy's idle time or +//! the pool's spin budget, whichever ends first, then leaves (`ABSENT`). +//! Claimable by a compare-and-swap, with no waiting. +//! * `YIELD`: an idle member inside `rayon::yield_now`; not claimable. //! //! A leader publishes a [`Section`] (a stack object) by writing its address //! and an argument into the member's slot and bumping `gen`; the member runs @@ -23,7 +29,6 @@ //! futex round per section that grew with the number of unused members). //! //! No thread-local state is used: a member's identity is its slot index. -#![cfg_attr(not(feature = "parallel"), allow(dead_code))] use crate::error::Refused; use crate::park::Parker; @@ -41,9 +46,11 @@ macro_rules! trace { }; } -#[cfg(feature = "parallel")] -use crate::park::{BUSY, DETACH}; -use crate::park::{LENT, RESERVED}; +const ABSENT: usize = 0; +const IDLE: usize = 1; +#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +const YIELD: usize = 2; +const RESERVED: usize = 3; /// Pure spin for a while, then yield the CPU between polls. /// @@ -63,67 +70,45 @@ pub(crate) fn backoff(spins: &mut u32) { } } -/// One worker's slot: its state word, its team-member dispatch state, and -/// where it sleeps. +/// One member's dispatch slot. #[repr(align(128))] -pub(crate) struct Slot { - /// `RESERVED`/`LENT`/`BUSY`/`DETACH` (see [`crate::park::SlotState`]). - pub(crate) state: crate::park::SlotState, +struct Slot { + state: AtomicUsize, /// Bumped by the current leader per published section. gen: AtomicUsize, ptr: AtomicPtr
, arg: AtomicUsize, - /// Idle policy written by the releasing leader before the slot is freed. + /// Idle policy written by the releasing leader before `IDLE` is stored. idle_spin_ns: AtomicU64, + idle_yield: AtomicBool, /// Spin budget, in polls, of this reserved member's waits for work (see /// `docs/design/waiting.md`); written by the leader at root-scope entry. spin_budget: AtomicU32, - /// The mask this member saved before pinning itself; only the worker - /// thread of this slot touches it (see [`crate::pin::SavedMask`]). + /// Set by the leader while the member's thread is lent to Rayon + /// (`Exec::external` on a team token). + lent: AtomicBool, + /// The mask this member saved before pinning itself; only the member thread + /// of this slot touches it (see [`crate::pin::SavedMask`]). #[cfg_attr(not(feature = "parallel"), allow(dead_code))] - pub(crate) saved: crate::pin::SavedMask, - /// Where this worker sleeps: idle, as a member between sections, or while - /// it waits for a batch (`help_while`). An `Arc`, so that a completion can - /// keep it alive across its final decrement. - pub(crate) park: std::sync::Arc, - /// The worker thread leased to this slot (set when it attaches). - pub(crate) thread: std::sync::OnceLock, - /// The worker's broadcast cursor (only the worker touches it). - bseq: AtomicU64, - /// Set while the worker waits for a batch and serves its pool meanwhile. - pub(crate) helping: AtomicBool, - /// Broadcast shares on this worker's stack (only the worker touches it). - /// While one runs, the worker takes no other share: a later broadcast run - /// above an unfinished one could wait at its barrier for a worker that - /// waits at the earlier one's. - in_bcast: AtomicUsize, - /// A released member that has not gone to sleep since (it still spins, - /// claimable without a wake): what `Pool::resident_members` counts besides - /// reserved members. - lingering: AtomicBool, - /// The worker attached to its lease / acknowledged its detach. - pub(crate) attached: AtomicBool, - pub(crate) acked: AtomicBool, + saved: crate::pin::SavedMask, + /// Where this member sleeps once its spin and yields are spent; woken by + /// the leader after it changed `gen`, `state` or `lent` of this slot. + park: Parker, } impl Slot { fn new() -> Self { Slot { - state: crate::park::SlotState::new(), + state: AtomicUsize::new(ABSENT), gen: AtomicUsize::new(0), ptr: AtomicPtr::new(core::ptr::null_mut()), arg: AtomicUsize::new(0), - idle_spin_ns: AtomicU64::new(u64::MAX), + idle_spin_ns: AtomicU64::new(0), + idle_yield: AtomicBool::new(false), spin_budget: AtomicU32::new(crate::park::THROTTLED_POLLS), + lent: AtomicBool::new(false), saved: crate::pin::SavedMask::new(), - park: std::sync::Arc::new(Parker::new()), - thread: std::sync::OnceLock::new(), - bseq: AtomicU64::new(0), - helping: AtomicBool::new(false), - in_bcast: AtomicUsize::new(0), - lingering: AtomicBool::new(false), - attached: AtomicBool::new(false), - acked: AtomicBool::new(false), + park: Parker::new(), } } } @@ -132,9 +117,8 @@ impl Slot { pub(crate) struct Waits { /// Where leaders sleep in `join` and participants in `barrier`. One per /// pool: a member may wake it after its last decrement, when the section - /// may already be gone, and the pool outlives every member loop. Also - /// where a plain outside thread waits for a batch of this pool. - pub(crate) sync: std::sync::Arc, + /// may already be gone, and the pool outlives every member loop. + pub(crate) sync: Parker, /// Spin budget of a wait, in polls (`PoolBuilder::spin_polls`). pub(crate) spin_polls: u32, /// Diagnostics (`Pool::wait_counters`): sleeps of leaders in joins, of @@ -146,14 +130,11 @@ pub(crate) struct Waits { /// Shared state of the awake members of one pool. pub(crate) struct Awake { - pub(crate) slots: Box<[Slot]>, - /// The pool's queue of task batches (plain `fork`, `for_ranges`, `external`). - pub(crate) queue: crate::park::Queue, - /// The pool's broadcasts. - pub(crate) bcast: crate::park::BroadcastList, + slots: Box<[Slot]>, /// What the joins and barriers of this pool's sections sleep on, and the /// pool's spin budget (`docs/design/waiting.md`). pub(crate) waits: Waits, + pub(crate) shutdown: AtomicBool, /// Bumped by `Pool::release_idle`; members that started earlier leave when idle. pub(crate) release_epoch: AtomicUsize, pub(crate) defaults: WaitPolicy, @@ -177,15 +158,14 @@ impl Awake { ) -> Self { Awake { slots: (0..width).map(|_| Slot::new()).collect(), - queue: crate::park::Queue::new(), - bcast: crate::park::BroadcastList::new(), waits: Waits { - sync: std::sync::Arc::new(Parker::new()), + sync: Parker::new(), spin_polls, join_sleeps: AtomicU64::new(0), barrier_sleeps: AtomicU64::new(0), member_parks: AtomicU64::new(0), }, + shutdown: AtomicBool::new(false), release_epoch: AtomicUsize::new(0), defaults, costs, @@ -196,23 +176,23 @@ impl Awake { } } - /// Members kept awake for teams: reserved ones, and released ones that - /// have not gone to sleep since. + /// Workers currently running a member loop, reserved or idle. pub(crate) fn resident(&self) -> usize { self.slots .iter() - .filter(|s| s.state.load() & RESERVED != 0 || s.lingering.load(Ordering::Acquire)) + .filter(|s| s.state.load(Ordering::Acquire) != ABSENT) .count() } - /// Hands members back: they become free under `policy`. + /// Hands members back: they become idle under `policy`. pub(crate) fn release(&self, ids: &[usize], policy: WaitPolicy) { let ns = u64::try_from(policy.idle_spin().as_nanos()).unwrap_or(u64::MAX); for &id in ids { let s = &self.slots[id]; s.idle_spin_ns.store(ns, Ordering::Relaxed); - s.lingering.store(true, Ordering::Release); - s.state.release(); + s.idle_yield + .store(policy.idle_runs_rayon(), Ordering::Relaxed); + s.state.store(IDLE, Ordering::SeqCst); s.park.wake(); } } @@ -293,87 +273,66 @@ impl Awake { trace!(self, None, Publish); } - /// Lends (or takes back) the members' threads to the pool's queue: while - /// lent, a reserved member runs the pool's tasks instead of idling. Taking - /// it back lets a task it is running finish; the member serves its team - /// again after that. + /// Lends (or takes back) the members' threads to Rayon: while lent, a + /// reserved member runs the pool's jobs instead of idling. #[cfg_attr(not(feature = "parallel"), allow(dead_code))] pub(crate) fn lend(&self, ids: &[usize], on: bool) { for &id in ids { let s = &self.slots[id]; - s.state.set_lent(on); - s.park.wake(); - } - } - - /// Wakes the workers that can take work of the queue or a broadcast: free - /// ones, lent members and workers helping while they wait. - pub(crate) fn wake_for_work(&self) { - for s in &*self.slots { - let st = s.state.load(); - if st == 0 || st == RESERVED | LENT || s.helping.load(Ordering::SeqCst) { + s.lent.store(on, Ordering::SeqCst); + if on { s.park.wake(); } } } - /// The slot of the calling thread in this pool, if it is one of its - /// workers (a scan of the slots: entry points only). - pub(crate) fn slot_of_current(&self) -> Option { - let me = std::thread::current().id(); - self.slots.iter().position(|s| s.thread.get() == Some(&me)) - } - - /// Reserves `need` free workers by compare-and-swap, waiting at most the - /// pool's acquisition timeout for busy ones to become free; on a shortfall - /// every tentative member is released and `Unavailable` returned. A worker - /// that runs a task is not free, so a reservation never shares a thread - /// with a running task. + /// Reserves `need` members, or refuses without waiting for a Rayon job. + /// + /// Idle resident members are claimed with a compare-and-swap. Missing ones + /// are requested with claim jobs; the leader waits at most the pool's + /// acquisition timeout for *idle workers to wake up*, never for a worker + /// that is busy: when the members are not all there by then the claim is + /// closed, every tentative member is released and `Unavailable` returned. pub(crate) fn acquire( self: &std::sync::Arc, + pool: &crate::pool::Pool, need: usize, policy: WaitPolicy, spin_budget: u32, ) -> Result { let mut ids = Vec::with_capacity(need); - let me = self.slot_of_current(); + // Idle members are claimed with a compare-and-swap. A member that is + // inside `yield_now` is not claimable at that instant, but it is only + // polling unless it found a job; retry for at most the timeout while + // such members exist, instead of treating them as busy at once. let started = std::time::Instant::now(); - let mut spins = 0u32; loop { - // Members of an earlier team that are still awake first (warm, - // claimable without a wake-up), then any other free worker. - let order = self - .slots - .iter() - .enumerate() - .filter(|(_, s)| s.lingering.load(Ordering::Acquire)) - .chain( - self.slots - .iter() - .enumerate() - .filter(|(_, s)| !s.lingering.load(Ordering::Acquire)), - ); - for (i, s) in order { + let mut yielding = false; + for (i, s) in self.slots.iter().enumerate() { if ids.len() == need { break; } - if Some(i) != me - && !ids.contains(&i) - && s.attached.load(Ordering::Acquire) - && s.state.try_reserve() + match s + .state + .compare_exchange(IDLE, RESERVED, Ordering::AcqRel, Ordering::Relaxed) { - s.lingering.store(false, Ordering::Release); - ids.push(i); + Ok(_) => ids.push(i), + Err(YIELD) => yielding = true, + Err(_) => {} } } - if ids.len() == need || started.elapsed() >= self.acquire_timeout { + if ids.len() == need || !yielding || started.elapsed() >= self.acquire_timeout { break; } - backoff(&mut spins); - } - if ids.len() < need { - self.release(&ids, policy); - return Err(Refused::Unavailable); + std::hint::spin_loop(); + } + let missing = need - ids.len(); + if missing > 0 { + self.claim_cold(pool, missing, &mut ids); + if ids.len() < need { + self.release(&ids, policy); + return Err(Refused::Unavailable); + } } self.set_spin_budget(&ids, spin_budget); Ok(Reservation { @@ -383,81 +342,46 @@ impl Awake { }) } - /// Runs one broadcast share or one queued task on worker `id`, if there is - /// one; the caller holds the right to run work (it won `BUSY`, or it is a - /// worker waiting for a batch). `true` when something ran. - #[cfg(feature = "parallel")] - pub(crate) fn run_one(&self, id: usize) -> bool { - let slot = &self.slots[id]; - let mut seq = slot.bseq.load(Ordering::Relaxed); - if slot.in_bcast.load(Ordering::Relaxed) == 0 && self.bcast.has_after(seq) { - let next = self.bcast.next_for(id, &mut seq); - slot.bseq.store(seq, Ordering::Relaxed); - if let Some(b) = next { - slot.in_bcast.fetch_add(1, Ordering::Relaxed); - // SAFETY: a linked broadcast that targets `id`; its caller - // keeps it alive until this share completed. `run_share` - // catches a panic of the share. - unsafe { crate::park::Broadcast::run_share(b, id) }; - slot.in_bcast.fetch_sub(1, Ordering::Relaxed); - return true; - } - } - if let Some((b, i)) = self.queue.claim() { - // SAFETY: index `i` of `b` was claimed here. - unsafe { crate::park::Batch::run_claimed(b, i) }; - return true; - } - false - } - - /// Whether worker `id` has work it could run now (a hint). #[cfg(feature = "parallel")] - fn work_for(&self, id: usize) -> bool { - let slot = &self.slots[id]; - self.queue.has_work() - || (slot.in_bcast.load(Ordering::Relaxed) == 0 - && self.bcast.has_after(slot.bseq.load(Ordering::Relaxed))) - } - - /// A free worker or a lent member takes one piece of work: it wins `BUSY` - /// first, so a team cannot reserve it meanwhile, then runs it. - #[cfg(feature = "parallel")] - fn try_work(&self, id: usize) -> bool { - if !self.work_for(id) { - return false; + fn claim_cold( + self: &std::sync::Arc, + pool: &crate::pool::Pool, + missing: usize, + ids: &mut Vec, + ) { + use std::sync::Arc; + use std::time::Instant; + let acq = Arc::new(Acq::new(missing)); + for _ in 0..missing { + let (aw, a) = (Arc::clone(self), Arc::clone(&acq)); + pool.inner().spawn(move || claim_job(&aw, &a)); + } + let deadline = Instant::now() + self.acquire_timeout; + let mut spins = 0; + while acq.joined() < missing && Instant::now() < deadline { + backoff(&mut spins); } - let slot = &self.slots[id]; - if !slot.state.try_take() { - return false; + let n = acq.close(); + for pos in 0..n { + let mut spins = 0; + loop { + let id = acq.ids[pos].load(Ordering::Acquire); + if id != usize::MAX { + ids.push(id); + break; + } + backoff(&mut spins); + } } - let ran = self.run_one(id); - slot.state.end_busy(); - ran } - /// Waits until `done` reports completion. A worker of this pool (`me`) - /// serves this pool's queue and broadcasts meanwhile, so nested and - /// cross-pool waits always make progress; it sleeps on its own parker, - /// which the batch's completion and new work of this pool wake. - #[cfg(feature = "parallel")] - pub(crate) fn help_while(&self, me: usize, done: impl Fn() -> bool) { - let slot = &self.slots[me]; - // Waits nest (a task run while helping waits in turn): the outer wait - // stays wakeable after the inner one returns. - let outer = slot.helping.swap(true, Ordering::SeqCst); - let mut waiter = crate::park::Waiter::new(self.waits.spin_polls); - while !done() { - if self.run_one(me) { - waiter = crate::park::Waiter::new(self.waits.spin_polls); - continue; - } - if waiter.step() == crate::park::Step::Sleep { - slot.park.park_while(|| !done() && !self.work_for(me)); - waiter = crate::park::Waiter::new(self.waits.spin_polls); - } - } - slot.helping.store(outer, Ordering::SeqCst); + #[cfg(not(feature = "parallel"))] + fn claim_cold( + self: &std::sync::Arc, + _pool: &crate::pool::Pool, + _missing: usize, + _ids: &mut Vec, + ) { } } @@ -476,137 +400,236 @@ impl Drop for Reservation { } } -/// The native loop of worker `id` of a pool, from its attachment until the -/// pool detaches it (`DETACH`, while it is not busy). See -/// `docs/design/own-threads.md` and `docs/design/waiting.md`. +/// One acquisition: how many claim jobs found a worker, and which. +#[cfg(feature = "parallel")] +struct Acq { + joined: AtomicUsize, + ids: Vec, +} + +#[cfg(feature = "parallel")] +impl Acq { + const CLOSED: usize = 1 << (usize::BITS - 1); + + fn new(capacity: usize) -> Self { + Acq { + joined: AtomicUsize::new(0), + ids: (0..capacity) + .map(|_| AtomicUsize::new(usize::MAX)) + .collect(), + } + } + + fn joined(&self) -> usize { + self.joined.load(Ordering::Acquire) & !Self::CLOSED + } + + /// A claim job asks for a position; `None` when closed or full. + fn join(&self) -> Option { + let mut cur = self.joined.load(Ordering::Acquire); + loop { + if cur & Self::CLOSED != 0 || cur >= self.ids.len() { + return None; + } + match self.joined.compare_exchange_weak( + cur, + cur + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Some(cur), + Err(now) => cur = now, + } + } + } + + /// Closes the claim; later jobs do nothing. Returns the joined count. + fn close(&self) -> usize { + self.joined.fetch_or(Self::CLOSED, Ordering::AcqRel) & !Self::CLOSED + } +} + +/// Runs on a pool worker: becomes a member for the acquisition, or does nothing. +#[cfg(feature = "parallel")] +fn claim_job(aw: &std::sync::Arc, acq: &std::sync::Arc) { + let Some(w) = rayon::current_thread_index() else { + return; + }; + let Some(slot) = aw.slots.get(w) else { + return; + }; + if slot + .state + .compare_exchange(ABSENT, RESERVED, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + // Stolen by a resident member that is serving Rayon (idle_runs_rayon): + // its thread cannot host a second loop. Hand the job back to the pool + // unless the acquisition is already over. + if acq.joined.load(Ordering::Acquire) & Acq::CLOSED == 0 { + let (aw, acq) = (std::sync::Arc::clone(aw), std::sync::Arc::clone(acq)); + rayon::spawn(move || claim_job(&aw, &acq)); + } + return; + } + let seen = slot.gen.load(Ordering::Acquire); + let epoch = aw.release_epoch.load(Ordering::Relaxed); + match acq.join() { + Some(pos) => { + acq.ids[pos].store(w, Ordering::Release); + member_loop(aw, w, seen, epoch); + } + None => slot.state.store(ABSENT, Ordering::Release), + } +} + +/// The member loop. Returns when the idle policy, `release_idle` or shutdown +/// returns the thread to Rayon. /// -/// * Reserved (and not lent): a team member. It waits for its slot's `gen` -/// in staged episodes (spin for the slot's budget, yield, park) and runs the -/// sections its leader publishes; it serves nothing else. -/// * Free, or a lent member: it takes broadcast shares and queued tasks, -/// winning `BUSY` before it removes anything, and otherwise waits in a staged -/// episode (the policy's idle spin) and parks until work, a reservation or a -/// detach arrives. +/// A reserved member waits for work in episodes (`docs/design/waiting.md`): an +/// episode starts when it first finds nothing to do after its last share (or +/// after it started, or was taken back from Rayon), spins for the slot's spin +/// budget in polls, yields a few times, then parks and stays reserved. The +/// budget is a count, so a member that is descheduled does not spend it while +/// it waits for its CPU. /// -/// A member restores its own affinity mask when it stops being reserved, and -/// before it serves foreign work as a lent member. +/// An unreserved (`IDLE`) member returns to Rayon after the policy's +/// `idle_spin` of wall time or after the pool's spin budget of polls, whichever +/// comes first, so a team released between calls does not keep every CPU busy +/// for the whole `idle_spin`. #[cfg(feature = "parallel")] -pub(crate) fn serve(aw: &Awake, id: usize) { +fn member_loop(aw: &Awake, id: usize, mut seen: usize, epoch: usize) { use crate::park::{Step, Waiter}; + use std::time::Instant; let slot = &aw.slots[id]; - let mut seen = slot.gen.load(Ordering::Acquire); - // Claimable only from now on: a leader that reserves this slot publishes - // its first section after the baseline above. - slot.attached.store(true, Ordering::SeqCst); + let mut idle_since: Option = None; + let mut idle_polls = 0u64; let mut waiter: Option = None; let mut was_lent = false; - let mut reserved = false; - let mut epoch = aw.release_epoch.load(Ordering::Relaxed); - // Whether the last iteration served as a member; a wait episode does not - // carry over between the member's and the free worker's waits. - let mut as_member = false; + let mut reserved = true; + let mut spins = 0u32; trace!(aw, Some(id), Enter); loop { - let st = slot.state.load(); - // Pinning ends with the reservation, and it ends on this thread. - if st & RESERVED != 0 { + let state = slot.state.load(Ordering::Acquire); + // Pinning ends with the reservation, and it ends on this thread: a + // member restores its own mask before it leaves `RESERVED` and before + // it serves Rayon work (a lend restores it below). + if state == RESERVED { reserved = true; } else if reserved { reserved = false; slot.saved.restore(); } - if st & DETACH != 0 && st & BUSY == 0 { - trace!(aw, Some(id), Leave); - return; - } - if st & RESERVED != 0 && st & LENT == 0 { - if !as_member { - as_member = true; - waiter = None; - } - if was_lent { - was_lent = false; + match state { + RESERVED if slot.lent.load(Ordering::Acquire) => { + if !was_lent { + was_lent = true; + // The thread serves the pool's jobs now: undo our pin first. + slot.saved.restore(); + trace!(aw, Some(id), LentBegin); + } + idle_since = None; waiter = None; - trace!(aw, Some(id), LentEnd); + // Serve the pool's jobs; resume team service when taken back. + if !matches!(rayon::yield_now(), Some(rayon::Yield::Executed)) { + for _ in 0..32 { + std::hint::spin_loop(); + } + } } - let g = slot.gen.load(Ordering::Acquire); - if g == seen { - let w = waiter - .get_or_insert_with(|| Waiter::new(slot.spin_budget.load(Ordering::Relaxed))); - if w.step() == Step::Sleep { - aw.waits.member_parks.fetch_add(1, Ordering::Relaxed); - trace!(aw, Some(id), ParkEnter); - park_member(slot, seen); - trace!(aw, Some(id), ParkExit); + RESERVED => { + if was_lent { + was_lent = false; + trace!(aw, Some(id), LentEnd); + } + idle_since = None; + let g = slot.gen.load(Ordering::Acquire); + if g == seen { + let w = waiter.get_or_insert_with(|| { + Waiter::new(slot.spin_budget.load(Ordering::Relaxed)) + }); + if w.step() == Step::Sleep { + aw.waits.member_parks.fetch_add(1, Ordering::Relaxed); + trace!(aw, Some(id), ParkEnter); + park_member(slot, seen); + trace!(aw, Some(id), ParkExit); + waiter = None; + } + } else { + seen = g; waiter = None; + let ptr = slot.ptr.load(Ordering::Relaxed); + let arg = slot.arg.load(Ordering::Relaxed); + // SAFETY: the leader keeps the section alive until its + // completion counter reaches zero, which happens only + // after the call below decremented it. + if unsafe { (*ptr).pin } { + // A pin section: this member saves its previous mask + // here, before it pins itself in its share. + slot.saved.save_if_unpinned(); + } + trace!(aw, Some(id), SectionBegin); + // SAFETY: as above; `aw` outlives this loop, so the wake + // after the last decrement does not touch the section. + unsafe { Section::execute_member(ptr, arg) }; + trace!(aw, Some(id), SectionEnd); } - } else { - seen = g; + } + IDLE => { waiter = None; - let ptr = slot.ptr.load(Ordering::Relaxed); - let arg = slot.arg.load(Ordering::Relaxed); - // SAFETY: the leader keeps the section alive until its - // completion counter reaches zero, which happens only after - // the call below decremented it. - if unsafe { (*ptr).pin } { - // A pin section: save the previous mask before pinning. - slot.saved.save_if_unpinned(); + if idle_since.is_none() { + idle_polls = 0; + } + let since = *idle_since.get_or_insert_with(Instant::now); + idle_polls += 1; + let spin_ns = slot.idle_spin_ns.load(Ordering::Relaxed); + let leave = aw.shutdown.load(Ordering::Relaxed) + || aw.release_epoch.load(Ordering::Relaxed) != epoch + || idle_polls > u64::from(aw.waits.spin_polls) + || since.elapsed().as_nanos() >= u128::from(spin_ns); + if leave { + if slot + .state + .compare_exchange(IDLE, ABSENT, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + trace!(aw, Some(id), Leave); + return; + } + continue; // claimed meanwhile + } + if slot.idle_yield.load(Ordering::Relaxed) + && slot + .state + .compare_exchange(IDLE, YIELD, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + let ran = rayon::yield_now(); + slot.state.store(IDLE, Ordering::Release); + if !matches!(ran, Some(rayon::Yield::Executed)) { + // Nothing to run: stay claimable for a moment. + for _ in 0..64 { + std::hint::spin_loop(); + } + } + } else { + backoff(&mut spins); } - trace!(aw, Some(id), SectionBegin); - // SAFETY: as above; `aw` outlives this loop, so the wake after - // the last decrement does not touch the section. - unsafe { Section::execute_member(ptr, arg) }; - trace!(aw, Some(id), SectionEnd); } - continue; - } - // Free (`0`) or lent (`RESERVED|LENT`): serve the pool. - if as_member { - as_member = false; - waiter = None; - } - if st & LENT != 0 && !was_lent { - was_lent = true; - // The thread serves foreign work now: undo our pin first. - slot.saved.restore(); - trace!(aw, Some(id), LentBegin); - } - if aw.try_work(id) { - waiter = None; - continue; - } - let now = aw.release_epoch.load(Ordering::Relaxed); - if now != epoch { - // `Pool::release_idle`: stop spinning now. - epoch = now; - waiter = Some(Waiter::new(0)); - } - let w = waiter.get_or_insert_with(|| { - let idle = slot.idle_spin_ns.load(Ordering::Relaxed); - Waiter::new(if idle == 0 { 0 } else { aw.waits.spin_polls }) - }); - if w.step() == Step::Sleep { - slot.lingering.store(false, Ordering::Release); - let qgen = aw.queue.gen(); - let bgen = aw.bcast.gen(); - slot.park.park_while(|| { - slot.state.load() == st - && aw.queue.gen() == qgen - && aw.bcast.gen() == bgen - && !aw.work_for(id) - }); - waiter = None; + _ => backoff(&mut spins), } } } -/// Sleeps (still reserved) until a section is published, the reservation ends, -/// the member is lent, or the pool detaches it. The handshake lives in -/// [`Parker`]. +/// Sleeps (still reserved) until a section is published, the reservation ends +/// or the thread is lent. The handshake lives in [`Parker`]. #[cfg(feature = "parallel")] fn park_member(slot: &Slot, seen: usize) { - slot.park - .park_while(|| slot.gen.load(Ordering::SeqCst) == seen && slot.state.load() == RESERVED); + slot.park.park_while(|| { + slot.gen.load(Ordering::SeqCst) == seen + && slot.state.load(Ordering::SeqCst) == RESERVED + && !slot.lent.load(Ordering::SeqCst) + }); } /// A unit of dispatched work, living on the leader's stack. @@ -729,7 +752,7 @@ impl Section { } crate::park::Completion::finish( core::ptr::addr_of!((*ptr).done), - waits.as_ref().map(|w| &*w.sync), + waits.as_ref().map(|w| &w.sync), ); } @@ -798,7 +821,7 @@ impl Section { return; } let mut waiter = self.waiter(); - self.done.wait(self.waits().map(|w| &*w.sync), &mut waiter); + self.done.wait(self.waits().map(|w| &w.sync), &mut waiter); if waiter.slept() { if let Some(w) = self.waits() { w.join_sleeps.fetch_add(1, Ordering::Relaxed); @@ -813,7 +836,7 @@ impl Section { let mut waiter = self.waiter(); let passed = self.barrier - .arrive(&self.cancelled, self.waits().map(|w| &*w.sync), &mut waiter); + .arrive(&self.cancelled, self.waits().map(|w| &w.sync), &mut waiter); if waiter.slept() { if let Some(w) = self.waits() { w.barrier_sleeps.fetch_add(1, Ordering::Relaxed); diff --git a/src/exec.rs b/src/exec.rs index 9a76549..e033e6e 100644 --- a/src/exec.rs +++ b/src/exec.rs @@ -637,10 +637,9 @@ impl<'p> Exec<'p> { /// This is the only place a bare width exists. The token is consumed, the /// pool is entered once (inline when the token is serial, has a budget of /// one, or the caller already is a worker of the pool) and `f` receives the - /// budget, for example to set a vendor BLAS thread count. The callback runs - /// on a worker of this pool (inline when the caller is one); a library that - /// schedules with Rayon needs `external_rayon` - /// (feature `rayon`) instead. How the library divides the width is + /// budget, for example to build `faer::Par::rayon(width)` or to set a vendor + /// BLAS thread count. Whatever the library schedules with that width runs on + /// this pool, never on an ambient one. How the library divides the width is /// its own responsibility. Without the `parallel` feature, `f` runs inline /// and receives a width of one. /// @@ -668,7 +667,7 @@ impl<'p> Exec<'p> { let width = self.budget; let _released = self.track.release(); // A team token lends its reserved members to the pool for the - // call: tasks the callback queues on this pool run on them too. + // call: the library's Rayon work runs on them as well. let _lent = self.team.map(|t| LendGuard::new(t.awake, t.ids)); // The calling thread may itself be a pinned participant of this // pool (the member that leads a `fork` child, or a token used @@ -681,48 +680,20 @@ impl<'p> Exec<'p> { f(NonZeroUsize::MIN) } - /// Runs `f(width)` inside a private Rayon pool of this pool's width, so that - /// a callback whose library parallelizes with Rayon (`rayon::join`, - /// parallel iterators) gets `width` threads without starting Rayon's - /// process-global pool. The opt-in Rayon interop layer (feature `rayon`; - /// `docs/design/own-threads.md`): the Rayon pool's threads are separate - /// from this pool's workers, built once per pool when first needed. - /// A serial or budget-one token calls `f(1)` inline. - /// - /// # Examples - /// - /// ``` - /// # #[cfg(feature = "rayon")] { - /// let pool = threadteam::Pool::builder().threads(2).build().unwrap(); - /// let w = threadteam::Exec::new(&pool, 2).unwrap().external_rayon(|w| w.get()); - /// assert_eq!(w, 2); - /// # } - /// ``` - #[cfg(feature = "rayon")] - pub fn external_rayon(self, f: impl FnOnce(NonZeroUsize) -> R + Send) -> R { - match self.pool { - Some(pool) if self.budget.get() > 1 => { - let width = self.budget; - let _released = self.track.release(); - pool.rayon().install(move || f(width)) - } - _ => f(NonZeroUsize::MIN), - } - } - /// Reserves `budget - 1` pool members for the scope of `f` and hands `f` /// the exclusive [`TeamExec`] authority over them and the calling thread. /// /// The calling thread is participant 0 (the leader); the members are - /// pool workers that stay reserved, spinning and serving no other task, + /// pool workers that stay reserved, spinning and serving no Rayon job, /// until `f` returns (or unwinds). A serial or budget-one token yields a /// team of one without touching the pool. /// - /// Acquisition never takes a worker that runs a task: free workers are - /// claimed with a compare-and-swap, and the call waits at most the pool's - /// acquisition timeout + /// Acquisition never waits for a Rayon job that is already running. Idle + /// members of an earlier team (see [`WaitPolicy`]) are claimed with a + /// compare-and-swap; missing ones are requested from the pool, and the + /// call waits at most the pool's acquisition timeout /// ([`PoolBuilder::acquire_timeout`](crate::PoolBuilder::acquire_timeout)) - /// for busy ones to become free. If not all members are there by then, every + /// for idle workers to wake. If not all members are there by then, every /// tentative claim is released and nothing runs. Before `f` is called, /// every participant has acknowledged on its own thread. /// @@ -831,6 +802,7 @@ impl<'p> Exec<'p> { // oversubscribed, whatever the pool's pinning setting. let oversubscribed = crate::pin::oversubscribed(self.budget.get()); let reservation = pool.awake.acquire( + pool, self.budget.get() - 1, policy, pool.awake.spin_budget(oversubscribed), diff --git a/src/ledger.rs b/src/ledger.rs index 8af7edc..20ba8ad 100644 --- a/src/ledger.rs +++ b/src/ledger.rs @@ -145,16 +145,14 @@ pub fn installs(root: u64) -> usize { /// assert!(ledger::global_pool_unstarted()); /// ``` pub fn global_pool_unstarted() -> bool { - // threadteam starts no Rayon pool at all unless the opt-in `rayon` feature - // is on; then the process-global one must still be unstarted. - #[cfg(feature = "rayon")] + #[cfg(feature = "parallel")] { rayon::ThreadPoolBuilder::new() .num_threads(1) .build_global() .is_ok() } - #[cfg(not(feature = "rayon"))] + #[cfg(not(feature = "parallel"))] { true } diff --git a/src/lib.rs b/src/lib.rs index ddbfe7c..01746d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,16 +9,12 @@ //! * [`Exec::new`] mints a token from a pool; [`Exec::serial`] needs none. //! * A token is not `Clone`/`Copy`; entries consume it; splits conserve slots. //! * Serial, budget-one and one-task entries run inline and never reach any -//! scheduler; no worker thread starts before a pool is built, and Rayon is -//! not used at all unless the opt-in `rayon` feature is on. +//! scheduler, so the process-global Rayon pool is never started implicitly. //! //! # Features //! -//! * `parallel` (default): run on the pool's own worker threads, leased from a -//! process-wide reserve (`docs/design/own-threads.md`). Without it the same -//! types exist, every pool has width 1 and every entry runs inline. -//! * `rayon` (opt-in): `Exec::external_rayon`, a private Rayon pool per pool -//! for callbacks that parallelize with Rayon. +//! * `parallel` (default): run on a private pool. Without it the same types +//! exist, every pool has width 1 and every entry runs inline. //! * `ledger`: test support, see the `ledger` module (compiled only with this feature). //! //! # Examples @@ -64,11 +60,6 @@ pub mod test_hooks; #[cfg(feature = "trace")] pub mod trace; mod track; -#[cfg(feature = "parallel")] -mod worker; -#[cfg(feature = "parallel")] -#[doc(hidden)] -pub use worker::spawned_workers; pub use cost::{pays_at, width_for_by, DispatchCosts, DispatchKind, HostClass, WidthCosts, PAYS}; pub use error::{Error, Refused}; @@ -78,11 +69,11 @@ pub use exec::Exec; pub use park::Parker; #[cfg(loom)] #[doc(hidden)] -pub use park::{Barrier, Batch, Broadcast, BroadcastList, Completion, Queue, SlotState, Waiter}; +pub use park::{Barrier, Completion, Waiter}; pub use policy::{raise_current_thread_qos, Qos, WaitOverride, WaitPolicy}; #[doc(hidden)] pub use pool::WaitCounters; -pub use pool::{Pool, PoolBuilder}; +pub use pool::{spawned_workers, Pool, PoolBuilder}; pub use reserve::{IntoTeamError, Team}; pub use team::{Cancelled, Participant, TeamExec}; diff --git a/src/park.rs b/src/park.rs index f59f87a..4fddb34 100644 --- a/src/park.rs +++ b/src/park.rs @@ -19,7 +19,6 @@ //! sequences, not a total order of SeqCst accesses, because `loom` does not model //! the latter and the exact code is model-checked. With nobody parked a wake is //! one uncontended atomic read-modify-write. -#![cfg_attr(not(feature = "parallel"), allow(dead_code))] use crate::sync::{AtomicBool, AtomicUsize, Condvar, Mutex, Ordering}; @@ -241,12 +240,6 @@ impl Completion { self.remaining.0.load(Ordering::Acquire) == 0 } - /// Whether every share has finished, read with the ordering of the - /// `Parker` protocol (for predicates evaluated under its lock). - pub fn is_done_now(&self) -> bool { - self.remaining.0.load(Ordering::SeqCst) == 0 - } - /// One share finished; the last one wakes a sleeping leader through `sync`. /// /// # Safety @@ -338,496 +331,3 @@ impl Barrier { } } } - -// --------------------------------------------------------------------------- -// The native scheduler's shared state (`docs/design/own-threads.md`): a slot's -// state word, the pool's queue of stack batches, and its broadcast list. The -// exact code is what `loom` checks. - -/// The slot is reserved by a team. -pub(crate) const RESERVED: usize = 1; -/// The reserved member is lent to the pool's queue (`Exec::external`). -pub(crate) const LENT: usize = 2; -/// The worker runs a task or a broadcast share. -pub(crate) const BUSY: usize = 4; -/// The pool is being dropped: the worker acknowledges and leaves. -pub(crate) const DETACH: usize = 8; - -/// The state word of one worker slot: bits [`RESERVED`], [`LENT`], [`BUSY`], -/// [`DETACH`]. A team reserves a slot only from `0`; a worker takes work only -/// from `0` (free) or `RESERVED|LENT` (lent), before it removes anything from a -/// queue; ending work clears `BUSY` only, releasing clears `RESERVED|LENT` -/// only, so neither erases the other's bits or `DETACH`. -#[doc(hidden)] -pub struct SlotState(AtomicUsize); - -impl Default for SlotState { - fn default() -> Self { - Self::new() - } -} - -impl SlotState { - /// A free slot. - pub fn new() -> Self { - SlotState(AtomicUsize::new(0)) - } - - /// The current bits. - pub fn load(&self) -> usize { - self.0.load(Ordering::Acquire) - } - - /// Reserves a free, idle slot for a team (`0 -> RESERVED`). - pub fn try_reserve(&self) -> bool { - self.0 - .compare_exchange(0, RESERVED, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - } - - /// The worker takes work: `0 -> BUSY`, or `RESERVED|LENT -> RESERVED|LENT|BUSY`. - pub fn try_take(&self) -> bool { - let cur = self.0.load(Ordering::Acquire); - if cur != 0 && cur != RESERVED | LENT { - return false; - } - self.0 - .compare_exchange(cur, cur | BUSY, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - } - - /// The worker finished its work (clears `BUSY` only). - pub fn end_busy(&self) { - self.0.fetch_and(!BUSY, Ordering::SeqCst); - } - - /// The team releases the slot (clears `RESERVED|LENT` only); a worker still - /// running a lent task stays `BUSY` and is claimable only once it finishes. - pub fn release(&self) { - self.0.fetch_and(!(RESERVED | LENT), Ordering::SeqCst); - } - - /// Lends a reserved member to the pool's queue, or takes it back. - pub fn set_lent(&self, on: bool) { - if on { - self.0.fetch_or(LENT, Ordering::SeqCst); - } else { - self.0.fetch_and(!LENT, Ordering::SeqCst); - } - } - - /// The pool is being dropped. - pub fn detach(&self) { - self.0.fetch_or(DETACH, Ordering::SeqCst); - } -} - -/// A batch of `tasks` tasks published on a pool's [`Queue`]: a stack object of -/// its owner, linked into the queue until its last index is claimed, alive -/// until its last task completed (the owner waits for that). -#[doc(hidden)] -pub struct Batch { - run: unsafe fn(*const (), usize), - data: *const (), - tasks: usize, - /// Next unclaimed index; read and written under the queue lock only. - next: AtomicUsize, - /// Next batch in the queue; under the queue lock only. - link: core::cell::Cell<*const Batch>, - /// Tasks not yet completed. - pub done: Completion, - /// Whom the last completion wakes (the owner's own parker, or the pool's). - waker: *const std::sync::Arc, - /// First panic payload, boxed twice so that it fits an `AtomicPtr`. - panic: core::sync::atomic::AtomicPtr>, -} - -// SAFETY: `data` points to a `Sync` closure of the owner that outlives the -// batch's tasks; `waker` points to an `Arc` the owner keeps alive while it -// waits; `link` is touched under the queue lock only. -unsafe impl Sync for Batch {} -unsafe impl Send for Batch {} - -impl Batch { - /// A batch of `tasks` calls `run(data, index)`, waking `waker` when the - /// last one completes. - /// - /// # Safety - /// - /// `run` and `data` belong together and stay valid until the batch is - /// done; `waker` outlives the owner's wait for the batch. - pub unsafe fn new( - run: unsafe fn(*const (), usize), - data: *const (), - tasks: usize, - waker: *const std::sync::Arc, - ) -> Self { - Batch { - run, - data, - tasks, - next: AtomicUsize::new(0), - link: core::cell::Cell::new(core::ptr::null()), - done: Completion::new(tasks), - waker, - panic: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), - } - } - - /// Runs claimed task `index` and completes it; a panic is caught and kept. - /// - /// # Safety - /// - /// `this` is a batch whose task `index` the caller claimed. It is not - /// dereferenced after the final decrement. - pub unsafe fn run_claimed(this: *const Batch, index: usize) { - let b = &*this; - let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (b.run)(b.data, index))); - if let Err(p) = r { - let fresh = Box::into_raw(Box::new(p)); - if b.panic - .compare_exchange( - core::ptr::null_mut(), - fresh, - core::sync::atomic::Ordering::AcqRel, - core::sync::atomic::Ordering::Acquire, - ) - .is_err() - { - drop(Box::from_raw(fresh)); - } - } - // Keep the waker alive across the final decrement: after it the owner - // may return and free the batch. - let waker = std::sync::Arc::clone(&*b.waker); - Completion::finish(core::ptr::addr_of!((*this).done), Some(&waker)); - } - - /// The first panic of a task, once the batch is done. - pub fn take_panic(&self) -> Option> { - let p = self - .panic - .swap(core::ptr::null_mut(), core::sync::atomic::Ordering::AcqRel); - // SAFETY: a non-null value was stored by `run_claimed` from - // `Box::into_raw` and `swap` hands it to exactly one taker. - (!p.is_null()).then(|| *unsafe { Box::from_raw(p) }) - } -} - -impl Drop for Batch { - fn drop(&mut self) { - drop(self.take_panic()); - } -} - -/// A pool's queue of [`Batch`]es: an intrusive list under a lock, newest -/// first. Claims happen under the lock, and the claim of a batch's last index -/// unlinks it, so nobody reaches a batch after it was unlinked; its owner frees -/// it only once all of its tasks completed. -#[doc(hidden)] -pub struct Queue { - head: Mutex, - /// Bumped (SeqCst) on every publication: what sleeping workers watch. - gen: AtomicUsize, - /// Linked batches, for a lock-free emptiness hint. - linked: AtomicUsize, -} - -struct QueueHead(*const Batch); -// SAFETY: the pointer is only dereferenced under the lock (see `Queue`). -unsafe impl Send for QueueHead {} - -impl Default for Queue { - fn default() -> Self { - Self::new() - } -} - -impl Queue { - /// An empty queue. The lock is touched once here: some platforms allocate - /// it lazily, and no later operation should allocate. - pub fn new() -> Self { - let q = Queue { - head: Mutex::new(QueueHead(core::ptr::null())), - gen: AtomicUsize::new(0), - linked: AtomicUsize::new(0), - }; - drop(q.head.lock().unwrap_or_else(|e| e.into_inner())); - q - } - - /// The publication counter. - pub fn gen(&self) -> usize { - self.gen.load(Ordering::SeqCst) - } - - /// Whether any batch has unclaimed tasks (a hint). - pub fn has_work(&self) -> bool { - self.linked.load(Ordering::Acquire) != 0 - } - - /// Links `b` (with at least one task); the caller then wakes the workers. - pub fn publish(&self, b: &Batch) { - debug_assert!(b.tasks > 0); - let mut head = self.head.lock().unwrap_or_else(|e| e.into_inner()); - b.link.set(head.0); - head.0 = b as *const Batch; - self.linked.fetch_add(1, Ordering::AcqRel); - drop(head); - self.gen.fetch_add(1, Ordering::SeqCst); - } - - /// Claims the next index of the newest batch; the caller must run it with - /// [`Batch::run_claimed`]. - pub fn claim(&self) -> Option<(*const Batch, usize)> { - let mut head = self.head.lock().unwrap_or_else(|e| e.into_inner()); - let b = head.0; - if b.is_null() { - return None; - } - // SAFETY: a linked batch is alive (see `Queue`). - let batch = unsafe { &*b }; - let i = batch.next.load(Ordering::Relaxed); - batch.next.store(i + 1, Ordering::Relaxed); - if i + 1 == batch.tasks { - head.0 = batch.link.get(); - self.linked.fetch_sub(1, Ordering::AcqRel); - } - Some((b, i)) - } - - /// Claims the next index of the caller's own batch `b`, wherever it is in - /// the list; `None` once all of its indices are claimed. - pub fn claim_own(&self, b: &Batch) -> Option { - let mut head = self.head.lock().unwrap_or_else(|e| e.into_inner()); - let i = b.next.load(Ordering::Relaxed); - if i >= b.tasks { - return None; - } - b.next.store(i + 1, Ordering::Relaxed); - if i + 1 == b.tasks { - // Unlink `b`: walk the (short) list to its predecessor. - let me = b as *const Batch; - if head.0 == me { - head.0 = b.link.get(); - } else { - let mut cur = head.0; - // SAFETY: linked batches are alive; `b` is linked (it still - // had unclaimed indices). - unsafe { - while (*cur).link.get() != me { - cur = (*cur).link.get(); - } - (*cur).link.set(b.link.get()); - } - } - self.linked.fetch_sub(1, Ordering::AcqRel); - } - Some(i) - } -} - -/// A broadcast published on a pool's [`BroadcastList`]: share `i` for every -/// worker `i < budget`. A stack object of its caller, linked until its caller -/// unlinks it after all shares completed. -#[doc(hidden)] -pub struct Broadcast { - run: unsafe fn(*const (), usize), - data: *const (), - budget: usize, - /// Publication number; written under the list lock before linking. - seq: core::cell::Cell, - link: core::cell::Cell<*const Broadcast>, - /// Shares not yet completed. - pub done: Completion, - waker: *const std::sync::Arc, - panic: core::sync::atomic::AtomicPtr>, -} - -// SAFETY: as for `Batch`; `seq` and `link` are touched under the list lock. -unsafe impl Sync for Broadcast {} -unsafe impl Send for Broadcast {} - -impl Broadcast { - /// A broadcast of `run(data, i)` to workers `0..budget`. - /// - /// # Safety - /// - /// As [`Batch::new`]. - pub unsafe fn new( - run: unsafe fn(*const (), usize), - data: *const (), - budget: usize, - waker: *const std::sync::Arc, - ) -> Self { - Broadcast { - run, - data, - budget, - seq: core::cell::Cell::new(0), - link: core::cell::Cell::new(core::ptr::null()), - done: Completion::new(budget), - waker, - panic: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), - } - } - - /// Runs worker `i`'s share and completes it; a panic is caught and kept. - /// - /// # Safety - /// - /// `this` is a linked broadcast that targets `i` and whose share `i` the - /// caller took; it is not dereferenced after the final decrement. - pub unsafe fn run_share(this: *const Broadcast, i: usize) { - let b = &*this; - let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (b.run)(b.data, i))); - if let Err(p) = r { - let fresh = Box::into_raw(Box::new(p)); - if b.panic - .compare_exchange( - core::ptr::null_mut(), - fresh, - core::sync::atomic::Ordering::AcqRel, - core::sync::atomic::Ordering::Acquire, - ) - .is_err() - { - drop(Box::from_raw(fresh)); - } - } - let waker = std::sync::Arc::clone(&*b.waker); - Completion::finish(core::ptr::addr_of!((*this).done), Some(&waker)); - } - - /// The first panic of a share, once the broadcast is done. - pub fn take_panic(&self) -> Option> { - let p = self - .panic - .swap(core::ptr::null_mut(), core::sync::atomic::Ordering::AcqRel); - // SAFETY: as in `Batch::take_panic`. - (!p.is_null()).then(|| *unsafe { Box::from_raw(p) }) - } -} - -impl Drop for Broadcast { - fn drop(&mut self) { - drop(self.take_panic()); - } -} - -/// A pool's broadcasts in publication order. Every worker runs the broadcasts -/// that target it in that order (its own sequence cursor), so concurrent -/// broadcasts reach every worker in the same order. Publication and unlinking -/// take the lock; completion does not, so nobody waits while holding it. -#[doc(hidden)] -pub struct BroadcastList { - inner: Mutex, - /// Bumped (SeqCst) on every publication. - gen: AtomicUsize, -} - -struct BroadcastHead { - head: *const Broadcast, - tail: *const Broadcast, - next_seq: u64, -} -// SAFETY: the pointers are only dereferenced under the lock. -unsafe impl Send for BroadcastHead {} - -impl Default for BroadcastList { - fn default() -> Self { - Self::new() - } -} - -impl BroadcastList { - /// An empty list (the lock is touched once, as in [`Queue::new`]). - pub fn new() -> Self { - let l = BroadcastList { - inner: Mutex::new(BroadcastHead { - head: core::ptr::null(), - tail: core::ptr::null(), - next_seq: 0, - }), - gen: AtomicUsize::new(0), - }; - drop(l.inner.lock().unwrap_or_else(|e| e.into_inner())); - l - } - - /// The publication counter. - pub fn gen(&self) -> usize { - self.gen.load(Ordering::SeqCst) - } - - /// Appends `b`; the caller then wakes the workers. - pub fn publish(&self, b: &Broadcast) { - let mut l = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - b.seq.set(l.next_seq); - l.next_seq += 1; - b.link.set(core::ptr::null()); - let me = b as *const Broadcast; - if l.tail.is_null() { - l.head = me; - } else { - // SAFETY: linked broadcasts are alive. - unsafe { (*l.tail).link.set(me) }; - } - l.tail = me; - drop(l); - self.gen.fetch_add(1, Ordering::SeqCst); - } - - /// The next broadcast for worker `i` at or after its cursor `seq`, which - /// moves past every broadcast it inspects (past every publication so far - /// when none is left for `i`). The caller runs the returned - /// share with [`Broadcast::run_share`]. - pub fn next_for(&self, i: usize, seq: &mut u64) -> Option<*const Broadcast> { - let l = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - let mut cur = l.head; - while !cur.is_null() { - // SAFETY: linked broadcasts are alive. - let b = unsafe { &*cur }; - if b.seq.get() >= *seq { - *seq = b.seq.get() + 1; - if i < b.budget { - return Some(cur); - } - } - cur = b.link.get(); - } - // Broadcasts published before and already unlinked are past too, or - // `has_after` would keep reporting them. - *seq = (*seq).max(l.next_seq); - None - } - - /// Whether a broadcast at or after `seq` exists (a hint, lock-free reads - /// of the counter only). - pub fn has_after(&self, seq: u64) -> bool { - (self.gen.load(Ordering::Acquire) as u64) > seq - } - - /// Unlinks `b` once all of its shares completed. - pub fn unlink(&self, b: &Broadcast) { - let mut l = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - let me = b as *const Broadcast; - let mut prev: *const Broadcast = core::ptr::null(); - let mut cur = l.head; - while cur != me { - prev = cur; - // SAFETY: `b` is linked, so the walk reaches it; links are alive. - cur = unsafe { (*cur).link.get() }; - } - let next = b.link.get(); - if prev.is_null() { - l.head = next; - } else { - // SAFETY: as above. - unsafe { (*prev).link.set(next) }; - } - if l.tail == me { - l.tail = prev; - } - } -} diff --git a/src/pin.rs b/src/pin.rs index 28c81d4..b1ae886 100644 --- a/src/pin.rs +++ b/src/pin.rs @@ -126,11 +126,6 @@ impl SavedMask { } } - /// Whether a pin is still in effect (a restore failed or is pending). - pub(crate) fn is_pinned(&self) -> bool { - self.pinned.load(Ordering::Relaxed) - } - /// Restores the saved mask on the calling thread, if this member is pinned. /// /// Best effort: when the kernel refuses the saved mask (a cpuset changed diff --git a/src/policy.rs b/src/policy.rs index 08112c7..64d2742 100644 --- a/src/policy.rs +++ b/src/policy.rs @@ -4,19 +4,24 @@ use core::time::Duration; /// How an idle member behaves after its reservation ended. /// -/// A member that is *reserved* neither runs unrelated tasks nor leaves +/// A member that is *reserved* neither runs unrelated Rayon jobs nor leaves /// on a timeout, whatever the policy says: a policy only governs the idle /// interval after the root reservation of a team ends. The interval is /// measured from that moment, not from the last time the member computed. /// -/// * `idle_spin`: whether an idle member keeps spinning, ready to be claimed -/// again without a wake-up, before it sleeps. Zero puts it to sleep at once; -/// any other value lets it spin for the pool's spin budget of polls -/// ([`PoolBuilder::spin_polls`](crate::PoolBuilder::spin_polls)). -/// * `idle_runs_rayon`: **no effect** since threadteam runs its own worker -/// threads (`docs/design/own-threads.md`): an idle worker always serves the -/// pool's queued tasks, and a worker that runs one is not claimable until it -/// finishes. Kept so that existing code compiles. +/// * `idle_spin`: how long an idle member keeps spinning, ready to be claimed +/// again without a Rayon wake-up, before it returns its thread to Rayon. +/// Zero returns it immediately. The pool's spin budget of polls +/// ([`PoolBuilder::spin_polls`](crate::PoolBuilder::spin_polls)) bounds it +/// too, whichever ends first. +/// * `idle_runs_rayon`: whether the spinning member runs queued Rayon jobs in +/// the meantime. When it does, a job it picked up runs to completion +/// (including nested work) before the member is claimable again, and an +/// acquisition that finds the member in such a job refuses instead of +/// waiting for it. +/// +/// While a member spins with `idle_runs_rayon == false`, its thread serves no +/// Rayon job of the pool for up to `idle_spin`. /// /// # Examples /// @@ -39,7 +44,7 @@ pub struct WaitPolicy { } impl WaitPolicy { - /// The pool default: an idle spin of 200 ms (bounded by the spin budget). + /// The pool default: spin for 200 ms, run no Rayon jobs while idle. pub const DEFAULT: WaitPolicy = WaitPolicy { idle_spin: Duration::from_millis(200), idle_runs_rayon: false, @@ -53,12 +58,12 @@ impl WaitPolicy { } } - /// Spin time of an unreserved idle member before it sleeps. + /// Spin time of an unreserved idle member before it returns to Rayon. pub const fn idle_spin(&self) -> Duration { self.idle_spin } - /// The `idle_runs_rayon` setting (no effect; see [`WaitPolicy`]). + /// Whether an unreserved idle member runs Rayon jobs while it spins. pub const fn idle_runs_rayon(&self) -> bool { self.idle_runs_rayon } @@ -117,7 +122,7 @@ impl WaitOverride { self } - /// Overrides `idle_runs_rayon` (no effect; see [`WaitPolicy`]). + /// Overrides whether idle members run Rayon jobs. #[must_use] pub const fn idle_runs_rayon(mut self, on: bool) -> Self { self.idle_runs_rayon = Some(on); diff --git a/src/pool.rs b/src/pool.rs index 9658d30..b8ebdd1 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -61,20 +61,30 @@ const SPIN_POLLS_DEFAULT: u32 = if cfg!(target_arch = "aarch64") { /// caller), where each call acquires its members with /// [`Exec::team`](crate::Exec::team) or uses plain tokens. /// -/// Dropping a `Pool` (or the `Team` that owns it) detaches its workers and -/// waits until each one acknowledged; after that no worker touches the pool. -/// Workers of a pool with default thread settings are *shared*: they go back, -/// asleep, to a process-wide reserve that the next pool reuses, so their -/// threads keep running. A pool with a [`thread_name`](PoolBuilder::thread_name), -/// an [`on_start`](PoolBuilder::on_start) hook, a stack size or a QoS of its -/// own has *dedicated* workers, which `drop` joins: when it returns, their -/// thread-local destructors (including state set up by the hook) have run. +/// Dropping a `Pool` (or the `Team` that owns it) ends its team members. What +/// happens to the worker threads depends on how the pool was built: +/// +/// * A pool with a [`thread_name`](PoolBuilder::thread_name), an +/// [`on_start`](PoolBuilder::on_start) hook, a +/// [`stack_size`](PoolBuilder::stack_size) or a [`qos`](PoolBuilder::qos) of +/// its own has **dedicated** workers: the drop shuts them down and joins +/// them, so when `drop` returns every worker thread has exited and its +/// thread-local destructors (including state set up by the hook) have run. +/// * A pool with the default thread settings has **shared** workers: the drop +/// waits until its members left, then keeps the idle workers in a +/// process-wide cache, keyed by width. The next `build` of a default pool of +/// the same width takes them instead of starting threads, and gives each +/// worker the calling thread's CPU set. A cached set of workers belongs to +/// one pool at a time. The cache holds at most as many threads as the +/// larger of the available parallelism and the widest pool built so far; +/// the oldest entries beyond that are joined. +/// /// Without the `parallel` feature there are no workers and nothing is joined. /// /// A pool dropped from one of its own workers cannot wait for itself: that drop -/// neither waits for nor joins the calling worker, which acknowledges and leaves -/// after its task returns. Drop the pool from a plain thread when the guarantee -/// matters. +/// signals shutdown and returns without joining, so the workers exit shortly +/// after instead of before the return. Drop the pool from a plain thread when +/// the guarantee matters. /// /// # Examples /// @@ -86,33 +96,26 @@ const SPIN_POLLS_DEFAULT: u32 = if cfg!(target_arch = "aarch64") { /// # Ok::<(), threadteam::Error>(()) /// ``` pub struct Pool { - /// The leased workers (`docs/design/own-threads.md`). + /// `None` only inside `drop`, which moves the workers to the cache. #[cfg(feature = "parallel")] - workers: Vec>, - /// Join handles of dedicated workers (empty for shared ones). + workers: Option, + /// Default thread settings: the workers go to the cache on drop. #[cfg(feature = "parallel")] - joins: std::sync::Mutex>>, + shared: bool, pub(crate) width: NonZeroUsize, pub(crate) awake: Arc, - /// The private Rayon pool of `Exec::external_rayon` (feature `rayon`). - #[cfg(feature = "rayon")] - rayon: std::sync::OnceLock, } -#[cfg(feature = "rayon")] impl Pool { - /// This pool's private Rayon pool, built on first use. - pub(crate) fn rayon(&self) -> &rayon::ThreadPool { - self.rayon.get_or_init(|| { - rayon::ThreadPoolBuilder::new() - .num_threads(self.width.get()) - .build() - .expect("a private Rayon pool for external_rayon") - }) + /// The private Rayon pool. + #[cfg(feature = "parallel")] + pub(crate) fn inner(&self) -> &rayon::ThreadPool { + match &self.workers { + Some(w) => &w.inner, + None => unreachable!("the workers leave the pool only in drop"), + } } -} -impl Pool { /// Starts configuring a pool. /// /// # Examples @@ -159,10 +162,10 @@ impl Pool { self.awake.defaults } - /// Asks idle team members to stop spinning and sleep now. + /// Asks idle team members to return their threads to Rayon now. /// - /// Members that are idle sleep at once; members that are reserved stop - /// spinning after their reservation ends, so this never revokes a live + /// Members that are idle leave at once; members that are reserved leave + /// when their reservation ends, so this never revokes a live /// [`TeamExec`](crate::TeamExec). The call does not wait for anything, /// so it is safe to make from inside a reservation. A team acquired later /// starts its members afresh. @@ -226,7 +229,7 @@ impl Pool { } /// Number of workers currently kept awake as team members, reserved or - /// idle and still awake. Zero once released members went to sleep. A diagnostic: the value + /// idle. Zero once idle members returned to Rayon. A diagnostic: the value /// can change at any moment. /// /// # Examples @@ -243,37 +246,124 @@ impl Pool { impl Drop for Pool { fn drop(&mut self) { + // Idle members leave their loops so that the workers can be joined. + self.awake.shutdown.store(true, Ordering::Release); #[cfg(feature = "parallel")] - { - // Detach every worker, then wait for their acknowledgements: after - // its acknowledgement a worker touches nothing of this pool. A - // worker of this pool that drops it (its own task owned the pool) - // acknowledges once that task returns; nobody waits for it. - for s in &*self.awake.slots { - s.state.detach(); - s.park.wake(); - } - let me = self.awake.slot_of_current(); - for (i, s) in self.awake.slots.iter().enumerate() { - // A worker that has not attached yet (a dedicated one still in - // its start hook) attaches, sees `DETACH` and acknowledges too. - if Some(i) == me { - continue; + if let Some(workers) = self.workers.take() { + // A pool dropped on one of its own workers cannot wait for that + // worker's member loop: its workers are not reused. + if self.shared && workers.inner.current_thread_index().is_none() { + // No member loop of this pool may run on the threads when the + // next pool takes them. + let mut spins = 0u32; + while self.awake.resident() > 0 { + if spins < 64 { + spins += 1; + std::thread::yield_now(); + } else { + std::thread::sleep(Duration::from_micros(50)); + } } - self.awake - .waits - .sync - .park_while(|| !s.acked.load(Ordering::SeqCst)); + cache::put(workers); } - let joins = std::mem::take(&mut *self.joins.lock().unwrap_or_else(|e| e.into_inner())); - let my_thread = std::thread::current().id(); - for j in joins { - if j.thread().id() != my_thread { - // A worker that panicked has still exited. - let _ = j.join(); - } + } + } +} + +/// A Rayon pool and its threads. Field order matters: `inner` is dropped +/// (shutdown signalled) before `_joiner` waits for the threads. +#[cfg(feature = "parallel")] +struct Workers { + inner: rayon::ThreadPool, + _joiner: Joiner, +} + +/// Idle shared workers, kept between pools of the same width. +#[cfg(feature = "parallel")] +mod cache { + use super::Workers; + use core::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Mutex, PoisonError}; + + static IDLE: Mutex> = Mutex::new(Vec::new()); + /// The widest shared pool built so far. + static WIDEST: AtomicUsize = AtomicUsize::new(0); + + /// Takes the idle workers of a pool of `width` threads, if any are cached. + pub(super) fn take(width: usize) -> Option { + WIDEST.fetch_max(width, Ordering::Relaxed); + let mut idle = IDLE.lock().unwrap_or_else(PoisonError::into_inner); + let at = idle + .iter() + .position(|w| w.inner.current_num_threads() == width)?; + Some(idle.remove(at)) + } + + /// Keeps `workers` for a later pool; joins the oldest entries beyond the + /// cap (the larger of the available parallelism and the widest pool). + pub(super) fn put(workers: Workers) { + let cap = std::thread::available_parallelism() + .map_or(1, core::num::NonZeroUsize::get) + .max(WIDEST.load(Ordering::Relaxed)); + let evicted: Vec = { + let mut idle = IDLE.lock().unwrap_or_else(PoisonError::into_inner); + idle.push(workers); + let mut total: usize = idle.iter().map(|w| w.inner.current_num_threads()).sum(); + let mut n = 0; + while total > cap && n + 1 < idle.len() { + total -= idle[n].inner.current_num_threads(); + n += 1; } - drop(core::mem::take(&mut self.workers)); + idle.drain(..n).collect() + }; + // Joined outside the lock. + drop(evicted); + } + + /// The widest shared pool built so far (tests). + #[cfg(test)] + pub(super) fn widest() -> usize { + WIDEST.load(Ordering::Relaxed) + } + + /// Threads currently cached (tests). + #[cfg(test)] + pub(super) fn cached_threads() -> usize { + let idle = IDLE.lock().unwrap_or_else(PoisonError::into_inner); + idle.iter().map(|w| w.inner.current_num_threads()).sum() + } +} + +/// Worker threads started by shared and dedicated pools since the process +/// started; a diagnostic for the reuse tests and benchmarks. +#[doc(hidden)] +pub fn spawned_workers() -> usize { + SPAWNED.load(Ordering::Relaxed) +} + +static SPAWNED: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + +/// The worker threads of a pool, joined when it is dropped. +#[cfg(feature = "parallel")] +struct Joiner { + handles: std::sync::Arc>>>, +} + +#[cfg(feature = "parallel")] +impl Drop for Joiner { + fn drop(&mut self) { + let handles = match self.handles.lock() { + Ok(mut guard) => std::mem::take(&mut *guard), + Err(poisoned) => std::mem::take(&mut *poisoned.into_inner()), + }; + let me = std::thread::current().id(); + // A worker of this pool cannot join itself: skip joining entirely. + if handles.iter().any(|h| h.thread().id() == me) { + return; + } + for handle in handles { + // A worker that panicked has still exited; nothing to report. + let _ = handle.join(); } } } @@ -398,9 +488,10 @@ impl PoolBuilder { self } - /// Default idle spin of an unreserved team member before it sleeps. - /// Defaults to 200 ms; zero puts it to sleep immediately. Any nonzero value - /// lets it spin for the pool's spin budget ([`spin_polls`](Self::spin_polls)). + /// Default spin time of an unreserved idle team member before it returns + /// its thread to Rayon. Defaults to 200 ms; zero returns it immediately. + /// The member also returns once it has polled for the pool's spin budget + /// ([`spin_polls`](Self::spin_polls)), whichever comes first. /// /// Operations may override it with [`WaitOverride`](crate::WaitOverride). /// A reserved member never leaves on a timeout. @@ -420,9 +511,11 @@ impl PoolBuilder { self } - /// **No effect** since threadteam runs its own worker threads - /// (`docs/design/own-threads.md`): an idle worker always serves the pool's - /// queued tasks. Kept so that existing code compiles. + /// Default for whether unreserved idle members run Rayon jobs while they + /// spin. Defaults to `false`: native team execution keeps them for the + /// next reservation. `true` is the compatibility mode in which an idle + /// member serves Rayon work (an acquisition then refuses a member that is + /// inside such a job instead of waiting for it). /// /// # Examples /// @@ -531,8 +624,7 @@ impl PoolBuilder { /// coordinating thread, so that thread must keep its CPU set. /// /// A member saves its previous mask in its own slot and restores it - /// *itself* before it stops being reserved: before it serves the pool's - /// queue again, before it leaves the pool, and + /// *itself* before it stops being reserved: before it returns to Rayon, and /// before its thread serves work that is not its team's — while it is lent /// by [`Exec::external`](crate::Exec::external), and when it runs such work /// itself (a member that leads a `fork` child and calls `external`). No @@ -597,8 +689,10 @@ impl PoolBuilder { /// and then sleeps until woken (`docs/design/waiting.md`; the shape of /// libgomp's `GOMP_SPINCOUNT`). A count, so a waiter that is descheduled /// does not spend it while it waits for its CPU. Zero sleeps after the - /// yields. An oversubscribed scope uses at most 100 polls. It also bounds - /// how long an idle worker keeps polling before it sleeps. + /// yields. An oversubscribed scope, or a pool whose waiters were recently + /// descheduled, uses at most 100 polls. It also bounds how long an idle + /// member released by a team keeps polling before it returns to Rayon, + /// together with [`idle_spin`](Self::idle_spin). /// /// # Examples /// @@ -638,60 +732,93 @@ impl PoolBuilder { } #[cfg(feature = "parallel")] - fn build_checked(self) -> Result { + fn build_checked(mut self) -> Result { let n = self .threads .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, NonZeroUsize::get)); - // Workers of the reserve have the default stack, the platform's QoS - // and no hooks; any other setting gets dedicated workers. let shared = self.thread_name.is_none() && self.on_start.is_none() && self.stack_size == DEFAULT_STACK_BYTES && self.qos.is_none(); - let on_start: Option> = self.on_start.map(Arc::from); - let how = crate::worker::Spawn { - stack: self.stack_size, - name: self.thread_name.as_deref(), - on_start, - qos: self.qos, - shared, - }; - let (workers, joins) = - crate::worker::obtain(n, &how).map_err(|e| Error::PoolBuild(e.to_string()))?; - let width = NonZeroUsize::new(n).unwrap_or(NonZeroUsize::MIN); - let awake = Arc::new(Awake::new( - width.get(), - self.wait, - self.acquire_timeout, - self.costs, - self.pin_participants, - self.spin_polls, - )); - // A reused worker takes the caller's CPU set, as a thread it spawned - // would. A dedicated worker was spawned by this caller and inherited - // it already; resetting it would undo the pinning of an `on_start` hook. - let mask = if shared { - crate::pin::read_here() - } else { - None + let workers = match shared.then(|| cache::take(n)).flatten() { + Some(workers) => { + // A reused worker takes the caller's CPU set, as a thread the + // caller spawned would. + if let Some(mask) = crate::pin::read_here() { + workers.inner.broadcast(|_| { + crate::pin::set_here(&mask); + }); + } + workers + } + None => self.spawn(n)?, }; - for (id, w) in workers.iter().enumerate() { - w.assign(crate::worker::Lease { - aw: Arc::clone(&awake), - id, - mask, - }); - } - // Workers attach on their own (after a start hook, for dedicated - // ones); until then a slot is not claimable by a team, and queued work - // waits for them. + let width = + NonZeroUsize::new(workers.inner.current_num_threads()).unwrap_or(NonZeroUsize::MIN); Ok(Pool { - workers, - joins: std::sync::Mutex::new(joins), + workers: Some(workers), + shared, width, - awake, - #[cfg(feature = "rayon")] - rayon: std::sync::OnceLock::new(), + awake: Arc::new(Awake::new( + width.get(), + self.wait, + self.acquire_timeout, + self.costs, + self.pin_participants, + self.spin_polls, + )), + }) + } + + /// Starts `n` new worker threads. + #[cfg(feature = "parallel")] + fn spawn(&mut self, n: usize) -> Result { + let handles = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let sink = std::sync::Arc::clone(&handles); + let qos = self.qos.or(Qos::PLATFORM_DEFAULT); + let mut builder = rayon::ThreadPoolBuilder::new() + .num_threads(n) + .stack_size(self.stack_size) + .spawn_handler(move |thread| { + let mut b = std::thread::Builder::new(); + if let Some(name) = thread.name() { + b = b.name(name.to_owned()); + } + if let Some(size) = thread.stack_size() { + b = b.stack_size(size); + } + SPAWNED.fetch_add(1, Ordering::Relaxed); + let handle = b.spawn(move || { + if let Some(q) = qos { + q.apply_to_current_thread(); + } + thread.run() + })?; + match sink.lock() { + Ok(mut guard) => guard.push(handle), + Err(poisoned) => poisoned.into_inner().push(handle), + } + Ok(()) + }); + if let Some(name) = self.thread_name.take() { + builder = builder.thread_name(name); + } + if let Some(hook) = self.on_start.take() { + builder = builder.start_handler(hook); + } + // On failure `builder` drops its handler, but threads already started + // are signalled by rayon and detached; join them here for determinism. + let built = builder.build(); + let inner = match built { + Ok(inner) => inner, + Err(e) => { + drop(Joiner { handles }); + return Err(Error::PoolBuild(e.to_string())); + } + }; + Ok(Workers { + inner, + _joiner: Joiner { handles }, }) } @@ -730,124 +857,39 @@ mod tests { use std::time::Duration; /// The last reference is dropped on a worker of the pool itself: the drop - /// must not wait for its own thread (it detaches the others, waits for - /// their acknowledgements and skips itself; the worker acknowledges and - /// leaves after its task). Public entries always borrow the pool while - /// their tasks run, so the task here is published on the pool's queue - /// directly, as the Rayon version used `spawn`. + /// must not wait for its own thread. #[test] fn drop_from_a_worker_of_the_pool_does_not_hang() { - for shared in [true, false] { - let mut b = Pool::builder().threads(2); - if !shared { - b = b.thread_name(|i| format!("self-drop-{i}")); - } - let pool = Arc::new(b.build().unwrap()); - let aw = Arc::clone(&pool.awake); - let last = std::sync::Mutex::new(Some(Arc::clone(&pool))); - let task = |_: usize| { - // The last reference: runs `Pool::drop` on this worker. - drop(last.lock().unwrap().take()); - }; - unsafe fn call(data: *const (), i: usize) { - (*data.cast::())(i) - } - fn call_of(_: &F) -> unsafe fn(*const (), usize) { - call:: - } - // SAFETY: `task` and the waker (held by `aw`) outlive the batch, - // which this test waits for before they go out of scope. - let batch = unsafe { - crate::park::Batch::new( - call_of(&task), - (&task as *const _ as *const ()).cast(), - 1, - &aw.waits.sync, - ) - }; - aw.queue.publish(&batch); - aw.wake_for_work(); - drop(pool); - let (tx, rx) = mpsc::channel::<()>(); - std::thread::scope(|s| { - s.spawn(|| { - let mut w = crate::park::Waiter::new(0); - batch.done.wait(Some(&aw.waits.sync), &mut w); - tx.send(()).unwrap(); - }); - rx.recv_timeout(Duration::from_secs(10)) - .expect("drop on a worker hung"); - }); - assert!( - last.lock().unwrap().is_none(), - "the pool was dropped by its worker" - ); - } - } - - /// Waits nest: an inner wait that returns leaves the outer one wakeable - /// for new work of the pool (`wake_for_work` looks at `helping`). - #[test] - fn a_nested_wait_keeps_the_outer_wait_wakeable() { - let pool = Pool::builder().threads(1).build().unwrap(); - let aw = &pool.awake; - let helping = || aw.slots[0].helping.load(Ordering::SeqCst); - // Both conditions hold at once, so neither wait runs work of slot 0 - // (whose worker serves it meanwhile). - aw.help_while(0, || { - aw.help_while(0, || true); - assert!(helping(), "the inner wait cleared the outer one's flag"); - true - }); - assert!(!helping()); - } - - /// A broadcast that completed and was unlinked before a worker outside - /// its budget looked: that worker's cursor still moves past it, so it - /// does not see pending work forever (and never parks). - #[test] - fn a_cursor_moves_past_a_broadcast_unlinked_before_it_looked() { - unsafe fn nothing(_: *const (), _: usize) {} - let waker = Arc::new(crate::park::Parker::new()); - let list = crate::park::BroadcastList::new(); - // SAFETY: `waker` outlives the broadcast; `nothing` reads no data. - let b = unsafe { crate::park::Broadcast::new(nothing, core::ptr::null(), 2, &waker) }; - list.publish(&b); - for i in 0..2 { - let mut seq = 0; - let share = list.next_for(i, &mut seq).expect("a share for i"); - // SAFETY: linked, alive, and targets `i`. - unsafe { crate::park::Broadcast::run_share(share, i) }; - } - list.unlink(&b); - let mut seq = 0; - assert!(list.has_after(seq)); - assert!(list.next_for(2, &mut seq).is_none()); - assert!(!list.has_after(seq), "the cursor stayed before it"); - } - - /// A pool built, used and dropped inside a task of another pool: the drop - /// waits for the inner pool's workers only, never for the outer worker it - /// runs on (a pool cannot be dropped from one of its own workers: every - /// task's caller borrows the pool). - #[test] - fn a_pool_dropped_inside_a_task_of_another_pool_does_not_hang() { - let outer = Pool::builder().threads(2).build().unwrap(); + let pool = Arc::new(Pool::builder().threads(2).build().unwrap()); + let last = Arc::clone(&pool); + let (gate_tx, gate_rx) = mpsc::channel::<()>(); let (done_tx, done_rx) = mpsc::channel::<()>(); - let done_tx = std::sync::Mutex::new(Some(done_tx)); - crate::Exec::new(&outer, 2).unwrap().fork(2, |i, _| { - if i == 0 { - let inner = Arc::new(Pool::builder().threads(2).build().unwrap()); - crate::Exec::new(&inner, 2).unwrap().fork(2, |_, _| {}); - drop(inner); - if let Some(tx) = done_tx.lock().unwrap().take() { - tx.send(()).unwrap(); - } - } + pool.inner().spawn(move || { + gate_rx.recv().unwrap(); + drop(last); // the last reference: runs `Pool::drop` on this worker + done_tx.send(()).unwrap(); }); + drop(pool); + gate_tx.send(()).unwrap(); done_rx .recv_timeout(Duration::from_secs(10)) - .expect("a drop inside another pool's task hung"); + .expect("drop on a worker hung"); + } + + /// The cache never holds more threads than its cap, whatever the number + /// of pools dropped. + #[test] + fn the_cache_keeps_at_most_its_cap_of_threads() { + let cap = std::thread::available_parallelism().map_or(1, NonZeroUsize::get); + let pools: Vec = (0..cap + 2) + .map(|_| Pool::builder().threads(1).build().unwrap()) + .collect(); + drop(pools); + // Other tests of this binary build wider pools concurrently, which + // can only raise the cap. + let cached = cache::cached_threads(); + let cap = cap.max(cache::widest()); + assert!(cached <= cap, "{cached} cached threads, cap {cap}"); } } diff --git a/src/reserve.rs b/src/reserve.rs index b5253d7..23a57b2 100644 --- a/src/reserve.rs +++ b/src/reserve.rs @@ -36,21 +36,22 @@ impl Drop for LeaderGuard<'_> { /// [`exec`](Team::exec). A minted token is an ordinary [`Exec`] that carries /// the team's authority: /// -/// * `fork` and `for_ranges` dispatch as team sections: no queue entry, no +/// * `fork` and `for_ranges` dispatch as team sections: no Rayon entry, no /// wake-up, no allocation. Children of `fork` carry the sub-team of their /// share, so nested forks stay on reserved members. /// * `broadcast` runs on the reserved members (a `fork` child may broadcast /// too: its members are its own); `can_broadcast` is always true. /// * `team` returns the carried (sub)team at once, without acquisition. /// * `external` hands the width as on any token and **lends the token's -/// members to the pool for the call**: they run the pool's queued tasks -/// instead of idling, and resume team service when the call returns. A lent -/// task that is still running delays the next section until it finishes. +/// members to the pool for the call**: they run the pool's Rayon jobs (the +/// library's work) instead of idling, and resume team service when the call +/// returns. A lent job that is still running delays the next section until it +/// finishes. /// /// Reserved members wait for work in staged episodes: they spin for the pool's /// spin budget ([`PoolBuilder::spin_polls`](crate::PoolBuilder::spin_polls)), -/// yield a few times, then **park** while staying reserved; they serve nothing -/// else until the `Team` is dropped (`docs/design/waiting.md`). The next +/// yield a few times, then **park** while staying reserved; they do not return +/// to Rayon until the `Team` is dropped (`docs/design/waiting.md`). The next /// dispatch wakes a parked member through its slot's parker, which costs a /// syscall on the first dispatch after a park (`benches/wake_probe.rs`) and one /// atomic read-modify-write for a warm one. @@ -67,8 +68,8 @@ impl Drop for LeaderGuard<'_> { /// A `Team` **owns** its [`Pool`] and has no lifetime: an engine stores a `Team` /// (or a plain `Pool` when it wants no team). `Pool` is not `Clone`, no method /// hands out a second handle, and [`pool`](Team::pool) only borrows. Dropping -/// the `Team` releases the members and then drops the pool (see [`Pool`] for -/// what that does to its workers); [`into_pool`](Team::into_pool) releases the members and gives the +/// the `Team` releases the members and then drops the pool, which joins the +/// workers; [`into_pool`](Team::into_pool) releases the members and gives the /// pool back instead. Tokens borrow the `Team`, so none outlives it. /// /// # Examples @@ -150,7 +151,7 @@ impl Pool { /// long as it lives. /// /// `width` counts the leader and is clamped to the pool width; zero is - /// treated as one. Acquisition never takes a worker that runs a task (see + /// treated as one. Acquisition never waits for a running Rayon job (see /// [`Exec::team`](crate::Exec::team)). Members park after the pool's spin /// budget without work ([`PoolBuilder::spin_polls`](crate::PoolBuilder::spin_polls)). /// @@ -191,7 +192,7 @@ impl Pool { let reservation = match self .awake - .acquire(width - 1, release, self.awake.spin_budget(false)) + .acquire(&self, width - 1, release, self.awake.spin_budget(false)) { Ok(r) => r, Err(reason) => return Err(IntoTeamError { pool: self, reason }), diff --git a/src/sched.rs b/src/sched.rs index efa275f..9861edd 100644 --- a/src/sched.rs +++ b/src/sched.rs @@ -10,7 +10,7 @@ use crate::track::Track; /// Whether the calling thread is a worker of `pool`. #[cfg(feature = "parallel")] pub(crate) fn in_pool(pool: &Pool) -> bool { - pool.awake.slot_of_current().is_some() + pool.inner().current_thread_index().is_some() } #[cfg(not(feature = "parallel"))] @@ -31,108 +31,42 @@ pub(crate) fn in_pool(_pool: &Pool) -> bool { /// re-pins at the next pin section. #[cfg(feature = "parallel")] pub(crate) fn unpin_worker(pool: &Pool) { - if let Some(worker) = pool.awake.slot_of_current() { + if let Some(worker) = pool.inner().current_thread_index() { pool.awake.unpin_member(worker); } } -/// Who waits for a batch of `pool`, and how (`docs/design/own-threads.md`). -#[cfg(feature = "parallel")] -enum Waiter { - /// A worker of `pool` itself (slot index): it runs its own batch's tasks - /// first, then serves the pool while it waits. - Own(usize), - /// A worker of another pool: it serves its own pool while it waits, so a - /// cross-pool chain (A -> B -> A) always makes progress. - Foreign(std::sync::Arc, usize), - /// A plain thread: it sleeps on the pool's parker. - Outside, -} - -#[cfg(feature = "parallel")] -fn waiter_of(pool: &Pool) -> Waiter { - if let Some(i) = pool.awake.slot_of_current() { - Waiter::Own(i) - } else if let Some((aw, i)) = crate::worker::current_worker() { - Waiter::Foreign(aw, i) - } else { - Waiter::Outside - } -} - -#[cfg(feature = "parallel")] -impl Waiter { - /// The parker the last completion wakes. - fn parker<'a>(&'a self, pool: &'a Pool) -> &'a std::sync::Arc { - match self { - Waiter::Own(i) => &pool.awake.slots[*i].park, - Waiter::Foreign(aw, i) => &aw.slots[*i].park, - Waiter::Outside => &pool.awake.waits.sync, - } - } - - /// Waits until `done` holds. - fn wait(&self, pool: &Pool, done: &crate::park::Completion) { - match self { - Waiter::Own(i) => pool.awake.help_while(*i, || done.is_done_now()), - Waiter::Foreign(aw, i) => aw.help_while(*i, || done.is_done_now()), - Waiter::Outside => { - let mut w = crate::park::Waiter::new(pool.awake.waits.spin_polls); - done.wait(Some(&pool.awake.waits.sync), &mut w); - } - } - } -} - -/// Runs `task(0..tasks)` on `pool` and waits for all of them. +/// Runs `task(0..tasks)` concurrently on `pool` and waits for all of them. /// -/// The tasks form one batch on the caller's stack, published on the pool's -/// queue, so a steady-state call allocates nothing. +/// The tasks form a recursive binary `rayon::join` tree, whose jobs live on the +/// stack, so a steady-state call allocates nothing. /// -/// * Caller is a worker of `pool`: it runs the batch's unclaimed tasks itself, -/// then serves the pool until the others finish. -/// * Caller is a worker of another pool: the pool's workers run the tasks; the -/// caller serves its own pool meanwhile. -/// * Caller is a plain thread: every task runs on a worker of `pool` and the -/// caller sleeps until the last one finishes. +/// * Caller is a worker of `pool`: the tree runs directly on the caller. +/// Nothing is installed. +/// * Caller is a plain thread or a worker of another pool: one +/// `ThreadPool::install` (a stack job) around the tree; every task runs on a +/// worker of `pool` and the caller waits. /// -/// A panicking task does not stop the others; the first panic is resumed once -/// all of them finished. +/// `rayon::join` is only reached on a worker of `pool` (inside `install`, or +/// when the caller already is one), so it never touches the global registry. #[cfg(feature = "parallel")] pub(crate) fn run_tasks(pool: &Pool, tasks: usize, track: &Track, task: F) { - if tasks == 0 { - return; - } - unsafe fn call(data: *const (), i: usize) { - // SAFETY: `data` is the `&F` of `run_tasks`, alive until the batch is done. - (*data.cast::())(i) - } - let waiter = waiter_of(pool); - if !matches!(waiter, Waiter::Own(_)) { + if in_pool(pool) { + join_tree(0, tasks, &task); + } else { track.note_install(); + pool.inner().install(|| join_tree(0, tasks, &task)); } - // SAFETY: `call::` and `task` belong together; both outlive the batch, - // which this function waits for; the waker is kept alive by `waiter` (or - // the pool) until the wait below returns. - let batch = unsafe { - crate::park::Batch::new( - call::, - (&task as *const F).cast(), - tasks, - waiter.parker(pool), - ) - }; - pool.awake.queue.publish(&batch); - pool.awake.wake_for_work(); - if let Waiter::Own(_) = waiter { - while let Some(k) = pool.awake.queue.claim_own(&batch) { - // SAFETY: index `k` of our own batch was claimed here. - unsafe { crate::park::Batch::run_claimed(&batch, k) }; - } - } - waiter.wait(pool, &batch.done); - if let Some(payload) = batch.take_panic() { - std::panic::resume_unwind(payload); +} + +/// Runs `task(lo..hi)` as a balanced join tree. Must run on a pool worker. +#[cfg(feature = "parallel")] +fn join_tree(lo: usize, hi: usize, task: &F) { + if hi - lo == 1 { + task(lo); + } else { + let mid = lo + (hi - lo) / 2; + rayon::join(|| join_tree(lo, mid, task), || join_tree(mid, hi, task)); } } @@ -143,88 +77,32 @@ pub(crate) fn run_tasks(_pool: &Pool, tasks: usize, _track: } } -/// Runs `f` on a worker of `pool`: inline when the caller is one, otherwise as -/// a one-task batch while the caller waits. +/// Runs `f` on a worker of `pool`, installing only when the caller is not one. #[cfg(feature = "parallel")] pub(crate) fn enter(pool: &Pool, track: &Track, f: impl FnOnce() -> R + Send) -> R { if in_pool(pool) { - return f(); - } - // No lock: a `Mutex` allocates on its first lock on some platforms - // (macOS), and this runs on every call. - let job = Handoff(core::cell::UnsafeCell::new(Some(f))); - let out = Handoff(core::cell::UnsafeCell::new(None)); - run_tasks(pool, 1, track, |_| { - // SAFETY: the batch has one index, claimed exactly once (under the - // queue lock), so this body runs once; the caller reads `out` only - // after the batch's completion, which orders this write before it. - let f = unsafe { job.take() }; - if let Some(f) = f { - let r = f(); - // SAFETY: as above. - unsafe { out.put(r) }; - } - }); - let r = out.0.into_inner(); - r.expect("the task ran") -} - -/// A value passed into and out of the single task of [`enter`]'s batch: that -/// task is its only user until the batch completes. -#[cfg(feature = "parallel")] -struct Handoff(core::cell::UnsafeCell>); - -// SAFETY: one thread at a time touches the value: the task while it runs, the -// caller after the batch completed (see `enter`); `T: Send` lets it move. -#[cfg(feature = "parallel")] -unsafe impl Sync for Handoff {} - -#[cfg(feature = "parallel")] -impl Handoff { - /// # Safety - /// No other thread accesses the value meanwhile. - unsafe fn take(&self) -> Option { - (*self.0.get()).take() - } - - /// # Safety - /// As [`take`](Self::take). - unsafe fn put(&self, v: T) { - *self.0.get() = Some(v); + f() + } else { + track.note_install(); + pool.inner().install(f) } } -/// Runs `f(i)` for every worker index `i < budget` at once, each on worker -/// `i` of `pool`. The caller is never a worker of `pool` (`Exec::broadcast` -/// refuses that); it waits, serving its own pool if it is a worker of one. +/// Runs `f(i)` for every worker index `i < budget` at once, or reports that the +/// caller already is a worker of `pool`. #[cfg(feature = "parallel")] pub(crate) fn broadcast(pool: &Pool, track: &Track, budget: usize, f: impl Fn(usize) + Sync) { + // No gate is needed: rayon pushes the per-worker jobs of one broadcast + // under a single lock, so every worker sees concurrent broadcasts in the + // same order and two of them cannot each hold part of the workers. A gate + // would deadlock a foreign-pool caller that steals a second broadcast onto + // its own stack while waiting. track.note_install(); - let task = |i: usize| { - let _live = track.task(); - f(i); - }; - unsafe fn call(data: *const (), i: usize) { - // SAFETY: `data` is the `&F` of `broadcast`, alive until it is done. - (*data.cast::())(i) - } - fn data_of(f: &F) -> *const () { - (f as *const F).cast() - } - fn call_of(_: &F) -> unsafe fn(*const (), usize) { - call:: - } - let waiter = waiter_of(pool); - // SAFETY: as in `run_tasks`; the broadcast is unlinked below, after all of - // its shares completed and before it goes out of scope. - let b = unsafe { - crate::park::Broadcast::new(call_of(&task), data_of(&task), budget, waiter.parker(pool)) - }; - pool.awake.bcast.publish(&b); - pool.awake.wake_for_work(); - waiter.wait(pool, &b.done); - pool.awake.bcast.unlink(&b); - if let Some(payload) = b.take_panic() { - std::panic::resume_unwind(payload); - } + pool.inner().broadcast(|ctx| { + let i = ctx.index(); + if i < budget { + let _live = track.task(); + f(i); + } + }); } diff --git a/src/team.rs b/src/team.rs index bcb7fde..0dbb1f6 100644 --- a/src/team.rs +++ b/src/team.rs @@ -83,11 +83,12 @@ impl Participant<'_> { /// reserved member waiting for work, the leader in a section's join, and a /// participant in a barrier spin with one PAUSE per poll for the pool's spin /// budget ([`PoolBuilder::spin_polls`](crate::PoolBuilder::spin_polls)), yield -/// a few times, then sleep until woken. Members run no unrelated task and +/// a few times, then sleep until woken. Members run no unrelated Rayon job and /// do not leave on a timeout, also between sections and during a long serial /// phase of the leader; they are returned when the scope ends. In a root scope -/// with more participants than CPUs in the caller's affinity mask, the spin -/// is throttled to 100 polls, so waiters free their CPUs quickly. +/// with more participants than CPUs in the caller's affinity mask, or in a +/// pool whose waiters were recently descheduled, the spin is throttled to 100 +/// polls, so waiters free their CPUs quickly. /// * If the pool pins members /// ([`PoolBuilder::pin_participants`](crate::PoolBuilder::pin_participants)), /// member `k` of a root scope (participant `k >= 1`) runs on the `k`-th CPU of @@ -296,7 +297,7 @@ impl<'team> TeamExec<'team> { /// The same authority with `over` applied to the scoped policy. /// /// Children inherit the effective policy. An override never weakens a - /// reservation: members stay reserved and run no unrelated task + /// reservation: members stay reserved and run no unrelated Rayon job /// whatever the policy says. The policy of the *root* reservation governs /// the idle interval after it ends; a child's override expires when the /// child returns and cannot release the parent's members. diff --git a/src/worker.rs b/src/worker.rs deleted file mode 100644 index e5ed338..0000000 --- a/src/worker.rs +++ /dev/null @@ -1,266 +0,0 @@ -//! The worker threads and the process-wide reserve (`docs/design/own-threads.md`). -//! -//! A worker is a `std::thread` that serves one pool at a time: a [`Lease`] -//! names the pool's shared state and the worker's slot in it. Between leases a -//! *shared* worker waits, asleep, in the process-wide reserve, so that a pool -//! built after another one was dropped (faer's `with_team` builds one per -//! call) reuses running threads instead of spawning fresh ones. A *dedicated* -//! worker (a pool with a thread name, a start hook, a stack size or a QoS of -//! its own) serves one pool and exits; that pool joins it. -//! -//! No thread-local state is used: a worker is found by its `ThreadId`. - -use crate::awake::Awake; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Condvar, Mutex, OnceLock}; -use std::thread::ThreadId; - -/// Worker threads spawned by this process so far (shared and dedicated). -static SPAWNED: AtomicUsize = AtomicUsize::new(0); - -/// How many worker threads this process has spawned so far: a diagnostic, to -/// check that pools built one after another reuse their workers. -#[doc(hidden)] -pub fn spawned_workers() -> usize { - SPAWNED.load(Ordering::Relaxed) -} - -/// What a worker serves: slot `id` of the pool whose shared state is `aw`, -/// with the leasing caller's CPU set (Linux) applied first. -pub(crate) struct Lease { - pub(crate) aw: Arc, - pub(crate) id: usize, - pub(crate) mask: Option, -} - -/// What a waiting worker is told next. -enum Next { - Lease(Lease), - /// Exit without serving (a dedicated worker of a pool whose build failed). - Exit, -} - -/// One worker thread's handle in this crate. -pub(crate) struct WorkerCtl { - next: Mutex>, - cv: Condvar, - /// A worker of the reserve (as opposed to a dedicated one). - shared: bool, - /// The pool and slot it serves now, for cross-pool lookups. - current: Mutex, usize)>>, - thread: OnceLock, -} - -impl WorkerCtl { - fn new(shared: bool) -> Self { - WorkerCtl { - next: Mutex::new(None), - cv: Condvar::new(), - shared, - current: Mutex::new(None), - thread: OnceLock::new(), - } - } - - /// Hands the worker its next lease. - pub(crate) fn assign(&self, lease: Lease) { - let mut next = self.next.lock().unwrap_or_else(|e| e.into_inner()); - *next = Some(Next::Lease(lease)); - drop(next); - self.cv.notify_one(); - } - - /// Tells an unleased worker to exit. - fn exit(&self) { - let mut next = self.next.lock().unwrap_or_else(|e| e.into_inner()); - *next = Some(Next::Exit); - drop(next); - self.cv.notify_one(); - } -} - -struct Reserve { - /// Shared workers waiting for a lease. - idle: Vec>, - /// The most idle workers kept: the largest of the available parallelism - /// and every width leased so far. - cap: usize, - /// Every live worker, for finding the calling thread's own lease. - live: Vec>, -} - -static RESERVE: OnceLock> = OnceLock::new(); - -fn reserve() -> &'static Mutex { - RESERVE.get_or_init(|| { - Mutex::new(Reserve { - idle: Vec::new(), - cap: std::thread::available_parallelism().map_or(1, |n| n.get()), - live: Vec::new(), - }) - }) -} - -fn lock(r: &Mutex) -> std::sync::MutexGuard<'_, Reserve> { - r.lock().unwrap_or_else(|e| e.into_inner()) -} - -/// The pool and slot the calling thread serves, when it is a worker of any -/// pool (the lookup of a cross-pool wait; entry points only). -pub(crate) fn current_worker() -> Option<(Arc, usize)> { - let r = RESERVE.get()?; - let me = std::thread::current().id(); - let ctl = lock(r) - .live - .iter() - .find(|c| c.thread.get() == Some(&me)) - .cloned()?; - let cur = ctl.current.lock().unwrap_or_else(|e| e.into_inner()); - cur.clone() -} - -/// How a pool's workers are started. -pub(crate) struct Spawn<'a> { - pub(crate) stack: usize, - pub(crate) name: Option<&'a dyn Fn(usize) -> String>, - pub(crate) on_start: Option>, - pub(crate) qos: Option, - /// Whether these settings are the reserve's (default stack, platform - /// QoS, no hooks): then the workers are shared. - pub(crate) shared: bool, -} - -/// The workers obtained for a pool, and the join handles of dedicated ones. -pub(crate) type Obtained = (Vec>, Vec>); - -/// Workers for a pool of `n`: idle ones of the reserve and freshly spawned -/// ones (shared), or `n` fresh dedicated ones with their join handles. -pub(crate) fn obtain(n: usize, how: &Spawn<'_>) -> std::io::Result { - let mut ctls = Vec::with_capacity(n); - let mut joins = Vec::new(); - if how.shared { - let mut r = lock(reserve()); - r.cap = r.cap.max(n); - while ctls.len() < n { - match r.idle.pop() { - Some(c) => ctls.push(c), - None => break, - } - } - } - while ctls.len() < n { - let index = ctls.len(); - match spawn(index, how) { - Ok((ctl, handle)) => { - ctls.push(ctl); - if !how.shared { - joins.push(handle); - } - } - Err(e) => { - // Hand back what was taken; dedicated workers spawned so far - // exit without a lease. - give_back_unleased(ctls, joins); - return Err(e); - } - } - } - Ok((ctls, joins)) -} - -/// Workers obtained for a pool whose build failed: shared ones go back to the -/// reserve, dedicated ones are stopped and joined. -fn give_back_unleased(ctls: Vec>, joins: Vec>) { - for c in ctls { - if c.shared { - lock(reserve()).idle.push(c); - } else { - c.exit(); - } - } - for j in joins { - let _ = j.join(); - } -} - -fn spawn( - index: usize, - how: &Spawn<'_>, -) -> std::io::Result<(Arc, std::thread::JoinHandle<()>)> { - let ctl = Arc::new(WorkerCtl::new(how.shared)); - let mut b = std::thread::Builder::new().stack_size(how.stack); - if let Some(name) = &how.name { - b = b.name(name(index)); - } - let qos = how.qos.or(crate::policy::Qos::PLATFORM_DEFAULT); - let on_start = how.on_start.clone(); - let c = Arc::clone(&ctl); - let handle = b.spawn(move || { - if let Some(q) = qos { - q.apply_to_current_thread(); - } - let _ = c.thread.set(std::thread::current().id()); - lock(reserve()).live.push(Arc::clone(&c)); - if let Some(hook) = on_start { - hook(index); - } - worker_main(&c); - let mut r = lock(reserve()); - if let Some(p) = r.live.iter().position(|w| Arc::ptr_eq(w, &c)) { - r.live.swap_remove(p); - } - })?; - SPAWNED.fetch_add(1, Ordering::Relaxed); - Ok((ctl, handle)) -} - -/// A worker's life: wait for a lease, attach, serve the pool until it is -/// detached, acknowledge, then wait in the reserve (shared) or exit. -fn worker_main(ctl: &Arc) { - loop { - let lease = { - let mut next = ctl.next.lock().unwrap_or_else(|e| e.into_inner()); - loop { - match next.take() { - Some(Next::Lease(l)) => break l, - Some(Next::Exit) => return, - None => next = ctl.cv.wait(next).unwrap_or_else(|e| e.into_inner()), - } - } - }; - let slot = &lease.aw.slots[lease.id]; - // The CPU set a thread spawned by the leasing caller would inherit. - if let Some(mask) = &lease.mask { - let _ = crate::pin::set_here(mask); - } - let _ = slot.thread.set(std::thread::current().id()); - *ctl.current.lock().unwrap_or_else(|e| e.into_inner()) = - Some((Arc::clone(&lease.aw), lease.id)); - // `serve` takes its section baseline before it marks the slot - // attached (claimable): a section published after that is seen. - crate::awake::serve(&lease.aw, lease.id); - // Epilogue: no access to the pool after the acknowledgement. - slot.saved.restore(); - let clean = !slot.saved.is_pinned(); - *ctl.current.lock().unwrap_or_else(|e| e.into_inner()) = None; - // Back in the reserve before the acknowledgement, so that a pool built - // right after this one was dropped finds the worker idle. A new lease - // only starts once this loop waits for it again, after the - // acknowledgement below. - let keep = ctl.shared && clean && { - let mut r = lock(reserve()); - if r.idle.len() < r.cap { - r.idle.push(Arc::clone(ctl)); - true - } else { - false - } - }; - slot.acked.store(true, Ordering::SeqCst); - lease.aw.waits.sync.wake(); - drop(lease); - if !keep { - return; - } - } -} diff --git a/tests/alloc_free.rs b/tests/alloc_free.rs index ee01111..28e73ff 100644 --- a/tests/alloc_free.rs +++ b/tests/alloc_free.rs @@ -2,67 +2,27 @@ #![cfg(all(feature = "parallel", not(feature = "ledger")))] use std::alloc::{GlobalAlloc, Layout, System}; -use std::cell::Cell; use std::num::NonZeroUsize; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; use threadteam::{Exec, Pool}; struct Counting; static ALLOCS: AtomicUsize = AtomicUsize::new(0); -/// While set, every counted allocation also records its size, its thread and -/// a backtrace, so that a failure says where it came from (a platform the -/// author cannot run, such as macOS on CI, is diagnosed from the log alone). -static RECORD: AtomicBool = AtomicBool::new(false); -static RECORDED: Mutex> = Mutex::new(Vec::new()); - -thread_local! { - /// The recording itself allocates: it is neither counted nor recorded. - static RECORDING: Cell = const { Cell::new(false) }; -} - -fn count(size: usize) { - if RECORDING.with(|r| r.get()) { - return; - } - ALLOCS.fetch_add(1, Ordering::SeqCst); - if RECORD.load(Ordering::SeqCst) { - RECORDING.with(|r| r.set(true)); - let entry = format!( - "{size} bytes on {:?}:\n{}", - std::thread::current().name(), - std::backtrace::Backtrace::force_capture() - ); - RECORDED - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(entry); - RECORDING.with(|r| r.set(false)); - } -} unsafe impl GlobalAlloc for Counting { unsafe fn alloc(&self, l: Layout) -> *mut u8 { - count(l.size()); + ALLOCS.fetch_add(1, Ordering::SeqCst); System.alloc(l) } unsafe fn dealloc(&self, p: *mut u8, l: Layout) { System.dealloc(p, l) } unsafe fn realloc(&self, p: *mut u8, l: Layout, n: usize) -> *mut u8 { - count(n); + ALLOCS.fetch_add(1, Ordering::SeqCst); System.realloc(p, l, n) } } -/// The allocations recorded so far, for a failure message. -fn recorded() -> String { - RECORDED - .lock() - .unwrap_or_else(|e| e.into_inner()) - .join("\n----\n") -} - #[global_allocator] static A: Counting = Counting; @@ -77,36 +37,27 @@ fn entries(pool: &Pool) { }); let w = Exec::new(pool, 3).unwrap().external(|w| w.get()); assert_eq!(w, 3); - if !threadteam::Exec::new(pool, 3).unwrap().can_broadcast() { - return; - } - Exec::new(pool, 3) - .unwrap() - .broadcast(|i| { - sink.fetch_add(i, Ordering::Relaxed); - }) - .unwrap(); assert!(sink.load(Ordering::Relaxed) > 0); } -/// Largest allocation count over `rounds` windows of one call each, after -/// warm-up: strict, every window counts (the native scheduler publishes stack -/// batches, so there is no amortized queue allocation to tolerate). +/// Smallest allocation count over `rounds` windows of one call each, after warm-up. +/// +/// From a plain thread every entry pushes one job into rayon's injector queue, +/// whose block allocation (one per 31 pushes) is amortized and outside this +/// crate; a per-call allocation shows up in every window, so the minimum +/// separates the two. fn measure(rounds: usize, f: impl Fn()) -> usize { for _ in 0..20 { f(); } - RECORD.store(true, Ordering::SeqCst); - let worst = (0..rounds) + (0..rounds) .map(|_| { let before = ALLOCS.load(Ordering::SeqCst); f(); ALLOCS.load(Ordering::SeqCst) - before }) - .max() - .unwrap_or(0); - RECORD.store(false, Ordering::SeqCst); - worst + .min() + .unwrap_or(0) } // One test: the counter is process-global. @@ -114,20 +65,7 @@ fn measure(rounds: usize, f: impl Fn()) -> usize { fn fork_for_ranges_external_do_not_allocate() { let pool = Pool::builder().threads(3).build().unwrap(); let plain = measure(10, || entries(&pool)); - assert_eq!(plain, 0, "plain-thread entries allocated:\n{}", recorded()); - // The same after the workers went to sleep: waking them allocates nothing. - let asleep = measure(5, || { - std::thread::sleep(std::time::Duration::from_millis(20)); - let before = ALLOCS.load(Ordering::SeqCst); - entries(&pool); - assert_eq!( - ALLOCS.load(Ordering::SeqCst) - before, - 0, - "entries after a park allocated:\n{}", - recorded() - ); - }); - let _ = asleep; + assert_eq!(plain, 0, "plain-thread entries allocated"); // Inside the pool: a worker of `pool` enters again, which installs nothing, // so every window is strict. @@ -140,8 +78,7 @@ fn fork_for_ranges_external_do_not_allocate() { assert_eq!( inside.load(Ordering::SeqCst), 0, - "in-pool entries allocated:\n{}", - recorded() + "in-pool entries allocated" ); // Participant pinning on: the extra pin section a root team scope runs must diff --git a/tests/loom_park.rs b/tests/loom_park.rs index d773623..e810464 100644 --- a/tests/loom_park.rs +++ b/tests/loom_park.rs @@ -203,170 +203,3 @@ fn a_cancellation_wakes_a_participant_asleep_in_a_barrier() { assert!(!waiter.join().unwrap()); }); } - -// --------------------------------------------------------------------------- -// The native scheduler (`docs/design/own-threads.md`): the slot state word, -// the queue of stack batches, the broadcast list, and an idle worker's sleep -// against a publication. The exact code, under `loom`. - -use threadteam::{Batch, Broadcast, BroadcastList, Queue, SlotState}; - -/// A team reservation and a worker taking work race for a free slot: exactly -/// one wins, and a release or a detach while the worker is busy neither frees -/// it early nor is erased when it finishes. -#[test] -fn a_free_slot_is_reserved_or_taken_never_both() { - model(|| { - let s = Arc::new(SlotState::new()); - let leader = { - let s = s.clone(); - thread::spawn(move || s.try_reserve()) - }; - let took = s.try_take(); - let reserved = leader.join().unwrap(); - assert!(took != reserved, "exactly one wins"); - if took { - // Busy: a detach arrives, then the work ends. - s.detach(); - s.end_busy(); - assert_eq!(s.load(), 8, "DETACH survives the end of the work"); - } else { - s.release(); - assert_eq!(s.load(), 0); - } - }); -} - -/// A lent member that is running a task when its reservation is released -/// stays busy (not claimable) until the task ends. -#[test] -fn a_release_does_not_free_a_busy_lent_member() { - model(|| { - let s = Arc::new(SlotState::new()); - assert!(s.try_reserve()); - s.set_lent(true); - assert!(s.try_take()); - let releaser = { - let s = s.clone(); - thread::spawn(move || s.release()) - }; - let claimer = { - let s = s.clone(); - thread::spawn(move || s.try_reserve()) - }; - releaser.join().unwrap(); - assert!(!claimer.join().unwrap(), "a busy worker is not reservable"); - s.end_busy(); - assert!(s.try_reserve(), "free again once the task ended"); - }); -} - -unsafe fn count_task(data: *const (), _i: usize) { - (*data.cast::()).fetch_add(1, Ordering::SeqCst); -} - -/// An owner publishes a batch of two tasks; two workers claim and run them -/// while the owner claims its own and then sleeps until the last completion: -/// every task runs exactly once and the owner wakes (no lost wake-up). -#[test] -fn a_batch_runs_every_task_once_and_wakes_its_sleeping_owner() { - model(|| { - let q = Arc::new(Queue::new()); - let ran = Arc::new(AtomicUsize::new(0)); - let waker = std::sync::Arc::new(Parker::new()); - // SAFETY: `ran` and `waker` outlive the batch, which the owner waits - // for below. - let batch = - Arc::new(unsafe { Batch::new(count_task, Arc::as_ptr(&ran).cast(), 2, &waker) }); - q.publish(&batch); - let ws: Vec<_> = (0..2) - .map(|_| { - let q = q.clone(); - thread::spawn(move || { - if let Some((b, i)) = q.claim() { - unsafe { Batch::run_claimed(b, i) }; - } - }) - }) - .collect(); - while let Some(i) = q.claim_own(&batch) { - unsafe { Batch::run_claimed(Arc::as_ptr(&batch), i) }; - } - batch.done.wait(Some(&waker), &mut Waiter::new(0)); - assert_eq!(ran.load(Ordering::SeqCst), 2); - for w in ws { - w.join().unwrap(); - } - }); -} - -unsafe fn share_task(data: *const (), i: usize) { - (*data.cast::<[AtomicUsize; 2]>())[i].fetch_add(1, Ordering::SeqCst); -} - -/// A broadcast to two workers: each runs its own share exactly once, in the -/// order of its cursor, and the caller wakes before it unlinks the broadcast. -#[test] -fn a_broadcast_runs_each_share_once_and_wakes_its_caller() { - model(|| { - let list = Arc::new(BroadcastList::new()); - let shares = Arc::new([AtomicUsize::new(0), AtomicUsize::new(0)]); - let waker = std::sync::Arc::new(Parker::new()); - // SAFETY: as above. - let b = - Arc::new(unsafe { Broadcast::new(share_task, Arc::as_ptr(&shares).cast(), 2, &waker) }); - list.publish(&b); - let ws: Vec<_> = (0..2) - .map(|i| { - let list = list.clone(); - thread::spawn(move || { - let mut seq = 0; - while let Some(b) = list.next_for(i, &mut seq) { - unsafe { Broadcast::run_share(b, i) }; - } - }) - }) - .collect(); - b.done.wait(Some(&waker), &mut Waiter::new(0)); - list.unlink(&b); - assert_eq!(shares[0].load(Ordering::SeqCst), 1); - assert_eq!(shares[1].load(Ordering::SeqCst), 1); - for w in ws { - w.join().unwrap(); - } - }); -} - -/// An idle worker going to sleep on its parker against a publication: the -/// publisher links the batch and wakes; the worker either sees the new work -/// before it sleeps or is woken (no lost wake-up), then claims it. -#[test] -fn an_idle_worker_is_never_left_asleep_by_a_publication() { - model(|| { - let q = Arc::new(Queue::new()); - let park = Arc::new(Parker::new()); - let ran = Arc::new(AtomicUsize::new(0)); - let waker = std::sync::Arc::new(Parker::new()); - let worker = { - let (q, park) = (q.clone(), park.clone()); - thread::spawn(move || { - let g = q.gen(); - park.park_while(|| q.gen() == g && !q.has_work()); - if let Some((b, i)) = q.claim() { - unsafe { Batch::run_claimed(b, i) }; - } - }) - }; - // SAFETY: as above. - let batch = unsafe { Batch::new(count_task, Arc::as_ptr(&ran).cast(), 1, &waker) }; - q.publish(&batch); - park.wake(); - worker.join().unwrap(); - // The worker claimed it, or it is still linked for the owner. - while let Some(i) = q.claim_own(&batch) { - unsafe { Batch::run_claimed(&batch, i) }; - } - batch.done.wait(Some(&waker), &mut Waiter::new(0)); - assert_eq!(ran.load(Ordering::SeqCst), 1); - }); -} diff --git a/tests/own_threads.rs b/tests/own_threads.rs deleted file mode 100644 index 9ea8deb..0000000 --- a/tests/own_threads.rs +++ /dev/null @@ -1,318 +0,0 @@ -//! The native workers (`docs/design/own-threads.md`): reuse through the -//! process-wide reserve, cross-pool waits, broadcast order, affinity on lease, -//! and panics. -#![cfg(feature = "parallel")] - -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Mutex; -use std::time::Duration; -use threadteam::{Exec, Pool}; - -/// A hang is a failure: abort the whole process if the test body stalls. -fn watchdog() -> impl Drop { - struct Stop(std::sync::mpsc::Sender<()>); - impl Drop for Stop { - fn drop(&mut self) { - let _ = self.0.send(()); - } - } - let (tx, rx) = std::sync::mpsc::channel::<()>(); - std::thread::spawn(move || { - if rx.recv_timeout(Duration::from_secs(60)).is_err() { - eprintln!("watchdog: test stalled"); - std::process::exit(101); - } - }); - Stop(tx) -} - -/// The reserve and the spawn counter are process wide: the tests of this file -/// take turns. -static SERIAL: Mutex<()> = Mutex::new(()); - -#[test] -fn a_pool_built_after_another_was_dropped_reuses_its_workers() { - let _w = watchdog(); - let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let width = 4; - // A long acquisition timeout: a worker clears its busy bit just after its - // last task completed, and on a loaded host that can take a while. - let build = || { - Pool::builder() - .threads(width) - .acquire_timeout(Duration::from_secs(5)) - .build() - .unwrap() - }; - let first = build(); - Exec::new(&first, width).unwrap().fork(width, |_, _| {}); - drop(first); - for round in 0..5 { - let before = threadteam::spawned_workers(); - let pool = build(); - let n = AtomicUsize::new(0); - Exec::new(&pool, width).unwrap().fork(width, |_, _| { - n.fetch_add(1, Ordering::SeqCst); - }); - assert_eq!(n.load(Ordering::SeqCst), width); - // A team on the reused workers, as faer's `with_team` takes one. - let team = pool.into_team(width).expect("idle reused workers"); - assert_eq!(team.size().get(), width); - drop(team); - assert_eq!( - threadteam::spawned_workers() - before, - 0, - "round {round}: a pool of the same width spawned threads" - ); - } -} - -#[test] -fn a_pool_with_hooks_gets_dedicated_workers_and_joins_them() { - let _w = watchdog(); - let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let started = std::sync::Arc::new(AtomicUsize::new(0)); - let s = std::sync::Arc::clone(&started); - let before = threadteam::spawned_workers(); - let pool = Pool::builder() - .threads(2) - .thread_name(|i| format!("dedicated-{i}")) - .on_start(move |_| { - s.fetch_add(1, Ordering::SeqCst); - }) - .build() - .unwrap(); - let names = Mutex::new(Vec::new()); - Exec::new(&pool, 2).unwrap().fork(2, |_, _| { - names - .lock() - .unwrap() - .push(std::thread::current().name().map(String::from)); - }); - drop(pool); - assert_eq!(threadteam::spawned_workers() - before, 2); - assert_eq!( - started.load(Ordering::SeqCst), - 2, - "the hook ran on each worker" - ); - assert!(names - .into_inner() - .unwrap() - .iter() - .all(|n| n.as_deref().is_some_and(|n| n.starts_with("dedicated-")))); -} - -/// A worker of pool A waits for pool B, whose task waits for pool A again. -/// Every waiting worker serves its own pool meanwhile, so the chain completes -/// even when every worker of A is waiting on B. -#[test] -fn a_cross_pool_chain_a_b_a_completes() { - let _w = watchdog(); - let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let a = Pool::builder().threads(2).build().unwrap(); - let b = Pool::builder().threads(2).build().unwrap(); - let inner = AtomicUsize::new(0); - Exec::new(&a, 2).unwrap().fork(2, |_, _| { - Exec::new(&b, 2).unwrap().fork(2, |_, _| { - Exec::new(&a, 2).unwrap().fork(2, |_, _| { - inner.fetch_add(1, Ordering::SeqCst); - }); - }); - }); - assert_eq!(inner.load(Ordering::SeqCst), 2 * 2 * 2); -} - -/// Two plain threads broadcast on one pool at once: every worker runs both -/// broadcasts, and all workers see them in the same order. -#[test] -fn concurrent_broadcasts_reach_every_worker_in_the_same_order() { - let _w = watchdog(); - let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let width = 3; - let pool = Pool::builder().threads(width).build().unwrap(); - for _ in 0..20 { - let seen: Vec>> = (0..width).map(|_| Mutex::new(Vec::new())).collect(); - std::thread::scope(|s| { - for tag in [1u8, 2] { - let (pool, seen) = (&pool, &seen); - s.spawn(move || { - Exec::new(pool, width) - .unwrap() - .broadcast(|i| seen[i].lock().unwrap().push(tag)) - .unwrap(); - }); - } - }); - let orders: Vec> = seen.into_iter().map(|m| m.into_inner().unwrap()).collect(); - assert!(orders.iter().all(|o| o.len() == 2), "{orders:?}"); - assert!(orders.windows(2).all(|w| w[0] == w[1]), "{orders:?}"); - } -} - -/// A task panics: the others still run, the panic reaches the caller after -/// all of them finished, and the pool keeps working. -#[test] -fn a_task_panic_propagates_after_the_others_and_the_pool_survives() { - let _w = watchdog(); - let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let pool = Pool::builder().threads(3).build().unwrap(); - let done = AtomicUsize::new(0); - let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - Exec::new(&pool, 3).unwrap().fork(3, |i, _| { - if i == 1 { - panic!("boom"); - } - std::thread::sleep(Duration::from_millis(5)); - done.fetch_add(1, Ordering::SeqCst); - }); - })); - assert!(r.is_err()); - assert_eq!(done.load(Ordering::SeqCst), 2); - let n = AtomicUsize::new(0); - Exec::new(&pool, 3).unwrap().fork(3, |_, _| { - n.fetch_add(1, Ordering::SeqCst); - }); - assert_eq!(n.load(Ordering::SeqCst), 3); -} - -/// A reused worker takes the CPU set of the caller that leases it, as a thread -/// that caller spawned would, not the set of whoever spawned it first. -#[cfg(target_os = "linux")] -#[test] -fn a_reused_worker_takes_the_leasing_callers_cpu_set() { - let _w = watchdog(); - let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - fn cpus() -> Vec { - unsafe { - let mut set: libc::cpu_set_t = std::mem::zeroed(); - libc::sched_getaffinity(0, std::mem::size_of::(), &mut set); - (0..libc::CPU_SETSIZE as usize) - .filter(|&c| libc::CPU_ISSET(c, &set)) - .collect() - } - } - fn set(cpus: &[usize]) -> bool { - unsafe { - let mut s: libc::cpu_set_t = std::mem::zeroed(); - libc::CPU_ZERO(&mut s); - for &c in cpus { - libc::CPU_SET(c, &mut s); - } - libc::sched_setaffinity(0, std::mem::size_of::(), &s) == 0 - } - } - let all = cpus(); - if all.len() < 4 { - eprintln!("skipped: needs 4 allowed CPUs"); - return; - } - // Run on a helper thread so the test thread's mask is never changed. - std::thread::spawn(move || { - let width = 2; - assert!(set(&all[..2])); - drop(Pool::builder().threads(width).build().unwrap()); - assert!(set(&all[2..4])); - let pool = Pool::builder().threads(width).build().unwrap(); - let masks = Mutex::new(Vec::new()); - Exec::new(&pool, width) - .unwrap() - .broadcast(|_| masks.lock().unwrap().push(cpus())) - .unwrap(); - for m in masks.into_inner().unwrap() { - assert_eq!( - m, - all[2..4].to_vec(), - "a reused worker kept another caller's CPU set" - ); - } - }) - .join() - .unwrap(); -} - -/// A dedicated worker keeps the CPU set its `on_start` hook pinned it to: the -/// lease does not reset it to the caller's set. -#[cfg(target_os = "linux")] -#[test] -fn a_dedicated_worker_keeps_the_pin_of_its_start_hook() { - let _w = watchdog(); - let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - fn cpus() -> Vec { - unsafe { - let mut set: libc::cpu_set_t = std::mem::zeroed(); - libc::sched_getaffinity(0, std::mem::size_of::(), &mut set); - (0..libc::CPU_SETSIZE as usize) - .filter(|&c| libc::CPU_ISSET(c, &set)) - .collect() - } - } - let all = cpus(); - if all.len() < 2 { - eprintln!("skipped: needs 2 allowed CPUs"); - return; - } - let target = all[1]; - let pool = Pool::builder() - .threads(2) - .on_start(move |_| unsafe { - let mut s: libc::cpu_set_t = std::mem::zeroed(); - libc::CPU_ZERO(&mut s); - libc::CPU_SET(target, &mut s); - libc::sched_setaffinity(0, std::mem::size_of::(), &s); - }) - .build() - .unwrap(); - let masks = Mutex::new(Vec::new()); - Exec::new(&pool, 2) - .unwrap() - .broadcast(|_| masks.lock().unwrap().push(cpus())) - .unwrap(); - for m in masks.into_inner().unwrap() { - assert_eq!(m, vec![target], "the lease undid the hook's pin"); - } -} - -/// Broadcast X's share 0 waits on another pool while X's share 1 waits at a -/// barrier of X's shares; meanwhile broadcast Y, whose shares meet at a barrier -/// too, is published. The waiting share 0 must not start Y's share on top of -/// X's: Y's barrier would wait for worker 1, which waits for X. -#[test] -fn a_waiting_broadcast_share_does_not_start_a_later_broadcast() { - let _w = watchdog(); - let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let a = Pool::builder().threads(2).build().unwrap(); - let b = Pool::builder().threads(2).build().unwrap(); - for _ in 0..20 { - let x_bar = std::sync::Barrier::new(2); - let y_bar = std::sync::Barrier::new(2); - let x_waiting = std::sync::atomic::AtomicBool::new(false); - std::thread::scope(|s| { - s.spawn(|| { - Exec::new(&a, 2) - .unwrap() - .broadcast(|i| { - if i == 0 { - x_waiting.store(true, Ordering::SeqCst); - Exec::new(&b, 2).unwrap().fork(2, |_, _| { - std::thread::sleep(Duration::from_millis(20)); - }); - } - x_bar.wait(); - }) - .unwrap(); - }); - s.spawn(|| { - while !x_waiting.load(Ordering::SeqCst) { - std::thread::yield_now(); - } - Exec::new(&a, 2) - .unwrap() - .broadcast(|_| { - y_bar.wait(); - }) - .unwrap(); - }); - }); - } -} diff --git a/tests/rayon_reuse.rs b/tests/rayon_reuse.rs new file mode 100644 index 0000000..e5686e4 --- /dev/null +++ b/tests/rayon_reuse.rs @@ -0,0 +1,186 @@ +//! Reuse of the private Rayon pool between pools of the same width (the +//! comparison experiment against own worker threads, tensor4all/faer#107). +#![cfg(feature = "parallel")] + +use std::collections::HashSet; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Mutex; +use std::thread::ThreadId; +use std::time::Duration; +use threadteam::{Exec, Pool}; + +/// A hang is a failure: abort the whole process if the test body stalls. +fn watchdog() -> impl Drop { + struct Stop(std::sync::mpsc::Sender<()>); + impl Drop for Stop { + fn drop(&mut self) { + let _ = self.0.send(()); + } + } + let (tx, rx) = std::sync::mpsc::channel::<()>(); + std::thread::spawn(move || { + if rx.recv_timeout(Duration::from_secs(60)).is_err() { + eprintln!("watchdog: test stalled"); + std::process::exit(101); + } + }); + Stop(tx) +} + +/// The cache and the spawn counter are process wide: the tests of this file +/// take turns. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn worker_ids(pool: &Pool) -> HashSet { + let ids = Mutex::new(HashSet::new()); + Exec::new(pool, pool.width().get()) + .unwrap() + .broadcast(|_| { + ids.lock().unwrap().insert(std::thread::current().id()); + }) + .unwrap(); + ids.into_inner().unwrap() +} + +#[test] +fn a_pool_built_after_another_was_dropped_reuses_its_workers() { + let _w = watchdog(); + let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let width = 4; + let build = || { + Pool::builder() + .threads(width) + .acquire_timeout(Duration::from_secs(5)) + .build() + .unwrap() + }; + let first = build(); + Exec::new(&first, width).unwrap().fork(width, |_, _| {}); + let ids = worker_ids(&first); + drop(first); + for round in 0..5 { + let before = threadteam::spawned_workers(); + let pool = build(); + let n = AtomicUsize::new(0); + Exec::new(&pool, width).unwrap().fork(width, |_, _| { + n.fetch_add(1, Ordering::SeqCst); + }); + assert_eq!(n.load(Ordering::SeqCst), width); + assert_eq!(worker_ids(&pool), ids, "round {round}: other threads"); + // A team on the reused workers, as faer's `with_team` takes one. + let team = pool.into_team(width).expect("idle reused workers"); + assert_eq!(team.size().get(), width); + drop(team); + assert_eq!( + threadteam::spawned_workers() - before, + 0, + "round {round}: a pool of the same width spawned threads" + ); + } +} + +#[test] +fn two_live_pools_of_one_width_never_share_workers() { + let _w = watchdog(); + let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + drop(Pool::builder().threads(2).build().unwrap()); + let a = Pool::builder().threads(2).build().unwrap(); + let b = Pool::builder().threads(2).build().unwrap(); + let (ia, ib) = (worker_ids(&a), worker_ids(&b)); + assert_eq!(ia.len(), 2); + assert_eq!(ib.len(), 2); + assert!(ia.is_disjoint(&ib), "{ia:?} {ib:?}"); +} + +#[test] +fn a_pool_with_hooks_gets_dedicated_workers_and_joins_them() { + let _w = watchdog(); + let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let started = std::sync::Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let s = std::sync::Arc::clone(&started); + let before = threadteam::spawned_workers(); + let pool = Pool::builder() + .threads(2) + .thread_name(|i| format!("dedicated-{i}")) + .on_start(move |_| { + s.fetch_add(1, Ordering::SeqCst); + }) + .build() + .unwrap(); + let names = Mutex::new(Vec::new()); + Exec::new(&pool, 2).unwrap().fork(2, |_, _| { + names + .lock() + .unwrap() + .push(std::thread::current().name().map(String::from)); + }); + drop(pool); + assert_eq!(threadteam::spawned_workers() - before, 2); + assert!(names + .into_inner() + .unwrap() + .iter() + .all(|n| n.as_deref().is_some_and(|n| n.starts_with("dedicated-")))); + } + // The drop joined the workers: their hooks ran once each, never again. + assert_eq!(started.load(Ordering::SeqCst), 4); +} + +/// A reused worker takes the CPU set of the caller that builds the pool, as a +/// thread that caller spawned would, not the set of whoever spawned it first. +#[cfg(target_os = "linux")] +#[test] +fn a_reused_worker_takes_the_building_callers_cpu_set() { + let _w = watchdog(); + let _turn = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + fn cpus() -> Vec { + unsafe { + let mut set: libc::cpu_set_t = std::mem::zeroed(); + libc::sched_getaffinity(0, std::mem::size_of::(), &mut set); + (0..libc::CPU_SETSIZE as usize) + .filter(|&c| libc::CPU_ISSET(c, &set)) + .collect() + } + } + fn set(cpus: &[usize]) -> bool { + unsafe { + let mut s: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_ZERO(&mut s); + for &c in cpus { + libc::CPU_SET(c, &mut s); + } + libc::sched_setaffinity(0, std::mem::size_of::(), &s) == 0 + } + } + let all = cpus(); + if all.len() < 4 { + eprintln!("skipped: needs 4 allowed CPUs"); + return; + } + // Run on a helper thread so the test thread's mask is never changed. + std::thread::spawn(move || { + let width = 3; + assert!(set(&all[..2])); + let first = Pool::builder().threads(width).build().unwrap(); + let ids = worker_ids(&first); + drop(first); + assert!(set(&all[2..4])); + let pool = Pool::builder().threads(width).build().unwrap(); + assert_eq!(worker_ids(&pool), ids, "the workers were not reused"); + let masks = Mutex::new(Vec::new()); + Exec::new(&pool, width) + .unwrap() + .broadcast(|_| masks.lock().unwrap().push(cpus())) + .unwrap(); + for m in masks.into_inner().unwrap() { + assert_eq!( + m, + all[2..4].to_vec(), + "a reused worker kept another caller's CPU set" + ); + } + }) + .join() + .unwrap(); +}