diff --git a/apps/website/content/docs/headless/getting-started.mdx b/apps/website/content/docs/headless/getting-started.mdx index 1a0562206..bd832bea8 100644 --- a/apps/website/content/docs/headless/getting-started.mdx +++ b/apps/website/content/docs/headless/getting-started.mdx @@ -6,7 +6,7 @@ nav: Headless engine A headless renderer starts with `createLocalRowModel`. Add `createGrid` only when your renderer needs UI state. The grid below is exactly that: 75 services rendered from a plain ``, with `createLocalRowModel` driving sort and filter and `createGrid` driving row selection. -`setQuery` — triggered here by typing into the filter — does not settle synchronously. The model rebuilds cooperatively, yielding between slices so a large query cannot block the frame, and publishes progress and failures through `status`. (Sorting a column is the exception: a sort-only change on ungrouped data re-orders rows the model has already indexed, so it settles synchronously.) Two things follow, and the example does both: **select** what you subscribe to, or you re-render on every slice, and **read `status`**, or a rebuild that fails leaves stale rows on screen with nothing to say so. +`setQuery` does not settle synchronously in the general case — the model rebuilds cooperatively, yielding between slices so a large query cannot block the frame, and publishes progress and failures through `status`. Sort-only and filter-only changes on ungrouped data are the exception, including typing into the filter here: each re-orders or re-selects rows the model has already indexed, so it settles synchronously with no `rebuilding` phase. Grouped and mixed changes still rebuild cooperatively — see [Snapshot & subscribe](/docs/headless/state-model) for a demo. Two things are still worth doing, and the example does both: **select** what you subscribe to, so a cooperative rebuild elsewhere doesn't re-render you on every slice, and **read `status`**, so a failed rebuild doesn't leave stale rows on screen with nothing to say so. diff --git a/apps/website/content/docs/headless/state-model.mdx b/apps/website/content/docs/headless/state-model.mdx index 29d39fba3..a613b727f 100644 --- a/apps/website/content/docs/headless/state-model.mdx +++ b/apps/website/content/docs/headless/state-model.mdx @@ -6,7 +6,7 @@ nav: Headless engine The row model and UI grid are independent observable stores. Subscribe only to the state your renderer uses. -A `setQuery` that changes the filter does not settle synchronously — the model rebuilds cooperatively, yielding between slices so a large query cannot block the frame. A sort-only change on ungrouped data is the one exception: it re-orders rows the model has already indexed, so it settles synchronously and never publishes a `rebuilding` phase — a plain sort needs no progress UI. On a small dataset even the cooperative rebuild is over before a human (or React) can see it happen, which is why the button below filters 150,000 rows instead of 75: watch `status` cycle through `rebuilding` with a live percentage, then settle back to `ready`. +A `setQuery` that changes row grouping does not settle synchronously — the model rebuilds cooperatively, yielding between slices so a large query cannot block the frame. Sort-only and filter-only changes on ungrouped data are the exception: each re-orders or re-selects rows the model has already indexed, so it settles synchronously and never publishes a `rebuilding` phase — plain sorting and filtering need no progress UI. Grouped and mixed changes still rebuild cooperatively. On a small dataset even the cooperative rebuild is over before a human (or React) can see it happen, which is why the button below groups 150,000 rows instead of 75: watch `status` cycle through `rebuilding` with a live percentage, then settle back to `ready`. @@ -43,7 +43,7 @@ committed, and mutations keep committing into it meanwhile: `setRows`, `applyTransaction` and both expansion paths publish a new snapshot while a rebuild runs. A renderer that stops re-reading the snapshot during a rebuild drops those, which is exactly the streaming-plus-filter case. The table in the -example above stays on the last committed filter result throughout, exactly +example above stays on the last committed result throughout, exactly like this. `completedRows` and `totalRows` count the rebuild's **work units, not rows**. diff --git a/apps/website/content/examples/headless-rebuild-progress/RebuildProgressDemo.tsx b/apps/website/content/examples/headless-rebuild-progress/RebuildProgressDemo.tsx index 78585d7d1..857f03fce 100644 --- a/apps/website/content/examples/headless-rebuild-progress/RebuildProgressDemo.tsx +++ b/apps/website/content/examples/headless-rebuild-progress/RebuildProgressDemo.tsx @@ -19,8 +19,8 @@ export function RebuildProgressDemo() { // Selecting `snapshot` (not the whole state) means this component bails // out on identity between rebuild slices — it only renders once, when the - // filter actually lands. `RebuildProgress` above is the one re-rendering on - // every slice in the meantime. + // grouping change actually lands. `RebuildProgress` above is the one + // re-rendering on every slice in the meantime. const readSnapshot = useCallback( () => rowModel.getState().snapshot, [rowModel], @@ -31,28 +31,29 @@ export function RebuildProgressDemo() { readSnapshot, ); - const [filtered, setFiltered] = useState(false); + const [grouped, setGrouped] = useState(false); - // A FILTER change, not a sort: a sort-only change on ungrouped data - // settles synchronously and never publishes a `rebuilding` phase, so it - // could not demonstrate the progress readout at all. - const toggleFilter = () => { - const next = !filtered; - setFiltered(next); + // A GROUPING change, not a filter or a sort: both of those settle + // synchronously on ungrouped data (the sort fast path and the filter fast + // path each require `rowGroups.length === 0`), so neither could + // demonstrate the progress readout anymore. Grouping never takes a fast + // path — it always rebuilds cooperatively — which is exactly why it is the + // vehicle here. + const toggleGrouped = () => { + const next = !grouped; + setGrouped(next); rowModel.setQuery({ ...snapshot.query, - filters: next - ? [{ columnId: "region", operator: "equals", value: "west" }] - : [], + rowGroups: next ? [{ columnId: "region" }] : [], }); }; return (
-

@@ -72,14 +73,21 @@ export function RebuildProgressDemo() {

{snapshot .range(0, Math.min(PREVIEW_ROWS, snapshot.visibleRowCount)) - .filter((entry) => entry.kind === "data") - .map(({ rowId, row }) => ( - - {columns.map((c) => ( - - ))} - - ))} + .map((entry) => + entry.kind === "data" ? ( + + {columns.map((c) => ( + + ))} + + ) : ( + + + + ), + )}
{String(c.accessor(row))}
{String(c.accessor(entry.row))}
+ {String(entry.value)} ({entry.childCount}) +
diff --git a/apps/website/content/examples/headless-rebuild-progress/__tests__/RebuildProgressDemo.test.tsx b/apps/website/content/examples/headless-rebuild-progress/__tests__/RebuildProgressDemo.test.tsx index ad46a4707..a264038f5 100644 --- a/apps/website/content/examples/headless-rebuild-progress/__tests__/RebuildProgressDemo.test.tsx +++ b/apps/website/content/examples/headless-rebuild-progress/__tests__/RebuildProgressDemo.test.tsx @@ -43,15 +43,12 @@ describe("RebuildProgressDemo", () => { }); fireEvent.click( - screen.getByRole("button", { name: /filter 150,000 orders/i }), + screen.getByRole("button", { name: /group 150,000 orders/i }), ); await waitFor( () => { expect(status).toHaveTextContent("Ready."); - // The filter landed: only the 30,000 west-region orders survive, - // and every preview row is one of them. - expect(screen.getByText(/30,000 rows indexed/)).toBeInTheDocument(); }, { timeout: REBUILD_TIMEOUT }, ); @@ -60,22 +57,36 @@ describe("RebuildProgressDemo", () => { // Proves the rebuild actually published at least one intermediate // `rebuilding` slice before landing on `ready` — the whole reason this // example exists. On the small 75-row custom-renderer example this - // would be a coin flip; at 150,000 rows it is not. A sort-only change - // could never pass this: on ungrouped data it settles synchronously - // with no `rebuilding` phase at all. + // would be a coin flip; at 150,000 rows it is not. A sort-only or + // filter-only change could never pass this: on ungrouped data both + // settle synchronously with no `rebuilding` phase at all. Grouping is + // the one change vehicle that is cooperative by design, not omission. expect(sawRebuilding).toBe(true); + // The grouping landed: every visible region group has surfaced as its + // own row (5 regions), distinct from the plain data rows. const previewRows = screen.getAllByRole("row").slice(1); expect(previewRows.length).toBeGreaterThan(0); - for (const row of previewRows) { - expect(row).toHaveTextContent("west"); - } + const groupRows = previewRows.filter((row) => + /\(\d+\)/.test(row.textContent ?? ""), + ); + expect(groupRows.length).toBeGreaterThan(0); + + // Group rows sit alongside the 150,000 data rows in the indexed + // count, so it goes up, not down, once grouping lands. + const rowsIndexedText = screen.getByText(/rows indexed/).textContent; + const indexedCount = Number( + rowsIndexedText + ?.match(/^([\d,]+) rows indexed/)?.[1] + ?.replace(/,/g, ""), + ); + expect(indexedCount).toBeGreaterThan(150_000); }, REBUILD_TIMEOUT + 5_000, ); it( - "clears the filter cooperatively on the second click", + "ungroups cooperatively on the second click", async () => { render(); await waitFor(() => screen.getByText(/150,000 rows indexed/), { @@ -83,11 +94,15 @@ describe("RebuildProgressDemo", () => { }); fireEvent.click( - screen.getByRole("button", { name: /filter 150,000 orders/i }), + screen.getByRole("button", { name: /group 150,000 orders/i }), + ); + await waitFor( + () => { + const status = screen.getByRole("status"); + expect(status).toHaveTextContent("Ready."); + }, + { timeout: REBUILD_TIMEOUT }, ); - await waitFor(() => screen.getByText(/30,000 rows indexed/), { - timeout: REBUILD_TIMEOUT, - }); let sawRebuilding = false; const status = screen.getByRole("status"); @@ -102,9 +117,7 @@ describe("RebuildProgressDemo", () => { subtree: true, }); - fireEvent.click( - screen.getByRole("button", { name: /show all 150,000 orders/i }), - ); + fireEvent.click(screen.getByRole("button", { name: /ungroup/i })); await waitFor( () => { @@ -115,10 +128,17 @@ describe("RebuildProgressDemo", () => { ); observer.disconnect(); - // Removing a filter re-runs the same cooperative path over all + // Removing the grouping re-runs the same cooperative path over all // 150,000 source rows, so the toggle demonstrates progress in both // directions. expect(sawRebuilding).toBe(true); + + // Ungrouped: no group rows remain, only plain data rows. + const previewRows = screen.getAllByRole("row").slice(1); + const groupRows = previewRows.filter((row) => + /\(\d+\)/.test(row.textContent ?? ""), + ); + expect(groupRows.length).toBe(0); }, REBUILD_TIMEOUT + 5_000, ); diff --git a/apps/website/content/examples/headless-rebuild-progress/data.ts b/apps/website/content/examples/headless-rebuild-progress/data.ts index a173c2562..222580fac 100644 --- a/apps/website/content/examples/headless-rebuild-progress/data.ts +++ b/apps/website/content/examples/headless-rebuild-progress/data.ts @@ -8,7 +8,7 @@ export interface Order { const REGIONS = ["north", "south", "east", "west", "central"]; // Deliberately large and deterministic (no Math.random): big enough that a -// filter change cannot settle inside one animation frame, so the rebuild +// grouping change cannot settle inside one animation frame, so the rebuild // really does publish multiple `rebuilding` slices instead of jumping // straight to `ready` — see the note on the smaller custom-renderer example. export const ORDER_COUNT = 150_000; diff --git a/apps/website/content/examples/headless-rebuild-progress/example.ts b/apps/website/content/examples/headless-rebuild-progress/example.ts index d4558f711..b04bf4226 100644 --- a/apps/website/content/examples/headless-rebuild-progress/example.ts +++ b/apps/website/content/examples/headless-rebuild-progress/example.ts @@ -3,7 +3,7 @@ import { defineExample } from "../../../lib/docs/examples/define"; export default defineExample({ title: "Watching a rebuild", description: - "Filtering 150,000 rows cannot settle inside one animation frame, so status.kind cycles through rebuilding with a live completedRows/totalRows progress readout before returning to ready.", + "Grouping 150,000 rows cannot settle inside one animation frame, so status.kind cycles through rebuilding with a live completedRows/totalRows progress readout before returning to ready.", files: [ "RebuildProgressDemo.tsx", "RebuildProgress.tsx", diff --git a/docs/superpowers/plans/2026-08-19-filter-subset-rebuild.md b/docs/superpowers/plans/2026-08-19-filter-subset-rebuild.md new file mode 100644 index 000000000..8e6e37584 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-filter-subset-rebuild.md @@ -0,0 +1,61 @@ +# Filter Subset Rebuild Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Filter-only `setQuery` on ungrouped data settles at TanStack parity (50k: ~240ms → ≤ same-run TanStack ~58ms) via a synchronous subset rebuild that carries unflipped records by identity and never sorts the full set. + +**Architecture:** `isFilterOnlyChange` classifier gate → new `packages/row-model/src/filter-rebuild.ts` (verdict diff; O(flipped) record + map updates; merge of surviving visible order with the sorted flipped-in subset; bulk tree build) → wired into `setQuery` beside the sort fast path, publishing an ordinary barrier. + +**Spec:** `docs/superpowers/specs/2026-08-19-filter-subset-rebuild-design.md` — read first. Baseline + attribution: scratchpad `filter-baseline.md`. + +**Conventions:** as the #457 arc (TDD, mutation-harden, constraints-only comments, prettier/lint/typecheck per commit, verify HEAD before amends, never touch ~/repos/pretable). Branch: `blove/filter-fast-path` off `a438efb0`. + +**Grounding facts (verify only if contradicted):** classifier facets live in `classifyQueryDelta` (compiled-query.ts); `isSortOnlyChange` is the pattern to mirror. Sort-rebuild (`sort-rebuild.ts`) shows: guard style, `fillSortKeysFromPrevious` per-row fill, decorated `{record, keys}` entries feeding `createOrderStatisticTreeFromSortedEntries`, `publishCommittedRoot` publish, error semantics. The merged per-row cache entry holds sortKeys (unguarded) + metadata (guarded). `#finalizeMetadata` is the one metadata constructor. Filter predicates run in `evaluate` via `evaluateFilter(filter, column, value)` per active filter — locate and reuse, do not duplicate predicate semantics. `visible.rows` tree entries are `OrderedRowEntry {record, keys}`; in-order walk via `entries()`/`range`. Counters wire through diagnostics.ts (interface + init + reset list). + +--- + +### Task F1: `isFilterOnlyChange` classifier + +**Files:** `packages/row-model/src/compiled-query.ts`; test `__tests__/query-delta.test.ts`. + +- [ ] TDD: mirror the `isSortOnlyChange` suite — true only when runtime filters differ and nothing else does; false for each other facet changing alongside; false when authorities differ; false under external filter authority both sides (runtime filters empty twice); false for foreign plans. Export `isFilterOnlyChange` beside `isSortOnlyChange`, derived from the same delta. +- [ ] Mutation: flip the derivation to ignore `sortChanged` → a combined sort+filter case must fail. +- [ ] Suite green (413 baseline), commit: `feat(row-model): classify filter-only plan changes`. + +### Task F2: `filter-rebuild.ts` — the subset rebuild + +**Files:** create `packages/row-model/src/filter-rebuild.ts`; modify `diagnostics.ts` (counters: `filterRebuilds`, `filterRowsFlipped`, `filterMergeSortedInsertions`, and `filterRebuildMs` or reuse — report choice); test `__tests__/filter-fast-path.test.ts` (create). + +Signature mirrors `rebuildRootForSortOnlyChange`: `rebuildRootForFilterOnlyChange({captured, nextPlan, revision, now, instrumentation})`. Guards: TypeError unless `isFilterOnlyChange`; TypeError if grouped. + +Algorithm (spec Design §subset rebuild — follow it exactly): +- One pass over `captured.sourceOrder.entries()`: per record — `fillSortKeysFromPrevious(nextPlan, captured.queryPlan, record, instrumentation)` (100% carries); compute the NEW verdict via the plan's predicate machinery over the row (reuse `evaluateFilter` + accessor reads — expose a plan-internal `filterVerdict(plan, row)` static+free function if needed; do NOT re-implement predicate semantics); diff vs `record.metadata.filterPasses`. +- Unflipped: carry. Flipped: build the new record through the existing metadata construction (a plan-internal helper that rebuilds metadata with a new `filterPasses` around carried values — the `#finalizeMetadata` seam; keys already in the store). Transient-set flipped rows only; zero flips → rows map carries by identity (pin this). +- Visible: in-order walk of `captured.visible.rows` collecting still-passing entries (reuse entry objects when the record carried; flipped-out skipped); flipped-in rows become fresh decorated entries of their NEW records, sorted among themselves with `compareWithSortKeys` + id tiebreak; single merge (both sequences already strictly sorted by the same total order) → `createOrderStatisticTreeFromSortedEntries`. +- Root: revision/parentRevision/cause as sort-rebuild; `sourceOrder`/`expansion` carried. Instrumentation: counts + wall time. + +- [ ] TDD red-first per the spec's Testing list items 2–5 (equivalence incl. filter-to-empty/empty-to-filter/zero-flip; identity; merge fixture with interleaved + tied flip-ins; counters). Cold-model oracles throughout; fixture controls asserted (flip-ins interleave, tie pair's source order opposes id order). +- [ ] Mutations: (a) merge order broken (append instead of merge) → bulk constructor throw or equivalence fail; (b) rebuild ALL records → identity test fails; (c) verdict diff inverted → equivalence fails. +- [ ] Suite green, typecheck, lint, prettier. Commit: `feat(row-model): synchronous subset rebuild for filter-only changes`. + +### Task F3: wire into `setQuery` + +**Files:** `packages/row-model/src/create-local-row-model.ts`; test `__tests__/filter-fast-path.test.ts`. + +- [ ] Branch beside the sort fast path (after it, same shape): `isFilterOnlyChange && ungrouped` → cancelActive("superseded"), rebuild, `publishCommittedRoot(root, prev, rev)` (default barrier reason — NOT "reorder"), resolved transition; error path mirrors the sort branch exactly (shared error-construction helper if extraction is clean — implementer judgment, report). +- [ ] TDD: mirror the sort fast-path model-level suite — synchronous (no scheduler entries), notify-once, supersede in-flight cooperative, accessor/predicate failure shape pinned against the slow path first, recovery, setRows-after behaves, equivalence vs cold model, `changesSince` reports a NORMAL barrier reason (pin "bulk-replace", NOT "reorder" — a wrong reorder here would corrupt the renderer; this is the highest-stakes assertion in the cycle, mutation-verify it: publish "reorder" → the test must fail). +- [ ] Existing cooperative-path tests that used filter-only changes as their subject vehicle now take the fast path — same edit discipline as cycle-1 Task 5 (justify each; keep cooperative subjects on cooperative changes, e.g. sort+filter combined). +- [ ] Full package suite green; root build+typecheck. Commit: `feat(row-model): filter-only setQuery completes synchronously on flat queries`. + +### Task F4: verification + +- [ ] Full repo: build, typecheck, lint, test, api (no report drift expected — nothing public changes). +- [ ] Bench protocol (filter-baseline.md method): both filter scripts × both scales × both adapters, repeats 3; PLUS the no-regression sweep: sort both scales, grouped gate (3 quiet runs or the paired-control method if loaded), mount metric. Evaluate the four spec bars. +- [ ] One trace of 50k filter-metadata: confirm the settle tail collapsed and the renderer replacement still runs its cooperative retained-state path. +- [ ] Write `/filter-cycle-results.md` with verdicts. STOP: merge decision to the user with numbers. + +## Self-review notes (applied) + +- Spec coverage: classifier → F1; rebuild+counters → F2; wiring+journal pin → F3; bars → F4. Reserve lever and rejected approach have no tasks. +- Names: `isFilterOnlyChange`, `rebuildRootForFilterOnlyChange`, `filterRebuilds`/`filterRowsFlipped`/`filterMergeSortedInsertions`, plan-internal `filterVerdict` (working name; implementer may improve, reporting it). +- The one semantic decision delegated with a pin requirement: zero-flip behavior (new revision, wholesale carry) — F2 decides and pins it; the spec's Testing item 2 names it. diff --git a/docs/superpowers/plans/2026-08-24-columnar-verdicts.md b/docs/superpowers/plans/2026-08-24-columnar-verdicts.md new file mode 100644 index 000000000..b37f57e2c --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-columnar-verdicts.md @@ -0,0 +1,54 @@ +# Columnar Verdict Cache Implementation Plan (Amendment J) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the per-row verdict pass in the filter fast path with compiled predicates scanning slot-indexed value vectors, per Amendment J (`docs/superpowers/specs/2026-08-24-dense-handle-amendment-j-columnar-verdicts.md` — read it first; its five design deltas are binding). + +**Architecture:** All inside `packages/row-model`. `CompiledRowInput` gains `slot`; the plan's shared evaluation cache gains a per-filter-column `SlotVector` written wherever metadata is evaluated (the freshness invariant), adopted by reference across filter-only changes; `compileFilterPredicate` hoists operator dispatch out of the row loop; `filter-rebuild` consumes a bulk verdict scan. Per-row `filterVerdict` keeps its exact semantics for k-sized/grouped paths. + +**Worktree:** `/Users/blove/repos/pretable/.claude/worktrees/homepage-hero-demo-3878ef`, branch `blove/filter-fast-path`. Test filter: `pnpm --filter @pretable-internal/row-model test` (557 green). + +**Verified anchors** (re-verify; lines drift): `compiled-query.ts` — `CompiledRowInput` ~80; `CachedEvaluation` + cache comment ~262-300; constructor/active-columns ~1440-1470; `evaluate` ~1471 (collects `values` Map over `#active`, calls `#filterVerdict((id) => values.get(id))`); `#filterVerdict` ~1623 (the one predicate loop, `evaluateFilter(filter, #byId.get(...), valueOf(...))`); static `filterVerdict` ~1650 (memo guard then live accessor reads); `adoptEvaluationCache` static ~1900-1930 (`nextPlan.#evaluationCache = previousPlan.#evaluationCache`); `evaluateFilter` + `FILTER_OPERATORS` (search; operand validation at ~735-800 shows the operator/type space: number/date/text/enum/boolean, between/dateBetween ranges, string ops). `filter-rebuild.ts` — the `forEachSlotEntry` walk calling `filterVerdict(nextPlan, previous as never)` per row. Callers of `evaluate`: `row-store.ts` (buildRowStore, rebuildRowStoreForQuery — the latter is DEAD code, do not thread it), `transaction-draft.ts` `createRecord` (~290) and its callers, `cooperative-transition.ts`. Callers of `filterVerdict`: `filter-rebuild.ts`, `transaction-draft.ts` (`passesNext` memo), possibly `distinct-values.ts`/`group-index.ts` — grep and list them in your report. + +**Bars:** Amendment J §Bars. Grouped and k-sized paths byte-identical in behavior; zero public API drift (`pnpm api` in the final task must show NOTHING — `CompiledRowInput` is internal; if it turns out to be re-exported through a governed barrel, STOP and report). + +--- + +### Task 1: `CompiledRowInput.slot` + threading + +**Files:** `compiled-query.ts` (the input type), `row-store.ts`, `transaction-draft.ts`, `cooperative-transition.ts`, `filter-rebuild.ts`, `sort-rebuild.ts`, plus every other `evaluate`/`filterVerdict`/`sortKeysOf`/`fillSortKeysFromPrevious` call site the compiler finds. + +- Add `readonly slot: number` to `CompiledRowInput` (REQUIRED — the compiler enumerates the call sites; every caller has the record or just allocated the slot). Records passed `as never` (whole-record inputs) already carry `.slot`, so most sites compile untouched — verify which literal-input sites need an explicit `slot`. +- No behavior change; the field is unread this task. Tests: full suite green with zero existing-test edits (except fixtures that build `CompiledRowInput` literals — sanctioned, list them). Commit: `feat(row-model): thread slots into compiled-query inputs`. + +### Task 2: compiled predicates + +**Files:** `compiled-query.ts` (+ its test file) + +- `compileFilterPredicate(filter, column): (value: unknown) => boolean` — one specialized closure per runtime filter, built at plan construction (`#compiledPredicates: readonly ((value: unknown) => boolean)[]` parallel to `#runtimeQuery.filters`), hoisting operand normalization out of the loop. It must reproduce `evaluateFilter`'s semantics EXACTLY — implement it by refactoring `evaluateFilter` into "compile" + "apply" so the semantics exist once (the current `evaluateFilter(filter, column, value)` becomes `compileFilterPredicate(filter, column)(value)` internally, or delegates — no duplicated predicate logic). +- `#filterVerdict` switches to the compiled array (same `every` shape, no `#byId` get per row). +- Tests: an exhaustive operator-semantics equivalence sweep — for every (column type, operator) pair in `FILTER_OPERATORS` with representative + edge values (empty string, NaN-adjacent, null/undefined cells, between bounds inclusive/exclusive as today), assert compiled predicate ≡ the pre-refactor behavior (pin against expected outcomes, not against the refactored code). Red-first for the new API; full suite green. Commit: `perf(row-model): compile filter predicates once per plan`. + +### Task 3: columnar value store + freshness writes + +**Files:** `compiled-query.ts` (+ tests) + +- The shared-cache object (whatever `adoptEvaluationCache` moves — currently the `#evaluationCache` WeakMap reference) grows a sibling `#columnarValues: Map}>` for FILTER columns, adopted in the same static call. Because plans currently share the WeakMap by direct field assignment, decide the container: wrap both in one `#sharedEvaluationState` object so adoption stays ONE assignment — refactor `adoptEvaluationCache` accordingly (its doc comment updates). +- **REVISED per Amendment J §3 (rev. 2026-08-24)**: `evaluate` NEVER writes cells (two ingest paths hand it a `-1` placeholder slot, and aborted drafts must not leave cell writes). The bulk scan (Task 4) is the only writer. THIS task implements the store plus the commit-side CLEARS: in `applyFlatTransactionDraft`'s success path (beside the `slotWrites` block), clear the cells of every prepared and removed row's slot on the current plan's shared state; `replaceFlatRowsDraft` and the initial build start from empty vectors (fresh shared state or a wholesale reset — pick the shape that matches how shared state is created there and document it). The Task-1 review's placeholder-site table (`d64fba85` review) is the authority on which slots are real. The vectors are COW (`slotVectorWithAll`) — but per-row writes during an O(n) ingest would copy chunks per row; use the transient pattern instead: accumulate writes per evaluation BURST... simpler and correct: `evaluate` calls land during bulk builds and k-sized updates alike, so give the store an explicit mutable-fill discipline mirroring the codebase's transient/freeze idiom: cells are written into a MUTABLE chunk representation owned by the shared state (plans on the same shared state never race — the model is single-threaded and the store is not revision-scoped; it is a CACHE, not a source of truth — old snapshots never read it). Document exactly that: the columnar store is cache-not-truth, mutable-in-place, keyed by (columnId, slot), correct because of the freshness invariant, and NEVER consulted by snapshot reads. This avoids COW entirely — record the deviation from the plan-of-record wording (Amendment J §2 says chunked COW; in-place-mutable-cache is simpler and sound because nothing revision-scoped reads it; note it in the amendment via a one-line edit in this task's commit). +- Slot-capacity growth: vectors grow like the allocator (chunk table extension on demand). +- Tests: freshness invariant oracle — a scripted model (ingest → filter commit → update changed values → remove+add reusing a slot → filter commit → a transaction whose accessor THROWS mid-draft (aborted) → filter commit) asserting the columnar answer equals a fresh accessor read for EVERY (filter column, live slot) after every step. Mutation (perform, restore, report): skip the commit-side clear for prepared rows → the update step's oracle fails. Commit: `feat(row-model): columnar filter-value cache with write-through freshness`. + +### Task 4: bulk verdict scan + filter-rebuild consumption + +**Files:** `compiled-query.ts`, `filter-rebuild.ts` (+ tests) + +- New internal `bulkFilterVerdicts(plan, recordsBySlot, slotCapacity): MembershipBitset` — per filter: loop live slots (holes-aware walk over `recordsBySlot`), read the cell (hole → live accessor read + write-through fill), apply the compiled predicate, AND across filters into the bitset (first filter sets, subsequent filters clear — or evaluate all filters per slot in one pass; CHOOSE the one-pass-per-slot shape so a row's cells are read with locality and the fallback fill happens at most once per row, and document the choice). +- `filter-rebuild.ts`: the walk keeps its shape but the per-row `filterVerdict` call is replaced by a bitset lookup from ONE `bulkFilterVerdicts` call before the walk (or fold the flip-diff into the scan loop — keep the existing walk + `testMembershipBit(nextVisibleSlots, slot)` reads; simplest coherent shape wins, document it). Zero-flip identity carry, merge, `derivedById`, instrumentation counters all unchanged; add a `columnarVerdictScans` work counter. +- Per-row `filterVerdict` (static) is UNTOUCHED for k-sized/grouped callers. +- Tests: the columnar-vs-per-row equivalence oracle on randomized query scripts (reuse the Task 3 fixture style; include newly-referenced columns mid-script to exercise the hole-fill path); all existing filter fast-path pins (order-independence, zero record rebuilds, `filterRowsFlipped` counts) must pass UNCHANGED. Mutation: make the scan skip the hole-fill fallback → the newly-referenced-column script step fails. Commit: `perf(row-model): filter rebuild verdicts from columnar scan`. + +### Task 5: gates + measurement + +- `pnpm build && pnpm api` — expect ZERO `.api.md` drift (all internal). Full root `pnpm test` (react flake rule). Lint, prettier. +- Bench per the established protocol (M1+M2 plan Task 8 verbatim; scales `target` + `hypothesis`; baseline = the commit before this plan's Task 1; TanStack controls in band; interleaved paired sides; no grep|head; traced share run AFTER headlines for the verdict-share ≲3% bar). +- Deliverable: `docs/superpowers/specs/2026-08-24-columnar-verdicts-results.md` — table, deltas vs 108.3/116.8, both bar verdicts, fitness, conclusion (including what remains: rebuild-body ~17%, HAMT ~9%, render/commit ~23%). Commit. diff --git a/docs/superpowers/plans/2026-08-24-dense-handle-m0-probe.md b/docs/superpowers/plans/2026-08-24-dense-handle-m0-probe.md new file mode 100644 index 000000000..9a612a3b7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-dense-handle-m0-probe.md @@ -0,0 +1,343 @@ +# Dense-Handle M0 Pricing Probe Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Measure, in isolated Node microbenchmarks, the cost of the three new primitives from the dense-handle spec (membership bitsets, chunked COW slot vectors, columnar verdict scans) at 50k rows, and produce a go/no-go verdict against the spec's projected filter-commit window. + +**Architecture:** One self-contained Node script (no imports from the repo — these primitives don't exist yet; the probe IS their first draft) that implements minimal versions of each primitive, verifies them against naive oracles so we never time broken code, then times (a) each primitive in isolation, (b) a composed "filter-commit equivalent" that mirrors the model-side work of a 50k filter flip, and (c) chunk-COW maintenance under a streaming transaction mix. Results are compared against the trace attribution of the code being replaced (persistent-map 42.5ms, verdict pass 18.3ms, double-lookup 11.9ms — from `filter-final-results.md`). + +**Tech Stack:** Node (v20+, plain JS, no deps). Method precedent: `index-representation-probe.md`, whose Node prediction landed within 2ms of the browser number. + +**Context for the executor:** +- Spec: `docs/superpowers/specs/2026-08-24-dense-handle-core-design.md` (read it first). +- The probe file is throwaway measurement tooling — it lives in the session scratchpad, NOT in the repo. Only the results document is committed. +- Machine-load rule: before timing, run `uptime`; if 1-min load ≥ ~8 on this 10-core Mac, report that and rely on the spread check below rather than absolute trust. The fitness test is the control's spread: rep-to-rep spread of any median ≤ 20% or the run is invalid — rerun. +- Numbers rule: never report a single run; report median of 5 timed reps after 2 warmup reps, and report the min/max spread. + +**Scratchpad directory (create the probe here):** +`/private/tmp/claude-501/-Users-blove-repos-pretable--claude-worktrees-running-examples-component-d29c33/e6a8fc40-eb1a-438e-8b29-07506e9af41d/scratchpad` + +--- + +### Task 1: Probe primitives with correctness oracles + +**Files:** +- Create: `/m0-probe.mjs` + +- [ ] **Step 1: Write the primitives and their oracle checks** + +Create `m0-probe.mjs` with the following content (this is the complete file for Task 1; Task 2 appends to it): + +```js +// M0 pricing probe — dense-handle core primitives at 50k. +// Spec: docs/superpowers/specs/2026-08-24-dense-handle-core-design.md +// Throwaway measurement code; committed artifact is the results doc only. + +const N = 50_000; +const CHUNK = 1024; +const WARMUP = 2, REPS = 5; + +// ---------- membership bitset (immutable-by-copy) ---------- +const bsNew = (n) => new Uint32Array((n + 31) >>> 5); +const bsClone = (b) => b.slice(); +const bsSet = (b, i) => { b[i >>> 5] |= 1 << (i & 31); }; +const bsTest = (b, i) => (b[i >>> 5] >>> (i & 31)) & 1; +const bsXor = (a, b) => { const out = new Uint32Array(a.length); for (let i = 0; i < a.length; i++) out[i] = a[i] ^ b[i]; return out; }; +const bsPopcount = (b) => { let c = 0; for (let i = 0; i < b.length; i++) { let w = b[i]; w -= (w >>> 1) & 0x55555555; w = (w & 0x33333333) + ((w >>> 2) & 0x33333333); c += (((w + (w >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24; } return c; }; +// iterate set bits, calling f(index); returns count +function bsForEach(b, f) { + let count = 0; + for (let w = 0; w < b.length; w++) { + let word = b[w]; + const base = w << 5; + while (word !== 0) { + const t = word & -word; + f(base + (31 - Math.clz32(t))); + word ^= t; + count++; + } + } + return count; +} + +// ---------- chunked COW slot vector ---------- +function vecFromArray(arr) { + const chunks = []; + for (let i = 0; i < arr.length; i += CHUNK) chunks.push(arr.slice(i, i + CHUNK)); + return { chunks, length: arr.length }; +} +const vecGet = (v, i) => v.chunks[i >>> 10][i & 1023]; +// COW write: copies the chunk table + the one touched chunk +function vecWith(v, i, val) { + const chunks = v.chunks.slice(); + const c = chunks[i >>> 10].slice(); + c[i & 1023] = val; + chunks[i >>> 10] = c; + return { chunks, length: v.length }; +} +// batched COW write for one commit: copies table once, each touched chunk once +function vecWithAll(v, writes /* [i, val][] */) { + const chunks = v.chunks.slice(); + const copied = new Set(); + for (const [i, val] of writes) { + const ci = i >>> 10; + if (!copied.has(ci)) { chunks[ci] = chunks[ci].slice(); copied.add(ci); } + chunks[ci][i & 1023] = val; + } + return { chunks, length: v.length }; +} + +// ---------- fixture ---------- +// Mirrors the S2 bench shape loosely: numeric column + string column. +function mulberry32(seed) { return () => { let t = (seed += 0x6d2b79f5); t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; } +const rnd = mulberry32(42); +const records = new Array(N); +const priceCol = new Float64Array(N); // columnar numeric cache +const nameCol = new Array(N); // columnar string cache +for (let s = 0; s < N; s++) { + const price = Math.floor(rnd() * 10_000) / 10; + const name = `instrument-${(s * 7919) % N}-${s % 97}`; + records[s] = { id: `row-${s}`, slot: s, price, name }; + priceCol[s] = price; + nameCol[s] = name; +} +const live = bsNew(N); +for (let s = 0; s < N; s++) bsSet(live, s); + +// ---------- oracles (run before ANY timing; abort on mismatch) ---------- +function assertEq(actual, expected, label) { + if (actual !== expected) { console.error(`ORACLE FAIL: ${label}: ${actual} !== ${expected}`); process.exit(1); } +} + +// Oracle 1: numeric columnar scan matches naive filter +{ + const bs = bsNew(N); + for (let s = 0; s < N; s++) if (priceCol[s] >= 250 && priceCol[s] < 500) bsSet(bs, s); + let naive = 0; + for (let s = 0; s < N; s++) if (records[s].price >= 250 && records[s].price < 500) { naive++; assertEq(bsTest(bs, s), 1, `numeric bit ${s}`); } + assertEq(bsPopcount(bs), naive, "numeric scan popcount"); +} +// Oracle 2: xor-diff enumerates exactly the flipped slots +{ + const a = bsNew(N), b = bsNew(N); + for (let s = 0; s < N; s++) { if (s % 3 === 0) bsSet(a, s); if (s % 5 === 0) bsSet(b, s); } + const diff = bsXor(a, b); + const flipped = []; + bsForEach(diff, (s) => flipped.push(s)); + const expected = []; + for (let s = 0; s < N; s++) if ((s % 3 === 0) !== (s % 5 === 0)) expected.push(s); + assertEq(flipped.length, expected.length, "xor flip count"); + for (let i = 0; i < expected.length; i++) assertEq(flipped[i], expected[i], `xor flip order at ${i}`); +} +// Oracle 3: COW vector — writes land, snapshot validity holds under overwrite +{ + const v0 = vecFromArray(records); + const v1 = vecWithAll(v0, [[5, { id: "row-X", slot: 5 }], [40_000, { id: "row-Y", slot: 40_000 }]]); + assertEq(vecGet(v1, 5).id, "row-X", "cow write 5"); + assertEq(vecGet(v1, 40_000).id, "row-Y", "cow write 40000"); + assertEq(vecGet(v0, 5).id, "row-5", "snapshot validity 5"); // old snapshot unchanged + assertEq(vecGet(v0, 40_000).id, "row-40000", "snapshot validity 40000"); + assertEq(vecGet(v1, 6).id, "row-6", "untouched neighbor"); +} +console.log("oracles: PASS"); +``` + +- [ ] **Step 2: Run the oracles** + +Run: `node /m0-probe.mjs` +Expected: `oracles: PASS`, exit code 0. If any `ORACLE FAIL` prints, fix the primitive before proceeding — do not time broken code. + +--- + +### Task 2: Timed sections — isolation, composed commit, streaming maintenance + +**Files:** +- Modify: `/m0-probe.mjs` (append after the oracle block) + +- [ ] **Step 1: Append the timing harness and measurement sections** + +```js +// ---------- timing harness ---------- +function bench(label, fn) { + for (let i = 0; i < WARMUP; i++) fn(); + const times = []; + for (let i = 0; i < REPS; i++) { const t0 = performance.now(); fn(); times.push(performance.now() - t0); } + times.sort((x, y) => x - y); + const med = times[REPS >> 1], min = times[0], max = times[REPS - 1]; + const spreadPct = med > 0 ? ((max - min) / med) * 100 : 0; + console.log(`${label}: median ${med.toFixed(3)}ms (min ${min.toFixed(3)} / max ${max.toFixed(3)}, spread ${spreadPct.toFixed(0)}%)`); + return { med, min, max, spreadPct }; +} +let sink = 0; // defeat dead-code elimination + +const results = {}; + +// --- A. isolation: primitives --- +results.numericScan = bench("A1 numeric columnar scan -> bitset (50k)", () => { + const bs = bsNew(N); + for (let s = 0; s < N; s++) if (priceCol[s] >= 250 && priceCol[s] < 500) bs[s >>> 5] |= 1 << (s & 31); + sink += bs[0]; +}); +results.stringScan = bench("A2 string columnar scan .includes -> bitset (50k)", () => { + const bs = bsNew(N); + for (let s = 0; s < N; s++) if (nameCol[s].includes("7")) bs[s >>> 5] |= 1 << (s & 31); + sink += bs[0]; +}); +// old membership: ~25% visible; new: shifted band flips roughly half in/half out +const oldBs = bsNew(N); +for (let s = 0; s < N; s++) if (priceCol[s] >= 250 && priceCol[s] < 500) bsSet(oldBs, s); +const newBs = bsNew(N); +for (let s = 0; s < N; s++) if (priceCol[s] >= 375 && priceCol[s] < 625) bsSet(newBs, s); +results.xorDiff = bench("A3 xor + enumerate flipped slots", () => { + const diff = bsXor(oldBs, newBs); + let acc = 0; + bsForEach(diff, (s) => { acc += s; }); + sink += acc; +}); +const recordsVec = vecFromArray(records); +results.vecReads = bench("A4 50k vecGet reads (records by slot)", () => { + let acc = 0; + for (let s = 0; s < N; s++) acc += vecGet(recordsVec, s).price; + sink += acc; +}); +// baseline for comparison: same reads through a string-keyed Map (NOT the HAMT, +// which the trace prices at 42.5ms — this bounds how much of that is stringiness) +const byIdMap = new Map(records.map((r) => [r.id, r])); +const idList = records.map((r) => r.id); +results.mapReads = bench("A5 50k string-keyed Map.get reads (baseline)", () => { + let acc = 0; + for (let s = 0; s < N; s++) acc += byIdMap.get(idList[s]).price; + sink += acc; +}); + +// --- B. composed filter-commit equivalent (model-side work, minus tree build) --- +// scan -> xor -> enumerate -> resolve flip-in records -> sort flip-ins by key -> merge survivors +results.composed = bench("B composed filter-commit equivalent", () => { + // 1. verdict scan + const next = bsNew(N); + for (let s = 0; s < N; s++) if (priceCol[s] >= 375 && priceCol[s] < 625) next[s >>> 5] |= 1 << (s & 31); + // 2. diff + const diff = bsXor(oldBs, next); + const flippedIn = [], flippedOut = []; + bsForEach(diff, (s) => { if ((next[s >>> 5] >>> (s & 31)) & 1) flippedIn.push(s); else flippedOut.push(s); }); + // 3. resolve + sort flip-ins by sort key (price asc, mirroring an active sort) + flippedIn.sort((x, y) => priceCol[x] - priceCol[y]); + // 4. merge survivors (walk old membership in slot order as a stand-in for the + // old-tree range walk) with sorted flip-ins into the new visible order + const survivors = []; + bsForEach(oldBs, (s) => { if ((next[s >>> 5] >>> (s & 31)) & 1) survivors.push(s); }); + survivors.sort((x, y) => priceCol[x] - priceCol[y]); + const merged = new Array(survivors.length + flippedIn.length); + let i = 0, j = 0, k = 0; + while (i < survivors.length && j < flippedIn.length) + merged[k++] = priceCol[survivors[i]] <= priceCol[flippedIn[j]] ? survivors[i++] : flippedIn[j++]; + while (i < survivors.length) merged[k++] = survivors[i++]; + while (j < flippedIn.length) merged[k++] = flippedIn[j++]; + // touch resolved records so the read isn't elided + let acc = 0; + for (let m = 0; m < merged.length; m++) acc += vecGet(recordsVec, merged[m]).slot; + sink += acc + merged.length; +}); +// NOTE for results doc: the real path does NOT re-sort survivors (order comes +// proven from the old tree walk); the survivors.sort here is deliberate +// overcounting — call it out as slack in the projection. + +// --- C. streaming maintenance: chunk-COW under a transaction mix --- +const TXN_COUNT = 1000, TXN_SIZE = 100; +const txns = []; +{ + const r2 = mulberry32(7); + for (let t = 0; t < TXN_COUNT; t++) { + const writes = []; + for (let w = 0; w < TXN_SIZE; w++) { const s = Math.floor(r2() * N); writes.push([s, records[s]]); } + txns.push(writes); + } +} +results.streaming = bench(`C ${TXN_COUNT} commits x ${TXN_SIZE} random writes (chunk-COW)`, () => { + let v = recordsVec; + for (const writes of txns) v = vecWithAll(v, writes); + sink += v.chunks.length; +}); +console.log(` per-commit: ${(results.streaming.med / TXN_COUNT * 1000).toFixed(1)}µs`); +results.bitsetClone = bench("C2 1000 whole bitset clones (6.25KB each)", () => { + let b = oldBs; + for (let t = 0; t < TXN_COUNT; t++) b = bsClone(b); + sink += b[0]; +}); + +// --- verdict --- +const newParts = results.numericScan.med + results.xorDiff.med + results.composed.med; +console.log("---"); +console.log(`replaced (trace attribution): persistent-map 42.5 + verdict 18.3 + double-lookup 11.9 = 72.7ms`); +console.log(`new (probe, overcounted): scan+diff+composed = ${newParts.toFixed(1)}ms`); +console.log(`sink: ${sink}`); // keep the JIT honest +``` + +- [ ] **Step 2: Check machine load, then run the full probe** + +Run: `uptime`, then `node /m0-probe.mjs > /m0-probe-output.txt 2>&1; echo "exit=$?"` and then print the output file. +Expected: `oracles: PASS`, all sections print medians, `exit=0`. Redirect to a file and check the exit code — never pipe through `grep|head` (SIGPIPE truncates gates). + +- [ ] **Step 3: Validate the run's fitness** + +Check every reported spread. If any section's spread exceeds 20%, or `uptime` 1-min load was ≥ 8: rerun the probe once; if still noisy, report the noise explicitly in the results doc rather than hiding it. + +--- + +### Task 3: Results document, go/no-go, commit + +**Files:** +- Create: `docs/superpowers/specs/2026-08-24-dense-handle-m0-results.md` (in the `homepage-hero-demo-3878ef` worktree — results are load-bearing conclusions, so they live with the specs, per the "if the session directory is gone, the specs carry the conclusions" rule) + +- [ ] **Step 1: Write the results document** + +Structure (fill with the actual measured numbers — no placeholders may survive): + +```markdown +# M0 pricing probe — results + +Date: 2026-08-24. Probe: `m0-probe.mjs` (session scratchpad, throwaway). +Machine load at run: . Fitness: all spreads ≤ 20%? . + +## Isolation numbers (median of 5, 2 warmups) + +| section | median | spread | +|---|---|---| +| A1 numeric columnar scan → bitset | ... | ... | +| A2 string columnar scan → bitset | ... | ... | +| A3 xor + enumerate flips | ... | ... | +| A4 50k records-by-slot vecGet | ... | ... | +| A5 50k string-keyed Map.get (baseline) | ... | ... | +| B composed filter-commit equivalent | ... | ... | +| C streaming: per-commit chunk-COW (100 writes) | ...µs | ... | +| C2 whole-bitset clone | ... | ... | + +## Read-across + +- Replaced work (trace attribution, `filter-final-results.md`): 72.7ms. +- New-structure equivalent (overcounted — includes a survivors sort the real + path does not perform): ms. +- Streaming regression check: per-commit COW cost µs vs the HAMT's + structural sharing — verdict on whether 60Hz streaming survives: . +- Caveat: Node numbers; the precedent probe landed within 2ms of the browser, + but browser certification is M7's job, not M0's. + +## Go/no-go + + +``` + +- [ ] **Step 2: Commit** + +```bash +cd /Users/blove/repos/pretable/.claude/worktrees/homepage-hero-demo-3878ef +git add docs/superpowers/specs/2026-08-24-dense-handle-m0-results.md +git commit -m "docs: M0 pricing probe results for the dense-handle core + +Co-Authored-By: Claude Opus 5 " +``` + +Expected: clean commit on `blove/filter-fast-path`. diff --git a/docs/superpowers/plans/2026-08-24-dense-handle-m1-m2.md b/docs/superpowers/plans/2026-08-24-dense-handle-m1-m2.md new file mode 100644 index 000000000..2309f2dfd --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-dense-handle-m1-m2.md @@ -0,0 +1,855 @@ +# Dense-Handle M1+M2 Implementation Plan (slots + membership bitsets) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give every row a dense integer slot for its lifetime, carry a per-revision `recordsBySlot` chunked-COW vector and a `visibleSlots` membership bitset, and rewrite the filter-only rebuild to consume them — eliminating the 18.5ms records-HAMT walk and the 11.9ms old-verdict double lookup at 50k. + +**Architecture:** Three new row-model-internal primitives (spec: `docs/superpowers/specs/2026-08-24-dense-handle-core-design.md`; pricing: `...m0-results.md`, GO) sit UNDER the existing persistent structures. `RevisionRoot` gains two REQUIRED fields so the TypeScript compiler enumerates every construction site; a decision table in Task 5 says what each site does. String ids remain the public currency; public API diff must be zero. + +**Tech Stack:** TypeScript, vitest (`pnpm --filter @pretable/row-model test`), existing bench harness (`scripts/bench-matrix.mjs`). + +**Worktree:** `/Users/blove/repos/pretable/.claude/worktrees/homepage-hero-demo-3878ef`, branch `blove/filter-fast-path`. All paths below are relative to it. + +**Plan-level deviation from the spec (deliberate, record it in the results doc):** the spec names a per-revision "live set" bitset; it is NOT built. Hole-skipping iteration over `recordsBySlot` (undefined = free slot) serves as the live domain, which removes a whole structure and its maintenance. The visible set IS built (M2). + +**Design invariants (repeat in code comments where noted):** +1. A slot binds to one row for that row's lifetime; release only on permanent removal; reuse allowed afterwards. +2. Old snapshots stay valid under slot reuse because every revision holds its own immutable chunk table — revision N's `recordsBySlot` still binds slot s to whatever row owned s at revision N. +3. `visibleSlots` is REAL only for flat (ungrouped) roots; grouped roots carry the `EMPTY_MEMBERSHIP` sentinel and keep answering membership from the group index (`filter-membership.ts` is unchanged). +4. Membership IS the verdict (H-cycle invariant) — the bitset is a faster index of the same structural answer, never a stored verdict. + +--- + +### Task 1: `membership-bitset.ts` + +**Files:** +- Create: `packages/row-model/src/membership-bitset.ts` +- Test: `packages/row-model/src/__tests__/membership-bitset.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import { + EMPTY_MEMBERSHIP, + cloneMembership, + createMembership, + clearMembershipBit, + setMembershipBit, + testMembershipBit, +} from "../membership-bitset"; + +describe("membership bitset", () => { + it("round-trips set/clear/test across word boundaries", () => { + const bits = createMembership(100); + for (const slot of [0, 31, 32, 63, 64, 99]) { + expect(testMembershipBit(bits, slot)).toBe(false); + setMembershipBit(bits, slot); + expect(testMembershipBit(bits, slot)).toBe(true); + } + clearMembershipBit(bits, 32); + expect(testMembershipBit(bits, 32)).toBe(false); + expect(testMembershipBit(bits, 31)).toBe(true); + expect(testMembershipBit(bits, 63)).toBe(true); + }); + + it("clone is independent of the original", () => { + const bits = createMembership(64); + setMembershipBit(bits, 10); + const copy = cloneMembership(bits, 64); + clearMembershipBit(copy, 10); + setMembershipBit(copy, 20); + expect(testMembershipBit(bits, 10)).toBe(true); + expect(testMembershipBit(bits, 20)).toBe(false); + }); + + it("clone can grow capacity, preserving low bits", () => { + const bits = createMembership(32); + setMembershipBit(bits, 31); + const grown = cloneMembership(bits, 200); + expect(testMembershipBit(grown, 31)).toBe(true); + setMembershipBit(grown, 199); + expect(testMembershipBit(grown, 199)).toBe(true); + }); + + it("reads beyond a bitset's words answer false (EMPTY sentinel contract)", () => { + expect(testMembershipBit(EMPTY_MEMBERSHIP, 0)).toBe(false); + expect(testMembershipBit(EMPTY_MEMBERSHIP, 12345)).toBe(false); + const bits = createMembership(32); + expect(testMembershipBit(bits, 500)).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @pretable/row-model test -- membership-bitset` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write the implementation** + +```ts +/** + * Membership bitsets: one bit per SLOT (see `slot-allocator`). A committed + * root's verdict is its membership (the filter-membership invariant); the + * bitset is a faster INDEX of that same structural answer for flat roots, + * never a stored verdict. Grouped roots carry `EMPTY_MEMBERSHIP` and keep + * answering from the group index. + * + * Mutable while a producer is building the next revision's set; frozen by + * convention once a root captures it (no Object.freeze — typed arrays do not + * support it; discipline is "producers build fresh or clone, never write a + * captured root's bitset", the same convention every persistent structure + * here relies on). + * + * Whole-copy on change is the point: 50k rows is 6.25KB, negligible per + * commit (M0 measured ~1µs), so no COW machinery exists at this layer. + */ + +export type MembershipBitset = Uint32Array; + +/** Shared sentinel for roots whose membership lives elsewhere (grouped). */ +export const EMPTY_MEMBERSHIP: MembershipBitset = new Uint32Array(0); + +export function createMembership(capacity: number): MembershipBitset { + return new Uint32Array((capacity + 31) >>> 5); +} + +/** Clone, growing to `capacity` when it exceeds the source's words. */ +export function cloneMembership( + bits: MembershipBitset, + capacity: number, +): MembershipBitset { + const words = Math.max(bits.length, (capacity + 31) >>> 5); + const next = new Uint32Array(words); + next.set(bits); + return next; +} + +export function setMembershipBit(bits: MembershipBitset, slot: number): void { + bits[slot >>> 5]! |= 1 << (slot & 31); +} + +export function clearMembershipBit( + bits: MembershipBitset, + slot: number, +): void { + bits[slot >>> 5]! &= ~(1 << (slot & 31)); +} + +/** Out-of-range slots read as false — the EMPTY sentinel relies on this. */ +export function testMembershipBit( + bits: MembershipBitset, + slot: number, +): boolean { + const word = bits[slot >>> 5]; + return word === undefined ? false : ((word >>> (slot & 31)) & 1) === 1; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @pretable/row-model test -- membership-bitset` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add packages/row-model/src/membership-bitset.ts packages/row-model/src/__tests__/membership-bitset.test.ts +git commit -m "feat(row-model): membership bitset primitive" +``` + +--- + +### Task 2: `slot-allocator.ts` + +**Files:** +- Create: `packages/row-model/src/slot-allocator.ts` +- Test: `packages/row-model/src/__tests__/slot-allocator.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import { createSlotAllocator } from "../slot-allocator"; + +describe("slot allocator", () => { + it("allocates dense sequential slots from zero", () => { + const slots = createSlotAllocator(); + expect([slots.allocate(), slots.allocate(), slots.allocate()]).toEqual([ + 0, 1, 2, + ]); + expect(slots.capacity).toBe(3); + }); + + it("reuses released slots before growing", () => { + const slots = createSlotAllocator(); + slots.allocate(); + const b = slots.allocate(); + slots.allocate(); + slots.release(b); + expect(slots.allocate()).toBe(b); + expect(slots.capacity).toBe(3); + }); + + it("capacity is monotonic and counts the high-water mark", () => { + const slots = createSlotAllocator(); + for (let i = 0; i < 10; i += 1) slots.allocate(); + for (let i = 0; i < 10; i += 1) slots.release(i); + expect(slots.capacity).toBe(10); + for (let i = 0; i < 10; i += 1) slots.allocate(); + expect(slots.capacity).toBe(10); + }); + + it("throws on double release", () => { + const slots = createSlotAllocator(); + const a = slots.allocate(); + slots.release(a); + expect(() => slots.release(a)).toThrow(/released|live/i); + }); + + it("throws on releasing a never-allocated slot", () => { + const slots = createSlotAllocator(); + expect(() => slots.release(5)).toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @pretable/row-model test -- slot-allocator` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write the implementation** + +```ts +/** + * Per-MODEL slot allocator: every row gets a small dense integer for its + * lifetime, assigned at ingest and released only on permanent removal. + * Mutable by design — this is instance state, not revision state; the + * revision-scoped structures (`slot-vector`, `membership-bitset`) are what + * keep old snapshots valid when a released slot is reused. + * + * Capacity is the high-water mark and never shrinks, so slot-indexed + * structures never renumber. Release is fail-loud (double release would hand + * one slot to two live rows, which corrupts every slot-indexed structure + * from that commit on). + */ + +export interface SlotAllocator { + readonly capacity: number; + allocate(): number; + release(slot: number): void; +} + +export function createSlotAllocator(): SlotAllocator { + const free: number[] = []; + let next = 0; + let live = new Uint8Array(1024); + const ensure = (slot: number) => { + if (slot < live.length) return; + const grown = new Uint8Array(Math.max(live.length * 2, slot + 1)); + grown.set(live); + live = grown; + }; + return { + get capacity() { + return next; + }, + allocate() { + const slot = free.length > 0 ? free.pop()! : next++; + ensure(slot); + live[slot] = 1; + return slot; + }, + release(slot) { + if (!Number.isInteger(slot) || slot < 0 || slot >= next) { + throw new RangeError(`Slot ${slot} was never allocated.`); + } + if (live[slot] !== 1) { + throw new RangeError(`Slot ${slot} is not live (double release).`); + } + live[slot] = 0; + free.push(slot); + }, + }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @pretable/row-model test -- slot-allocator` +Expected: PASS (5 tests). + +- [ ] **Step 5: Commit** + +```bash +git add packages/row-model/src/slot-allocator.ts packages/row-model/src/__tests__/slot-allocator.test.ts +git commit -m "feat(row-model): per-model slot allocator" +``` + +--- + +### Task 3: `slot-vector.ts` + +**Files:** +- Create: `packages/row-model/src/slot-vector.ts` +- Test: `packages/row-model/src/__tests__/slot-vector.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import { + SLOT_VECTOR_CHUNK, + emptySlotVector, + forEachSlotEntry, + slotVectorFromEntries, + slotVectorGet, + slotVectorWithAll, +} from "../slot-vector"; + +describe("slot vector", () => { + it("stores entries at their slots, holes read undefined", () => { + const vec = slotVectorFromEntries([[0, "a"], [2, "c"], [1500, "far"]], 2000); + expect(slotVectorGet(vec, 0)).toBe("a"); + expect(slotVectorGet(vec, 1)).toBeUndefined(); + expect(slotVectorGet(vec, 2)).toBe("c"); + expect(slotVectorGet(vec, 1500)).toBe("far"); + expect(slotVectorGet(vec, 1999)).toBeUndefined(); + }); + + it("withAll writes and clears land; result reports chunks copied", () => { + const base = slotVectorFromEntries([[0, "a"], [1, "b"]], 10); + const { next, chunksTouched } = slotVectorWithAll( + base, + [[0, "A"], [1, undefined], [5, "f"]], + 10, + ); + expect(slotVectorGet(next, 0)).toBe("A"); + expect(slotVectorGet(next, 1)).toBeUndefined(); + expect(slotVectorGet(next, 5)).toBe("f"); + expect(chunksTouched).toBe(1); // all three slots share chunk 0 + }); + + it("old snapshots survive later writes, including slot overwrite (COW pin)", () => { + const v0 = slotVectorFromEntries([[5, "old-5"], [1030, "old-1030"]], 2048); + const { next: v1 } = slotVectorWithAll( + v0, + [[5, "new-5"], [1030, undefined]], + 2048, + ); + // v1 sees the writes... + expect(slotVectorGet(v1, 5)).toBe("new-5"); + expect(slotVectorGet(v1, 1030)).toBeUndefined(); + // ...and v0 is byte-identical to before: the snapshot-validity invariant + // that makes slot REUSE safe for held revisions. + expect(slotVectorGet(v0, 5)).toBe("old-5"); + expect(slotVectorGet(v0, 1030)).toBe("old-1030"); + }); + + it("a commit touching k slots in one chunk copies exactly one chunk", () => { + const entries: [number, string][] = []; + for (let s = 0; s < 4096; s += 1) entries.push([s, `v${s}`]); + const base = slotVectorFromEntries(entries, 4096); + const writes: [number, string][] = []; + for (let s = 100; s < 150; s += 1) writes.push([s, `w${s}`]); + const { next, chunksTouched } = slotVectorWithAll(base, writes, 4096); + expect(chunksTouched).toBe(1); + // untouched chunks are carried by reference, not copied + expect(next.chunks[1]).toBe(base.chunks[1]); + expect(next.chunks[0]).not.toBe(base.chunks[0]); + }); + + it("withAll can grow capacity for slots beyond the old table", () => { + const base = slotVectorFromEntries([[0, "a"]], 1); + const { next } = slotVectorWithAll(base, [[5000, "far"]], 5001); + expect(slotVectorGet(next, 5000)).toBe("far"); + expect(slotVectorGet(next, 0)).toBe("a"); + expect(slotVectorGet(base, 5000)).toBeUndefined(); + }); + + it("forEachSlotEntry skips holes and visits every live entry once", () => { + const vec = slotVectorFromEntries([[3, "c"], [SLOT_VECTOR_CHUNK + 1, "x"]], 3000); + const seen: Array<[number, string]> = []; + forEachSlotEntry(vec, (value, slot) => seen.push([slot, value])); + expect(seen).toEqual([[3, "c"], [SLOT_VECTOR_CHUNK + 1, "x"]]); + }); + + it("emptySlotVector reads undefined everywhere", () => { + expect(slotVectorGet(emptySlotVector(), 0)).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @pretable/row-model test -- slot-vector` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write the implementation** + +```ts +/** + * Immutable chunked slot-indexed vector: a chunk table over + * `SLOT_VECTOR_CHUNK`-element chunks, copy-on-write per commit. A commit + * touching k slots copies the table plus each touched chunk once — this is + * what keeps old snapshots valid under slot reuse: every revision holds its + * own table, so revision N still binds slot s to whatever row owned s at + * revision N no matter what later commits do (M0 priced maintenance at + * ~33–98µs per 100-write commit). + * + * Holes (`undefined`) are free slots. Iteration hole-skips, which is why no + * separate "live" bitset exists (recorded plan deviation from the spec). + */ + +export const SLOT_VECTOR_CHUNK = 1024; + +export interface SlotVector { + /** Sparse table: a missing/undefined chunk reads as all holes. */ + readonly chunks: ReadonlyArray | undefined>; +} + +const EMPTY: SlotVector = Object.freeze({ chunks: Object.freeze([]) }); + +export function emptySlotVector(): SlotVector { + return EMPTY; +} + +export function slotVectorFromEntries( + entries: Iterable, + capacity: number, +): SlotVector { + const chunks: Array | undefined> = new Array( + Math.ceil(capacity / SLOT_VECTOR_CHUNK), + ); + for (const [slot, value] of entries) { + const index = (slot / SLOT_VECTOR_CHUNK) | 0; + let chunk = chunks[index]; + if (chunk === undefined) { + chunk = new Array(SLOT_VECTOR_CHUNK); + chunks[index] = chunk; + } + chunk[slot % SLOT_VECTOR_CHUNK] = value; + } + return { chunks }; +} + +export function slotVectorGet( + vector: SlotVector, + slot: number, +): T | undefined { + const chunk = vector.chunks[(slot / SLOT_VECTOR_CHUNK) | 0]; + return chunk === undefined ? undefined : chunk[slot % SLOT_VECTOR_CHUNK]; +} + +/** + * One commit's writes (`undefined` value = clear the slot), COW: table copied + * once, each touched chunk copied once. `capacity` may exceed the old + * table's reach (allocator growth). + */ +export function slotVectorWithAll( + vector: SlotVector, + writes: ReadonlyArray, + capacity: number, +): { readonly next: SlotVector; readonly chunksTouched: number } { + const tableSize = Math.max( + vector.chunks.length, + Math.ceil(capacity / SLOT_VECTOR_CHUNK), + ); + const chunks: Array | ReadonlyArray | undefined> = + new Array(tableSize); + for (let i = 0; i < vector.chunks.length; i += 1) chunks[i] = vector.chunks[i]; + const copied = new Set(); + for (const [slot, value] of writes) { + const index = (slot / SLOT_VECTOR_CHUNK) | 0; + if (!copied.has(index)) { + const existing = chunks[index]; + chunks[index] = + existing === undefined + ? new Array(SLOT_VECTOR_CHUNK) + : existing.slice(); + copied.add(index); + } + (chunks[index] as Array)[slot % SLOT_VECTOR_CHUNK] = value; + } + return { next: { chunks }, chunksTouched: copied.size }; +} + +/** Hole-skipping walk in slot order. */ +export function forEachSlotEntry( + vector: SlotVector, + callback: (value: T, slot: number) => void, +): void { + for (let index = 0; index < vector.chunks.length; index += 1) { + const chunk = vector.chunks[index]; + if (chunk === undefined) continue; + const base = index * SLOT_VECTOR_CHUNK; + for (let offset = 0; offset < chunk.length; offset += 1) { + const value = chunk[offset]; + if (value !== undefined) callback(value, base + offset); + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @pretable/row-model test -- slot-vector` +Expected: PASS (7 tests). + +- [ ] **Step 5: Commit** + +```bash +git add packages/row-model/src/slot-vector.ts packages/row-model/src/__tests__/slot-vector.test.ts +git commit -m "feat(row-model): chunked copy-on-write slot vector" +``` + +--- + +### Task 4: Stamp `slot` on `RowRecord`; thread the allocator through record creation + +**Files:** +- Modify: `packages/row-model/src/internal-types.ts` (RowRecord, ~line 22) +- Modify: `packages/row-model/src/row-store.ts` (`buildRowStore`, `BuildRowStoreInput`; `rebuildRowStoreForQuery` needs NO change — its `{ ...previous, metadata }` spread carries `slot`) +- Modify: `packages/row-model/src/transaction-draft.ts` (`prepareRecord` ~line 290; `applyFlatTransactionDraft` ~line 753; `replaceFlatRowsDraft` ~line 1247) +- Modify: `packages/row-model/src/create-local-row-model.ts` (create the allocator, pass it down) +- Test: `packages/row-model/src/__tests__/slot-lifecycle.test.ts` (new) + +**The lifecycle rules (put this decision logic exactly where each case lives):** + +| Event | Slot action | +|---|---| +| New row ingested (initial build, set-rows add, transaction add) | `slots.allocate()`, stamped on the frozen record | +| Row updated (transaction update, set-rows carry, metadata re-evaluation) | carry `previous.slot` | +| Row permanently removed (transaction remove, set-rows drop) | `slots.release(previous.slot)` — AFTER the draft is known effective (see abandon rule) | +| Draft abandoned (`effective: false` return, or the `catch`→`remap` path) | release every slot the draft allocated; release NO removed slot | + +**Abandon rule rationale (comment it):** a draft allocates slots while preparing records, but `applyFlatTransactionDraft` can still return ineffective or throw after that; leaked allocations would pin free-list slots forever. Track `allocatedSlots: number[]` in the draft; on the two failure paths release them; on the success path release `removedSlots` instead. + +- [ ] **Step 1: Write the failing test** + +The row-model package's test suite constructs models via `createLocalRowModel` (see any existing test in `packages/row-model/src/__tests__/` for the fixture idiom — reuse the simplest existing fixture columns/rows pattern; do not invent a new fixture style). The assertions below reach the committed root via the model's internals: check how existing tests access roots — if none do, export a test-only accessor from the module under `/** @internal test-only */` or assert through `getState().snapshot` plus instrumentation counters, whichever existing tests already use. The behaviors to pin: + +```ts +// slot-lifecycle.test.ts — behavior pins (adapt fixture idiom from existing tests): +// 1. "slots are dense from zero at initial build": model with rows A,B,C → +// records carry slots {0,1,2} (order = source order). +// 2. "update carries the slot": transaction updating B → B's new record has +// B's old slot; A and C untouched (same record identity). +// 3. "remove releases; a later add reuses": transaction removing B, then a +// separate transaction adding D → D's record carries B's former slot, and +// allocator capacity stays 3. +// 4. "set-rows replacement carries intersecting ids": setRows([B', E]) after +// {A,B,C} → B' keeps B's slot; E gets a released slot (0 or 2), capacity +// stays 3. +// 5. "abandoned draft leaks nothing": a transaction that is entirely +// ineffective (e.g. removing a nonexistent id) followed by an add — the +// add's slot shows no gap (capacity grew by exactly the rows actually +// added since build). +``` + +Write them as real vitest tests with the fixture idiom you found. Every pin must be able to fail: e.g. for pin 3, verify it fails if you temporarily make release a no-op (mutation check — actually run it once, then restore). + +- [ ] **Step 2: Run to verify the suite fails to compile / assert** + +Run: `pnpm --filter @pretable/row-model test -- slot-lifecycle` +Expected: FAIL — `slot` does not exist on RowRecord yet. + +- [ ] **Step 3: Implement** + +3a. `internal-types.ts` — add to `RowRecord`: + +```ts + /** + * Dense integer handle, assigned at ingest, stable for the row's lifetime + * (updates carry it; only permanent removal releases it). Slot-indexed + * structures (`recordsBySlot`, `visibleSlots`) are the array-resident fast + * path that replaces string-keyed lookups on O(n) walks. + */ + readonly slot: number; +``` + +3b. `row-store.ts` — `BuildRowStoreInput` gains `readonly slots: SlotAllocator;`. In `buildRowStore`'s record loop: `const slot = previous !== undefined ? previous.slot : input.slots.allocate();` and add `slot` to the frozen record literal. After the loop, when `input.previous !== undefined`, release dropped ids: + +```ts + if (input.previous !== undefined) { + for (const [rowId, record] of input.previous.entries()) { + if (!seen.has(rowId)) input.slots.release(record.slot); + } + } +``` + +(`seen` is the existing duplicate-id Set — it already holds exactly the new id set.) + +3c. `transaction-draft.ts` — `prepareRecord` gains a `slot: number` parameter and stamps it in the frozen record. Call sites: adds allocate (`input.slots.allocate()`, recorded in `allocatedSlots`), updates pass `input.root.rows.get(rowId)!.slot` (the previous record is already fetched adjacent to every prepare call — reuse it, do not add a second `get`). `applyFlatTransactionDraft` and `replaceFlatRowsDraft` inputs gain `readonly slots: SlotAllocator`. Apply the lifecycle table: success path releases removed slots just before the `effective: true` return; the `effective: false` return (~line 982) and the `catch (error) { return remap(error); }` tail release `allocatedSlots`. + +3d. `create-local-row-model.ts` — `const slots = createSlotAllocator();` beside the other instance state; pass to `buildRowStore` (initial + any other call) and into every draft-input literal (the compiler finds them once the input types require it). + +3e. `rebuildRowStoreForQuery` in `row-store.ts`: verify the `{ ...previous, metadata }` spread now carries `slot` (it does — no code change; leave a one-line comment noting slot carries by spread). + +- [ ] **Step 4: Run the full package suite** + +Run: `pnpm --filter @pretable/row-model test` +Expected: PASS — including all 515 pre-existing tests (slot threading must not disturb any behavior) and the new lifecycle pins. Also run `pnpm --filter @pretable/row-model typecheck`. + +- [ ] **Step 5: Commit** + +```bash +git add -A packages/row-model/src +git commit -m "feat(row-model): stamp lifetime slots on row records" +``` + +--- + +### Task 5: `RevisionRoot.recordsBySlot` — required field, all sites + +**Files:** +- Modify: `packages/row-model/src/internal-types.ts` (RevisionRoot) +- Modify: `packages/row-model/src/row-store.ts`, `transaction-draft.ts`, `create-local-row-model.ts`, `filter-rebuild.ts`, `sort-rebuild.ts`, `cooperative-transition.ts` +- Test: `packages/row-model/src/__tests__/records-by-slot.test.ts` (new) + +- [ ] **Step 1: Add the field and let the compiler enumerate sites** + +`RevisionRoot` gains: + +```ts + /** + * Slot-indexed view of `rows` — same records, array-resident. Per-revision + * immutable (chunked COW), which is what keeps THIS root's bindings valid + * when the allocator later reuses a slot. Invariant, test-pinned: + * slotVectorGet(recordsBySlot, record.slot) === record for every record in + * `rows`, at every committed root. + */ + readonly recordsBySlot: SlotVector>; + /** + * The slot-space size this root's slot-indexed structures were built for + * (the allocator's capacity at commit time). A root must be + * SELF-DESCRIBING: readers size bitsets and walks from this field, never + * from the live allocator — reading the allocator would let later growth + * leak into a held snapshot's domain. + */ + readonly slotCapacity: number; +``` + +`slotCapacity` per site: wherever the table below says "carried", carry it; drafted/built sites stamp `input.slots.capacity` (or the threaded capacity value, for the cooperative transition) at commit time. + +Run `pnpm --filter @pretable/row-model typecheck` — the errors are the site list. Decision table: + +| Site | `recordsBySlot` | +|---|---| +| `create-local-row-model.ts:609` (initial) | from `buildRowStore` result (see 5a) | +| `create-local-row-model.ts` expansion/spread sites (~944, ~1400) | carried by `...previousRoot` — no edit | +| `create-local-row-model.ts` drafted sites (~1073, ~1137) | from the draft result (see 5b) | +| `filter-rebuild.ts:188` | `captured.recordsBySlot` (rows carried by identity) | +| `sort-rebuild.ts:115` | `captured.recordsBySlot` (rows carried by identity) | +| `cooperative-transition.ts:710` (`finish`) | from the transition state (see 5c) | + +5a. `BuiltRowStore` gains `readonly recordsBySlot: SlotVector<...>`; `buildRowStore` computes it after the loop: + +```ts + recordsBySlot: slotVectorFromEntries( + records.map((record) => [record.slot, record] as const), + input.slots.capacity, + ), +``` + +5b. `TransactionDraftResult` and `RowsReplacementDraftResult` gain the field. In `applyFlatTransactionDraft`, build the write list on the success path — removals clear, prepared records write: + +```ts + const slotWrites: Array | undefined]> = [ + ...removedRecords.map((record) => [record.slot, undefined] as const), + ...prepared.map((record) => [record.slot, record] as const), + ]; + const { next: recordsBySlot, chunksTouched } = slotVectorWithAll( + input.root.recordsBySlot, + slotWrites, + input.slots.capacity, + ); +``` + +(`removedRecords` = the records behind `effectiveRemoves`, already fetched for `groupedRemovals` — hoist ONE array instead of fetching twice.) Add `chunksTouched` to instrumentation: `input.instrumentation.work.slotChunksTouched += chunksTouched` (add the counter to `LocalRowModelInstrumentation.work` in `diagnostics.ts`, initialized 0, alongside its neighbors). `replaceFlatRowsDraft`: same pattern over its own removed/added/carried records; if it internally rebuilds wholesale, `slotVectorFromEntries` over the final records is acceptable there (set-rows is O(n) already). + +5c. `cooperative-transition.ts`: `rebuildRowStoreForQuery`'s `Pick<...>` return gains `recordsBySlot` built with `slotVectorFromEntries` over its `records` (capacity: pass `slots.capacity` in — thread the allocator reference or just `capacity: number` through the existing options of the transition; prefer passing the allocator's capacity value at capture time, since the transition must NOT observe later growth). `finish()` reads it off the retained state. + +- [ ] **Step 2: Write the invariant test** + +`records-by-slot.test.ts` — one scripted sequence covering: initial build → update transaction → remove+add transaction (slot reuse) → set-rows replacement → filter-only setQuery → sort-only setQuery. After EVERY committed revision assert, for every record in the root's `rows`: `slotVectorGet(root.recordsBySlot, record.slot) === record` (identity, not equality), and count(records) === count(live entries via `forEachSlotEntry`). Plus the held-snapshot pin: capture the root BEFORE the remove+add, run the remove+add (slot reused), assert the CAPTURED root still resolves the old record at that slot. Use the same root-access idiom as Task 4. + +- [ ] **Step 3: Run** + +`pnpm --filter @pretable/row-model test` and `typecheck` — all green, zero remaining compile errors (that's the proof all sites are handled). + +- [ ] **Step 4: Commit** + +```bash +git add -A packages/row-model/src +git commit -m "feat(row-model): per-revision recordsBySlot slot vector" +``` + +--- + +### Task 6: `RevisionRoot.visibleSlots` — required field, flat-real / grouped-empty + +**Files:** +- Modify: `packages/row-model/src/internal-types.ts`, plus the same six files as Task 5 +- Modify: `packages/row-model/src/visible-index.ts` (helper) +- Test: `packages/row-model/src/__tests__/visible-slots.test.ts` (new) + +- [ ] **Step 1: Add the field and the helper** + +`RevisionRoot` gains: + +```ts + /** + * Flat roots: one bit per slot, set iff the row is a member of + * `visible.rows` — the same structural verdict `filter-membership` + * resolves, indexed for O(1)/word-scan access. Grouped roots carry + * `EMPTY_MEMBERSHIP` (their membership lives in the group index) and every + * reader must treat it per that module's contract. Never mutated after the + * root commits. + */ + readonly visibleSlots: MembershipBitset; +``` + +`visible-index.ts` gains: + +```ts +/** Membership bitset of a FLAT visible tree: one pass, entry.record.slot. */ +export function membershipFromFlatTree<...>( + rows: VisibleIndexRoot<...>["rows"], + capacity: number, +): MembershipBitset { + const bits = createMembership(capacity); + for (const entry of rows.range(0, rows.size)) { + setMembershipBit(bits, entry.record.slot); + } + return bits; +} +``` + +Decision table (same compiler-driven enumeration): + +| Site | `visibleSlots` | +|---|---| +| initial (609) | grouped query → `EMPTY_MEMBERSHIP`; flat → `membershipFromFlatTree(visibleTree, slots.capacity)` | +| spread sites | carried automatically — CHECK each: the two expansion sites only re-attach group indexes over the same flat tree/groups, membership unchanged → correct to carry | +| drafted sites | from the draft result (Step 2) | +| `filter-rebuild.ts` | Task 7 rewrites this producer — for THIS task make it compile honestly: zero-flip arm carries `captured.visibleSlots`; non-zero arm `membershipFromFlatTree(newTree, capacity)` (temporary; Task 7 replaces it with the verdict-pass bitset) | +| `sort-rebuild.ts` | `captured.visibleSlots` — a sort-only change keeps the member SET identical | +| cooperative `finish` | grouped → `EMPTY_MEMBERSHIP`; flat → `membershipFromFlatTree(state.flatRows, capacity)` | + +- [ ] **Step 2: Draft maintenance in `applyFlatTransactionDraft`** + +On the success path, flat roots (`previousGroups === undefined`): + +```ts + const visibleSlots = cloneMembership(input.root.visibleSlots, input.slots.capacity); + for (const record of removedRecords) clearMembershipBit(visibleSlots, record.slot); + for (const record of prepared) { + if (passesNext(record)) setMembershipBit(visibleSlots, record.slot); + else clearMembershipBit(visibleSlots, record.slot); + } +``` + +Grouped roots: `const visibleSlots = EMPTY_MEMBERSHIP;`. `replaceFlatRowsDraft`: flat → `membershipFromFlatTree` over its final flat tree (O(n), and set-rows is O(n) already); grouped → sentinel. + +- [ ] **Step 3: Write the oracle test** + +`visible-slots.test.ts`: (1) equivalence oracle — after each step of a scripted sequence (build with a filter active → transaction flipping some rows across the filter boundary → remove a visible row → filter-only setQuery), assert for a FLAT root that `testMembershipBit(root.visibleSlots, record.slot) === (root.visible.rows.get(record.rowId) !== undefined)` for EVERY record; (2) grouped roots carry the `EMPTY_MEMBERSHIP` sentinel by identity; (3) mutation-hardening: run the oracle once with `clearMembershipBit` in the removal loop commented out and confirm it FAILS, then restore (do this as a one-time verification, note it in the task report — do not leave a permanently-mutated test). + +- [ ] **Step 4: Run** + +`pnpm --filter @pretable/row-model test` and `typecheck` — green. + +- [ ] **Step 5: Commit** + +```bash +git add -A packages/row-model/src +git commit -m "feat(row-model): per-revision visibleSlots membership bitset" +``` + +--- + +### Task 7: Rewrite the filter-only rebuild on slots + bitsets + +**Files:** +- Modify: `packages/row-model/src/filter-rebuild.ts` (the walk, ~lines 78–110, and the visibleSlots wiring from Task 6) +- Test: existing `filter-*` suites must pass unchanged; add one order-independence pin + +- [ ] **Step 1: Replace the walk** + +The current walk iterates `captured.sourceOrder.range(...)` and pays `captured.rows.get(rowId)` (HAMT, 18.5ms at 50k) plus `rowPassesFilter(captured, rowId)` (`visible.rows.get`, 11.9ms). Replace with a hole-skipping slot walk that computes the new bitset as it goes: + +```ts + const nextVisibleSlots = createMembership(capacity); // capacity: see note + const flippedIn: OrderedRowEntry[] = []; + const flippedOut = new Set(); + // Slot order, not source order — sound because nothing downstream reads + // this walk's order: flippedIn is comparator-sorted below, flippedOut is a + // set, and the merge consumes the OLD TREE's walk. recordsBySlot replaces + // the rows-HAMT get; visibleSlots replaces the old-verdict membership get. + forEachSlotEntry(captured.recordsBySlot, (previous) => { + const passes = filterVerdict(nextPlan, previous as never); + if (passes) setMembershipBit(nextVisibleSlots, previous.slot); + if (passes === testMembershipBit(captured.visibleSlots, previous.slot)) return; + if (passes) { + const keys = sortKeysOf(nextPlan, previous as never) as readonly CompiledSortKey[]; + flippedIn.push(Object.freeze({ record: previous, keys })); + } else { + flippedOut.add(previous.rowId); + } + }); +``` + +Capacity note: the rebuild has no allocator reference and must not read one (roots are self-describing — see `slotCapacity`, added in Task 5). Use `captured.slotCapacity` as `capacity` here; the new root carries the same value. + +Zero-flip arm: `visibleSlots: captured.visibleSlots` (drop `nextVisibleSlots`). Non-zero arm: `visibleSlots: nextVisibleSlots`, replacing Task 6's temporary `membershipFromFlatTree` call. Everything else — flip sort, merge, `orderIsProven`, `derivedById`, journal reason, instrumentation — is UNCHANGED. + +- [ ] **Step 2: Order-independence pin** + +Add to the existing filter fast-path suite: a model whose transaction history makes slot order ≠ source order ≠ visible order (e.g. build A,B,C,D; remove B; add E — E reuses B's slot), then a filter-only setQuery flipping E and A. Assert the visible sequence equals what the public API contract requires (compare against a freshly-built model with the same final rows and query — same visible order). This is the pin that fails if anyone later makes the walk order-sensitive. + +- [ ] **Step 3: Run the full package suite + the repo's react suite** + +`pnpm --filter @pretable/row-model test` (all 515+ green) and `pnpm --filter @pretable/react test` (the 1246-test suite exercises setQuery end to end; local flake rule — a timed-out test re-runs once before it counts). + +- [ ] **Step 4: Commit** + +```bash +git add -A packages/row-model/src +git commit -m "perf(row-model): filter-only rebuild walks slots and diffs membership bitsets" +``` + +--- + +### Task 8: Node A/B + browser bench measurement + +**Files:** +- Create: results section appended to `docs/superpowers/specs/2026-08-24-dense-handle-m0-results.md` (new `## M1+M2 measured` section) — or a sibling `...-m1-m2-results.md` if the section grows past a screen + +- [ ] **Step 1: Rebuild and A/B in the browser bench** + +Protocol (violations produced three wrong conclusions in the prior arc — follow exactly): +1. `lsof -i :4173` — if held by another process, STOP and report; never kill the holder (parallel session). +2. Baseline side = this branch BEFORE Task 1 (`git log` — the commit before the first M1 commit; use a throwaway worktree at that commit if the machine is loaded, interleaving paired runs). Variant side = HEAD. ONE variable: the M1+M2 commits. +3. Per side: `pnpm --filter @pretable/app-bench build` (rebuilds dependency dists via prepare:deps), then `pnpm --filter @pretable/app-bench preview:bench` (background), then: + `node scripts/bench-matrix.mjs --adapters=pretable,tanstack --scenarios=S2 --scale=target --scripts=filter-metadata,filter-text --repeats=3` + (Check `shared/bench-adapter-families.js` for the exact TanStack adapter id before running; `--scale=target` is the 50k tier — confirm against existing `status/*-s2-target-*` summary filenames. Also run `--scale=dev` for the 3k tier.) +4. Read the summary JSONs the run writes under `status/` — field names as in the existing `chromium-pretable-default-s2-target-filter-*.summary.json` files (settle, `post_interaction_long_tasks_ms`, interaction latency). Redirect any gate/script output to files and check exit codes — no `grep|head` pipelines. +5. Fitness: TanStack same-run numbers must sit in the historical band (compare to the baseline side's TanStack numbers — that IS the band); if they moved, the regime changed — rerun, don't conclude. + +- [ ] **Step 2: Write the results** + +Table: baseline-side vs variant-side for 50k settle (both filter scripts), 3k settle, block/long-tasks, interaction latency, TanStack controls. Name the delta against the branch's pre-M1 numbers (158.3/157.5ms @50k, 34.5/33.6ms @3k) and against the est. −30ms. State plainly if the gain is smaller than estimated and where the time went (trace only if needed: `node scripts/analyze-cdp.mjs --window=interaction ` — traces skew absolutes ~2×, shares only). + +- [ ] **Step 3: Full gates** + +`pnpm build && pnpm api` (expect ZERO `.api.md` drift — everything here is internal; any drift is a defect, stop and fix), `pnpm lint`, full `pnpm test` at repo root (react vitest flake rule: 1–2 random timeouts per full run locally — re-run before believing). + +- [ ] **Step 4: Commit** + +```bash +git add docs/superpowers/specs/ +git commit -m "docs: M1+M2 measured results (slots + membership bitsets)" +``` diff --git a/docs/superpowers/plans/2026-08-24-dense-layout-seam.md b/docs/superpowers/plans/2026-08-24-dense-layout-seam.md new file mode 100644 index 000000000..106659810 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-dense-layout-seam.md @@ -0,0 +1,132 @@ +# Dense-Identity Layout Seam Implementation Plan (Amendment I) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give `RowHeightIndex` a dense lane keyed by row-model slots so a 50k refilter performs zero per-survivor string work, and feed it through the renderer-dom controller from new `ɵ`-internal row-model snapshot reads — attacking the ~33% layout share of the remaining filter-settle window. + +**Architecture:** Per Amendment I (`docs/superpowers/specs/2026-08-24-dense-handle-amendment-i-layout-seam.md` — read it first). Dense mode is a PER-GENERATION, all-or-nothing property of the index: a generation whose every entry carries a `denseKey` maintains a visible-slots bitset instead of the `visibleKeys` HAMT and runs slot-indexed refilter/reorder; any input that cannot supply dense keys drops the whole index back to the string lane via the controller's existing full-replacement fallback. Measurements and tombstones stay string-identity-keyed in BOTH lanes (slot reuse must never touch retention — the amendment's §3 trap). + +**Tech Stack:** TypeScript; vitest (`pnpm --filter @pretable-internal/layout-core test`, `pnpm --filter @pretable-internal/row-model test`, renderer-dom + react suites — check each package.json for exact names); bench harness per M1+M2 Task 8. + +**Worktree:** `/Users/blove/repos/pretable/.claude/worktrees/homepage-hero-demo-3878ef`, branch `blove/filter-fast-path`. + +**Verified code anchors** (re-verify line numbers before editing; they drift): +- `packages/layout-core/src/row-height-index.ts`: `refilter` at ~1582; `apply` at ~1326; `retainMeasurement` at ~1273; `reorder` ends ~1563; replacement builder class begins ~1841 (`#visibleKeys` at 1841, ingest hashSet at ~2161); `#next(...)` generation constructor ~1804; `HeightValue` carries `ref/identity/estimatedHeight/height/measured`. +- `packages/layout-core/src/types.ts`: `RowHeightReplacementSource` (~line 259 area), operation types. +- `packages/renderer-dom/src/row-layout-controller.ts`: `rowRef` 297 (allocates a frozen ref PER CALL), `identityOf` 305, `replacementSourceOf` 1444 (per-row `rowAt` = O(log n) rank descent each), apply-ops construction ~831/~877, refilter/reorder dispatch ~1794–1855 (fallback-on-throw contract lives here). +- `packages/row-model/src/create-local-row-model.ts`: `createSnapshot` 86 (wraps `createFlatSnapshot(root)`; instrumented variant spreads and re-wraps reads — new reads must be added to BOTH). The snapshot's flat implementation is in `visible-index.ts` or adjacent (find `createFlatSnapshot`). +- Row-model facts: `RevisionRoot` has `recordsBySlot`, `visibleSlots`, `slotCapacity`; records carry `.slot`; `ɵ` prefix is the repo's internal-export convention (see `ɵfilterAuthority`). + +**Bars (from Amendment I):** untraced 50k filter settle ≤ ~95ms; refilterFallbackCount 0 on the happy path; 3k no regression; TanStack controls in band; every `.api.md` line of drift reviewed and intended; docs api-surface guard green (update registered tables in the same commit if it fires). + +--- + +### Task 1: layout-core dense primitives + type surface + +**Files:** +- Create: `packages/layout-core/src/dense-membership.ts` +- Modify: `packages/layout-core/src/types.ts` +- Test: `packages/layout-core/src/__tests__/dense-membership.test.ts` + +- [ ] **Step 1: Failing test for the bitset** — mirror row-model's `membership-bitset.test.ts` behaviors (set/clear/test across word boundaries; out-of-range reads false; clone-with-growth). layout-core cannot import row-model, so this is a deliberate ~40-line duplicate; the module header must say so and name the original. +- [ ] **Step 2:** Red run: `pnpm --filter @pretable-internal/layout-core test -- dense-membership` (verify the package's real name first). +- [ ] **Step 3:** Implement `dense-membership.ts` (same function shapes as row-model's `membership-bitset.ts`: `createDenseMembership(capacity)`, `setDenseBit`, `clearDenseBit`, `testDenseBit`, `cloneDenseMembership`). Header comment: duplicated from `@pretable-internal/row-model` `membership-bitset.ts` by design — layout-core stays dependency-free; keep the two in sync by hand. +- [ ] **Step 4:** Green run. +- [ ] **Step 5: Types.** In `types.ts`: `RowHeightReplacementSource` gains `readonly denseCapacity?: number`, and its `entryAt` row shape gains `readonly denseKey?: number`. Every `RowHeightOperation` variant's ref-bearing shape gains `readonly denseKey?: number`. Doc comments state the contract from Amendment I §1: dense keys are OPTIONAL; a generation is dense only when EVERY entry carries one and `denseCapacity` is present; a dense key is the row's CURRENT model slot, valid only while the model binds that slot (the caller owns that currency); mixed input falls back to the string lane wholesale. +- [ ] **Step 6:** Typecheck + lint the package; commit: + +```bash +git add packages/layout-core/src/dense-membership.ts packages/layout-core/src/types.ts packages/layout-core/src/__tests__/dense-membership.test.ts +git commit -m "feat(layout-core): dense-membership primitive and dense-key type surface" +``` + +--- + +### Task 2: dense generations in `RowHeightIndex` — state, builder ingest, guards + +**Files:** +- Modify: `packages/layout-core/src/row-height-index.ts` +- Test: `packages/layout-core/src/__tests__/row-height-index.test.ts` (extend) + +**Design (decision-complete):** +- `HeightValue` gains `readonly denseKey: number | undefined` — stamped at every ingest site from the input's `denseKey`. +- The index gains two generation fields threaded through `#next` and the builder: `#denseCapacity: number | undefined` and `#visibleSlots: DenseMembership | undefined`. INVARIANT (comment it at the field): `#visibleSlots !== undefined` ⇔ this generation is dense ⇔ every sequence entry has a `denseKey` < `#denseCapacity`. A dense generation does NOT maintain `#visibleKeys` (it stays `null`); a string generation never allocates `#visibleSlots`. +- The replacement BUILDER decides the lane at `begin`: source has `denseCapacity` AND every ingested entry carries `denseKey` → dense (build the bitset as it ingests, skip `hashSet(visibleKeys, ...)` entirely); the first entry missing a `denseKey` when `denseCapacity` was declared → throw `RowHeightReplacementLifecycleError` (the controller's existing catch → full string replacement... no: throw would loop). Instead: missing `denseKey` with declared capacity is a CALLER BUG — throw; the controller only declares `denseCapacity` when the snapshot guarantees slots (Task 4 makes that guarantee). No capacity declared → string lane, exactly today's code. +- Guards on a dense index: `apply` insert dup-check and `retainMeasurement`'s visible-check use `testDenseBit(#visibleSlots, op.denseKey)`. An op WITHOUT `denseKey` reaching a dense index throws `RowHeightReplacementLifecycleError` with a message naming the contract ("dense index requires dense-keyed operations; fall back to a full replacement") — the dispatch sites in the controller already treat throws as fallback (verify: the ~1794–1855 block and the apply call site; if apply throws are NOT already routed to fallback, Task 5 wires that). +- `apply` insert/remove maintain `#visibleSlots` (set/clear by denseKey) in dense mode, `#visibleKeys` in string mode — never both. +- Measurements/tombstones: UNTOUCHED in both lanes (string identity). `measure()` keeps working by identity on both lanes. + +- [ ] **Step 1: Failing tests first** (extend the existing suite in its own describe): (a) a dense-built index answers `apply` insert-dup and `retainMeasurement` guards correctly (both accept/reject cases); (b) an op without `denseKey` on a dense index throws the lifecycle error; (c) a string-built index is bit-for-bit unaffected (run a representative existing scenario through both construction styles and compare observables); (d) dense builder with a missing entry denseKey under declared capacity throws. +- [ ] **Step 2:** Red run. **Step 3:** Implement. **Step 4:** Green run + full layout-core suite (133+ tests) — string-lane tests must pass UNCHANGED (zero expectation edits; if one needs editing, stop: the lane leaked). +- [ ] **Step 5:** Commit `feat(layout-core): dense generations — builder ingest, bitset membership, guarded ops`. + +--- + +### Task 3: dense `refilter` and `reorder` + +**Files:** +- Modify: `packages/layout-core/src/row-height-index.ts` (refilter ~1582, reorder ending ~1563) +- Test: `packages/layout-core/src/__tests__/row-height-index.test.ts` (extend) + +**Design (decision-complete) — dense refilter:** +- Old pass (in-order walk, unchanged shape): additionally build `unconsumedBySlot: (HeightValue | undefined)[]` sized `#denseCapacity` (one allocation per refilter; 50k pointers ≈ 400KB transient — fine) instead of the string `unconsumed` Map. `previousValues` stays. +- New-order walk, per row: `denseKey` REQUIRED (absent → throw lifecycle error → controller fallback); dup-check via a local `seen` DenseMembership; `nextVisibleSlots` bit set per row; survivor = `unconsumedBySlot[denseKey]` — reuse verbatim, clear the array cell, NO identity string computed. Entrant: compute `identity` NOW (only entrants pay the string), then the existing measured/tombstone lookups verbatim; stamp `denseKey` on the new HeightValue. +- Leavers: cells still set in `unconsumedBySlot` — walk it (or keep a parallel count/list; simplest: iterate the array once, `for (let s = 0; s < capacity; s++)`) and run the existing retire logic (identity is already ON the HeightValue). ORDER CAUTION: today's leaver pass follows the OLD-SEQUENCE order via the Map's insertion order, and tombstone tickets are assigned in that order (comment at ~1668 says so, and ticket order is observable via cap eviction). A slot-index walk breaks that order. Preserve it: iterate `previousValues` in order and retire those whose `unconsumedBySlot[value.denseKey]` cell is still occupied (then clear it). Pin this with a test: cap-limited tombstones + a narrowing refilter → eviction order identical between lanes. +- `unchanged` detection, work counters, and every observable stay as today. +- **Dense reorder:** same substitution — the `unconsumed` Map in `reorder` becomes `unconsumedBySlot`; keys resolve by `denseKey`; missing key → lifecycle throw. +- refilter on a dense index yields a dense generation (`nextVisibleSlots` becomes its `#visibleSlots`); on a string index, today's code verbatim. + +- [ ] **Step 1: Failing tests:** (a) **lane-equivalence oracle**: a randomized script (seeded PRNG, ~200 rows, 30 steps of narrowing/widening/reordering refilters + measures) run through a string-lane index and a dense-lane index in lockstep; after every step compare heights sequence, `totalHeight`, work-counter observables that are lane-independent (`refilterEntriesReused/Inserted/Retired`), tombstone count. (b) the ticket-order pin above. (c) **the Amendment §3 slot-reuse trap**: measure row X; refilter X out (X tombstoned, slot kept); simulate permanent removal + slot reuse by presenting a NEW identity Y with X's old denseKey in the next FULL replacement (dense builder), then refilter Y in and out: Y must ingest at estimate (never X's measurement), X's measurement must return only for X's identity. Mutation-harden: key the hot measurement path by denseKey on purpose (temporary) and watch (c) fail; restore. +- [ ] **Step 2:** Red. **Step 3:** Implement. **Step 4:** Green + full package suite, string tests untouched. +- [ ] **Step 5:** Commit `perf(layout-core): slot-indexed refilter and reorder for dense generations`. + +--- + +### Task 4: row-model `ɵ` snapshot dense reads + +**Files:** +- Modify: wherever `createFlatSnapshot` lives (find it: `grep -rn "createFlatSnapshot" packages/row-model/src`), `create-local-row-model.ts` (createSnapshot wrapper at 86 — add the new reads to the instrumented spread too), `packages/row-model/src/index.ts` if snapshot type is exported there +- Test: `packages/row-model/src/__tests__/` (extend an existing snapshot-adjacent suite or add one) + +**Design:** the snapshot gains three `ɵ`-prefixed reads, documented as the renderer seam (flat roots only): +- `ɵvisibleSlotRange(start, end): readonly number[]` — slots of visible rows in order, from the tree's materialized `range` walk (entries carry `record.slot`). Grouped root → returns `undefined` (type: `readonly number[] | undefined`) — the caller must fall back. +- `ɵslotOfRowId(rowId): number | undefined` — one HAMT get (`root.rows.get`); for k-sized op stamping only (doc comment MUST say "k-sized paths only — never call per visible row"). +- `ɵslotCapacity(): number | undefined` — `root.slotCapacity` for flat roots, `undefined` for grouped. +Wire the instrumented wrapper (createSnapshot at 86) to pass them through (count `snapshotOutputRowsRead` for the range read, mirroring `range`). + +- [ ] Steps: failing test (flat: slots match `rowAt(i)`-resolved records' slots across a filter change; grouped: all three return undefined; `ɵslotOfRowId` on missing id → undefined) → red → implement → green (full row-model suite; expect NO existing-test edits) → commit `feat(row-model): internal dense snapshot reads for the renderer seam`. + +--- + +### Task 5: renderer-dom controller — dense sources, ops stamping, bulk walk, pooled refs + +**Files:** +- Modify: `packages/renderer-dom/src/row-layout-controller.ts` +- Test: renderer-dom's controller suite (find it; 154 tests exist in the package) + +**Design (decision-complete):** +- `replacementSourceOf` (1444): resolve `const slots = target.ɵvisibleSlotRange(0, target.visibleRowCount)` and `const capacity = target.ɵslotCapacity()` ONCE; when both defined, the source declares `denseCapacity: capacity` and `entryAt` returns `{ key, denseKey: slots[index] }`. ALSO replace the per-row `target.rowAt(index)` with a bulk `target.range(0, rowCount)` materialized ONCE per source construction (kills the O(n log n) rank descents) — entryAt indexes the array; keep the same omitted-row error on a hole. Grouped/undefined → today's shape verbatim (string lane). +- Apply-ops construction (~831/~877): stamp `denseKey: target.ɵslotOfRowId(row.rowId)` — when it returns undefined on a dense index the op will throw in layout-core and the dispatch's existing catch routes to full replacement; VERIFY the apply call site is inside the fallback-on-throw protection (the refilter dispatch at ~1794–1855 is; if the apply path isn't, wrap it to the same contract and count it in `refilterFallbackCount`-adjacent diagnostics — read how the controller currently handles apply throws first and match that convention). +- Pooled refs (spec M5's "rowRefs pooled by slot"): `rowRef` (297) allocates a frozen object per call. Add a slot-indexed pool (plain array on the controller instance, grown to capacity) so a data-row ref for slot s is created once and reused while the rowId matches (`pool[s]?.rowId === row.rowId ? pool[s] : (pool[s] = freeze({...}))`). Group refs unpooled. This is safe because refs are value-compared via `identityOf` everywhere (sameRef at ~317) — but VERIFY no consumer relies on ref allocation identity per call; grep the controller for `===` comparisons on refs before pooling; if any exist, report NEEDS_CONTEXT rather than guessing. +- `identityOf` calls on the n-sized paths should now be rare; do NOT micro-optimize further here. + +- [ ] Steps: failing tests (dense source construction: a fake snapshot with slots → source entries carry denseKeys and bulk range is called once — spy/count; fallback: grouped snapshot → no denseCapacity; pooled refs: two source constructions reuse ref objects for unchanged rows — assert identity) → red → implement → green (renderer-dom suite + react suite `pnpm --filter @pretable/react test`) → commit `perf(renderer-dom): dense-keyed layout sources, bulk visible walk, slot-pooled refs`. + +--- + +### Task 6: end-to-end pins + API surface + docs guard + +**Files:** +- Modify: `.api.md` reports via `pnpm build && pnpm api`; docs tables ONLY if the guard fires +- Test: react integration suite; website suite if docs tables changed + +- [ ] **Step 1:** An end-to-end react test (extend the existing filter fast-path integration suite in packages/react if one exists — find where `refilterPathCount` is asserted): drive a 200-row grid through filter-on → narrow → widen → filter-off; assert `refilterPathCount` advanced and `refilterFallbackCount === 0` (the dense path didn't silently fall back), and row heights survive (a measured row keeps its height across a flip-out/flip-in). Mutation: break denseKey stamping (stamp undefined) → the test must catch it via `refilterFallbackCount > 0` — this is the pin that keeps the dense lane from silently rotting back to fallback. +- [ ] **Step 2:** `pnpm build && pnpm api` — REVIEW the `.api.md` diff line by line: expected drift is layout-core types (`denseKey`/`denseCapacity`/HeightValue if exported) and row-model `ɵ` snapshot reads. Anything else → stop and fix. Commit the reports with the code that caused them if not already. +- [ ] **Step 3:** Full repo `pnpm test` (react flake rule applies) + `pnpm lint`. If the docs api-surface guard fails, update the registered tables per the guard's own error message in the same commit. +- [ ] **Step 4:** Commit `test(react): dense layout seam end-to-end pins` (+ api/docs updates). + +--- + +### Task 7: measurement + +Same protocol as M1+M2 Task 8 (`docs/superpowers/plans/2026-08-24-dense-handle-m1-m2.md` Task 8 — reuse verbatim: port check, load, interleaved paired sides, TanStack controls, medians of 3, no grep|head, no tracing for headlines). Baseline side = the commit before this plan's Task 1. Scales: `target` (50k) and `hypothesis` (3k — NOT `dev`, which is 750 rows; verified in the M1+M2 run). Scripts: filter-metadata, filter-text. Deliverable: `docs/superpowers/specs/2026-08-24-dense-layout-seam-results.md` with the table, deltas vs 116.8/125.6ms, the ≤~95ms bar verdict, fitness statement, and what remains (columnar cache next). Optionally one traced run AFTER the headline numbers for the share re-attribution (layout share ≤10% bar). Commit the doc. diff --git a/docs/superpowers/specs/2026-08-19-filter-subset-rebuild-design-amendment-g.md b/docs/superpowers/specs/2026-08-19-filter-subset-rebuild-design-amendment-g.md new file mode 100644 index 000000000..45e37f984 --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-filter-subset-rebuild-design-amendment-g.md @@ -0,0 +1,103 @@ +# Amendment G: renderer membership path + model levers + +**Amends:** `2026-08-19-filter-subset-rebuild-design.md` (its "Renderer: +untouched" section is superseded — decision 2026-08-19 after F4's NO-GO). +**Evidence:** `blank-viewport-diagnosis.md`, `reingest-composition.md` +(scratchpad). + +## What F4 + diagnosis established + +1. A latent controller defect (scroll during any active replacement leaves + the stale window unpainted → blank) was made deterministic by the + synchronous filter commit. **Fixed in G1** (commit 0deb7e90): the + active-branch `setViewport` republishes a stale-but-visible window, + preserving `rebuilding` status and the old observedRevision. +2. The settle attribution was corrected by profile: the controller's + cooperative replacement costs ~4µs/survivor (~45–63ms); the rest is the + model's own synchronous rebuild. Journal-ops lose to a subset layout + path above k≈2–9k flips; `refilter()` ceiling ≈ 20–25ms at 50k. + +## G-workstream design + +### G2 — RebuildProgressDemo retarget (website) + +The demo's filter toggle is synchronous now; its subject is cooperative +progress. Retarget the toggle to a GROUPED change (rowGroups on region) — +grouped stays cooperative by design and reads naturally in the demo's +prose. Same migration discipline as the cycle-1 sort→filter retarget: +demo + test + embedding-page prose all truthful. + +### G3a — journal reason `"refilter"` + +Reset reason union gains `"refilter"`: asserts the visible row ORDER of +surviving rows is unchanged and row identities are stable — membership +changed (rows entered/left), nothing else. Published by the filter fast +path instead of `"bulk-replace"`. Fail-closed exactly like `"reorder"` +(unaware consumers treat any reset as full replacement — grid-core pin +extended; api reports regenerated; docs guards checked — the B1+B2 +playbook). + +### G3b — layout-core `refilter(source)` + +Sibling of `reorder()`: walks the new order; reuses existing entries by +key (measurements ride); keys ABSENT from existing entries are inserted +with the estimate-or-default height rule; existing keys absent from the +new order leave (tombstone per the retention policy — read what the +cooperative path does with measured leavers and match it). Synchronous; +same immutability/counters/diagnostics discipline as `reorder`. Throws +only on structural impossibilities (duplicate keys, bad rowCount); +membership deltas are its PURPOSE, not an error. + +### G3c — controller refilter path + +`synchronize`: reset reason `"refilter"` with aligned revision → capture +anchor → `rowHeights.refilter(replacementSourceOf(target))` → anchor +restore → publishReady; ANY throw → `startReplacement` fallback +(counters: `refilterPathCount`, `refilterFallbackCount` on the existing +seam). Mid-replacement: fail-closed (restart) this cycle — no compose +(membership + pending catch-up is exactly the complexity the reorder +compose rule excluded; the FINAL-retarget machinery is not extended). + +### G3d — model store sharing (the −17ms lever) + +On a filter-only change the new plan's sort configuration is identical +(classifier-guaranteed), so `filter-rebuild` ADOPTS the previous plan's +sort-key store rather than refilling per row: a plan-internal seam +(`adoptSortKeyStore(nextPlan, previousPlan)`, guarded by TypeError unless +the caller holds a filter-only delta — document caller-owned precondition) +points the new plan's `#sortKeys`-bearing cache at the previous plan's +map... **Design constraint:** the merged cache entry also holds guarded +metadata keyed to the OLD plan's evaluations; adopting the map wholesale +would leak OLD metadata into NEW-plan cache hits, which is WRONG (verdicts +changed). Resolution: adopt only if the metadata-hit guard also checks +plan epoch, OR share at the sortKeys level only — the implementer +proposes the minimal sound mechanism (options: entry-level plan tag; +separate keys map shared by reference while metadata cache stays fresh — +note this re-splits what the rehash fix merged, so it must NOT reintroduce +a second per-row WeakMap fill on the hot path: sharing BY REFERENCE means +zero new sets). If no sound mechanism exists without re-growing per-row +work, drop G3d and record why — it is an optimization, not a bar +requirement. + +## Bar (unchanged) and projection + +50k settle ≤ same-run TanStack (~58ms band): model rebuild at realistic +flip counts (~36ms verdicts + merge/build + O(k)) + refilter ~10–25ms + +frame. The bench's 75%-flip cell is the worst case and may land above +TanStack's own worst case — evaluate the bar on the MEASURED numbers and +report honestly; the reserve lever (per-filter verdict decomposition) +remains named for a miss. + +## Testing + +G2: demo tests assert the cooperative cycle over the grouped toggle. +G3a: B1's journal-suite pattern (all-refilter range → "refilter"; mixed → +degrade; unaware-consumer pin in grid-core). +G3b: reorder's suite pattern + membership cases (enter/leave/disjoint, +measured leavers tombstoned, estimates for entrants; equivalence oracle vs +full replace at every rank; mutation-hardened). +G3c: B4's suite pattern (happy path, fallback flavors, anchor semantics, +mid-replacement restart). +G3d (if kept): cache-correctness adversarial tests — a NEW-plan evaluate +after adoption must NOT hit OLD metadata; keys resolve with zero refills. +End-to-end: F4 re-run, all four bars. diff --git a/docs/superpowers/specs/2026-08-19-filter-subset-rebuild-design.md b/docs/superpowers/specs/2026-08-19-filter-subset-rebuild-design.md new file mode 100644 index 000000000..336a61a96 --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-filter-subset-rebuild-design.md @@ -0,0 +1,147 @@ +# Filter subset rebuild: settling filters at TanStack parity + +**Follows:** the #457 arc (merged PR #479). This cycle applies the same +philosophy to filter-only changes. Baseline: `filter-baseline.md` +(scratchpad, 2026-08-19). + +**Date:** 2026-08-19 + +## Problem + +Filter interaction latency already beats TanStack (15–18ms vs 32–54ms at +50k, first changed frame in 1). The gap is the **settle tail**: ~240ms of +cooperative rebuild at 50k vs TanStack's ~58ms. Trace attribution: 76% +row-model — persistent-map rebuild 59ms, transition driver 37ms, predicate +re-evaluation 36ms, tree rebuild 26ms; renderer height re-ingest is only +13ms; paint negligible. + +A filter-only change produces a **subset-or-superset of an already-sorted +set**: values, sort keys, group paths, and order are unchanged; only +membership verdicts change. Rebuilding every record and both persistent +structures is work the change does not logically require. + +## Decisions (brainstorm, 2026-08-19) + +- **Bar:** 50k S2 filter settle ≤ same-run TanStack settle (~58ms band). +- **Approach 1 adopted:** synchronous filter-delta subset rebuild — records + rebuilt ONLY for verdict-flipped rows; merge-based visible tree; journal + stays a barrier (renderer share too small to chase). +- **Approach 2 rejected:** moving `filterPasses` out of metadata ripples + through `filteredLeaf`/aggregate machinery for savings Approach 1 already + captures when flip counts are small. +- **Approach 3 named as the reserve lever:** per-filter verdict bitsets so + an edit to one filter re-runs one predicate. Pulled only on a missed bar. + +## Success criteria + +1. 50k S2 filter-metadata AND filter-text: `completed ×3`, + settle ≤ same-run TanStack settle (~15% tolerance for run noise). +2. Interaction latency stays in its current band (15–18ms) — the fast + first frame must not regress. +3. No regressions: 3k bands, sort bands (15–17ms), grouped gate ≤ 8, + mount, full repo suites, api reports unchanged. +4. Work-based assertions: on a filter-only change, unflipped rows' records + carry by identity; the rows-map transient performs O(flipped) sets; the + visible tree is built without a comparator sort (merge + bulk build — + assert zero Array.sort of the full set, via instrumentation counters). + +## Design + +### Classifier + +`isFilterOnlyChange(previous, next)` derived from the existing +`classifyQueryDelta`: filtersChanged AND nothing else changed (runtime +facets; authorities equal). Fast-path gate additionally requires ungrouped +(both plans) and operation `set-query`. Conservative as always: any doubt → +cooperative path. + +### Synchronous subset rebuild (`filter-rebuild.ts`, sibling of sort-rebuild) + +Per source-order row (all records, not just visible): + +1. **Verdict:** run the NEW plan's filter predicates against stored values + where retained, accessors where not. Note: predicate inputs are column + VALUES — the sort-key store retains values only for sort columns; filter + columns' values are not retained (cycle-1 finding). So predicates re-run + with accessor reads per row — the measured ~36ms. (The reserve lever + attacks this; not this phase.) +2. **Diff:** compare against `previous.metadata.filterPasses`. + - Unflipped: record carries BY IDENTITY. No map write. + - Flipped: new record via the existing `#finalizeMetadata`-equivalent + path (new `filterPasses`, new `filteredLeaf` around the carried + aggregate values; sort keys carried from the plan store — the new plan + needs its store filled per row exactly as sort-rebuild does via + `fillSortKeysFromPrevious`; a filter-only change carries ALL sort + columns, so fills are 100% carries). + - Rows map: ONE transient over the captured map, `set` only flipped + rows. +3. **Visible tree by merge:** walk the OLD visible tree in order emitting + still-passing entries (their decorated `{record, keys}` entries carry — + flipped-out rows are skipped; flipped-in rows are NOT in the old tree); + merge with the newly-passing rows sorted by their stored keys + (`compareWithSortKeys` over the decorated pairs — sorting only the + flipped-in set, k log k); the merged strictly-sorted array feeds + `createOrderStatisticTreeFromSortedEntries`. Flipped-in entries must + reference the NEW records (rebuilt in step 2); still-passing entries + whose records carried MUST reference the carried records (entry reuse by + identity where possible — a reused entry object is valid because record + and keys are both unchanged). +4. **Publish:** `publishCommittedRoot` with the default barrier reason + (NOT "reorder" — the row set changed). Error semantics identical to + sort-rebuild (accessor/predicate failure → same error status shape, + state untouched). + +### The new plan's sort-key store + +Filled per row during the same pass (all carries — assert via the existing +`sortKeyCarries` counter). This keeps the A-invariant: any tree bound to +the new plan resolves keys fail-loud-safely. + +### Instrumentation + +`work.filterRebuilds`, `work.filterRowsFlipped`, and reuse of +`synchronousRebuildMs` (or a sibling `filterRebuildMs` — implementer picks, +consistently). The zero-full-sort assertion rides the merge design: add +`work.filterMergeSortedInsertions` (= flipped-in count) so tests can pin +that only the flipped-in subset was sorted. + +### Renderer + +Untouched. The barrier-driven cooperative replacement remains; its 13ms +share is accepted. (The bulk path from C2a does NOT apply — the base has +retained measurements — and must not: assert the retained-state path still +runs, no behavior change.) + +## Out of scope + +- Per-filter verdict decomposition (reserve lever). +- `filterPasses` ownership move (rejected). +- Grouped filter changes (cooperative, as before). +- Any renderer/journal change. +- External filter authority: under `filterAuthority: "external"` the + runtime filters are empty — a public filter change classifies as no + runtime change (same containment as sort authority in cycle 1). + +## Testing + +House standard (TDD, mutation-hardened, positive/negative twins): + +1. Classifier: table-driven, mirroring the sort classifier's suite. +2. Equivalence: fast-path result vs cold model — visible order, counts, + aggregates, distinct values — across: narrowing, widening, disjoint + flip (both directions at once), filter-to-empty, empty-to-filter, + no-op verdict change (filter changed but zero rows flip: still a new + revision, tree may carry wholesale — decide and pin). +3. Identity: unflipped records `toBe` across the change; rows-map root + changes ONLY when flips exist; flipped records new with correct + `filteredLeaf`. +4. The merge: fixture where flipped-in rows interleave arbitrarily with + survivors (ties included — sourceOrder resolution); mutation: break the + merge order → bulk constructor throws or equivalence fails. +5. Counters: flipped counts exact; merge-sorted-insertions == flipped-in; + sortKeyCarries == rowCount, evaluations == 0. +6. Stale-hazard heir: `setRows` after a filter fast path re-evaluates + correctly (the moved-row check and journal behavior unchanged). +7. Supersede/error paths: mirror the sort fast-path suite. +8. End-to-end: bench protocol, both filter scripts, both scales, plus the + full no-regression sweep (sort, grouped gate, mount). diff --git a/docs/superpowers/specs/2026-08-19-membership-verdicts-design.md b/docs/superpowers/specs/2026-08-19-membership-verdicts-design.md new file mode 100644 index 000000000..5103d745f --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-membership-verdicts-design.md @@ -0,0 +1,117 @@ +# Membership verdicts: cutting the filter commit's ordering rebuild + +**Follows:** `2026-08-19-filter-subset-rebuild-design.md` + Amendment G. +F4's re-run (scratchpad `filter-cycle-results-2.md`) passed three bars and +missed settle 4x: 50k filter settle 216–227ms vs TanStack ~57ms, with 69% +of the window in the row model — persistent-map 45ms, order-statistic-tree +35ms. + +**Date:** 2026-08-19 + +## What the exploration established (facts, cited in the map) + +- The 80ms splits into **disjoint halves**: record/metadata rebuild + rows-map + writes (~45ms, scales with flipped rows) and the visible tree + `byId` + HAMT rebuild (~35ms, scales with survivors). +- `filteredLeaf` is **not** a per-leaf flag — filtered aggregation is + membership in a separate aggregate tree (`aggregate-tree.ts:23-39` has no + flag). The wrapper exists only to tell `updateAggregateRoots` insert vs + remove, and both derivation sites hold a live plan. +- **The visible tree already is the membership set.** `nearestVisibleRef` + (`visible-index.ts:212-216`) uses `rankOf(rowId) === undefined` as its + visibility predicate today. +- Membership-as-tree-measure (considered, **rejected**): no measure-update + primitive exists — one flip costs remove+insert (2× O(log n) node copies + + a HAMT set), so at high flip counts it loses to the bulk build it would + replace; it also grows the **sort** path (all rows in the tree) and breaks + `nearestVisibleRef`. Recorded here so it is not re-proposed. + +## Success criteria + +1. 50k S2 filter-metadata and filter-text: settle ≤ same-run TanStack + (~57ms band), `completed ×3`, zero blank frames. +2. No regression: 3k filter (currently 42/50ms), sort both scales, grouped + gate ≤8, mount, full repo suites, api reports (this cycle DOES change the + public `CompiledRowMetadata` shape — the reason must be the only diff). +3. Work assertions: a filter-only change performs **zero** record + reconstructions and zero rows-map writes (counters); the visible tree's + `byId` is derived by removal, not refilled (counter). + +## Workstream H1 — verdicts come from root membership + +`CompiledRowMetadata` loses `filterPasses`. Nothing stores a per-row +verdict; membership in the root's visible structure IS the verdict. + +- **Resolution seam:** one internal helper, e.g. + `passesFilter(root, rowId)` — flat: `root.visible.rows.get(rowId) !== +undefined`; grouped: the group index's leaf membership (the grouped path + has its own leaf trees — the helper must answer correctly for both, or + grouped callers use a grouped-specific accessor; implementer picks one + coherent seam and documents it). +- **Producers** (`evaluate` / `#finalizeMetadata` / `refilterRecordMetadata`) + stop writing the field. The verdict a producer _computes_ still drives + where the row is inserted — it becomes a local, not stored state. + `#finalizeMetadata` stops allocating the per-aggregate-column wrapper + carrying `filteredLeaf`; `updateAggregateRoots` / + `updateMutableAggregates` decide insert-vs-remove from the computed + verdict passed alongside the record (both sites already receive plan + + record — thread the verdict explicitly rather than re-deriving). +- **Consumers** (the 19 sites in the map) reroute: filter-rebuild's diff + reads the CAPTURED root's membership; transaction-draft's six + old-verdict sites read `input.root` (previous) and its new-verdict sites + read the draft's own visible structure; distinct-values reads the root it + already holds; group-index's `filteredCount` accumulation takes the + computed verdict. +- **`refilterRecordMetadata` is deleted.** With `filterPasses` and the + `filteredLeaf` wrapper gone from metadata, a flipped row needs **no new + record at all** — filter-rebuild's rows map carries by identity in every + case, exactly as sort-rebuild's does. This is the 45ms. +- **`rebaseSourceOrder`** (the one plan-less metadata producer, + `transaction-draft.ts:700`) simply stops carrying the field. + +## Workstream H3 — cheaper bulk tree construction + +`createOrderStatisticTreeFromSortedEntries` gains an internal variant (or +options) for callers that can prove their input: + +- **Derived `byId`:** accept a base map and a leaver set — the new map is + the old one minus k leavers (k removes) instead of n inserts. Filter's + survivors are always a subset of the captured tree's entries. +- **Trusted order:** skip the n−1 verification comparisons when the caller + passes a proof token (filter's merge produces strictly-sorted output by + construction from two strictly-sorted sequences; sort's `Array.sort` + + tiebreak likewise). The verification stays the default and stays + unconditional for untrusted callers — the rationale comment at the + existing site explains why it exists; extend it with when it may be + skipped. +- Both are internal primitives; no package-index exposure. + +## Out of scope + +- Membership-as-measure (rejected above). +- G3d sort-key store adoption (still deferred; ~17ms, revisit only if the + bar is missed after H1+H3). +- Per-filter verdict decomposition (the standing reserve lever). +- Grouped fast paths (still cooperative). + +## Testing + +House standard; the sortKeys migration (cycle 2) is the template. + +H1: equivalence oracles vs cold models across flat and GROUPED paths +(grouped exercises the rerouted aggregate/filteredCount consumers even +though it takes no fast path); the transaction-draft old-verdict sites get +adversarial tests — same-reference mutation flipping a row's verdict must +still emit the correct remove/insert change ops (this is the case the +row-keyed store could not serve; membership resolution must); aggregates +correct under filtered and all populations; distinct-values `filtered` +population correct; identity assertions (zero record rebuilds on a filter +change — counter-pinned); `nearestVisibleRef` semantics unchanged. + +H3: byId-by-removal equals byId-by-refill (structural equality across every +key); trusted-order variant equals verified build; a deliberately +misordered trusted input is NOT silently accepted in tests (assert the +variant is only used where order is proven — a mutation that feeds it +misordered input must be caught by an equivalence oracle downstream). + +End-to-end: the F4 protocol, all bars, plus the api-report review. diff --git a/docs/superpowers/specs/2026-08-24-columnar-verdicts-results.md b/docs/superpowers/specs/2026-08-24-columnar-verdicts-results.md new file mode 100644 index 000000000..7bea36e4b --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-columnar-verdicts-results.md @@ -0,0 +1,273 @@ +# Columnar verdict cache measured results (Amendment J) — 2026-08-24 + +Browser-bench A/B of the columnar verdict cache: compiled per-plan filter +predicates, the mutable columnar filter-value store with commit-side clears, +and the filter rebuild consuming per-record columnar verdict scans. + +- **Variant**: `683ecd93` (HEAD of `blove/filter-fast-path` — plan Tasks 1–4: + `d64fba85`, `ec871e6f`, `30c43223`, `683ecd93`). +- **Baseline**: `eab3b893` (the commit before Task 1; verified via + `git log eab3b893..HEAD -- packages/` that exactly those four columnar + commits are the only `packages/` changes — one variable). +- **Design**: throwaway worktree at the baseline commit, fresh install + + `pnpm --filter @pretable/app-bench build` per side, one preview server on + 4173 at a time (port verified free first), interleaved paired rounds, 3 + repeats per cell, medians reported. Because round A's metadata cells + disagreed with its text cells, a second full 50k paired round was run + (rounds A and B below; 6 repeats per 50k cell pooled). No CDP tracing for + headline numbers; runner output redirected to files, exit codes checked. +- **Machine load**: heavy — 1-min load 35–110 across rounds (10-core Mac, + 8.7GB of 10GB swap used, parallel sessions active). Same regime as the + M1+M2 (33–54) and seam (24–46) runs, with a worse spike (110) during the + variant 3k round; interleaving plus TanStack same-run controls are the + fitness arbiter as before. +- **Scale note**: 3k tier is `--scale=hypothesis` (3,000 rows; `rowCount` + confirmed in the summaries). + +## Gates + +- `pnpm build`: PASS. `pnpm api`: PASS with **zero `.api.md` drift** — + everything this milestone touched is row-model-internal. +- Full root `pnpm test`: all 11 packages + both apps green. One react test + (`external-filter-authority.test.tsx` — an aria header-state assertion) + failed once inside the full run at 1-min load 35, then passed 3/3 in + isolation and a full `@pretable/react` re-run (1247/1247) — load flake, + not a regression. The `apps/*` phase (skipped by pnpm after that first + failure) was run separately: green. +- `pnpm lint`: PASS. `prettier --check .`: fails on 4 docs files + (`dense-handle-core-design`, `dense-handle-m0-results`, + `dense-handle-m1-m2-results`, `dense-layout-seam-results`) that fail + identically at the baseline commit — pre-existing on the branch, not + introduced here, left untouched per the commit-only-this-file rule. + +## Results (settle quantizes to ~8.3ms frame steps) + +### 50k rows (S2, `--scale=target`) — two paired rounds, medians of 3 + +| Metric | Script | Base A | Var A | Base B | Var B | Pooled base (6) | Pooled var (6) | +| --------------------------------- | --------------- | ----------- | ---------- | ----------- | ----------- | --------------- | -------------- | +| settle_duration_ms | filter-metadata | 107.4 | 108.9 | 98.9 | 107.1 | 107.3 | 108.6 | +| settle_duration_ms | filter-text | 108.3 | 100.0 | 108.4 | 108.3 | 108.4 | 104.3 | +| post_interaction_long_tasks_ms | metadata / text | 88 / 91 | 88 / 87 | 87 / 93 | 87 / 89 | — | — | +| interaction_latency_ms | metadata / text | 17.1 / 16.7 | 8.4 / 16.7 | 16.9 / 17.7 | 17.1 / 16.4 | — | — | +| post_interaction_blank_gap_frames | both | 0 | 0 | 0 | 0 | 0 | 0 | +| TanStack control settle | filter-metadata | 49.7 | 57.4 | 58.0 | 57.6 | — | — | +| TanStack control settle | filter-text | 57.4 | 50.1 | 51.7 | 57.1 | — | — | + +Repeat spreads tell the story: every 50k pretable cell on BOTH sides bounces +between the 12-frame (~99–100ms) and 13-frame (~107–109ms) quantization +bins — baseline metadata [98.3, 98.9, 107.1, 107.4, 107.4, 107.5], variant +metadata [99.5, 107.1, 108.3, 108.9, 109.1, 116.7], variant text [99.0, +100.0, 100.2, 108.3, 109.6, 109.7]. Round A's apparent text win (−8.3) did +not reproduce in round B (0.0); round B's apparent metadata loss (+8.2) is +the same bin-bounce in the other direction (round A: +1.5). The variant +metadata 8.4ms latency in round A likewise read 17.1 in round B. + +### 3k rows (S2, `--scale=hypothesis`) — one paired round + +| Metric | Script | Baseline | Variant | Δ | +| --------------------------------- | --------------- | ----------- | ----------- | ------- | +| settle_duration_ms | filter-metadata | 33.3 | 33.3 | 0 | +| settle_duration_ms | filter-text | 34.3 | 33.9 | ~0 | +| post_interaction_long_tasks_ms | both | 0 | 0 | 0 | +| post_interaction_blank_gap_frames | both | 0 | 0 | 0 | +| TanStack control settle | metadata / text | 24.9 / 24.3 | 25.7 / 25.1 | in band | + +## Deltas vs the seam and the cumulative arc + +- Vs the seam record (108.3 metadata / 116.8 text): pooled variant medians + are 108.6 / 104.3 — metadata unchanged, text ~one bin better than the + RECORD but indistinguishable from today's PAIRED baseline (108.4), which + itself sat a bin under the old text record. The paired comparison is the + honest one: **no reproducible settle change in either script.** +- Cumulative arc: pre-M1 166.6/158.4 → M1+M2 116.8/125.6 → seam + 108.3/116.8 → columnar **~108/~104–108 (flat)**. The branch's total gain + remains the ~1.4–1.5× of the first two milestones; this milestone added + none that the settle metric can resolve. + +## Bar verdicts + +- **50k settle improves in BOTH scripts vs 108.3/116.8: MISSED.** Both + scripts are flat within one frame across two interleaved paired rounds; + the only sub-frame signal (pooled text −4.1ms) is half a quantization bin + and did not survive the confirmation round as a per-round delta. +- **Traced verdict share ≲3%: MISSED — ~17% (vs 17.5% pre-columnar, + essentially unchanged).** Breakdown in the table below. +- **3k no regression: MET** (0 / −0.4ms). +- **Zero blank frames: MET** (0 in all 18 pretable summaries, both sides, + both scales). +- **TanStack controls in band: MET** — all 50k control cells straddle the + same 48–58ms one-frame band on both sides and both rounds (49.7–58.0); + 3k controls 24.3–25.7. The regime held; the flat result is trustworthy. + +## Traced share (variant, 50k filter-metadata, `--window=interaction`) + +One traced run after the headlines (`PLAYWRIGHT_PERF_TRACE=1`, repeats=1), +`analyze-cdp.mjs --window=interaction` with the build's sourcemap. Window +68.0ms (seam trace: 74.2ms). Traced absolutes skew ~2× — shares only. +Groups sum to ~81%; the rest is the sub-0.3% frame tail. + +| Subsystem (self time) | Share | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------- | +| **Verdict evaluation, columnar path** — `bulkFilterVerdictScan` 7.1% + exported wrapper 0.9% + `textCell` normalization 4.4% + compiled `contains` predicate 2.1% + `columnarGetCell` 2.2% + `assertRealSlot` 0.4% | **17.1%** | +| filter-rebuild walk body (`rebuildRootForFilterOnlyChange` 9.6% + walk callback 9.6%) | 19.2% | +| persistent HAMT (`persistent-map.js`) | 12.2% | +| layout-core dense refilter (`refilter` + `#refilterDense`) | 6.8% | +| order-statistic tree | 3.2% | +| slot-vector + visible-index + plan-equality misc | 2.5% | +| react render/commit + DOM (react pkg, react-dom, `getBoundingClientRect`, `measureText`, `querySelectorAll`, adapter) | ~15.5% | +| (program) + GC | 4.2% | + +Why the share did not move: the cache eliminated the accessor read and the +per-row values-Map get, but those were never the dominant term. What +remains is (a) **per-row dispatch** — the walk calls `bulkFilterVerdictScan` +once per record through the exported wrapper, paying wrapper + `instanceof` +guard + per-cell `assertRealSlot` for every one of 50k rows (~8% scan +machinery, plus its share of the 9.6% walk-callback self time), and (b) +**per-cell value normalization** — the compiled text predicate still runs +`String(value).toLocaleLowerCase()` on every raw cell (`textCell` 4.4% + +predicate 2.1%), because the columnar store caches RAW values, not +normalized ones. The compile step hoisted operand normalization; cell +normalization stayed in the loop. + +## Fitness statement + +- TanStack same-run controls agree across sides within one frame on every + cell in every round; the pretable repeat spreads overlap across sides. + The interleaved paired design plus in-band controls make the "flat" + conclusion trustworthy despite 1-min load 35–110. +- The round-B baseline run's matrix process exited 1: a side spec + (`row-height-error-applicability`, a 60s `waitForFunction`) timed out + under load AFTER the last cell's bench completed — all 12 summaries in + that round are `status: completed` and are used here. +- Settle cannot resolve sub-frame changes in this regime: both sides + bin-bounce between 12 and 13 frames. The traced share is the sharper + instrument, and it independently confirms the flat settle result. + +## Conclusion + +The columnar verdict cache **did not deliver**: 50k filter settle is flat +(~108ms metadata / ~104–108ms text vs the seam's 108.3/116.8) and the +traced verdict share is ~17%, unchanged from 17.5% pre-columnar — because +the cost it removed (accessor reads, Map gets) was not where the time was. +The gap to TanStack (~50–58ms same-run controls) remains ~50ms, and the +window now names its levers precisely: make the scan actually bulk (hoist +the per-row wrapper/guard dispatch into one loop inside the plan — the +plan-of-record's original one-call-before-the-walk shape; ~8% scan +machinery + part of the 19.2% walk body), cache normalized values for text +filters instead of raw ones (~6.5%), and only then the structural terms — +the HAMT (12.2%), the rebuild walk body itself, and render/commit (~15.5%). +The mechanism (store, freshness clears, compiled predicates) is sound, +tested, and API-silent; its payoff is gated on removing the per-row and +per-cell overheads that this measurement surfaced. + +## Fix cycle (one-call sweep + normalized cells) — 2026-08-25 + +Both levers the trace named were implemented and re-measured. + +### What changed + +- **One-call bulk sweep.** `bulkFilterVerdictScan` (one exported-wrapper + + `instanceof` + per-cell `assertRealSlot` call per record) is GONE, folded + into `CompiledQueryPlan.bulkFilterVerdictSweep`: ONE call per rebuild that + owns the `forEachSlotEntry` walk. Plan resolution, the `instanceof` guard, + filter columns, predicate arrays, normalizers, and each filter's column + vector are hoisted out of the row loop; cells are read through the new + assert-free `columnarGetCellTrusted` (walk slots are nonnegative integers + by construction — trust-by-construction documented at the sweep, and a + wrong-slot fill mutation is caught by the equivalence oracle). + `filter-rebuild.ts` passes a `(record, passes)` callback that keeps its + flip-set/bitset logic — still one closure call per row, but zero wrapper / + guard / assert / Map-get work per row. +- **Normalized cells.** The columnar store now caches the SCAN + representation, normalized once at fill (`normalizeCellForScan`): text + lowercased (`textCell`), dates as `toDayMs` day-ms, enum `String`-coerced, + boolean `booleanValue`-coerced, numbers raw. Predicates gained normalized + twins (`compileFilterPredicateForNormalized`) whose closures skip the + per-row normalization; the operator sweep gained twin tests pinned to the + same literal expectations. The per-row `filterVerdict`/`evaluate` paths + keep raw values and raw predicates, unchanged. +- **isEmpty/garbage-date resolution.** Emptiness is a RAW-value property the + normalized forms cannot preserve (raw `NaN` in a text column is empty but + normalizes to non-empty `"nan"`; garbage and empty dates both normalize to + `NaN`, and `isEmpty` must stay FALSE on garbage while comparisons fail on + both). Rather than a two-field cell, `isEmpty`/`isNotEmpty` filters stay + on live accessor reads through the raw predicate inside the sweep — + documented at `normalizeCellForScan`, pinned by an explicit + garbage-vs-empty date test (garbage fails `on`/`after` AND `isEmpty`; + empty fails comparisons AND passes `isEmpty`). + +Verification: 654 row-model tests green (616 + 38 added), full root +`pnpm test` green (exit 0, no flakes this run). Mutations (performed, +caught, restored): wrong-slot fill → 30 failures including the randomized +equivalence oracle; fill stores RAW value → both new warm-cell membership +pins fail; normalizer drops the lowercase → 8 failures across the +normalized twins and warm pins. + +### Paired 50k re-measure (round A; medians of 3; load 10.9–14.1) + +Same protocol: baseline throwaway worktree at `891d6cb4` (fresh install), +bench rebuilt per side, port 4173 verified free, one matrix-managed server +at a time (build-identity asserted), TanStack same-run controls. Machine +load 10.9–14.1 — far lighter than the original run's 35–110. + +| Metric | Script | Base `891d6cb4` | Fixed | Δ | +| ---------------------- | --------------- | --------------- | ----------- | ------- | +| settle_duration_ms | filter-metadata | 99.9 | 100.8 | +0.9 | +| settle_duration_ms | filter-text | 99.6 | 100.0 | +0.4 | +| interaction_latency_ms | metadata / text | 16.6 / 16.3 | 16.6 / 16.6 | ~0 | +| long tasks / blank | both | 84–87 / 0 | 82–88 / 0 | ~0 | +| TanStack control | metadata / text | 58.0 / 50.7 | 57.5 / 50.8 | in band | + +Both scripts' cells agree (both flat, sub-frame Δ), so the +disagreement-triggered second round was not needed. Note both SIDES sit in +the 12-frame bin (~99–101ms) that the heavier original session only +bounced into — the lighter machine, not the fix. + +### Traced share (fixed variant, filter-metadata, `--window=interaction`, 65.3ms window) + +Verdict machinery now: sweep row closure 1.9% + `forEachSlotEntry` 1.9% + +`columnarGetCellTrusted` 1.7% + fill-side `columnarSetCell` 2.0% + +`#readColumnValue` 1.1% + `normalizeCellForScan` 0.5% + `textCell` 3.3% + +normalized `contains` predicate 3.0% = **~15.4%** (was ~17.1%). The +rebuild walk body is 8.9% + 11.5% callback = 20.4% (was 19.2% — the +callback now absorbs frames the old per-row scan call held). HAMT, +tree, layout-core, react terms unchanged in kind. + +### Why it is STILL flat — the structural finding + +The bench scripts apply ONE filter commit against a COLD store: the +measured interaction IS the fill. So the fill-time normalization +(`textCell` 3.3%, `columnarSetCell` 2.0%, accessor reads 1.1%) runs inside +the same window it used to run in — it moved from "per predicate call" to +"per fill", but with exactly one commit those are the same count. The +dispatch overhead the sweep removed (wrapper + instanceof + per-cell +asserts, ~1.3–1.5% traced) was real but half a frame. The warm-path win — +repeat filter commits verdicting over already-normalized cells with ZERO +fills, which the new tests prove — is structurally invisible to a +single-commit script and to the settle metric. + +### Verdict + +**STILL FLAT.** Settle: +0.9ms metadata / +0.4ms text (sub-frame, controls +in band). Traced verdict share: ~15.4% vs ~17.1% — a real but small +reduction, and the remainder is cold-fill work the script's shape makes +unavoidable plus the walk/merge/HAMT structure the fix never targeted. Per +the arc's standard this does not clear the bar; the revert decision goes +back to the controller. + +## Decision: reverted + +The store, scan, and normalization — `30c43223` (columnar store + +commit-side clears), `683ecd93` (scan + filter-rebuild consumption + +setDerivations reset), `73f1ae24` (one-call sweep + normalized cells) — +are reverted in `revert(row-model): drop the columnar verdict store — +measured flat twice`. Kept: `ec871e6f` (compiled per-plan filter +predicates, operator sweep and malformed-operand pins included) and +`d64fba85` (`CompiledRowInput.slot` threading). Rationale: flat twice — +a warm-path saving of ~3ms inside a ~100ms settle does not buy the +machinery, and git history preserves it if the calculus changes. The +lesson worth carrying: the bench scripts apply one filter commit against +a COLD store, so the measured interaction IS the fill — any +cache-the-fill design is structurally invisible to a single-commit +script, and that must be checked before building the cache. diff --git a/docs/superpowers/specs/2026-08-24-dense-handle-amendment-i-layout-seam.md b/docs/superpowers/specs/2026-08-24-dense-handle-amendment-i-layout-seam.md new file mode 100644 index 000000000..a05d9b85d --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-dense-handle-amendment-i-layout-seam.md @@ -0,0 +1,101 @@ +# Amendment I — dense-identity layout seam + +Date: 2026-08-24. Status: design, pre-plan. Amends +`2026-08-24-dense-handle-core-design.md` (executes its "renderer seam" +milestone, promoted AHEAD of the columnar cache by measurement). + +## Why the promotion + +M1+M2 took 50k filter settle 166.6 → 116.8ms (`...m1-m2-results.md`). The +2026-08-24 re-attribution of the remaining ~117ms (traced interaction window, +shares): **layout `refilter` ≈ 33%** (16.8% walk + 13.6% string-HAMT +`hashString`/`lookupEntry`/`set` + 2.4% OST rank reads), row-model rebuild + +verdict pass ≈ 29%, renderer/react commit ≈ 25%. The columnar verdict cache +(spec M3) caps at ~14%. The layout seam is the same string-identity disease +M1+M2 cured in row-model, and it is now the single largest lever. Columnar +follows as the next milestone after this one. + +## What the seam pays today (measured anatomy, 50k refilter) + +1. `replacementSourceOf` (`renderer-dom/row-layout-controller.ts:1444`): + `entryAt(index)` → `snapshot.rowAt(index)` — an O(log n) order-statistic + rank descent PER ROW → O(n log n) for the walk, plus ref/identity + stringification per row. +2. `RowHeightIndex.refilter` (`layout-core/row-height-index.ts:1582`): per + row, ~4–5 string-hash operations — `unconsumed` Map set (old pass), `seen` + Set add, `visibleKeys` HAMT insert (rebuilt FROM SCRATCH every refilter), + `unconsumed.get`; entrants add `hashGet(measurements)` (+ tombstone + lookups). + +## Design + +### 1. Dense keys cross the seam as an OPTIONAL contract + +- Row-model snapshot gains an internal-facing visible-row read that exposes + the record's `slot` and the root's `slotCapacity` alongside the existing + ref (exact shape decided in the plan after reading the snapshot module; + the public `publicRow` object is NOT touched). It must also expose a bulk + in-order visible walk (the tree's materialized `range`) so the source stops + paying a rank descent per row. +- `RowHeightReplacementSource` entries gain optional `denseKey: number` and + the source gains optional `denseCapacity: number`. When every entry of a + replacement/refilter/reorder carries a dense key, `RowHeightIndex` runs its + dense lane; any entry without one falls back to the string lane wholesale + (no mixed-lane pass). String `identity` REMAINS the durable identity in + both lanes. + +### 2. Two-tier measurement store inside `RowHeightIndex` + +- **Hot lane (dense, slot-keyed):** the per-pass O(n) structures — the + refilter walk's `unconsumed`/`seen`, the `visibleKeys` membership, and the + current-measurement lookup for LIVE rows — become bitsets / dense arrays + sized by `denseCapacity`. +- **Cold lane (string-keyed, unchanged):** tombstones (`tombstones`, + `tombstoneOrder`) and any measurement whose row is not currently live. + Tombstones CANNOT go dense — a slot is lifetime-bound and REUSED after + permanent removal, while a tombstone's whole purpose is to outlive the row. + +### 3. The slot-reuse invalidation contract (THE trap of this milestone) + +A slot-keyed measurement is valid exactly as long as the slot binds the same +row. `refilter` never crosses a slot release (filter leavers keep their model +slots — they are hidden, not removed). Permanent removals reach the layout +index only through full replacements (`beginReplacement`) — the plan pins +that claim in code review before anything is built on it — so: + +- refilter/reorder may trust slot-keyed hot state unconditionally; +- a FULL replacement rebuilds hot state from its own walk and must fold + retiring measured rows into the cold (string) tier by identity, exactly as + today's retention semantics demand; +- the pin that keeps this honest: filter row X out (tombstoned), permanently + remove X, add row Y that REUSES X's slot, refilter Y visible — Y must + ingest at estimate height, and X's retained measurement must return only + on X's identity, never attach to Y. Mutation-hardened both ways. + +### 4. What stays put + +- Retention semantics, ticket order, cap eviction, the fallback-on-throw + contract, and every observable of `refilter`/`reorder` are UNCHANGED — + equivalence oracle: dense lane vs string lane must produce identical + observable sequences on randomized flip scripts. +- The blank-viewport latch and anchor semantics (G3c) are untouched. + +## Bars + +- 50k refilter's layout share (walk + its HAMT + OST reads, traced shares) + drops to ≲ 10% of the window; untraced 50k filter settle ≤ ~95ms + (from 116.8/125.6) with long-tasks reduced accordingly; 3k no regression; + TanStack controls in band; zero blank frames; refilterFallbackCount 0. +- Deliberate API surface: layout-core types and row-model internal snapshot + reads WILL move `.api.md` — every line of that diff is reviewed and + intended (M1+M2's zero-drift property ends here by design). The docs + api-surface guard is test-pinned to those reports; if it fires, the + registered tables update in the same commit, per the guard's contract. + +## Explicitly out of scope + +- The columnar verdict cache (next milestone). +- Renderer/react commit costs (~25%) — different levers (memoization, + windowed commit), different milestone. +- Any change to grouped replacement paths beyond keeping them compiling on + the string lane. diff --git a/docs/superpowers/specs/2026-08-24-dense-handle-amendment-j-columnar-verdicts.md b/docs/superpowers/specs/2026-08-24-dense-handle-amendment-j-columnar-verdicts.md new file mode 100644 index 000000000..918856d85 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-dense-handle-amendment-j-columnar-verdicts.md @@ -0,0 +1,97 @@ +# Amendment J — columnar verdict cache + +> **Status: CLOSED — reverted.** Built, measured flat twice (original run +> and the one-call-sweep + normalized-cells fix cycle), and reverted per the +> no-unmeasured-budget standard. Compiled predicates and +> `CompiledRowInput.slot` threading were kept; the store, scan, and +> normalization were dropped. Full record: +> `2026-08-24-columnar-verdicts-results.md`. + +Date: 2026-08-24. Status: design, pre-plan. Executes the core spec's +"Columnar evaluation cache" milestone (M3), narrowed to VERDICTS; amends it +with the mechanics the spec left open. Follows Amendment I (measured: +`...dense-layout-seam-results.md`). + +## Why now, and the target + +Post-seam re-attribution of the 50k filter window (74.2ms traced): verdict +evaluation (`#filterVerdict` + `evaluateFilter` accessor loop) ≈ **17.5%**, +the largest single row-model share. Per row it pays: a resolver closure, a +`#byId` Map get per filter, a live accessor call per filter, and +`evaluateFilter`'s per-row operator dispatch. M0 priced the replacement — a +monomorphic scan over a slot-indexed value vector — at 0.26ms (numeric) / +0.70ms (string `.includes`) per 50k pass. + +Target: verdict share → ≲3%; untraced 50k settle from 108.3/116.8 toward +~95–105ms. Sort keys are OUT of scope (they have their own store); grouped +paths keep the per-row route. + +## Design deltas beyond the core spec + +### 1. `CompiledRowInput` gains `slot` + +Every caller already holds the record (or is creating it and just allocated +the slot). `evaluate` and the verdict paths receive the slot so columnar +cells can be written and read without any string key. + +### 2. Columnar store: per-column `SlotVector` on the shared cache + +A `Map>` of RESOLVED accessor values for +FILTER columns, living beside the evaluation cache and adopted by reference +in the same `adoptEvaluationCache` call (same validity argument: a +filter-only change preserves every accessor's semantics). Storage revised +at implementation (`30c43223`): mutable-in-place chunked vectors with +per-chunk presence bitsets (`mutable-columnar.ts`), NOT the COW +`slot-vector` — sound because the store is cache-not-truth and nothing +revision-scoped ever reads it; the module header carries the argument. + +### 3. Freshness invariant (the load-bearing rule — REVISED) + +Revised 2026-08-24 during Task 1 review, which found the original wording +("written wherever metadata is evaluated") unsound: drafts evaluate rows +BEFORE the draft is known effective, so an aborted draft would leave cells +reflecting values that never committed, at slots the committed root still +owns. Also, two ingest paths present a `-1` placeholder slot to `evaluate` +(allocation is deliberately deferred past the throwing accessor — the +capacity-leak fix in `d64fba85`), so `evaluate` cannot be the writer anyway. + +**The bulk scan is the ONLY writer** (write-through on holes). Commit-side +maintenance only CLEARS: every committed transaction clears the cells of its +changed and removed rows (k-sized, beside the existing `slotWrites` block); +a full set-rows/initial build starts from empty vectors; a non-filter-only +plan change compiles fresh shared state (vectors start empty and refill on +the next scan — extending adoption to sort-only changes is a follow-up, not +this milestone). Aborted drafts never touched the cells; entrants and +updated rows are holes until the next scan reads them once. + +Pinned by: the equivalence oracle (columnar ≡ per-row verdicts on +randomized scripts including update, slot-reuse, AND a throwing-accessor +aborted-draft step), and a mutation test that skips the commit-side clear. + +### 4. Holes fall back per cell + +A vector missing a cell (column newly active, or a row ingested under a +plan that didn't reference the column) answers by live accessor read AND +fills the cell (write-through). First filter commit on a newly-referenced +column therefore pays one O(n) accessor pass — the same pass it pays today +every commit — and subsequent commits scan. + +### 5. Compiled predicates + +Per filter, resolve column + operator ONCE into a monomorphic +`(value) => boolean` closure (`compileFilterPredicate`), hoisting operand +normalization (e.g. between-bounds, lowercased needles) out of the row +loop. The bulk scan is: per filter, loop live slots over the vector, +AND into the verdict bitset. `filter-rebuild`'s walk consumes the bitset +instead of calling `filterVerdict` per row; the per-row `filterVerdict` +remains for k-sized and grouped paths, unchanged in semantics. + +## Bars + +- 50k filter-metadata AND filter-text settle improve vs the seam baseline + (108.3/116.8) with TanStack controls in band; traced verdict share ≲3%. +- 3k no regression; zero blank frames; grouped gate untouched. +- Zero public API drift (everything is row-model-internal; `CompiledRowInput` + is internal — verify, and if it leaks into a public type, stop and review). +- Equivalence oracle + freshness mutation + existing 557 row-model tests, all + work-count pins intact. diff --git a/docs/superpowers/specs/2026-08-24-dense-handle-core-design.md b/docs/superpowers/specs/2026-08-24-dense-handle-core-design.md new file mode 100644 index 000000000..6e7f2af16 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-dense-handle-core-design.md @@ -0,0 +1,190 @@ +# Dense-handle core: slots, bitsets, and columnar evaluation + +Date: 2026-08-24. Status: approved design, pre-plan. +Branch: continues `blove/filter-fast-path` (held unmerged by decision; the +filter fast-path branch folds into this arc and lands with it). + +## Problem + +The filter fast-path arc (F/G/H cycles + post-H commits, HEAD `27e8168c`) +took 50k S2 filter settle from 241.8ms to 158.3ms, but the spec bar — settle +≤ same-run TanStack (~67ms) — is missed at 2.4–2.7×, and the final trace +attribution (`filter-final-results.md` §4) says the bar is not reachable from +the current representation: the remaining 134ms window is dominated not by any +algorithm but by a single architectural fact — **string row ids are the +currency of every hot path**. Each per-row step pays string hash + HAMT +traversal, several times over: + +- `sourceOrder` entries are `{rowId, sourceOrder}` — resolving the record is + a 50k-lookup HAMT walk (18.5ms attributed to `filter-rebuild.ts` directly). +- The old verdict is a second lookup on the same string + (`rowPassesFilter` → `visible.rows.get` → byId HAMT, 11.9ms). +- `hashString` alone is 10.4ms — re-hashing strings whose hashes never change. +- The per-row verdict pass is row-at-a-time polymorphic evaluation (18.3ms). +- The renderer repeats the pattern: a `visibleKeys` HAMT rebuilt per + replacement (~7ms), a frozen rowRef allocated per row. + +Sort pays the same currencies: 50k sort settle is 266ms with a 247ms block on +this branch (300/279 on `main`). + +## Goal and bars + +Make the whole row-model commit pipeline — filter, sort, transactions, +grouping, and the renderer seam — speak **dense integer handles** internally, +so every O(n) pass is array-resident. String ids remain the public currency; +the public API is expected to be unchanged (any diff must be deliberate). + +Success bars (bench protocol: medians of 3, same-run TanStack controls in +band, port 4173 free, no `grep|head` on gates): + +1. **50k S2 filter-metadata AND filter-text settle ≤ same-run TanStack**, + completed ×3, zero blank frames. +2. **50k S2 sort settle ≤ same-run TanStack** (bar value read from the same + run's TanStack settle, not the latency band). +3. **Any single main-thread block ≤ 50ms** on every measured script. A path + that cannot meet this at some scale must size-gate to the retained + cooperative path at that scale rather than ship a longer block. +4. **No regressions**: 3k cells, mount, interaction latency (15–18ms band), + grouped gate (`rebuild_slice_max_ms` ≤ 8), anchor/focus/selection + preservation, full repo suites, `api:check`. + +## Non-goals + +- Replacing the persistent structures as source of truth (the full-columnar + / flat-array core was probed and rejected: old-snapshot validity is + load-bearing at four call sites and flat mutation measured 4.7–401ms per + commit vs the tree's 0.16ms — `index-representation-probe.md`). +- Refinement incrementality (monotone filter-as-you-type narrowing scans + only current members). Orthogonal, semantic, filed as follow-up. +- Worker offload, WASM, or SharedArrayBuffer anything. +- Re-adding stored per-record verdicts. Membership IS the verdict (H-cycle + invariant) — it just becomes a bitset. + +## Architecture + +Three new row-model-internal primitives sit UNDER the existing persistent +structures. The HAMT and order-statistic trees remain the identity/order +source of truth; rank queries, snapshot semantics, `orderIsProven`, and +`derivedById` all carry over. + +### Slot allocator (per model instance, not per revision) + +Every row gets a small integer slot for its lifetime: assigned at ingest, +stamped on the record (`record.slot`), released to a free-list on permanent +removal, reused. Capacity grows monotonically — streaming inserts never +renumber existing rows. + +### Slot vectors (immutable, chunked, copy-on-write) + +Two-level arrays: a chunk table over 1024-element chunks. A commit touching +k rows copies ~min(k, ceil(n/1024)) chunks plus the table; reads are two +indexed loads. Used for `recordsBySlot` and the columnar caches. + +**This is what keeps old snapshots valid under slot reuse**: each revision +holds its own immutable chunk table, so revision N's arrays still bind slot +s to whatever row owned s at revision N. A held snapshot's answers cannot +change when a later revision frees and reuses the slot. + +### Membership bitsets (immutable, whole-copied) + +50k rows = 6.25KB; whole-copy per commit is negligible even at streaming +rates — no COW machinery. Two bitsets matter per revision: the **live set** +(slots currently bound to rows — the scan domain) and the **visible set** +(filter membership, ungrouped). Old-vs-new verdict diff = XOR + word-scan of +set bits, replacing ~100k HAMT lookups. + +### Columnar evaluation cache + +`compiled-query`'s evaluation cache becomes per-referenced-column slot-indexed +vectors. The verdict pass becomes, per filter, a monomorphic scan over the +live-set domain producing a bitset, combined across filters by AND. Cache +adoption across filter-only plan changes generalizes the existing +`adoptEvaluationCache` (by-reference, tagged by the writing plan). Keys that +encode numerically take typed-array storage, which also gives sort a numeric +fast path. + +Memory character: ~n × referenced columns × 8B (~2MB at 50k × 5 columns). +M0 measures how much of this the current cache already pays. + +## Component inventory + +| Component | Change | +| --------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| `slot-allocator.ts` / `slot-vector.ts` / `membership-bitset.ts` | new, row-model internal | +| `compiled-query.ts` | columnar cache; monomorphic per-filter scans; bitset AND combine | +| `filter-rebuild.ts` | records via `recordsBySlot`; diff via bitset XOR; merge + bulk build unchanged in shape | +| `sort-rebuild.ts` | sort survivor slot arrays on columnar keys; bulk tree build | +| `transaction-draft.ts` / `row-store.ts` | slot assignment/release; chunk-COW writes; live-set maintenance | +| `cooperative-transition.ts` | RETAINED as the size-gate fallback; reads the same slot structures | +| `persistent/order-statistic-tree.ts` | entries carry slot; internal byId keyed by slot (no string hashing in `get`) | +| `group-index.ts` | leaf trees hold slots; `rowParents` a slot-indexed vector | +| react + layout-core seam | `visibleKeys` HAMT → bitset; rowRefs pooled by slot; refilter/reorder consume slot permutations | + +## Data flow — filter-only commit, ungrouped, 50k + +1. Classifier (existing) proves filter-only. +2. Next plan adopts the columnar cache by reference. +3. Per-filter columnar scan over the live set → AND → new visible bitset. +4. XOR old visible bitset; iterate flipped words → flippedIn / flippedOut. +5. Flip-ins resolve records by `recordsBySlot[slot]`, keys from columnar + vectors; k log k sort of flip-ins only. +6. Linear merge with the old tree walk (`range`, materialized), bulk build + with `orderIsProven` + `derivedById` gated on removals < survivors — all + as shipped today. +7. Publish with `"refilter"` journal reason; renderer permutes heights + bitset-driven. + +Estimated window from the trace attribution: ~55–75ms settle, model block +well under 50ms. Estimates are estimates; M0 and per-milestone measurement +are the authority. + +## Error handling / invariants + +- Dev-mode fail-loud: `recordsBySlot[record.slot] === record` per revision; + allocator double-release / out-of-range access throws. +- Test-time equivalence oracle: bitset membership ≡ tree membership on every + rebuilt path. +- Instrumentation counters extended: bitset diffs, columnar scans, chunk + copies — so work assertions stay pinnable and mutation-hardened. + +## Testing + +- **Property equivalence**: random transaction/query sequences run through + the new pipeline and compared exactly (visible order + membership) against + the current implementation's results. +- **Snapshot-validity pin**: hold a snapshot; remove a row; insert a new row + that REUSES its slot; assert the held snapshot's answers are unchanged. + Mutation twin: break the chunk-COW deliberately and watch it fail. +- **Existing work assertions survive**: filter-only commit rebuilds zero + records, writes zero rows-map nodes (`filter-verdicts.test.ts`, + `filter-fast-path.test.ts` pins carry forward). +- **Chunk-copy bound pinned**: a commit touching k rows copies ≤ f(k) chunks. +- Bench certification per the bars above, under the measurement protocol + (one variable at a time, rebuilt dist per side, controls in band, gates + redirected to files and exit codes checked). + +## Milestones (each independently measured; M0 gates the arc) + +- **M0 — Node pricing probe.** Bitset diff, columnar scan, and slot-vector + maintenance under a streaming transaction mix, at 50k, in isolation + (method proven by `index-representation-probe.md`, whose browser + prediction landed within 2ms). Go/no-go numbers before production code. +- **M1** — allocator + vectors + `recordsBySlot`; rebuild loops stop calling + `rows.get` (−18.5ms est.). +- **M2** — membership bitsets + XOR diff replace the double lookup (−22ms est.). +- **M3** — columnar verdict scan (−13ms est.). +- **M4** — sort on slot arrays + columnar keys. +- **M5** — renderer seam (bitset `visibleKeys`, pooled refs). +- **M6** — grouping conversion (leaf trees + `rowParents` on slots). +- **M7** — certification against all bars; size-gate any path missing the + 50ms block bar; PR of the combined arc (this + the held filter branch). + +## Carried context (do not re-litigate) + +- G3d (`fillSortKeysFromPrevious` adoption) was collected by `27e8168c`; + close it, don't execute it. +- The order-statistic tree is finished as a target (8.5ms). +- A verdict-Set probe and a memoized grouped comparator both measured flat + and were reverted; the waste is re-hashing, not the container. +- Membership-as-tree-measure rejected with numbers (H spec). +- `derivedById` is correctly declined at 50/50 flip ratios (pinned at 49/51). diff --git a/docs/superpowers/specs/2026-08-24-dense-handle-m0-results.md b/docs/superpowers/specs/2026-08-24-dense-handle-m0-results.md new file mode 100644 index 000000000..4e399c446 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-dense-handle-m0-results.md @@ -0,0 +1,69 @@ +# M0 pricing probe — results + +Date: 2026-08-24. Probe: `m0-probe.mjs` (session scratchpad, throwaway). + +Machine load at run: `load averages: 37.32 23.69 22.12` (run 1) and `22.49 21.92 21.57` (run 2) on a 10-core Mac — well above the ≥8 caution threshold both times; parallel sessions were saturating the machine. Fitness: **NOT all spreads ≤ 20%** in either run. Run 1 (reported below) had four sections over: A3 195%, A5 25%, B 79%, C2 539%. The mandated rerun (run 2, second column) was noisier still (A1 78%, A3 102%, A5 88%, B 91%, C 27%, C2 484%), so the noise is reported rather than hidden. All oracles printed `oracles: PASS` before any timing in both runs. + +Why the verdict survives the noise: the two runs' medians agree to within ~1.4x on every section, the _worst observed maximum_ of every relevant section is still 3–10x inside its go-threshold, and the noisy sections are the sub-millisecond ones where scheduler jitter dominates a tiny denominator. The noise widens the error bars; it cannot move any number across a decision boundary. + +## Isolation numbers (median of 5, 2 warmups) + +| section | run 1 median (spread) | run 2 median (spread) | +| ---------------------------------------------- | --------------------- | --------------------- | +| A1 numeric columnar scan → bitset | 0.263ms (4%) | 0.266ms (78%) | +| A2 string columnar scan `.includes` → bitset | 0.693ms (1%) | 0.701ms (1%) | +| A3 xor + enumerate flips | 0.131ms (195%) | 0.129ms (102%) | +| A4 50k records-by-slot vecGet | 0.100ms (12%) | 0.105ms (5%) | +| A5 50k string-keyed Map.get (baseline) | 1.218ms (25%) | 1.615ms (88%) | +| B composed filter-commit equivalent | 3.764ms (79%) | 5.227ms (91%) | +| C streaming: per-commit chunk-COW (100 writes) | 33.1µs (10%) | 42.1µs (27%) | +| C2 1000 whole-bitset clones | 1.005ms (539%) | 1.207ms (484%) | + +Notes on individual sections: + +- A4 vs A5: slot-indexed vecGet (0.10ms) is ~12x cheaper than string-keyed + `Map.get` (1.2–1.6ms) for the same 50k reads — and the Map baseline is itself + far cheaper than the HAMT the trace prices at 42.5ms. Stringiness alone does + not explain the HAMT cost; the dense-slot representation removes both the + string hashing and the persistent-tree pointer chasing. +- A3's absolute times are 0.08–0.34ms; the huge spread percentages are jitter + on a tiny denominator, not instability of the primitive. +- C2: cloning a 6.25KB bitset 1000 times costs ~1ms total (~1µs each) — a + whole-membership clone per commit is effectively free. + +## Read-across + +- Replaced work (trace attribution, `filter-final-results.md`): + persistent-map 42.5 + verdict pass 18.3 + double-lookup 11.9 = **72.7ms**. +- New-structure equivalent, deliberately **overcounted**: A1 + A3 + B = + **4.2ms** (run 1) / **5.6ms** (run 2). The overcount: section B re-sorts the + ~12.5k survivors from scratch (2 × `Array.prototype.sort` dominates its + 3.8–5.2ms), whereas the real path takes survivor order proven from the old + tree walk and only sorts the flip-ins. B alone is therefore an upper bound + on the model-side commit work, and it also already contains its own scan and + xor, so A1+A3+B double-counts those too. Even so: ≥13x under the replaced + 72.7ms. +- Streaming regression check: per-commit chunk-COW cost for 100 random writes + is **33–42µs** against the ≲500µs comfort bound — 12–15x headroom. Adding a + per-commit whole-bitset clone (~1µs) doesn't change that. The HAMT's + structural sharing is not being given up for anything close to its cost. +- Caveat: these are Node numbers, not browser numbers. The precedent probe + (`index-representation-probe.md`) landed within 2ms of the browser result; + browser certification remains M7's job, not M0's. + +## Go/no-go + +**GO.** + +- New-primitive total (A1 + A3 + B, overcounted): 4.2–5.6ms ≲ 15ms required. ✓ +- Per-commit streaming COW: 33–42µs ≲ 500µs required. ✓ +- Load sensitivity: the per-run maxima are observations under their own runs, + not ceilings. A third independent run (spec review, load 22.6) measured + A1+A3+B = 8.25ms median (B max 23.3ms) and 98.4µs per commit — medians + still ≥2× inside every bound, but expect the maxima to scale with load. +- Against the 72.7ms of replaced work, the new primitives price at roughly + 6–8% of the cost they displace, leaving the spec's ~55–75ms filter-commit + window dominated by the parts M0 did not model (tree build / render), which + is exactly where the budget was expected to live. + +No primitive priced out. Proceed to M1. diff --git a/docs/superpowers/specs/2026-08-24-dense-handle-m1-m2-results.md b/docs/superpowers/specs/2026-08-24-dense-handle-m1-m2-results.md new file mode 100644 index 000000000..7d463312e --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-dense-handle-m1-m2-results.md @@ -0,0 +1,90 @@ +# M1+M2 measured results — slots + membership bitsets (2026-08-24) + +Browser-bench A/B of the M1+M2 arc: filter-only rebuild walks `recordsBySlot` +and diffs membership bitsets, deleting two 50k string-HAMT lookup passes. + +- **Variant**: `e529d9d0` (HEAD of `blove/filter-fast-path`). +- **Baseline**: `8f2b63b2` (last commit before M1; verified via + `git log 8f2b63b2..HEAD -- packages/` that every intervening `packages/` + commit is M1+M2 row-model work — one variable). +- **Design**: throwaway worktree at the baseline commit, fresh + `pnpm install --frozen-lockfile` + bench build per side, interleaved paired + rounds (baseline 50k → variant 50k → baseline 3k → variant 3k), 3 repeats + per cell, medians reported. No CDP perf tracing (`PLAYWRIGHT_PERF_TRACE` + unset) — these are headline-grade absolutes. +- **Machine load**: heavy and falling — 1-min load 38.1 at start, 33–54 + during the four rounds (10-core Mac, parallel sessions active). Runs were + interleaved specifically so both sides saw the same regime; the TanStack + same-run controls below are the fitness arbiter. +- **Scale note**: the 3k tier is `--scale=hypothesis` (3,000 rows). The plan's + `--scale=dev` is the 750-row tier — confirmed against existing summary + `rowCount` fields before running. + +## Results (medians of 3; settle quantizes to ~8.3ms frame steps) + +### 50k rows (S2, `--scale=target`) + +| Metric | Script | Baseline | Variant (M1+M2) | Δ | +| ------------------------------ | --------------- | -------- | --------------- | --------- | +| settle_duration_ms | filter-metadata | 166.6 | 116.8 | **−49.8** | +| settle_duration_ms | filter-text | 158.4 | 125.6 | **−32.8** | +| post_interaction_long_tasks_ms | filter-metadata | 148 | 105 | −43 | +| post_interaction_long_tasks_ms | filter-text | 141 | 109 | −32 | +| interaction_latency_ms | filter-metadata | 16.7 | 16.6 | ~0 | +| interaction_latency_ms | filter-text | 16.6 | 16.6 | ~0 | +| TanStack control settle | filter-metadata | 58.3 | 58.4 | +0.1 | +| TanStack control settle | filter-text | 50.1 | 50.5 | +0.4 | + +Repeat spreads: baseline metadata [158.3, 166.6, 174.3]; variant metadata +[116.1, 116.8, 124.5]; variant filter-text had one 474.1ms outlier (load +spike — median unaffected), the other two repeats were 124.9/125.6. + +### 3k rows (S2, `--scale=hypothesis`) + +| Metric | Script | Baseline | Variant (M1+M2) | Δ | +| ------------------------------ | --------------- | ----------- | --------------- | ---------------- | +| settle_duration_ms | filter-metadata | 41.7 | 41.7 | 0 | +| settle_duration_ms | filter-text | 41.8 | 33.5 | −8.3 (one frame) | +| post_interaction_long_tasks_ms | both | 0 | 0 | 0 | +| interaction_latency_ms | both | 16.6 / 15.8 | 16.6 / 16.5 | ~0 | +| TanStack control settle | filter-metadata | 25.0 | 25.0 | 0 | +| TanStack control settle | filter-text | 32.6 | 25.0 | −7.6* | + +\* Baseline text repeats were [24.4, 32.6, 34.1] vs variant [24.8, 25.0, +32.9] — same one-frame band, median landed on different sides of a frame +boundary. Within normal spread, not a regime change. + +## Deltas vs the branch's pre-M1 record + +Pre-M1 (recorded earlier on this branch): 50k settle 158.3 (metadata) / +157.5 (text), long-tasks 141; 3k settle 34.5 / 33.6. + +- The baseline side reproduced those numbers under today's load (158.4–166.6 + @50k; 41.7–41.8 @3k, one frame above the recorded 33.6–34.5 — consistent + with load, and why the paired baseline, not the old record, is the + comparison basis). +- 50k improvement vs paired baseline: **−49.8ms (metadata) / −32.8ms + (text)** settle, −43/−32ms long-tasks. Against the trace-attribution + estimate of ~−30ms: **met on filter-text, exceeded on filter-metadata**. +- 3k: at most one frame of improvement — expected; the deleted HAMT passes + are small in absolute terms at 3k and settle is frame-quantized. + +## Fitness statement + +- TanStack same-run controls agree across sides within one frame on all four + cells (50k: 58.3→58.4, 50.1→50.5; 3k: 25.0→25.0, 32.6→25.0 with + overlapping repeat ranges). The regime did not move between sides. +- Load was high (1-min 33–54 on 10 cores) throughout; the interleaved paired + design plus in-band controls make the deltas trustworthy, but individual + absolutes carry load noise (see the 474ms variant outlier and the baseline + 3k 75.5ms outlier — both single repeats, both excluded by the median). + +## Conclusion + +M1+M2 delivered: 50k filter settle dropped from ~158–167ms to ~117–126ms +(−33 to −50ms), meeting the ~−30ms estimate, with interaction latency +unchanged and long-tasks down proportionally. Pretable still trails the +TanStack bar (~50–58ms settle in the same runs, vs the ~67ms historical +bar): roughly **60–75ms of remaining settle gap is M3's problem** — the +filter-commit path after M1+M2 is no longer dominated by row-record lookups, +so the next attribution pass starts from a fresh trace, not this doc. diff --git a/docs/superpowers/specs/2026-08-24-dense-layout-seam-results.md b/docs/superpowers/specs/2026-08-24-dense-layout-seam-results.md new file mode 100644 index 000000000..6a48fb23a --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-dense-layout-seam-results.md @@ -0,0 +1,142 @@ +# Dense layout seam measured results (Amendment I) — 2026-08-24 + +Browser-bench A/B of the dense layout seam: layout-core dense generations +(slot-indexed refilter/reorder, bitset membership), row-model internal dense +snapshot reads, and renderer-dom dense-keyed layout sources. + +- **Variant**: `a9cfac08` (HEAD of `blove/filter-fast-path` — the complete + seam, Tasks 1–6). +- **Baseline**: `93ff4054` (the commit before seam Task 1; verified via + `git log 93ff4054..HEAD -- packages/` that all six intervening `packages/` + commits are seam work — one variable). Note the baseline INCLUDES M1+M2, + so it should and does reproduce the 116.8/125.6ms record. +- **Design**: throwaway worktree at the baseline commit, fresh + `pnpm install --frozen-lockfile` + `pnpm --filter @pretable/app-bench build` + per side, one preview server on 4173 at a time (port verified free first), + interleaved paired rounds (baseline 50k → variant 50k → baseline 3k → + variant 3k), 3 repeats per cell, medians reported. No CDP tracing for the + headline numbers; runner output redirected to files, exit codes checked. +- **Machine load**: heavy — 1-min load 24–46 across the four rounds (10-core + Mac, 9GB of 10GB swap used, parallel sessions active). Interleaving plus + the TanStack same-run controls are the fitness arbiter, as in the M1+M2 + run which saw the same regime (33–54). +- **Scale note**: the 3k tier is `--scale=hypothesis` (3,000 rows; + `rowCount` confirmed in the summaries). `--scale=dev` is 750 rows and was + not used. + +## Results (medians of 3; settle quantizes to ~8.3ms frame steps) + +### 50k rows (S2, `--scale=target`) + +| Metric | Script | Baseline | Variant (seam) | Δ | +| --------------------------------- | --------------- | -------- | -------------- | --------- | +| settle_duration_ms | filter-metadata | 125.3 | 108.3 | **−17.0** | +| settle_duration_ms | filter-text | 125.0 | 116.8 | **−8.2** | +| post_interaction_long_tasks_ms | filter-metadata | 111 | 89 | −22 | +| post_interaction_long_tasks_ms | filter-text | 103 | 93 | −10 | +| interaction_latency_ms | filter-metadata | 16.3 | 17.4 | ~0 | +| interaction_latency_ms | filter-text | 16.6 | 15.7 | ~0 | +| post_interaction_blank_gap_frames | both | 0 | 0 | 0 | +| TanStack control settle | filter-metadata | 50.0 | 49.6 | −0.4 | +| TanStack control settle | filter-text | 50.8 | 50.0 | −0.8 | + +Repeat spreads: baseline metadata [125.0, 125.3, 140.8]; variant metadata +[108.0, 108.3, 108.4] — unusually tight for this machine; variant text +[109.0, 116.8, 123.4]. + +### 3k rows (S2, `--scale=hypothesis`) + +| Metric | Script | Baseline | Variant (seam) | Δ | +| --------------------------------- | --------------- | ----------- | -------------- | ---------------- | +| settle_duration_ms | filter-metadata | 33.5 | 33.4 | 0 | +| settle_duration_ms | filter-text | 41.6 | 33.7 | −7.9 (one frame) | +| post_interaction_long_tasks_ms | both | 0 | 0 | 0 | +| interaction_latency_ms | metadata / text | 16.8 / 17.0 | 16.6 / 24.0* | ~0 / +1 frame* | +| post_interaction_blank_gap_frames | both | 0 | 0 | 0 | +| TanStack control settle | filter-metadata | 33.0 | 25.0 | −8.0** | +| TanStack control settle | filter-text | 25.7 | 23.9 | −1.8 | + +\* Variant 3k text latency repeats straddled a frame boundary under load; +settle (the governing metric) improved a frame. Not a regression signal. + +\*\* Baseline metadata repeats [24.3, 33.0, 34.0] vs variant [24.7, 25.0, +34.1] — same one-frame band, medians landed on different sides of a frame +boundary. Same pattern (in the other direction) appeared in the M1+M2 run; +within normal spread, not a regime change. + +## Deltas vs M1+M2 and the cumulative arc + +- Today's baseline side (= the M1+M2 state) read 125.3/125.0 @50k and + 33.5/41.6 @3k — the M1+M2 record (116.8/125.6 @50k; 41.7/33.5 @3k) within + one frame under today's load, which is why the paired baseline, not the + record, is the comparison basis. +- Seam vs M1+M2 record: 116.8 → 108.3 (metadata, −8.5) and 125.6 → 116.8 + (text, −8.8). Vs paired baseline: **−17.0 / −8.2ms**, long-tasks −22/−10. +- Cumulative arc (pre-M1: 166.6/158.4): 50k filter settle is now + **108.3/116.8 — −58.3/−41.6ms total, a ~1.4–1.5× speedup** on the branch. + +## Bar verdicts + +- **Untraced 50k settle ≤ ~95ms: MISSED.** 108.3 (metadata) / 116.8 (text). + The improvement is real and the controls prove the regime held, but the + seam bought ~1–2 frames, not the ~3 the bar assumed. Long-tasks dropped + proportionally (111→89, 103→93), consistent with the win being genuine + main-thread work removed, not measurement drift. +- **Layout share ≲ 10% of the traced window: MET** (~7.4%, table below). + The two verdicts together say the amendment's attribution was right — + layout's walk is no longer the problem — but the pre-seam trace charged + more of the window to layout than the layout walk alone actually cost; + part of that share was the row-model rebuild it overlapped with. +- **3k no regression: MET** (0 / −7.9ms). +- **TanStack controls in band: MET** (all four cells within one frame across + sides; 50k controls 49.6–50.8 both sides). +- **Zero blank frames: MET** (`post_interaction_blank_gap_frames` 0 in all + 12 pretable summaries, both sides, both scales). +- `refilterFallbackCount === 0` is pinned by the react e2e suite (seam + Task 6, commit `a9cfac08`), mutation-hardened — not re-measured here. + +## Traced share re-attribution (variant, 50k filter-metadata) + +One traced run AFTER the headlines (`PLAYWRIGHT_PERF_TRACE=1`, repeats=1), +`analyze-cdp.mjs --window=interaction` with the build's sourcemap. Traced +absolutes skew ~2× — shares only. Window 74.2ms, 74.2ms sampled. + +| Subsystem (self time) | Share | +| ----------------------------------------------------------------------- | -------- | +| layout-core refilter walk (`refilter` + `#refilterDense` + window prep) | **7.4%** | +| row-model filter-rebuild walk (`filter-rebuild.js`) | 17.4% | +| compiled-query verdict evaluation (`compiled-query.js`) | 17.5% | +| persistent HAMT (`persistent-map.js`) | 9.2% | +| order-statistic tree | 3.6% | +| slot-vector | 2.9% | +| visible-index | 2.8% | +| change-journal | 0.5% | +| react render/commit + DOM (react-dom, measure, style/attr, grapheme) | ~23% | +| (program) + GC | 7.7% | + +The layout walk (dense lane) is 7.4% — under the 10% bar. The window is now +dominated by the row-model rebuild + verdict evaluation (~35% combined) and +the render/commit side (~23%). The HAMT share that remains (9.2%) belongs to +row-model's own persistent structures, not layout reads. + +## Fitness statement + +- TanStack same-run controls agree across sides within one frame on all four + cells; the 3k metadata −8ms is a frame-boundary artifact with overlapping + repeat ranges. The regime did not move between sides. +- Load was heavy (1-min 24–46 on 10 cores, swap nearly full) throughout; the + interleaved paired design plus in-band controls make the deltas + trustworthy. Individual absolutes carry load noise (baseline metadata's + 140.8 first repeat), excluded by the medians. + +## Conclusion + +The dense layout seam delivered a real but smaller-than-estimated win: 50k +filter settle dropped from ~125ms to **108.3 (metadata) / 116.8 (text)** — +−17.0/−8.2ms with long-tasks down proportionally and zero blank frames — +missing the ~95ms bar, while the traced layout share fell to ~7.4%, meeting +the ≤10% attribution bar. Layout is no longer where the time goes: the +remaining ~58–67ms gap to the TanStack bar (~50ms settle in these same +runs) now sits in the row-model rebuild and verdict evaluation (~35% of the +window) plus render/commit (~23%) — which is exactly the **columnar verdict +cache, the next milestone**. diff --git a/packages/core/core.api.md b/packages/core/core.api.md index 8bbc0098c..44b37e9bc 100644 --- a/packages/core/core.api.md +++ b/packages/core/core.api.md @@ -264,7 +264,7 @@ export type PretableChangeSequence = { } | { readonly kind: "reset"; readonly toRevision: number; - readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder"; + readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder" | "refilter"; }; // @public (undocumented) @@ -1177,6 +1177,12 @@ export interface PretableRowModelSnapshot { expect(dataRowReads).toBe(100_000); }); - test('handles a "reorder" reset exactly as a "bulk-replace" reset', () => { - // Fail-closed pin: grid-core is deliberately reorder-UNAWARE. A reset - // whose reason is "reorder" (or any reason this suite has never heard - // of) must take the same full-rebuild path as "bulk-replace" — the - // reason field is advisory for consumers that opt into it, never a - // requirement for correctness. + test('handles "reorder" and "refilter" resets exactly as a "bulk-replace" reset', () => { + // Fail-closed pin: grid-core is deliberately reorder- and + // refilter-UNAWARE. A reset whose reason is "reorder" or "refilter" (or + // any reason this suite has never heard of) must take the same + // full-rebuild path as "bulk-replace" — the reason field is advisory + // for consumers that opt into it, never a requirement for correctness. const rows = [1, 2, 3, 4].map((id) => ({ id, team: "a", score: id })); const model = createLocalRowModel({ rows, @@ -796,16 +796,19 @@ describe("indexed row selection", () => { model.setRows([rows[3]!, rows[2]!, rows[1]!, rows[0]!]); const snapshot = model.getState().snapshot; - const project = (reason: "reorder" | "bulk-replace" | "unknown-revision") => + const project = ( + reason: "reorder" | "refilter" | "bulk-replace" | "unknown-revision", + ) => projectIndexedSelection(selected, previous, snapshot, { kind: "reset", toRevision: snapshot.revision, reason, }); const viaReorder = project("reorder"); + const viaRefilter = project("refilter"); const viaBulkReplace = project("bulk-replace"); - for (const projected of [viaReorder, viaBulkReplace]) { + for (const projected of [viaReorder, viaRefilter, viaBulkReplace]) { expect(getIndexedSelectionSummary(projected, snapshot)).toEqual( getIndexedSelectionSummary(viaBulkReplace, snapshot), ); diff --git a/packages/layout-core/src/__tests__/dense-membership.test.ts b/packages/layout-core/src/__tests__/dense-membership.test.ts new file mode 100644 index 000000000..888eeb1a3 --- /dev/null +++ b/packages/layout-core/src/__tests__/dense-membership.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { + cloneDenseMembership, + clearDenseBit, + createDenseMembership, + setDenseBit, + testDenseBit, +} from "../dense-membership"; + +describe("dense membership bitset", () => { + it("round-trips set/clear/test across word boundaries", () => { + const bits = createDenseMembership(100); + for (const slot of [0, 31, 32, 63, 64, 99]) { + expect(testDenseBit(bits, slot)).toBe(false); + setDenseBit(bits, slot); + expect(testDenseBit(bits, slot)).toBe(true); + } + clearDenseBit(bits, 32); + expect(testDenseBit(bits, 32)).toBe(false); + expect(testDenseBit(bits, 31)).toBe(true); + expect(testDenseBit(bits, 63)).toBe(true); + }); + + it("clone is independent of the original", () => { + const bits = createDenseMembership(64); + setDenseBit(bits, 10); + const copy = cloneDenseMembership(bits, 64); + clearDenseBit(copy, 10); + setDenseBit(copy, 20); + expect(testDenseBit(bits, 10)).toBe(true); + expect(testDenseBit(bits, 20)).toBe(false); + }); + + it("clone can grow capacity, preserving low bits", () => { + const bits = createDenseMembership(32); + setDenseBit(bits, 31); + const grown = cloneDenseMembership(bits, 200); + expect(testDenseBit(grown, 31)).toBe(true); + setDenseBit(grown, 199); + expect(testDenseBit(grown, 199)).toBe(true); + }); + + it("reads beyond a bitset's words answer false", () => { + const bits = createDenseMembership(32); + expect(testDenseBit(bits, 500)).toBe(false); + expect(testDenseBit(createDenseMembership(0), 12345)).toBe(false); + }); +}); diff --git a/packages/layout-core/src/__tests__/row-height-index.test.ts b/packages/layout-core/src/__tests__/row-height-index.test.ts index 2b0e91daf..0430781b9 100644 --- a/packages/layout-core/src/__tests__/row-height-index.test.ts +++ b/packages/layout-core/src/__tests__/row-height-index.test.ts @@ -1722,3 +1722,1016 @@ describe("bulk replacement when the base holds no retained state", () => { expect(result.getHeight(0)).toBe(63); }); }); + +describe("synchronous refilter over existing height entries", () => { + /** + * A base index with mixed measured and estimated entries: rows 0..N-1 with + * varied estimates (including `undefined` → default height), every third row + * measured to a height its estimate could not predict. + */ + function refilterFixture(count = 25, maxRetainedMeasurements?: number) { + const keys = Array.from({ length: count }, (_, index) => + index % 5 === 0 ? group(String(index)) : data(String(index)), + ); + const estimates = keys.map((_, index) => + index % 4 === 3 ? undefined : 18 + (index % 7) * 3, + ); + let base = createIndex( + keys.map((key, index) => entry(key, estimates[index])), + 30, + maxRetainedMeasurements, + ); + for (let index = 0; index < count; index += 3) { + base = base.measure(index, keys[index]!, 51 + index); + } + const entries = keys.map((key, index) => entry(key, estimates[index])); + return { keys, estimates, entries, base, count }; + } + + function sourceOf( + rows: readonly RowHeightEntry[], + ): RowHeightReplacementSource { + return { rowCount: rows.length, entryAt: (index) => rows[index]! }; + } + + /** Every rank's offset, height, and key, plus the total: full geometry. */ + function rankTable(index: RowHeightIndex) { + return { + rowCount: index.rowCount, + total: index.getTotalHeight(), + keys: Array.from({ length: index.rowCount }, (_, rank) => + index.keyAt(rank), + ), + offsets: Array.from({ length: index.rowCount + 1 }, (_, rank) => + index.getOffsetForIndex(rank), + ), + heights: Array.from({ length: index.rowCount }, (_, rank) => + index.getHeight(rank), + ), + }; + } + + /** The retained-state observables: cache, tombstones, visible measurements. */ + function retainedState(index: RowHeightIndex) { + const diagnostics = getRowHeightIndexDiagnosticsForTesting(index); + return { + measurementCacheCount: diagnostics.measurementCacheCount, + tombstoneCount: diagnostics.tombstoneCount, + visibleMeasurementCount: diagnostics.visibleMeasurementCount, + }; + } + + function expectMatchesReplaceOracle( + base: RowHeightIndex, + rows: readonly RowHeightEntry[], + ): RowHeightIndex { + const refiltered = base.refilter(sourceOf(rows)); + const replaced = base.replace(rows); + expect(rankTable(refiltered)).toEqual(rankTable(replaced)); + expect(retainedState(refiltered)).toEqual(retainedState(replaced)); + return refiltered; + } + + test("pure shrink matches the full replacement oracle", () => { + const { entries, base } = refilterFixture(); + expectMatchesReplaceOracle( + base, + entries.filter((_, index) => index % 2 === 0), + ); + }); + + test("pure grow matches the full replacement oracle", () => { + const { entries, base } = refilterFixture(10); + expectMatchesReplaceOracle(base, [ + ...entries.slice(0, 4), + entry(data("entrant-a"), 44), + ...entries.slice(4), + entry(data("entrant-b")), + entry(group("entrant-c"), 61), + ]); + }); + + test("a disjoint same-count membership matches the oracle", () => { + const { base, count } = refilterFixture(10); + const disjoint = Array.from({ length: count }, (_, index) => + entry(data(`other-${index}`), 20 + index), + ); + const refiltered = expectMatchesReplaceOracle(base, disjoint); + expect(refiltered.rowCount).toBe(count); + }); + + test("empty→populated and populated→empty match the oracle", () => { + const { entries, base } = refilterFixture(8); + const emptied = expectMatchesReplaceOracle(base, []); + expect(emptied.rowCount).toBe(0); + const empty = createIndex([]); + expectMatchesReplaceOracle(empty, entries.slice(0, 5)); + }); + + test("a measured leaver's measurement survives and is restored on return", () => { + const { keys, entries, base } = refilterFixture(); + // Row 4 is unmeasured, row 6 is measured (51 + 6). Drop both. + const measuredLeaver = keys[6]!; + const unmeasuredLeaver = keys[4]!; + const without = entries.filter((_, index) => index !== 4 && index !== 6); + const shrunk = base.refilter(sourceOf(without)); + + // The measured leaver tombstones; the unmeasured one simply vanishes. + expect(shrunk.hasMeasurement(measuredLeaver)).toBe(true); + expect(shrunk.hasMeasurement(unmeasuredLeaver)).toBe(false); + expect(retainedState(shrunk)).toEqual(retainedState(base.replace(without))); + + // A later refilter that brings the measured leaver back restores its + // measurement — the retention rule, observed behaviorally. + const stillWithoutFour = entries.filter((_, index) => index !== 4); + const returned = shrunk.refilter(sourceOf(stillWithoutFour)); + expect(returned.getHeight(5)).toBe(51 + 6); + expect(retainedState(returned)).toEqual( + retainedState(base.replace(without).replace(stillWithoutFour)), + ); + }); + + test("estimate-carrying entrants use the estimate-or-default ingest rule", () => { + const { entries, base } = refilterFixture(6); + const grown = base.refilter( + sourceOf([ + entry(data("with-estimate"), 47), + ...entries, + entry(data("no-estimate")), + ]), + ); + expect(grown.getHeight(0)).toBe(47); + expect(grown.getHeight(grown.rowCount - 1)).toBe(30); + }); + + test("survivor entries are reused verbatim: measurements and estimates ride", () => { + const { keys, base } = refilterFixture(); + // Hand the source lying estimates for every surviving row: a refilter + // must not re-estimate or re-measure survivors, so original heights ride. + const lying = keys + .map((key) => entry(key, 999)) + .filter((_, index) => index % 2 === 0); + const shrunk = base.refilter(sourceOf(lying)); + expect(shrunk.rowCount).toBe(13); + for (let rank = 0; rank < shrunk.rowCount; rank += 1) { + expect(shrunk.getHeight(rank)).toBe(base.getHeight(rank * 2)); + } + }); + + test("counts reused, inserted, and retired entries exactly", () => { + const { entries, base, count } = refilterFixture(); + const survivors = entries.filter((_, index) => index % 2 === 0); + const entrants = [entry(data("new-1"), 21), entry(data("new-2"))]; + const next = base.refilter(sourceOf([...survivors, ...entrants])); + expect(getRowHeightIndexDiagnosticsForTesting(next)).toMatchObject({ + refilterEntriesReused: survivors.length, + refilterEntriesInserted: entrants.length, + refilterEntriesRetired: count - survivors.length, + }); + + const disjoint = base.refilter( + sourceOf(entries.map((_, index) => entry(data(`d-${index}`)))), + ); + expect(getRowHeightIndexDiagnosticsForTesting(disjoint)).toMatchObject({ + refilterEntriesReused: 0, + refilterEntriesInserted: count, + refilterEntriesRetired: count, + }); + }); + + test("an identical membership and order is a no-op returning the same index", () => { + const { entries, base } = refilterFixture(); + expect(base.refilter(sourceOf(entries))).toBe(base); + const empty = createIndex([]); + expect(empty.refilter(sourceOf([]))).toBe(empty); + }); + + test("duplicate keys throw; membership deltas do not", () => { + const { entries, base } = refilterFixture(); + const duplicated = [...entries.slice(0, 10), entries[4]!]; + expect(() => base.refilter(sourceOf(duplicated))).toThrow( + /duplicate stable row-height key/i, + ); + // A missing existing key is a LEAVER, not an error — refilter's purpose. + expect(() => base.refilter(sourceOf(entries.slice(1)))).not.toThrow(); + }); + + test("a bad rowCount throws a RangeError", () => { + const { entries, base } = refilterFixture(4); + for (const rowCount of [0.5, -1, Number.NaN]) { + let thrown: unknown; + try { + base.refilter({ rowCount, entryAt: (index) => entries[index]! }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(RangeError); + } + expect(() => + base.refilter({ + rowCount: 1, + entryAt: "nope" as unknown as (index: number) => RowHeightEntry, + }), + ).toThrow(TypeError); + }); + + test("maxRetainedMeasurements zero drops a measured leaver's measurement", () => { + const { keys, entries, base } = refilterFixture(9, 0); + const without = entries.filter((_, index) => index !== 6); + const shrunk = base.refilter(sourceOf(without)); + expect(shrunk.hasMeasurement(keys[6]!)).toBe(false); + expect(retainedState(shrunk)).toEqual(retainedState(base.replace(without))); + }); + + test("retention-cap pressure evicts the oldest tombstones like replace", () => { + const { entries, base } = refilterFixture(13, 2); + // Rows 0, 3, 6, 9, 12 are measured; dropping them all retires five + // measured leavers into a cap of two: only the two NEWEST tickets stay. + const survivors = entries.filter((_, index) => index % 3 !== 0); + const refiltered = base.refilter(sourceOf(survivors)); + const replaced = base.replace(survivors); + expect(retainedState(refiltered)).toEqual(retainedState(replaced)); + // Behavioral pin on WHICH measurements survived: bring every measured + // leaver back; both paths must restore the same subset. + expect(rankTable(refiltered.refilter(sourceOf(entries)))).toEqual( + rankTable(replaced.replace(entries)), + ); + }); + + test("leaves the old index untouched", () => { + const { entries, base } = refilterFixture(); + const before = rankTable(base); + const beforeState = retainedState(base); + const shrunk = base.refilter(sourceOf(entries.slice(0, 10))); + expect(shrunk).not.toBe(base); + expect(rankTable(base)).toEqual(before); + expect(retainedState(base)).toEqual(beforeState); + }); + + test("post-refilter mutations behave exactly like a replace-built index", () => { + const { keys, entries, base } = refilterFixture(); + const survivors = entries.filter((_, index) => index % 2 === 0); + const viaRefilter = base.refilter(sourceOf(survivors)); + const viaReplace = base.replace(survivors); + + // measure + const measuredA = viaRefilter.measure(2, keys[4]!, 83); + const measuredB = viaReplace.measure(2, keys[4]!, 83); + expect(rankTable(measuredA)).toEqual(rankTable(measuredB)); + + // reorder + const reversed = [...survivors].reverse(); + expect(rankTable(measuredA.reorder(sourceOf(reversed)))).toEqual( + rankTable(measuredB.reorder(sourceOf(reversed))), + ); + + // refilter again (chain), then a full replacement + const next = [...survivors.slice(3), entry(data("late"), 26)]; + const chainA = measuredA.refilter(sourceOf(next)); + const chainB = measuredB.refilter(sourceOf(next)); + expect(rankTable(chainA)).toEqual(rankTable(chainB)); + expect(retainedState(chainA)).toEqual(retainedState(chainB)); + const final = [entry(data("z-1"), 31), entry(data("z-2"))]; + expect(rankTable(chainA.replace(final))).toEqual( + rankTable(chainB.replace(final)), + ); + }); +}); + +describe("dense generations (Amendment I, Task 2)", () => { + const denseEntry = ( + key: Key, + denseKey: number | undefined, + estimatedHeight?: number, + ): RowHeightEntry => ({ key, estimatedHeight, denseKey }); + + function denseSource( + rows: readonly RowHeightEntry[], + denseCapacity: number | undefined, + ): RowHeightReplacementSource { + return { + rowCount: rows.length, + denseCapacity, + entryAt: (index) => rows[index]!, + }; + } + + function rebuild( + base: RowHeightIndex, + rows: readonly RowHeightEntry[], + denseCapacity: number | undefined, + ): RowHeightIndex { + const builder = base.beginReplacement(denseSource(rows, denseCapacity)); + while (!builder.done) builder.advance({ maxUnits: 256 }); + return builder.finish(); + } + + test("builds a dense generation whose apply guards work by slot", () => { + const a = data("a"); + const b = data("b"); + const base = rebuild( + createIndex([]), + [denseEntry(a, 0, 20), denseEntry(b, 2, 30)], + 8, + ); + expect(base.rowCount).toBe(2); + expect(base.getTotalHeight()).toBe(50); + + // Insert into a free slot is accepted; a duplicated slot is rejected. + const c = data("c"); + const inserted = base.apply([ + { kind: "insert", ref: c, index: 2, estimatedHeight: 40, denseKey: 5 }, + ]); + expect(inserted.rowCount).toBe(3); + expect(inserted.getTotalHeight()).toBe(90); + expect(() => + inserted.apply([ + { + kind: "insert", + ref: data("x"), + index: 0, + estimatedHeight: 10, + denseKey: 5, + }, + ]), + ).toThrow(/duplicate/i); + + // Remove clears the slot so it can be reinserted, and the measurement + // returns by STRING identity, exactly as on the string lane. + const measured = inserted.measure(1, b, 47); + const removed = measured.apply([ + { kind: "remove", ref: b, previousIndex: 1, denseKey: 2 }, + ]); + expect(removed.rowCount).toBe(2); + const restored = removed.apply([ + { + kind: "insert", + ref: data("b"), + index: 1, + estimatedHeight: 12, + denseKey: 2, + }, + ]); + expect(restored.getHeight(1)).toBe(47); + }); + + test("retainMeasurement guards visibility by slot on a dense generation", () => { + const a = data("a"); + const base = rebuild(createIndex([], 30, 4), [denseEntry(a, 1, 20)], 8); + // Absent row (slot 5 unoccupied): retained. + const gone = data("gone"); + const retained = base.retainMeasurement(gone, 73, 5); + expect(retained.hasMeasurement(gone)).toBe(true); + // Visible row (slot 1 occupied): rejected, as on the string lane. + expect(() => base.retainMeasurement(a, 80, 1)).toThrow(/visible row/i); + // No denseKey on a dense index: lifecycle error, not a silent accept. + expectReplacementLifecycleError( + () => base.retainMeasurement(gone, 73), + "failed", + ); + }); + + test("an operation without a denseKey on a dense generation throws the lifecycle error", () => { + const a = data("a"); + const base = rebuild(createIndex([]), [denseEntry(a, 0, 20)], 4); + expectReplacementLifecycleError( + () => + base.apply([ + { kind: "insert", ref: data("b"), index: 1, estimatedHeight: 10 }, + ]), + "failed", + ); + expectReplacementLifecycleError( + () => base.apply([{ kind: "remove", ref: a, previousIndex: 0 }]), + "failed", + ); + expectReplacementLifecycleError( + () => + base.apply([{ kind: "update", ref: a, index: 0, estimatedHeight: 25 }]), + "failed", + ); + }); + + test("a missing entry denseKey under a declared capacity throws (bulk and cooperative)", () => { + const rows: readonly RowHeightEntry[] = [ + denseEntry(data("a"), 0, 20), + { key: data("b"), estimatedHeight: 30 }, + ]; + // Bulk path: no retained state. + { + const builder = createIndex([]).beginReplacement(denseSource(rows, 8)); + expectReplacementLifecycleError(() => { + while (!builder.done) builder.advance({ maxUnits: 256 }); + }, "failed"); + } + // Cooperative path: a retained measurement forces the phased builder. + { + const base = createIndex([], 30, 4).retainMeasurement(data("gone"), 51); + const builder = base.beginReplacement(denseSource(rows, 8)); + expectReplacementLifecycleError(() => { + while (!builder.done) builder.advance({ maxUnits: 256 }); + }, "failed"); + } + // A denseKey at or above the declared capacity is the same broken promise. + { + const builder = createIndex([]).beginReplacement( + denseSource([denseEntry(data("a"), 8, 20)], 8), + ); + expectReplacementLifecycleError(() => { + while (!builder.done) builder.advance({ maxUnits: 256 }); + }, "failed"); + } + }); + + test("dense refilter narrows and widens by slot and stays dense", () => { + const a = data("a"); + const b = data("b"); + const c = data("c"); + const base = rebuild( + createIndex([]), + [denseEntry(a, 0, 20), denseEntry(b, 1, 30), denseEntry(c, 5, 40)], + 8, + ).measure(1, b, 47); + + // Narrow: b leaves (tombstoned), a and c survive with heights intact. + const narrowed = base.refilter( + denseSource([denseEntry(a, 0, 20), denseEntry(c, 5, 40)], 8), + ); + expect(narrowed.rowCount).toBe(2); + expect(narrowed.getTotalHeight()).toBe(60); + expect(narrowed.hasMeasurement(b)).toBe(true); + + // Widen: b returns as an entrant and gets its retained measurement back. + const widened = narrowed.refilter( + denseSource( + [denseEntry(a, 0, 20), denseEntry(b, 1, 12), denseEntry(c, 5, 40)], + 8, + ), + ); + expect(widened.rowCount).toBe(3); + expect(widened.getHeight(1)).toBe(47); + expect(getRowHeightIndexDiagnosticsForTesting(widened)).toMatchObject({ + tombstoneCount: 0, + }); + + // The result is a DENSE generation: its guards still demand dense keys. + expectReplacementLifecycleError( + () => + widened.apply([ + { kind: "insert", ref: data("d"), index: 3, estimatedHeight: 10 }, + ]), + "failed", + ); + // An identical membership and order is a no-op returning the same index. + expect( + widened.refilter( + denseSource( + [denseEntry(a, 0, 20), denseEntry(b, 1, 12), denseEntry(c, 5, 40)], + 8, + ), + ), + ).toBe(widened); + }); + + test("dense reorder permutes by slot and stays dense", () => { + const a = data("a"); + const b = data("b"); + const c = data("c"); + const base = rebuild( + createIndex([]), + [denseEntry(a, 2, 20), denseEntry(b, 4, 30), denseEntry(c, 7, 40)], + 8, + ).measure(2, c, 55); + + const reordered = base.reorder( + denseSource( + [denseEntry(c, 7, 40), denseEntry(a, 2, 20), denseEntry(b, 4, 30)], + 8, + ), + ); + expect([0, 1, 2].map((index) => reordered.keyAt(index))).toEqual([c, a, b]); + expect(reordered.getHeight(0)).toBe(55); + expect(reordered.getTotalHeight()).toBe(105); + + // Identity permutation is a no-op returning the same index. + expect( + base.reorder( + denseSource( + [denseEntry(a, 2, 20), denseEntry(b, 4, 30), denseEntry(c, 7, 40)], + 8, + ), + ), + ).toBe(base); + + // A slot that matches no existing row (missing or duplicated) throws. + expect(() => + base.reorder( + denseSource( + [denseEntry(c, 7, 40), denseEntry(a, 2, 20), denseEntry(b, 3, 30)], + 8, + ), + ), + ).toThrow(/does not match an existing row/i); + expect(() => + base.reorder( + denseSource( + [denseEntry(c, 7, 40), denseEntry(a, 2, 20), denseEntry(b, 7, 30)], + 8, + ), + ), + ).toThrow(/does not match an existing row/i); + // The permutation contract still demands equal row counts. + expect(() => base.reorder(denseSource([denseEntry(a, 2, 20)], 8))).toThrow( + RangeError, + ); + // The result is a DENSE generation: its guards still demand dense keys. + expectReplacementLifecycleError( + () => + reordered.apply([ + { kind: "insert", ref: data("d"), index: 3, estimatedHeight: 10 }, + ]), + "failed", + ); + }); + + test("dense refilter and reorder validate dense keys like the builder ingest", () => { + const a = data("a"); + const b = data("b"); + const base = rebuild( + createIndex([]), + [denseEntry(a, 0, 20), denseEntry(b, 1, 30)], + 4, + ); + // Missing key: lifecycle error (controller falls back to a replacement). + expectReplacementLifecycleError( + () => base.refilter(denseSource([denseEntry(a, 0, 20), entry(b, 30)], 4)), + "failed", + ); + expectReplacementLifecycleError( + () => base.reorder(denseSource([denseEntry(b, 1, 30), entry(a, 20)], 4)), + "failed", + ); + // Out-of-range and malformed keys: the same lifecycle error class. + for (const badKey of [4, -1, 1.5, Number.NaN]) { + expectReplacementLifecycleError( + () => + base.refilter( + denseSource([denseEntry(a, 0, 20), denseEntry(b, badKey, 30)], 4), + ), + "failed", + ); + expectReplacementLifecycleError( + () => + base.reorder( + denseSource([denseEntry(b, badKey, 30), denseEntry(a, 0, 20)], 4), + ), + "failed", + ); + } + // A duplicated slot in a refilter's new order throws like the builder. + expect(() => + base.refilter( + denseSource([denseEntry(a, 0, 20), denseEntry(data("c"), 0, 10)], 4), + ), + ).toThrow(/duplicate dense row-height slot/i); + }); + + test("retainMeasurement rejects malformed dense keys before reading the bitset", () => { + const a = data("a"); + const base = rebuild(createIndex([], 30, 4), [denseEntry(a, 1, 20)], 8); + const gone = data("gone"); + // A negative or fractional key must fail loud: 1.5's `&31` truncation + // would otherwise silently read a DIFFERENT row's bit. + for (const badKey of [-1, 1.5, Number.NaN, Number.MAX_SAFE_INTEGER + 2]) { + expect(() => base.retainMeasurement(gone, 73, badKey)).toThrow( + /non-negative safe integer/i, + ); + } + // The well-formed absent-slot retain still works after the guard. + expect(base.retainMeasurement(gone, 73, 5).hasMeasurement(gone)).toBe(true); + }); + + test("apply rejects an operation whose denseKey drifted from the entry's slot", () => { + const a = data("a"); + const b = data("b"); + const base = rebuild( + createIndex([]), + [denseEntry(a, 0, 20), denseEntry(b, 2, 30)], + 8, + ); + expectReplacementLifecycleError( + () => + base.apply([{ kind: "remove", ref: b, previousIndex: 1, denseKey: 3 }]), + "failed", + ); + expectReplacementLifecycleError( + () => + base.apply([ + { kind: "move", ref: a, previousIndex: 0, index: 1, denseKey: 2 }, + ]), + "failed", + ); + expectReplacementLifecycleError( + () => + base.apply([ + { + kind: "update", + ref: a, + index: 0, + estimatedHeight: 25, + denseKey: 1, + }, + ]), + "failed", + ); + // The matching key still works on all three variants. + const applied = base.apply([ + { kind: "update", ref: a, index: 0, estimatedHeight: 25, denseKey: 0 }, + { kind: "move", ref: a, previousIndex: 0, index: 1, denseKey: 0 }, + { kind: "remove", ref: b, previousIndex: 0, denseKey: 2 }, + ]); + expect(applied.rowCount).toBe(1); + expect(applied.getHeight(0)).toBe(25); + }); + + test("a source without denseCapacity runs the string lane, dense stamps and all", () => { + const rows = [entry(data("a"), 20), entry(data("b")), entry(data("c"), 40)]; + const stampedRows = [ + denseEntry(data("a"), 0, 20), + denseEntry(data("b"), 1), + denseEntry(data("c"), 2, 40), + ]; + const plain = rebuild(createIndex([]), rows, undefined); + const stamped = rebuild(createIndex([]), stampedRows, undefined); + for (const index of [plain, stamped]) { + expect(index.rowCount).toBe(3); + expect(index.getTotalHeight()).toBe(90); + } + expect(getRowHeightIndexDiagnosticsForTesting(stamped)).toEqual( + getRowHeightIndexDiagnosticsForTesting(plain), + ); + // The string lane still guards duplicates through the HAMT and still + // accepts refilter — nothing dense leaked in. + const refiltered = stamped.refilter( + denseSource([entry(data("b"))], undefined), + ); + expect(refiltered.rowCount).toBe(1); + expect(() => + stamped.apply([{ kind: "insert", ref: data("a"), index: 0 }]), + ).toThrow(/duplicate/i); + }); + + test("a dense rebuild with reassigned slots is not a no-op and guards by the NEW slots", () => { + const a = data("a"); + const b = data("b"); + const first = rebuild( + createIndex([]), + [denseEntry(a, 0, 20), denseEntry(b, 1, 30)], + 8, + ); + // Identical rows AND identical slots: a no-op returns the base generation. + const same = rebuild( + first, + [denseEntry(a, 0, 20), denseEntry(b, 1, 30)], + 8, + ); + expect(same).toBe(first); + // The same identities on NEW slots must build a new generation whose + // bitset answers for the new slots, never the stale ones. + const moved = rebuild( + first, + [denseEntry(a, 4, 20), denseEntry(b, 5, 30)], + 8, + ); + expect(moved).not.toBe(first); + const inserted = moved.apply([ + { + kind: "insert", + ref: data("c"), + index: 2, + estimatedHeight: 10, + denseKey: 0, + }, + ]); + expect(inserted.rowCount).toBe(3); + expect(() => + moved.apply([ + { + kind: "insert", + ref: data("c"), + index: 2, + estimatedHeight: 10, + denseKey: 4, + }, + ]), + ).toThrow(/duplicate/i); + }); + + test("a no-op replacement never crosses lanes: a dense source over a string base rebuilds", () => { + const rows = [denseEntry(data("a"), 0, 20), denseEntry(data("b"), 1, 30)]; + const stringBase = rebuild(createIndex([]), rows, undefined); + // Identical rows, but the source declares a capacity: the result must be + // a DENSE generation (its guards demand dense keys), not the string base. + const dense = rebuild(stringBase, rows, 8); + expect(dense).not.toBe(stringBase); + expectReplacementLifecycleError( + () => + dense.apply([ + { kind: "insert", ref: data("c"), index: 2, estimatedHeight: 10 }, + ]), + "failed", + ); + // And the reverse: a capacity-less source over a dense base returns to + // the string lane even when the rows are identical. + const stringAgain = rebuild(dense, rows, undefined); + expect(stringAgain).not.toBe(dense); + const applied = stringAgain.apply([ + { kind: "insert", ref: data("c"), index: 2, estimatedHeight: 10 }, + ]); + expect(applied.rowCount).toBe(3); + }); +}); + +describe("dense refilter and reorder (Amendment I, Task 3)", () => { + const denseEntry = ( + key: Key, + denseKey: number | undefined, + estimatedHeight?: number, + ): RowHeightEntry => ({ key, estimatedHeight, denseKey }); + + function sourceOf( + rows: readonly RowHeightEntry[], + denseCapacity?: number, + ): RowHeightReplacementSource { + return { + rowCount: rows.length, + denseCapacity, + entryAt: (index) => rows[index]!, + }; + } + + function rebuild( + base: RowHeightIndex, + rows: readonly RowHeightEntry[], + denseCapacity?: number, + ): RowHeightIndex { + const builder = base.beginReplacement(sourceOf(rows, denseCapacity)); + while (!builder.done) builder.advance({ maxUnits: 256 }); + return builder.finish(); + } + + /** Every rank's offset, height, and key, plus the total: full geometry. */ + function rankTable(index: RowHeightIndex) { + return { + rowCount: index.rowCount, + total: index.getTotalHeight(), + keys: Array.from({ length: index.rowCount }, (_, rank) => + index.keyAt(rank), + ), + offsets: Array.from({ length: index.rowCount + 1 }, (_, rank) => + index.getOffsetForIndex(rank), + ), + heights: Array.from({ length: index.rowCount }, (_, rank) => + index.getHeight(rank), + ), + }; + } + + /** The lane-INDEPENDENT observables the equivalence oracle compares. */ + function laneObservables(index: RowHeightIndex) { + const diagnostics = getRowHeightIndexDiagnosticsForTesting(index); + return { + refilterEntriesReused: diagnostics.refilterEntriesReused, + refilterEntriesInserted: diagnostics.refilterEntriesInserted, + refilterEntriesRetired: diagnostics.refilterEntriesRetired, + reorderEntriesReused: diagnostics.reorderEntriesReused, + tombstoneCount: diagnostics.tombstoneCount, + measurementCacheCount: diagnostics.measurementCacheCount, + visibleMeasurementCount: diagnostics.visibleMeasurementCount, + }; + } + + /** mulberry32 — a tiny deterministic PRNG for the oracle script. */ + function prng(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + } + + test("lane-equivalence oracle: a randomized flip script produces identical observables", () => { + // NOT the authority on leaver TICKET ORDER: the oracle's observables + // compare counts and heights, and a randomized script only occasionally + // drives tombstones through cap eviction, so a slot-ordered leaver pass + // could survive this test. The dedicated old-sequence ticket pin below + // ("dense leavers take tombstone tickets in OLD-SEQUENCE order") is the + // authority — never delete that pin in this oracle's favor. + const count = 200; + const random = prng(0xd15ea5e); + // Slots are a shuffled permutation of 0..count-1 so slot order and + // sequence order genuinely disagree (a slot-ordered bug cannot hide). + const slots = Array.from({ length: count }, (_, index) => index); + for (let index = count - 1; index > 0; index -= 1) { + const other = Math.floor(random() * (index + 1)); + [slots[index], slots[other]] = [slots[other]!, slots[index]!]; + } + const keys = Array.from({ length: count }, (_, index) => + data(`row-${index}`), + ); + const estimates = Array.from( + { length: count }, + (_, index) => 16 + (index % 9) * 4, + ); + const rowOf = (index: number): RowHeightEntry => + denseEntry(keys[index]!, slots[index]!, estimates[index]!); + + const cap = 8; + let stringLane: RowHeightIndex = createIndex([], 30, cap); + let denseLane: RowHeightIndex = createIndex([], 30, cap); + const all = Array.from({ length: count }, (_, index) => index); + stringLane = rebuild(stringLane, all.map(rowOf), undefined); + denseLane = rebuild(denseLane, all.map(rowOf), count); + + let visible = [...all]; + const compare = () => { + expect(rankTable(denseLane)).toEqual(rankTable(stringLane)); + expect(laneObservables(denseLane)).toEqual(laneObservables(stringLane)); + }; + compare(); + + for (let step = 0; step < 30; step += 1) { + const kind = Math.floor(random() * 4); + if (kind === 0) { + // Narrowing refilter: keep each visible row with p = 0.6. + const survivors = visible.filter(() => random() < 0.6); + visible = survivors; + const rows = visible.map(rowOf); + stringLane = stringLane.refilter(sourceOf(rows)); + denseLane = denseLane.refilter(sourceOf(rows, count)); + } else if (kind === 1) { + // Widening refilter: splice each hidden row back in with p = 0.4, + // at a random position, so entrants interleave with survivors. + const visibleSet = new Set(visible); + const next = [...visible]; + for (const index of all) { + if (visibleSet.has(index)) continue; + if (random() >= 0.4) continue; + next.splice(Math.floor(random() * (next.length + 1)), 0, index); + } + visible = next; + const rows = visible.map(rowOf); + stringLane = stringLane.refilter(sourceOf(rows)); + denseLane = denseLane.refilter(sourceOf(rows, count)); + } else if (kind === 2 && visible.length > 1) { + // Reorder: shuffle the visible order (pure permutation). + const next = [...visible]; + for (let index = next.length - 1; index > 0; index -= 1) { + const other = Math.floor(random() * (index + 1)); + [next[index], next[other]] = [next[other]!, next[index]!]; + } + visible = next; + const rows = visible.map(rowOf); + stringLane = stringLane.reorder(sourceOf(rows)); + denseLane = denseLane.reorder(sourceOf(rows, count)); + } else if (visible.length > 0) { + // Measure a random visible row to a height its estimate cannot guess. + const position = Math.floor(random() * visible.length); + const height = 20 + Math.round(random() * 320) / 4; + const key = keys[visible[position]!]!; + stringLane = stringLane.measure(position, key, height); + denseLane = denseLane.measure(position, key, height); + } + compare(); + } + // The script must have actually exercised retention pressure. + expect( + getRowHeightIndexDiagnosticsForTesting(denseLane).measurementCacheCount, + ).toBeGreaterThan(0); + }); + + test("dense leavers take tombstone tickets in OLD-SEQUENCE order, pinned via cap eviction", () => { + // THE authority on leaver ticket order. The lane-equivalence oracle + // above does not reliably cover it (its observables are counts and + // heights, and cap eviction rarely engages under its random script), so + // this pin must stay even though the oracle looks like it subsumes it. + const count = 10; + const cap = 2; + // Slots run OPPOSITE to sequence order: position p holds slot count-1-p. + // A leaver pass that iterated by slot index would assign tickets in + // exactly the reversed order, so cap eviction would keep the WRONG rows. + const keys = Array.from({ length: count }, (_, index) => + data(`t-${index}`), + ); + const rowOf = (index: number): RowHeightEntry => + denseEntry(keys[index]!, count - 1 - index, 20 + index); + const all = Array.from({ length: count }, (_, index) => index); + + let stringLane: RowHeightIndex = rebuild( + createIndex([], 30, cap), + all.map(rowOf), + undefined, + ); + let denseLane: RowHeightIndex = rebuild( + createIndex([], 30, cap), + all.map(rowOf), + count, + ); + // Measure positions 0..4; the narrowing refilter retires all five into a + // cap of two, so only the two NEWEST tickets survive — and tickets are + // assigned in old-sequence order, so the survivors are rows 3 and 4. + for (let position = 0; position < 5; position += 1) { + const height = 60 + position; + stringLane = stringLane.measure(position, keys[position]!, height); + denseLane = denseLane.measure(position, keys[position]!, height); + } + const survivors = all.slice(5).map(rowOf); + const narrowedString = stringLane.refilter(sourceOf(survivors)); + const narrowedDense = denseLane.refilter(sourceOf(survivors, count)); + + // Direct pin: rows 3 and 4 keep their measurements, rows 0..2 lost them. + for (const lane of [narrowedString, narrowedDense]) { + expect(lane.hasMeasurement(keys[3]!)).toBe(true); + expect(lane.hasMeasurement(keys[4]!)).toBe(true); + expect(lane.hasMeasurement(keys[0]!)).toBe(false); + expect(lane.hasMeasurement(keys[1]!)).toBe(false); + expect(lane.hasMeasurement(keys[2]!)).toBe(false); + } + // And the behavioral twin: widening everything back must restore the + // identical height table on both lanes (63 and 64 return, the rest + // re-enter at their estimates). + const restoredString = narrowedString.refilter(sourceOf(all.map(rowOf))); + const restoredDense = narrowedDense.refilter( + sourceOf(all.map(rowOf), count), + ); + expect(rankTable(restoredDense)).toEqual(rankTable(restoredString)); + expect(restoredDense.getHeight(3)).toBe(63); + expect(restoredDense.getHeight(4)).toBe(64); + expect(restoredDense.getHeight(0)).toBe(20); + }); + + test("slot reuse never leaks a measurement across identities (Amendment I §3)", () => { + const r0 = data("r0"); + const r1 = data("r1"); + const x = data("x"); + const y = data("y"); + const capacity = 8; + const base = rebuild( + createIndex([], 30, 4), + [denseEntry(r0, 0, 20), denseEntry(r1, 1, 30), denseEntry(x, 3, 25)], + capacity, + ); + + // Measure X, then refilter X out: X is tombstoned, slot 3 still X's. + const measured = base.measure(2, x, 77); + const withoutX = measured.refilter( + sourceOf([denseEntry(r0, 0, 20), denseEntry(r1, 1, 30)], capacity), + ); + expect(withoutX.hasMeasurement(x)).toBe(true); + + // Permanent removal + slot reuse: a FULL dense replacement presents a + // NEW identity Y on X's old denseKey. Y must ingest at its estimate — + // the retained 77 belongs to X's identity, not to slot 3. + const reused = rebuild( + withoutX, + [denseEntry(r0, 0, 20), denseEntry(r1, 1, 30), denseEntry(y, 3, 25)], + capacity, + ); + expect(reused.getHeight(2)).toBe(25); + expect(reused.hasMeasurement(y)).toBe(false); + expect(reused.hasMeasurement(x)).toBe(true); + + // Refilter Y out and back in: the dense entrant path must resolve the + // measurement lookup by IDENTITY, so Y still ingests at estimate. + const withoutY = reused.refilter( + sourceOf([denseEntry(r0, 0, 20), denseEntry(r1, 1, 30)], capacity), + ); + const withY = withoutY.refilter( + sourceOf( + [denseEntry(r0, 0, 20), denseEntry(r1, 1, 30), denseEntry(y, 3, 25)], + capacity, + ), + ); + expect(withY.getHeight(2)).toBe(25); + expect(withY.hasMeasurement(y)).toBe(false); + + // X's measurement returns ONLY for X's identity: presenting X again on a + // fresh slot restores 77. + const withXBack = withY.refilter( + sourceOf( + [ + denseEntry(r0, 0, 20), + denseEntry(r1, 1, 30), + denseEntry(y, 3, 25), + denseEntry(x, 5, 25), + ], + capacity, + ), + ); + expect(withXBack.getHeight(3)).toBe(77); + expect(withXBack.getHeight(2)).toBe(25); + }); +}); diff --git a/packages/layout-core/src/dense-membership.ts b/packages/layout-core/src/dense-membership.ts new file mode 100644 index 000000000..a1d00121b --- /dev/null +++ b/packages/layout-core/src/dense-membership.ts @@ -0,0 +1,41 @@ +/** + * Membership bitset: one bit per dense key (row-model SLOT). Duplicated from + * `@pretable-internal/row-model`'s `membership-bitset.ts` BY DESIGN — + * layout-core stays dependency-free (it cannot import row-model), so this + * ~40-line primitive is hand-copied rather than shared. Keep the two files + * in sync by hand; if one changes shape, mirror the change in the other. + * + * See `docs/superpowers/specs/2026-08-24-dense-handle-amendment-i-layout-seam.md` + * for the dense lane this backs. + */ + +export type DenseMembership = Uint32Array; + +export function createDenseMembership(capacity: number): DenseMembership { + return new Uint32Array((capacity + 31) >>> 5); +} + +/** Clone, growing to `capacity` when it exceeds the source's words. */ +export function cloneDenseMembership( + bits: DenseMembership, + capacity: number, +): DenseMembership { + const words = Math.max(bits.length, (capacity + 31) >>> 5); + const next = new Uint32Array(words); + next.set(bits); + return next; +} + +export function setDenseBit(bits: DenseMembership, slot: number): void { + bits[slot >>> 5]! |= 1 << (slot & 31); +} + +export function clearDenseBit(bits: DenseMembership, slot: number): void { + bits[slot >>> 5]! &= ~(1 << (slot & 31)); +} + +/** Out-of-range slots read as false. */ +export function testDenseBit(bits: DenseMembership, slot: number): boolean { + const word = bits[slot >>> 5]; + return word === undefined ? false : ((word >>> (slot & 31)) & 1) === 1; +} diff --git a/packages/layout-core/src/row-height-index.ts b/packages/layout-core/src/row-height-index.ts index 41222521f..901e150b5 100644 --- a/packages/layout-core/src/row-height-index.ts +++ b/packages/layout-core/src/row-height-index.ts @@ -1,3 +1,11 @@ +import { + clearDenseBit, + cloneDenseMembership, + createDenseMembership, + setDenseBit, + testDenseBit, + type DenseMembership, +} from "./dense-membership"; import type { CreateRowHeightIndexOptions, RowHeightAnchor, @@ -20,6 +28,9 @@ interface Work { sortComparisons: number; reorderEntriesReused: number; reorderEntriesRemeasured: number; + refilterEntriesReused: number; + refilterEntriesInserted: number; + refilterEntriesRetired: number; } function createWork(entriesVisited = 0): Work { @@ -33,6 +44,9 @@ function createWork(entriesVisited = 0): Work { sortComparisons: 0, reorderEntriesReused: 0, reorderEntriesRemeasured: 0, + refilterEntriesReused: 0, + refilterEntriesInserted: 0, + refilterEntriesRetired: 0, }; } @@ -42,6 +56,11 @@ interface HeightValue { readonly estimatedHeight: number | undefined; readonly height: number; readonly measured: boolean; + /** + * The row's model slot as stamped by the ingesting input (Amendment I §1). + * Defined on every entry of a dense generation; inert on the string lane. + */ + readonly denseKey: number | undefined; } interface SequenceNode { @@ -162,6 +181,16 @@ export interface RowHeightIndexDiagnostics { * asserting zero by fiat. */ readonly reorderEntriesRemeasured: number; + /** Surviving entries relinked as-is by `refilter` — measurements ride. */ + readonly refilterEntriesReused: number; + /** New rows `refilter` ingested with the estimate-or-default height rule. */ + readonly refilterEntriesInserted: number; + /** + * Rows `refilter` removed from the visible set. Measured leavers keep their + * measurement as tombstones (see `tombstoneCount`); unmeasured leavers had + * nothing to retain. + */ + readonly refilterEntriesRetired: number; readonly visibleMeasurementCount: number; readonly tombstoneCount: number; readonly measurementCacheCount: number; @@ -209,6 +238,8 @@ interface ReplacementBase { readonly getKey: (key: TKey) => string | number; readonly root: SequenceNode | null; readonly visibleKeys: HashNode | null; + /** The base generation's dense capacity, for lane-aware no-op detection. */ + readonly denseCapacity: number | undefined; readonly measurements: HashNode | null; readonly tombstones: HashNode | null; readonly tombstoneOrder: KeyMapNode | null; @@ -1072,6 +1103,17 @@ class PersistentRowHeightIndex implements RowHeightIndex { readonly #getKey: (key: TKey) => string | number; readonly #root: SequenceNode | null; readonly #visibleKeys: HashNode | null; + /** + * INVARIANT (Amendment I §1): `#visibleSlots !== undefined` ⇔ this + * generation is DENSE ⇔ every sequence entry carries a `denseKey` less + * than `#denseCapacity`. A dense generation does NOT maintain + * `#visibleKeys` (it stays `null`); a string generation never allocates + * `#visibleSlots`. Measurements, tombstones, and retention order stay + * string-identity-keyed in BOTH lanes — slot reuse must never touch + * retention (the amendment's §3 trap). + */ + readonly #denseCapacity: number | undefined; + readonly #visibleSlots: DenseMembership | undefined; readonly #measurements: HashNode | null; readonly #tombstones: HashNode | null; readonly #tombstoneOrder: KeyMapNode | null; @@ -1084,6 +1126,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { readonly getKey: (key: TKey) => string | number; readonly root: SequenceNode | null; readonly visibleKeys: HashNode | null; + readonly denseCapacity: number | undefined; + readonly visibleSlots: DenseMembership | undefined; readonly measurements: HashNode | null; readonly tombstones: HashNode | null; readonly tombstoneOrder: KeyMapNode | null; @@ -1095,6 +1139,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { this.#getKey = options.getKey; this.#root = options.root; this.#visibleKeys = options.visibleKeys; + this.#denseCapacity = options.denseCapacity; + this.#visibleSlots = options.visibleSlots; this.#measurements = options.measurements; this.#tombstones = options.tombstones; this.#tombstoneOrder = options.tombstoneOrder; @@ -1110,6 +1156,9 @@ class PersistentRowHeightIndex implements RowHeightIndex { previousEntriesScanned: options.work.previousEntriesScanned, sortComparisons: options.work.sortComparisons, reorderEntriesReused: options.work.reorderEntriesReused, + refilterEntriesReused: options.work.refilterEntriesReused, + refilterEntriesInserted: options.work.refilterEntriesInserted, + refilterEntriesRetired: options.work.refilterEntriesRetired, reorderEntriesRemeasured: options.work.reorderEntriesRemeasured, visibleMeasurementCount: hashCount(options.measurements) - hashCount(options.tombstones), @@ -1243,6 +1292,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { return this.#next( root, this.#visibleKeys, + this.#denseCapacity, + this.#visibleSlots, measurements, this.#tombstones, this.#tombstoneOrder, @@ -1251,11 +1302,38 @@ class PersistentRowHeightIndex implements RowHeightIndex { ); } - retainMeasurement(ref: TKey, height: number): RowHeightIndex { + retainMeasurement( + ref: TKey, + height: number, + denseKey?: number, + ): RowHeightIndex { const normalized = normalizeHeight(height, "Measured row height"); const identity = this.#identity(ref); const work = createWork(1); - if (hashGet(this.#visibleKeys, identity, work) !== undefined) { + if (this.#visibleSlots !== undefined) { + // Dense generation: `#visibleKeys` is null, so the string-lane guard + // below would be vacuous — visibility is answered by the slot bitset. + if (denseKey === undefined) { + throw new RowHeightReplacementLifecycleError( + "failed", + "A dense row-height index requires dense-keyed operations; " + + "fall back to a full replacement.", + ); + } + if (!Number.isSafeInteger(denseKey) || denseKey < 0) { + // Mirror the insert arm: a negative or fractional key would silently + // mis-read here (1.5's `&31` truncation reads a DIFFERENT row's bit). + throw new RangeError( + `Dense row-height key ${denseKey} must be a non-negative ` + + "safe integer.", + ); + } + if (testDenseBit(this.#visibleSlots, denseKey)) { + throw new RangeError( + "Cannot retain an absent measurement for a visible row.", + ); + } + } else if (hashGet(this.#visibleKeys, identity, work) !== undefined) { throw new RangeError( "Cannot retain an absent measurement for a visible row.", ); @@ -1295,6 +1373,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { return this.#next( this.#root, this.#visibleKeys, + this.#denseCapacity, + this.#visibleSlots, measurements, tombstones, tombstoneOrder, @@ -1307,18 +1387,44 @@ class PersistentRowHeightIndex implements RowHeightIndex { if (operations.length === 0) return this; let root = this.#root; let visibleKeys = this.#visibleKeys; + let denseCapacity = this.#denseCapacity; + let visibleSlots = this.#visibleSlots; + // The slot bitset is MUTABLE, so the persistent contract demands a clone + // before the first membership write of this call; reference equality with + // `this.#visibleSlots` doubles as the no-op signal below. + let slotsShared = true; let measurements = this.#measurements; let tombstones = this.#tombstones; let tombstoneOrder = this.#tombstoneOrder; - let nextTicket = this.#nextTicket; const work = createWork(); + let nextTicket = this.#nextTicket; for (const operation of operations) { work.entriesVisited += 1; + if (visibleSlots !== undefined && operation.denseKey === undefined) { + throw new RowHeightReplacementLifecycleError( + "failed", + "A dense row-height index requires dense-keyed operations; " + + "fall back to a full replacement.", + ); + } if (operation.kind === "insert") { assertInsertIndex(operation.index, nodeCount(root)); const identity = this.#identity(operation.ref); - if (hashGet(visibleKeys, identity, work) !== undefined) { + if (visibleSlots !== undefined) { + const denseKey = operation.denseKey!; + if (!Number.isSafeInteger(denseKey) || denseKey < 0) { + throw new RangeError( + `Dense row-height key ${denseKey} must be a non-negative ` + + "safe integer.", + ); + } + if (testDenseBit(visibleSlots, denseKey)) { + throw new Error( + `Duplicate dense row-height slot ${denseKey}: ${identity}`, + ); + } + } else if (hashGet(visibleKeys, identity, work) !== undefined) { throw new Error(`Duplicate stable row-height key: ${identity}`); } const estimatedHeight = this.#estimated(operation.estimatedHeight); @@ -1333,10 +1439,24 @@ class PersistentRowHeightIndex implements RowHeightIndex { estimatedHeight: operation.estimatedHeight, height: measuredHeight ?? estimatedHeight, measured: measuredHeight !== undefined, + denseKey: operation.denseKey, }, work, ); - visibleKeys = hashSet(visibleKeys, identity, true, work); + if (visibleSlots !== undefined) { + const denseKey = operation.denseKey!; + const neededCapacity = Math.max(denseCapacity!, denseKey + 1); + if (slotsShared) { + visibleSlots = cloneDenseMembership(visibleSlots, neededCapacity); + slotsShared = false; + } else if (denseKey >>> 5 >= visibleSlots.length) { + visibleSlots = cloneDenseMembership(visibleSlots, neededCapacity); + } + setDenseBit(visibleSlots, denseKey); + denseCapacity = neededCapacity; + } else { + visibleKeys = hashSet(visibleKeys, identity, true, work); + } if (retainedTicket !== undefined) { tombstones = hashDelete(tombstones, identity, work); tombstoneOrder = mapDelete( @@ -1356,10 +1476,36 @@ class PersistentRowHeightIndex implements RowHeightIndex { const identity = this.#identity(operation.ref); const current = sequenceAt(root, sourceIndex)!; this.#assertIdentity(current, identity); + // Dense slot-drift guard for every variant that resolves a current + // entry (remove, move, update — insert has no current entry; its slot + // is range-validated and duplicate-checked in its own arm instead): a + // caller whose slot view drifted from the entry's stamped slot must + // fail loud rather than silently clearing or riding the wrong bit. + if ( + visibleSlots !== undefined && + operation.denseKey !== current.denseKey + ) { + throw new RowHeightReplacementLifecycleError( + "failed", + `Dense slot drift: operation denseKey ${operation.denseKey} does ` + + `not match the entry's stamped slot ${current.denseKey} for ` + + `${identity}; fall back to a full replacement.`, + ); + } if (operation.kind === "remove") { root = removeSequence(root!, sourceIndex, work).root; - visibleKeys = hashDelete(visibleKeys, identity, work); + if (visibleSlots !== undefined) { + // Clear by the ENTRY's stamped slot — the one that set the bit — + // which the identity assertion above ties to the caller's ref. + if (slotsShared) { + visibleSlots = cloneDenseMembership(visibleSlots, denseCapacity!); + slotsShared = false; + } + clearDenseBit(visibleSlots, current.denseKey!); + } else { + visibleKeys = hashDelete(visibleKeys, identity, work); + } if (current.measured) { if (this.#maxRetainedMeasurements === 0) { measurements = hashDelete(measurements, identity, work); @@ -1427,6 +1573,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { if ( root === this.#root && visibleKeys === this.#visibleKeys && + visibleSlots === this.#visibleSlots && + denseCapacity === this.#denseCapacity && measurements === this.#measurements && tombstones === this.#tombstones && tombstoneOrder === this.#tombstoneOrder && @@ -1437,6 +1585,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { return this.#next( root, visibleKeys, + denseCapacity, + visibleSlots, measurements, tombstones, tombstoneOrder, @@ -1445,6 +1595,11 @@ class PersistentRowHeightIndex implements RowHeightIndex { ); } + /** + * Builds a source with NO `denseCapacity`, so on a dense generation this + * is a deliberate LANE EXIT: the result is a string-lane generation (the + * full-replacement path is where the lane is legally re-decided). + */ replace(rows: readonly RowHeightEntry[]): RowHeightIndex { const builder = this.beginReplacement({ rowCount: rows.length, @@ -1484,6 +1639,14 @@ class PersistentRowHeightIndex implements RowHeightIndex { } if (rowCount === 0) return this; const boundEntryAt = entryAt.bind(source); + if (this.#visibleSlots !== undefined) { + // Dense generation (Amendment I): resolve the permutation by slot. + // The string-lane body below stays INLINE and byte-identical to its + // pre-dense form deliberately (the Task 3 pin: the string lane must be + // provably untouched by diff alone): do not unify the lanes or hoist + // the shared walks into helpers. + return this.#reorderDense(boundEntryAt, rowCount); + } const work = createWork(); // One in-order pass over the current sequence: the by-identity lookup @@ -1534,6 +1697,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { return this.#next( root, this.#visibleKeys, + this.#denseCapacity, + this.#visibleSlots, this.#measurements, this.#tombstones, this.#tombstoneOrder, @@ -1542,6 +1707,458 @@ class PersistentRowHeightIndex implements RowHeightIndex { ); } + /** + * Dense-lane reorder (Amendment I): the permutation resolves each source + * row by its `denseKey` (model slot) against a slot-indexed array — ZERO + * identity strings on the hot path; identities are computed only to name + * a row in an error. Key validation mirrors the builder's + * `#ingestDenseKey` cases (missing / malformed / out-of-range → lifecycle + * error), which the controller's fallback-on-throw contract routes to a + * full replacement that re-decides the lane. Membership is unchanged by + * construction — reorder permutes the existing rows — so the generation + * carries `#visibleSlots` and `#denseCapacity` forward untouched. + * + * TRUST BOUNDARY: a resolved row's `row.key` is trusted to match the + * slot's stored identity and deliberately never re-verified — that + * verification IS the per-row string cost this lane exists to delete. + * `apply` guards slot drift instead, because its operations are k-sized. + */ + #reorderDense( + boundEntryAt: (index: number) => RowHeightEntry, + rowCount: number, + ): RowHeightIndex { + const capacity = this.#denseCapacity!; + const work = createWork(); + + // One in-order pass over the current sequence: the by-slot lookup table + // for the walk below, and the old order for no-op detection. + const previousValues: HeightValue[] = []; + const unconsumedBySlot: (HeightValue | undefined)[] = new Array< + HeightValue | undefined + >(capacity); + { + const stack: SequenceNode[] = []; + let cursor = this.#root; + while (cursor !== null || stack.length > 0) { + while (cursor !== null) { + stack.push(cursor); + cursor = cursor.left; + } + const node = stack.pop()!; + previousValues.push(node.value); + unconsumedBySlot[node.value.denseKey!] = node.value; + work.previousEntriesScanned += 1; + cursor = node.right; + } + } + + // Walk the new order, relinking each EXISTING entry verbatim. Estimates + // and measurements ride along untouched inside the reused entry objects, + // so the source's `estimatedHeight`s are deliberately ignored. + const values: HeightValue[] = new Array>(rowCount); + let unchanged = true; + for (let index = 0; index < rowCount; index += 1) { + const row = boundEntryAt(index); + work.entriesVisited += 1; + const denseKey = row.denseKey; + if (denseKey === undefined) { + throw new RowHeightReplacementLifecycleError( + "failed", + "A dense row-height index requires a denseKey on every source " + + `row (${this.#identity(row.key)} carries none); fall back to ` + + "a full replacement.", + ); + } + if ( + !Number.isSafeInteger(denseKey) || + denseKey < 0 || + denseKey >= capacity + ) { + throw new RowHeightReplacementLifecycleError( + "failed", + `Dense key ${denseKey} for row ${this.#identity(row.key)} is ` + + `outside the dense generation's capacity ${capacity}.`, + ); + } + const value = unconsumedBySlot[denseKey]; + if (value === undefined) { + throw new Error( + `Reorder slot does not match an existing row (missing, or ` + + `duplicated in the new order): ${denseKey}`, + ); + } + unconsumedBySlot[denseKey] = undefined; + values[index] = value; + work.reorderEntriesReused += 1; + if (value !== previousValues[index]) unchanged = false; + } + if (unchanged) return this; + + const root = buildBalancedSequence(values, 0, rowCount, work); + return this.#next( + root, + this.#visibleKeys, + this.#denseCapacity, + this.#visibleSlots, + this.#measurements, + this.#tombstones, + this.#tombstoneOrder, + this.#nextTicket, + work, + ); + } + + /** + * Synchronous BY DESIGN (Amendment G, G3b): a filter-only commit changes + * MEMBERSHIP while surviving rows keep their relative order and their + * already-measured heights, so the cooperative replacement's per-row + * re-ingest and its sliced interval — the window the blank-viewport defect + * lived in — are pure overhead here. Measured in Node against dist: + * a 50k→12.5k narrowing runs 15–26ms (the amendment's 20–25ms ceiling); + * a 12.5k→50k widening runs ~57–80ms, dominated by the fresh ingest of + * 37.5k entrants — above the cost model's 30–35ms projection but still one + * synchronous pass with no replacement interval, versus the 45–63ms sliced + * cooperative flush (`reingest-composition.md`). Retention semantics deliberately mirror the + * cooperative path observable-for-observable: entrants reuse a retained + * (tombstoned) measurement when one exists, measured leavers tombstone in + * old-sequence ticket order (or drop outright at cap 0), unmeasured + * leavers vanish, and cap pressure evicts oldest-first. The caller + * (renderer-dom's row-layout controller) falls back to a full replacement + * on ANY throw. + */ + refilter(source: RowHeightReplacementSource): RowHeightIndex { + const rowCount = source.rowCount; + if (!Number.isSafeInteger(rowCount) || rowCount < 0) { + throw new RangeError( + "Refilter source rowCount must be a non-negative safe integer.", + ); + } + const entryAt = source.entryAt; + if (typeof entryAt !== "function") { + throw new TypeError("Refilter source entryAt must be a function."); + } + const boundEntryAt = entryAt.bind(source); + if (this.#visibleSlots !== undefined) { + // Dense generation (Amendment I): resolve survivors by slot. + // The string-lane body below stays INLINE and byte-identical to its + // pre-dense form deliberately (the Task 3 pin: the string lane must be + // provably untouched by diff alone): do not unify the lanes or hoist + // the shared walks into helpers. + return this.#refilterDense(boundEntryAt, rowCount); + } + const work = createWork(); + + // One in-order pass over the current sequence: the by-identity lookup + // table for the walk below, and the old order for no-op detection. + const previousValues: HeightValue[] = []; + const unconsumed = new Map>(); + { + const stack: SequenceNode[] = []; + let cursor = this.#root; + while (cursor !== null || stack.length > 0) { + while (cursor !== null) { + stack.push(cursor); + cursor = cursor.left; + } + const node = stack.pop()!; + previousValues.push(node.value); + unconsumed.set(node.value.identity, node.value); + work.previousEntriesScanned += 1; + cursor = node.right; + } + } + + // Walk the new order: survivors relink verbatim (their measurements and + // estimates ride, exactly as in `reorder`), entrants ingest fresh under + // the cooperative path's rule — measured-lookup against the FULL cache, + // so a returning tombstoned key gets its retained measurement back and + // sheds its tombstone. + const values: HeightValue[] = new Array>(rowCount); + const seen = new Set(); + let visibleKeys: HashNode | null = null; + let measurements = this.#measurements; + let tombstones = this.#tombstones; + let tombstoneOrder = this.#tombstoneOrder; + let nextTicket = this.#nextTicket; + let unchanged = rowCount === previousValues.length; + for (let index = 0; index < rowCount; index += 1) { + const row = boundEntryAt(index); + const identity = this.#identity(row.key); + work.entriesVisited += 1; + work.identityLookups += 1; + if (seen.has(identity)) { + throw new Error(`Duplicate stable row-height key: ${identity}`); + } + seen.add(identity); + visibleKeys = hashSet(visibleKeys, identity, true, work); + const existing = unconsumed.get(identity); + if (existing !== undefined) { + unconsumed.delete(identity); + values[index] = existing; + work.refilterEntriesReused += 1; + if (existing !== previousValues[index]) unchanged = false; + continue; + } + unchanged = false; + const estimatedHeight = this.#estimated(row.estimatedHeight); + const measuredHeight = hashGet(measurements, identity, work); + if (measuredHeight !== undefined) { + work.measurementEntriesScanned += 1; + // A tombstone can only exist where a retained measurement does, so + // the lookup is skipped for the (common) genuinely-new entrant. + const retainedTicket = hashGet(tombstones, identity, work); + if (retainedTicket !== undefined) { + tombstones = hashDelete(tombstones, identity, work); + tombstoneOrder = mapDelete( + tombstoneOrder, + ticketKey(retainedTicket), + work, + ); + } + } + values[index] = { + ref: row.key, + identity, + estimatedHeight: row.estimatedHeight, + height: measuredHeight ?? estimatedHeight, + measured: measuredHeight !== undefined, + denseKey: row.denseKey, + }; + work.refilterEntriesInserted += 1; + } + + // Whatever the walk left unconsumed has LEFT the visible set. The Map + // preserves old-sequence order, so measured leavers take new tickets in + // exactly the order the cooperative scan-visible phase would assign them. + for (const value of unconsumed.values()) { + unchanged = false; + work.refilterEntriesRetired += 1; + if (!value.measured) continue; + if (this.#maxRetainedMeasurements === 0) { + measurements = hashDelete(measurements, value.identity, work); + continue; + } + const ticket = nextTicket; + nextTicket = takeNextTicket(nextTicket); + tombstones = hashSet(tombstones, value.identity, ticket, work); + tombstoneOrder = mapSet( + tombstoneOrder, + ticketKey(ticket), + value.identity, + work, + ); + } + while (hashCount(tombstones) > this.#maxRetainedMeasurements) { + const oldest: KeyMapNode | undefined = + minimumMapEntry(tombstoneOrder); + if (oldest === undefined) { + throw new Error("Removed-measurement retention is inconsistent."); + } + tombstoneOrder = mapDelete(tombstoneOrder, oldest.key, work); + tombstones = hashDelete(tombstones, oldest.value, work); + measurements = hashDelete(measurements, oldest.value, work); + } + if (unchanged) return this; + + const root = buildBalancedSequence(values, 0, rowCount, work); + return this.#next( + root, + visibleKeys, + this.#denseCapacity, + this.#visibleSlots, + measurements, + tombstones, + tombstoneOrder, + nextTicket, + work, + ); + } + + /** + * Dense-lane refilter (Amendment I): survivors resolve by `denseKey` + * (model slot) against a slot-indexed array — ZERO identity strings for + * the (dominant) survivor population. Only ENTRANTS compute an identity, + * because measurements and tombstones stay string-identity-keyed in both + * lanes: a slot is lifetime-bound and reused after permanent removal, so + * slot reuse must never touch retention (the amendment's §3 trap). Key + * validation mirrors the builder's `#ingestDenseKey` cases (missing / + * malformed / out-of-range → lifecycle error → the controller's + * fallback-on-throw contract), and the duplicate check shares one bitset + * with the next generation's membership, exactly as the builder does. + * + * TRUST BOUNDARY: a survivor's `row.key` is trusted to match the slot's + * stored identity and deliberately never re-verified — that verification + * IS the per-row string cost this lane exists to delete. `apply` guards + * slot drift instead, because its operations are k-sized. + * + * LEAVER ORDER IS LOAD-BEARING: measured leavers take tombstone tickets + * in OLD-SEQUENCE order (the string lane inherits this from its Map's + * insertion order), and ticket order is observable through cap eviction. + * The leaver pass therefore walks `previousValues` in order and consumes + * the slot cells — NEVER the slot array by index, whose order is + * unrelated to the sequence. + */ + #refilterDense( + boundEntryAt: (index: number) => RowHeightEntry, + rowCount: number, + ): RowHeightIndex { + const capacity = this.#denseCapacity!; + const work = createWork(); + + // One in-order pass over the current sequence: the by-slot lookup table + // for the walk below, and the old order for no-op detection and the + // leaver pass. + const previousValues: HeightValue[] = []; + const unconsumedBySlot: (HeightValue | undefined)[] = new Array< + HeightValue | undefined + >(capacity); + { + const stack: SequenceNode[] = []; + let cursor = this.#root; + while (cursor !== null || stack.length > 0) { + while (cursor !== null) { + stack.push(cursor); + cursor = cursor.left; + } + const node = stack.pop()!; + previousValues.push(node.value); + unconsumedBySlot[node.value.denseKey!] = node.value; + work.previousEntriesScanned += 1; + cursor = node.right; + } + } + + // Walk the new order: survivors relink verbatim by slot (measurements + // and estimates ride, exactly as in `reorder`), entrants ingest fresh + // under the cooperative path's rule — measured-lookup against the FULL + // cache BY IDENTITY, so a returning tombstoned key gets its retained + // measurement back and sheds its tombstone. `nextVisibleSlots` doubles + // as the duplicate check and the next generation's membership. + const values: HeightValue[] = new Array>(rowCount); + const nextVisibleSlots = createDenseMembership(capacity); + let measurements = this.#measurements; + let tombstones = this.#tombstones; + let tombstoneOrder = this.#tombstoneOrder; + let nextTicket = this.#nextTicket; + let unchanged = rowCount === previousValues.length; + for (let index = 0; index < rowCount; index += 1) { + const row = boundEntryAt(index); + work.entriesVisited += 1; + const denseKey = row.denseKey; + if (denseKey === undefined) { + throw new RowHeightReplacementLifecycleError( + "failed", + "A dense row-height index requires a denseKey on every source " + + `row (${this.#identity(row.key)} carries none); fall back to ` + + "a full replacement.", + ); + } + if ( + !Number.isSafeInteger(denseKey) || + denseKey < 0 || + denseKey >= capacity + ) { + throw new RowHeightReplacementLifecycleError( + "failed", + `Dense key ${denseKey} for row ${this.#identity(row.key)} is ` + + `outside the dense generation's capacity ${capacity}.`, + ); + } + if (testDenseBit(nextVisibleSlots, denseKey)) { + throw new Error( + `Duplicate dense row-height slot ${denseKey}: ` + + this.#identity(row.key), + ); + } + setDenseBit(nextVisibleSlots, denseKey); + const existing = unconsumedBySlot[denseKey]; + if (existing !== undefined) { + unconsumedBySlot[denseKey] = undefined; + values[index] = existing; + work.refilterEntriesReused += 1; + if (existing !== previousValues[index]) unchanged = false; + continue; + } + unchanged = false; + const identity = this.#identity(row.key); + work.identityLookups += 1; + const estimatedHeight = this.#estimated(row.estimatedHeight); + const measuredHeight = hashGet(measurements, identity, work); + if (measuredHeight !== undefined) { + work.measurementEntriesScanned += 1; + // A tombstone can only exist where a retained measurement does, so + // the lookup is skipped for the (common) genuinely-new entrant. + const retainedTicket = hashGet(tombstones, identity, work); + if (retainedTicket !== undefined) { + tombstones = hashDelete(tombstones, identity, work); + tombstoneOrder = mapDelete( + tombstoneOrder, + ticketKey(retainedTicket), + work, + ); + } + } + values[index] = { + ref: row.key, + identity, + estimatedHeight: row.estimatedHeight, + height: measuredHeight ?? estimatedHeight, + measured: measuredHeight !== undefined, + denseKey, + }; + work.refilterEntriesInserted += 1; + } + + // Whatever the walk left unconsumed has LEFT the visible set. Iterate + // the OLD SEQUENCE (see the doc comment above — ticket order is + // observable via cap eviction) and consume each leaver's slot cell. + for (const value of previousValues) { + const slot = value.denseKey!; + if (unconsumedBySlot[slot] === undefined) continue; + unconsumedBySlot[slot] = undefined; + unchanged = false; + work.refilterEntriesRetired += 1; + if (!value.measured) continue; + if (this.#maxRetainedMeasurements === 0) { + measurements = hashDelete(measurements, value.identity, work); + continue; + } + const ticket = nextTicket; + nextTicket = takeNextTicket(nextTicket); + tombstones = hashSet(tombstones, value.identity, ticket, work); + tombstoneOrder = mapSet( + tombstoneOrder, + ticketKey(ticket), + value.identity, + work, + ); + } + while (hashCount(tombstones) > this.#maxRetainedMeasurements) { + const oldest: KeyMapNode | undefined = + minimumMapEntry(tombstoneOrder); + if (oldest === undefined) { + throw new Error("Removed-measurement retention is inconsistent."); + } + tombstoneOrder = mapDelete(tombstoneOrder, oldest.key, work); + tombstones = hashDelete(tombstones, oldest.value, work); + measurements = hashDelete(measurements, oldest.value, work); + } + if (unchanged) return this; + + const root = buildBalancedSequence(values, 0, rowCount, work); + return this.#next( + root, + null, + this.#denseCapacity, + nextVisibleSlots, + measurements, + tombstones, + tombstoneOrder, + nextTicket, + work, + ); + } + beginReplacement( source: RowHeightReplacementSource, ): RowHeightReplacementBuilder { @@ -1555,7 +2172,17 @@ class PersistentRowHeightIndex implements RowHeightIndex { if (typeof entryAt !== "function") { throw new TypeError("Replacement source entryAt must be a function."); } + const denseCapacity = source.denseCapacity; + if ( + denseCapacity !== undefined && + (!Number.isSafeInteger(denseCapacity) || denseCapacity < 0) + ) { + throw new RangeError( + "Replacement source denseCapacity must be a non-negative safe integer.", + ); + } return new PersistentRowHeightReplacementBuilder({ + denseCapacity, // With no retained state every ingest lookup would miss (see // `hasRetainedState`'s derivation), so the builder may build everything // in one synchronous O(n) pass instead of cooperative phases. @@ -1566,6 +2193,7 @@ class PersistentRowHeightIndex implements RowHeightIndex { getKey: this.#getKey, root: this.#root, visibleKeys: this.#visibleKeys, + denseCapacity: this.#denseCapacity, measurements: this.#measurements, tombstones: this.#tombstones, tombstoneOrder: this.#tombstoneOrder, @@ -1628,6 +2256,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { #next( root: SequenceNode | null, visibleKeys: HashNode | null, + denseCapacity: number | undefined, + visibleSlots: DenseMembership | undefined, measurements: HashNode | null, tombstones: HashNode | null, tombstoneOrder: KeyMapNode | null, @@ -1639,6 +2269,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { getKey: this.#getKey, root, visibleKeys, + denseCapacity, + visibleSlots, measurements, tombstones, tombstoneOrder, @@ -1665,6 +2297,9 @@ class PersistentRowHeightReplacementBuilder< #sequenceBuildStack: SequenceBuildFrame[] | null = []; #retentionBuildStack: RetentionBuildFrame[] | null = []; #visibleKeys: HashNode | null = null; + /** Dense lane (Amendment I §1): declared by the source at `begin`. */ + readonly #denseCapacity: number | undefined; + #visibleSlots: DenseMembership | null; #measurements: HashNode | null; #tombstones: HashNode | null = null; #tombstoneOrder: KeyMapNode | null = null; @@ -1701,10 +2336,16 @@ class PersistentRowHeightReplacementBuilder< readonly rowCount: number; readonly entryAt: (index: number) => RowHeightEntry; readonly bulk: boolean; + readonly denseCapacity: number | undefined; }) { this.#base = options.base; this.#entryAt = options.entryAt; this.#sourceRowCount = options.rowCount; + this.#denseCapacity = options.denseCapacity; + this.#visibleSlots = + options.denseCapacity === undefined + ? null + : createDenseMembership(options.denseCapacity); this.#measurements = options.base.measurements; this.#nextTicket = options.base.nextTicket; this.#bulk = options.bulk; @@ -1888,6 +2529,8 @@ class PersistentRowHeightReplacementBuilder< getKey: base.getKey, root: this.#root, visibleKeys: this.#visibleKeys, + denseCapacity: this.#denseCapacity, + visibleSlots: this.#visibleSlots ?? undefined, measurements: this.#measurements, tombstones: this.#tombstones, tombstoneOrder: this.#tombstoneOrder, @@ -1955,6 +2598,53 @@ class PersistentRowHeightReplacementBuilder< } } + /** + * The dense lane's per-row ingest (Amendment I §1). With no declared + * capacity the input's `denseKey` is stamped verbatim and stays inert + * (string lane). Under a declared capacity every row MUST carry an + * in-range `denseKey` — the source promised dense coverage, so a missing + * or out-of-range key is a CALLER BUG and throws rather than silently + * falling back (the controller only declares `denseCapacity` when the + * snapshot guarantees slots). The visible-slot bitset is built here, in + * place of the `#visibleKeys` HAMT the dense lane skips entirely; the + * string-identity `#identities` duplicate check still runs first at both + * call sites, because retention stays identity-keyed in both lanes. + */ + #ingestDenseKey( + row: RowHeightEntry, + identity: string, + ): number | undefined { + const capacity = this.#denseCapacity; + if (capacity === undefined) return row.denseKey; + const denseKey = row.denseKey; + if (denseKey === undefined) { + throw new RowHeightReplacementLifecycleError( + "failed", + `Replacement source declared denseCapacity ${capacity} but a row ` + + `(${identity}) carries no denseKey; the source broke its dense ` + + "promise.", + ); + } + if ( + !Number.isSafeInteger(denseKey) || + denseKey < 0 || + denseKey >= capacity + ) { + throw new RowHeightReplacementLifecycleError( + "failed", + `Dense key ${denseKey} for row ${identity} is outside the declared ` + + `denseCapacity ${capacity}.`, + ); + } + if (testDenseBit(this.#visibleSlots!, denseKey)) { + throw new Error( + `Duplicate dense row-height slot ${denseKey}: ${identity}`, + ); + } + setDenseBit(this.#visibleSlots!, denseKey); + return denseKey; + } + /** * The whole replacement in one pass, valid only when the base has no * retained state (`hasRetainedState === false`, checked by @@ -1984,28 +2674,38 @@ class PersistentRowHeightReplacementBuilder< row.estimatedHeight === undefined ? base.defaultHeight : normalizeHeight(row.estimatedHeight, "Estimated row height"); - this.#visibleKeys = hashSet( - this.#visibleKeys, - identity, - true, - this.#work, - ); + const denseKey = this.#ingestDenseKey(row, identity); + if (this.#denseCapacity === undefined) { + this.#visibleKeys = hashSet( + this.#visibleKeys, + identity, + true, + this.#work, + ); + } values.push({ ref: row.key, identity, estimatedHeight: row.estimatedHeight, height: estimatedHeight, measured: false, + denseKey, }); this.#ingestIndex += 1; } this.#entryAt = null; // Same no-op predicate as `#stepVisibleTraversal`: every prior visible - // entry matches its candidate's identity and estimate, and the counts - // agree. A count mismatch skips the scan entirely, so a true mount (empty + // entry matches its candidate's identity, estimate, AND dense stamp, and + // the counts and lanes agree. A no-op finishes to the BASE generation, so + // lane disagreement (or a reassigned slot) must rebuild even when the + // identities match — otherwise a stale bitset would answer for new slots. + // A count mismatch skips the scan entirely, so a true mount (empty // base, populated source) pays nothing here. - if (nodeCount(base.root) === values.length) { + if ( + nodeCount(base.root) === values.length && + this.#denseCapacity === base.denseCapacity + ) { let equal = true; const stack: SequenceNode[] = []; let cursor = base.root; @@ -2020,7 +2720,8 @@ class PersistentRowHeightReplacementBuilder< this.#work.previousEntriesScanned += 1; if ( candidate.identity !== node.value.identity || - candidate.estimatedHeight !== node.value.estimatedHeight + candidate.estimatedHeight !== node.value.estimatedHeight || + candidate.denseKey !== node.value.denseKey ) { equal = false; } @@ -2061,18 +2762,22 @@ class PersistentRowHeightReplacementBuilder< if (measuredHeight !== undefined) { this.#work.measurementEntriesScanned += 1; } - this.#visibleKeys = hashSet( - this.#visibleKeys, - identity, - true, - this.#work, - ); + const denseKey = this.#ingestDenseKey(row, identity); + if (this.#denseCapacity === undefined) { + this.#visibleKeys = hashSet( + this.#visibleKeys, + identity, + true, + this.#work, + ); + } values.push({ ref: row.key, identity, estimatedHeight: row.estimatedHeight, height: measuredHeight ?? estimatedHeight, measured: measuredHeight !== undefined, + denseKey, }); this.#ingestIndex += 1; return; @@ -2127,7 +2832,8 @@ class PersistentRowHeightReplacementBuilder< if (frame === undefined) { if ( this.#equalVisible && - this.#work.previousEntriesScanned === this.#values!.length + this.#work.previousEntriesScanned === this.#values!.length && + this.#denseCapacity === this.#base!.denseCapacity ) { this.#noOp = true; this.#phase = "done"; @@ -2156,7 +2862,8 @@ class PersistentRowHeightReplacementBuilder< if ( candidate === undefined || candidate.identity !== value.identity || - candidate.estimatedHeight !== value.estimatedHeight + candidate.estimatedHeight !== value.estimatedHeight || + candidate.denseKey !== value.denseKey ) { this.#equalVisible = false; } @@ -2336,6 +3043,7 @@ class PersistentRowHeightReplacementBuilder< this.#sequenceBuildStack = null; this.#retentionBuildStack = null; this.#visibleKeys = null; + this.#visibleSlots = null; this.#measurements = null; this.#tombstones = null; this.#tombstoneOrder = null; @@ -2365,6 +3073,8 @@ export function createRowHeightIndex( getKey: options.getKey, root: null, visibleKeys: null, + denseCapacity: undefined, + visibleSlots: undefined, measurements: null, tombstones: null, tombstoneOrder: null, diff --git a/packages/layout-core/src/types.ts b/packages/layout-core/src/types.ts index 567b2d9f9..84f4d8959 100644 --- a/packages/layout-core/src/types.ts +++ b/packages/layout-core/src/types.ts @@ -35,10 +35,22 @@ export interface RowMetricsIndex extends RowMetricsReader { updateHeight(index: number, height: number): void; } -/** One visible row and its optional unmeasured height estimate. @internal */ +/** + * One visible row and its optional unmeasured height estimate. + * + * `denseKey` is the OPTIONAL dense-lane contract (Amendment I §1): when + * present it is the row's CURRENT row-model slot, valid only while the + * model binds that slot — the caller (the renderer-dom controller) owns + * that currency, not this package. A replacement source runs the dense + * lane only when EVERY entry carries a `denseKey` and the source declares + * `denseCapacity`; a single entry missing one falls the whole generation + * back to the string lane. + * @internal + */ export interface RowHeightEntry { readonly key: TKey; readonly estimatedHeight?: number; + readonly denseKey?: number; } /** @@ -60,6 +72,12 @@ export interface CreateRowHeightIndexOptions { * root. Moves and removals retain measurements by stable identity; updates * invalidate the affected measurement so the estimate is used until the row * is measured again. + * + * Every variant carries an OPTIONAL `denseKey` (Amendment I §1): the row's + * CURRENT row-model slot, valid only while the model binds that slot — the + * caller owns that currency. An op reaching a dense-lane index without one + * is a lifecycle error (the dense lane requires every op to be dense-keyed); + * an op reaching a string-lane index ignores it. * @internal */ export type RowHeightOperation = @@ -68,23 +86,27 @@ export type RowHeightOperation = readonly ref: TKey; readonly index: number; readonly estimatedHeight?: number; + readonly denseKey?: number; } | { readonly kind: "remove"; readonly ref: TKey; readonly previousIndex: number; + readonly denseKey?: number; } | { readonly kind: "move"; readonly ref: TKey; readonly previousIndex: number; readonly index: number; + readonly denseKey?: number; } | { readonly kind: "update"; readonly ref: TKey; readonly index: number; readonly estimatedHeight?: number; + readonly denseKey?: number; }; /** A captured pixel position within a stable logical row. @internal */ @@ -93,9 +115,21 @@ export interface RowHeightAnchor { readonly offset: number; } -/** Indexed replacement input; rows are read lazily by cooperative builders. @internal */ +/** + * Indexed replacement input; rows are read lazily by cooperative builders. + * + * `denseCapacity` is the OPTIONAL dense-lane declaration (Amendment I §1): + * when present, the builder requires every ingested entry's `entryAt(...)` + * to carry a `denseKey` less than this capacity and builds the dense + * (slot-bitset) lane; a missing `denseKey` under a declared capacity is a + * caller bug (the source promised dense coverage and broke the promise) and + * throws rather than silently falling back. Omitting `denseCapacity` runs + * today's string lane unconditionally. + * @internal + */ export interface RowHeightReplacementSource { readonly rowCount: number; + readonly denseCapacity?: number; entryAt(index: number): RowHeightEntry; } @@ -174,8 +208,19 @@ export interface RowHeightIndex extends RowMetricsReader { */ getMeasuredHeightMean(): number | undefined; measure(index: number, ref: TKey, height: number): RowHeightIndex; - /** Retains a bounded measured height for a stable key absent from the view. */ - retainMeasurement(ref: TKey, height: number): RowHeightIndex; + /** + * Retains a bounded measured height for a stable key absent from the view. + * + * `denseKey` follows the Amendment I §1 op contract: on a dense generation + * it is REQUIRED — the visible-row guard is answered by the slot bitset, + * so a call without one throws the replacement lifecycle error and the + * caller falls back to a full replacement. A string generation ignores it. + */ + retainMeasurement( + ref: TKey, + height: number, + denseKey?: number, + ): RowHeightIndex; apply(operations: readonly RowHeightOperation[]): RowHeightIndex; replace(rows: readonly RowHeightEntry[]): RowHeightIndex; /** @@ -186,6 +231,19 @@ export interface RowHeightIndex extends RowMetricsReader { * permutation of the current rows; callers fall back to `beginReplacement`. */ reorder(source: RowHeightReplacementSource): RowHeightIndex; + /** + * Rebuilds the ordered structure for a MEMBERSHIP change, synchronously: + * surviving keys reuse their existing entries verbatim (measurements and + * estimates ride; source `estimatedHeight`s for survivors are ignored, as + * with `reorder`), keys absent from the existing entries are ingested fresh + * with the estimate-or-default rule (a returning key's retained measurement + * is restored), and existing keys absent from the new order leave under the + * cooperative path's retention policy (measured leavers tombstone, + * unmeasured leavers vanish). Membership deltas are the purpose, not an + * error — throws only on structural impossibilities (duplicate keys, bad + * rowCount); callers fall back to `beginReplacement` on any throw. + */ + refilter(source: RowHeightReplacementSource): RowHeightIndex; beginReplacement( source: RowHeightReplacementSource, ): RowHeightReplacementBuilder; diff --git a/packages/react/react.api.md b/packages/react/react.api.md index 02e57f2bf..a242282cb 100644 --- a/packages/react/react.api.md +++ b/packages/react/react.api.md @@ -533,7 +533,7 @@ export type PretableChangeSequence = { } | { readonly kind: "reset"; readonly toRevision: number; - readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder"; + readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder" | "refilter"; }; // @public (undocumented) @@ -1877,6 +1877,12 @@ export interface PretableRowModelSnapshot(); +const columns = [ + column.accessor("score", { type: "number" }), + column.accessor("label", { type: "text", wrap: true, widthPx: 200 }), +] as const; + +const rows: readonly Row[] = Array.from({ length: 200 }, (_, index) => ({ + id: index, + score: index, + label: `row ${index}`, +})); + +type Controller = RowLayoutController; + +const controllers: Controller[] = []; + +vi.mock("@pretable-internal/renderer-dom", async (importOriginal) => { + const actual = + await importOriginal(); + const createRowLayoutController: typeof actual.createRowLayoutController = ( + options, + ) => { + const controller = actual.createRowLayoutController(options); + controllers.push(controller as unknown as Controller); + return controller; + }; + return { ...actual, createRowLayoutController }; +}); + +const { PretableSurface } = await import("../pretable-surface"); + +const dataRef = (rowId: number): PretableVisibleRowRef => ({ + kind: "data", + rowId, +}); + +/** + * Layout-core's dense contract, used as a lane probe: a DENSE generation + * refuses any operation that arrives without a `denseKey`; a string-lane + * generation accepts it. `apply` is persistent and throws before producing + * anything, so probing never perturbs the published index. + */ +function isDenseIndex(controller: Controller): boolean { + const rowHeights = controller.getState().rowHeights; + try { + rowHeights.apply([{ kind: "update", ref: dataRef(0), index: 0 }]); + return false; + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch(/dense/i); + return true; + } +} + +afterEach(() => { + cleanup(); + controllers.length = 0; + vi.clearAllMocks(); +}); + +describe("dense layout seam, end to end", () => { + test("filter transitions ride the dense refilter path and measured heights survive a flip-out/flip-in", async () => { + const model = createLocalRowModel({ rows, columns }); + const { container } = render( + , + ); + await waitFor(() => { + expect( + container.querySelectorAll("[data-pretable-row]").length, + ).toBeGreaterThan(0); + }); + expect(controllers).toHaveLength(1); + const controller = controllers[0]!; + await waitFor(() => { + expect(controller.getState().status.kind).toBe("ready"); + }); + + // The react-mounted grid actually engaged the DENSE lane — see the module + // docblock for why the counters alone cannot vouch for this. + expect(isDenseIndex(controller)).toBe(true); + + // Measured OUTSIDE the planned viewport, so jsdom's zero-height DOM + // measurements of the visible rows never contest it. + const measured = dataRef(150); + act(() => { + controller.measure(measured, 63); + }); + expect(controller.getState().rowHeights.hasMeasurement(measured)).toBe( + true, + ); + + const base = getRowLayoutControllerDiagnosticsForTesting(controller); + + const setFilter = async ( + value: number | undefined, + expectedVisible: number, + ) => { + await act(async () => { + const transition = model.setQuery({ + filters: + value === undefined + ? [] + : [{ columnId: "score", operator: "gt", value }], + sort: [], + rowGroups: [], + }); + await transition.finished; + }); + await waitFor(() => { + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.visibleRowCount).toBe(expectedVisible); + }); + }; + + // filter-on → narrow (row 150 flips OUT) → widen (row 150 flips back + // IN) → filter-off. Scores are 0..199, `gt` is strict. + await setFilter(100, 99); + await setFilter(180, 19); + await setFilter(100, 99); + + // The measurement survived the flip-out/flip-in — restored from the + // identity-keyed retention, never re-estimated. + const widened = controller.getState(); + expect(widened.rowHeights.hasMeasurement(measured)).toBe(true); + const rank = widened.snapshot!.indexOf(measured); + expect(rank).toBeGreaterThanOrEqual(0); + expect(widened.rowHeights.getHeight(rank)).toBe(63); + + await setFilter(undefined, 200); + const final = controller.getState(); + expect(final.rowHeights.getHeight(final.snapshot!.indexOf(measured))).toBe( + 63, + ); + + // Every transition rode the in-place refilter path, and none fell back + // to a full replacement — the dense fast path ran, silently-fallback + // free. + const diagnostics = getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.refilterPathCount - base.refilterPathCount).toBe(4); + expect(diagnostics.refilterFallbackCount).toBe(0); + expect(diagnostics.reorderFallbackCount).toBe(0); + + // And the index is STILL dense — no transition quietly dropped the + // generation to the string lane. + expect(isDenseIndex(controller)).toBe(true); + + model.dispose(); + }, 30_000); +}); diff --git a/packages/react/src/__tests__/filter-menu-row-model-boundary.test.ts b/packages/react/src/__tests__/filter-menu-row-model-boundary.test.ts index 1e2eb95f7..17feaea97 100644 --- a/packages/react/src/__tests__/filter-menu-row-model-boundary.test.ts +++ b/packages/react/src/__tests__/filter-menu-row-model-boundary.test.ts @@ -18,6 +18,7 @@ import { describe, expect, it } from "vitest"; import { compileQuery, createColumnHelper, + filterVerdict, type PretableQueryFor, } from "@pretable-internal/row-model"; import type { ColumnType, FilterOperator } from "@pretable/core"; @@ -314,18 +315,20 @@ describe("filter menu -> row-model boundary", () => { }); expect( - plan.evaluate({ + filterVerdict(plan, { rowId: 1, row: { id: 1, value: matchValue }, sourceOrder: 0, - }).filterPasses, + slot: 0, + }), ).toBe(true); expect( - plan.evaluate({ + filterVerdict(plan, { rowId: 2, row: { id: 2, value: nonMatchValue }, sourceOrder: 0, - }).filterPasses, + slot: 1, + }), ).toBe(false); }, ); diff --git a/packages/react/vitest.config.ts b/packages/react/vitest.config.ts index 24823895c..a834d2033 100644 --- a/packages/react/vitest.config.ts +++ b/packages/react/vitest.config.ts @@ -24,6 +24,16 @@ export default defineConfig({ __dirname, "../layout-core/src/index.ts", ), + // Longer key FIRST: alias entries match in order, and the barrel entry + // below would otherwise swallow this subpath. The diagnostics seam is a + // direct-module export deliberately kept off renderer-dom's barrel; the + // dense-layout-seam end-to-end pin reads it through this alias (typecheck + // resolves it through tsconfig.typecheck.json's + // `@pretable-internal/renderer-dom/*` mapping). + "@pretable-internal/renderer-dom/row-layout-controller": resolve( + __dirname, + "../renderer-dom/src/row-layout-controller.ts", + ), "@pretable-internal/renderer-dom": resolve( __dirname, "../renderer-dom/src/index.ts", diff --git a/packages/renderer-dom/src/__tests__/indexed-renderer.test.ts b/packages/renderer-dom/src/__tests__/indexed-renderer.test.ts index b9d60e72f..548504506 100644 --- a/packages/renderer-dom/src/__tests__/indexed-renderer.test.ts +++ b/packages/renderer-dom/src/__tests__/indexed-renderer.test.ts @@ -1225,10 +1225,20 @@ describe("indexed DOM row layout controller", () => { expect(estimate.mock.calls.length).toBeLessThan(12); expect(estimate.mock.calls.length).toBeGreaterThan(0); expect(state.window.length).toBeLessThan(12); - expect(rangeCalls.length).toBeLessThanOrEqual(2); + // The replacement SOURCE materializes the visible set in BOUNDED chunked + // `range` walks (the dense seam's bulk read — it replaced 1,000 per-row + // `rowAt` rank descents WITHOUT ever reading the whole dataset in one + // call); every OTHER range read stays window-sized, which is the + // projection claim under test. + const buildWalks = rangeCalls.filter(([start, end]) => end - start > 12); + expect(buildWalks.length).toBeGreaterThan(0); expect( - Math.max(...rangeCalls.map(([start, end]) => end - start)), - ).toBeLessThan(12); + Math.max(...buildWalks.map(([start, end]) => end - start)), + ).toBeLessThanOrEqual(256); + expect(buildWalks[0]).toEqual([0, 256]); + expect(buildWalks[buildWalks.length - 1]).toEqual([768, 1_000]); + const windowReads = rangeCalls.filter(([start, end]) => end - start <= 12); + expect(windowReads.length).toBeLessThanOrEqual(2); rangeCalls.length = 0; const render = createDomRenderSnapshot({ @@ -2124,7 +2134,13 @@ describe("indexed DOM row layout controller", () => { expect(empty.range).toEqual({ start: 0, end: 0 }); }); - test("defers viewport publication during a reset until matching geometry is ready", () => { + // This test previously asserted that a mid-reset viewport request published + // NOTHING until the replacement finished — which is exactly the defect the + // stale republish fixes: the last-published window was planned for the OLD + // scroll position, so a scroll outside it left the grid blank for the whole + // replacement. The deferral semantics it pinned (anchor dropped, the finish + // honors the requested global scrollTop) are unchanged and still asserted. + test("repaints a stale window at a mid-reset scroll position instead of going blank", () => { const model = createModel( Array.from({ length: 20 }, (_, index) => ({ id: index, @@ -2135,7 +2151,7 @@ describe("indexed DOM row layout controller", () => { ); const { controller, scheduler } = createReadyController(model); // Retained state keeps the reset cooperative, so there is a rebuilding - // interval for the viewport request to be deferred into. + // interval for the viewport request to land inside. controller.measure(data(0), 45); const notifications = vi.fn(); controller.subscribe(notifications); @@ -2150,8 +2166,18 @@ describe("indexed DOM row layout controller", () => { const rebuilding = controller.getState(); notifications.mockClear(); controller.setViewport({ scrollTop: 440, viewportHeight: 88, overscan: 1 }); - expect(controller.getState()).toBe(rebuilding); - expect(notifications).not.toHaveBeenCalled(); + // Stale-but-visible: the OLD snapshot re-projected at the NEW scrollTop, + // published immediately, with the in-flight rebuild still reported. + const repainted = controller.getState(); + expect(repainted).not.toBe(rebuilding); + expect(notifications).toHaveBeenCalled(); + expect(repainted.snapshot).toBe(rebuilding.snapshot); + expect(repainted.observedRevision).toBe(rebuilding.observedRevision); + expect(repainted.scrollTop).toBe(440); + expect(repainted.viewport.scrollTop).toBe(440); + expect(repainted.window.length).toBeGreaterThan(0); + expect(repainted.window.map((entry) => entry.ref)).toContainEqual(data(10)); + expect(repainted.status).toMatchObject({ kind: "rebuilding" }); model.setRows( Array.from({ length: 10_001 }, (_, index) => ({ id: index, @@ -2160,8 +2186,7 @@ describe("indexed DOM row layout controller", () => { label: `superseding ${index}`, })), ); - expect(controller.getState().viewport.scrollTop).toBe(0); - expect(controller.getState().window).toBe(rebuilding.window); + expect(controller.getState().viewport.scrollTop).toBe(440); scheduler.flushAll(); expect(controller.getState()).toMatchObject({ scrollTop: 440, @@ -2171,6 +2196,113 @@ describe("indexed DOM row layout controller", () => { expect(controller.getState().range.start).toBeGreaterThan(0); }); + test("scrolling during an unabsorbable filter reset keeps a stale window visible", () => { + const model = createModel( + Array.from({ length: 40 }, (_, index) => ({ + id: index, + team: index < 20 ? "A" : "B", + score: index, + label: `row ${index}`, + })), + ); + const { controller, scheduler } = createReadyController(model); + // Measured base: retained state keeps the bulk-replace reset cooperative, + // reproducing the shipped blank-viewport shape (the filter commit lands + // its barrier synchronously, so the controller is mid-replacement BEFORE + // any same-commit reveal scroll can land). + controller.measure(data(0), 44); + // The live filter fast path now publishes a "refilter" reset the + // controller absorbs synchronously — no replacement interval exists to + // scroll into (the refilter-path suite pins that). This test keeps the + // deferred-viewport republish covered for the resets that still replace: + // degrade the reason so the same commit is unabsorbable, the shape any + // fallback or unaware-consumer path produces. + const realChangesSince = model.changesSince.bind(model); + vi.spyOn(model, "changesSince").mockImplementation((revision) => { + const sequence = realChangesSince(revision); + return sequence.kind === "reset" + ? { ...sequence, reason: "bulk-replace" as const } + : sequence; + }); + const before = controller.getState(); + model.setQuery({ + filters: [{ columnId: "team", operator: "equals", value: "B" }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }); + expect(controller.getState().status.kind).toBe("rebuilding"); + // The reveal scroll, outside the published window (rows 0-3 at scrollTop + // 0). Nothing has driven the scheduler: pre-fix the grid stayed blank + // here until the replacement flushed. + controller.setViewport({ scrollTop: 440, viewportHeight: 88, overscan: 1 }); + const stale = controller.getState(); + expect(stale.observedRevision).toBe(before.observedRevision); + expect(stale.snapshot).toBe(before.snapshot); + expect(stale.scrollTop).toBe(440); + expect(stale.window.length).toBeGreaterThan(0); + // Row 10 is team A — filtered OUT of the new set — so only the stale + // (pre-filter) snapshot can put it on screen. + expect(stale.window.map((entry) => entry.ref)).toContainEqual(data(10)); + expect(stale.status.kind).toBe("rebuilding"); + scheduler.flushAll(); + const final = controller.getState(); + expect(final.status.kind).toBe("ready"); + expect(final.observedRevision).toBe(model.getState().snapshot.revision); + // Anchor-less global-scroll semantics: the deferred flag mandates the + // requested scrollTop verbatim, not an anchor restoration to the old + // position. + expect(final.scrollTop).toBe(440); + expect(final.viewport.scrollTop).toBe(440); + // And the final window comes from the NEW (filtered) snapshot: ids 20..39 + // sorted ascending, index 10 at 44px rows -> id 30. + expect(final.window.length).toBeGreaterThan(0); + expect(final.window.map((entry) => entry.ref)).toContainEqual(data(30)); + for (const entry of final.window) { + expect(entry.ref.kind).toBe("data"); + expect( + entry.ref.kind === "data" && (entry.ref.rowId as number) >= 20, + ).toBe(true); + } + }); + + test("scrolling during a combined sort+filter replacement keeps a stale window visible", async () => { + // The latent pre-fast-path shape: a combined sort+filter change takes the + // model's cooperative transition, whose commit still lands a bulk-replace + // barrier while a same-frame scroll can arrive mid-replacement. + const model = createModel( + Array.from({ length: 40 }, (_, index) => ({ + id: index, + team: index < 20 ? "A" : "B", + score: index, + label: `row ${index}`, + })), + ); + const { controller, scheduler } = createReadyController(model); + controller.measure(data(0), 44); + const before = controller.getState(); + const transition = model.setQuery({ + filters: [{ columnId: "team", operator: "equals", value: "B" }], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], + }); + await transition.finished; + expect(controller.getState().status.kind).toBe("rebuilding"); + controller.setViewport({ scrollTop: 440, viewportHeight: 88, overscan: 1 }); + const stale = controller.getState(); + expect(stale.observedRevision).toBe(before.observedRevision); + expect(stale.snapshot).toBe(before.snapshot); + expect(stale.scrollTop).toBe(440); + expect(stale.window.map((entry) => entry.ref)).toContainEqual(data(10)); + expect(stale.status.kind).toBe("rebuilding"); + scheduler.flushAll(); + const final = controller.getState(); + expect(final.status.kind).toBe("ready"); + expect(final.observedRevision).toBe(model.getState().snapshot.revision); + expect(final.scrollTop).toBe(440); + // Descending over the filtered ids 20..39: index 10 -> id 29. + expect(final.window.map((entry) => entry.ref)).toContainEqual(data(29)); + }); + test("rolls a deferred viewport back after reset failure so the same request can retry", () => { const model = createModel( Array.from({ length: 40 }, (_, index) => ({ @@ -2180,7 +2312,7 @@ describe("indexed DOM row layout controller", () => { label: `old ${index}`, })), ); - let failNextEstimate = false; + let failingEstimates = 0; const scheduler = new ManualScheduler(); const controller = createRowLayoutController({ model, @@ -2188,8 +2320,8 @@ describe("indexed DOM row layout controller", () => { viewport: { scrollTop: 0, viewportHeight: 88, overscan: 0 }, scheduler, estimateRowHeight(row) { - if (failNextEstimate && row.id === 10) { - failNextEstimate = false; + if (failingEstimates > 0 && row.id === 10) { + failingEstimates -= 1; throw new Error("estimate exploded"); } return 44; @@ -2198,9 +2330,13 @@ describe("indexed DOM row layout controller", () => { }); scheduler.flushAll(); // Retained state keeps the reset cooperative, so the viewport request - // below is deferred into an ACTIVE replacement — the rollback under test. + // below lands inside an ACTIVE replacement — the rollback under test. controller.measure(data(1), 44); - failNextEstimate = true; + // Two failures: the first is consumed by the mid-replacement stale + // republish (row 10 sits in the requested window), the second by the + // replacement's own builder — only a replacement that FAILS exercises the + // rollback, since a successful one publishes the deferred scrollTop. + failingEstimates = 2; model.setRows( Array.from({ length: 40 }, (_, index) => ({ id: index, @@ -2211,6 +2347,12 @@ describe("indexed DOM row layout controller", () => { ); controller.measure(data(0), 123); controller.setViewport({ scrollTop: 440, viewportHeight: 88, overscan: 0 }); + // The stale republish failed, so the viewport was never committed: the + // published state still shows the old position, with an error status. + expect(controller.getState()).toMatchObject({ + viewport: { scrollTop: 0 }, + status: { kind: "error" }, + }); scheduler.flushAll(); expect(controller.getState()).toMatchObject({ viewport: { scrollTop: 0 }, @@ -2251,20 +2393,34 @@ describe("indexed DOM row layout controller", () => { ...modelState, snapshot: Object.freeze({ ...snapshot, - rowAt(index: number) { - if (!superseded) { - superseded = true; - source.setRows( - Array.from({ length: 301 }, (_, rowIndex) => ({ - id: rowIndex, - team: "C", - score: rowIndex, - label: `latest ${rowIndex}`, - })), - ); - throw new Error("stale source exploded"); + // The replacement source reads the model in chunked bulk + // `range` walks and its scheduled slices only index the + // results, so the hostile mid-slice access lives on the FIRST + // chunk's returned ARRAY: the first element read from a slice + // supersedes the build and explodes. (It used to live on + // `rowAt`, which the build no longer calls per row.) + range(start: number, end: number) { + const result = snapshot.range(start, end); + if (start !== 0) { + return result; } - return snapshot.rowAt(index); + return new Proxy(result, { + get(target, property, receiver) { + if (property === "0" && !superseded) { + superseded = true; + source.setRows( + Array.from({ length: 301 }, (_, rowIndex) => ({ + id: rowIndex, + team: "C", + score: rowIndex, + label: `latest ${rowIndex}`, + })), + ); + throw new Error("stale source exploded"); + } + return Reflect.get(target, property, receiver); + }, + }); }, }), }; @@ -3330,7 +3486,7 @@ describe("indexed DOM row layout controller", () => { expect(controller.getState()).toBe(before); }); - describe("sort-only reorder permutation path", () => { + describe("synchronous reset fast paths (reorder/refilter)", () => { const tenRows = Array.from({ length: 10 }, (_, index) => ({ id: index + 1, team: "A", @@ -3361,12 +3517,21 @@ describe("indexed DOM row layout controller", () => { // structural cast reads the same frozen object the seam returns. const heightIndexDiagnostics = ( index: unknown, - ): { reorderEntriesReused: number; reorderEntriesRemeasured: number } => + ): { + reorderEntriesReused: number; + reorderEntriesRemeasured: number; + refilterEntriesReused: number; + refilterEntriesInserted: number; + refilterEntriesRetired: number; + } => ( index as { diagnostics: { reorderEntriesReused: number; reorderEntriesRemeasured: number; + refilterEntriesReused: number; + refilterEntriesInserted: number; + refilterEntriesRetired: number; }; } ).diagnostics; @@ -3891,6 +4056,349 @@ describe("indexed DOM row layout controller", () => { expect(oracle.controller.getState().scrollTop).toBe(after.scrollTop); }); }); + + describe("filter-only refilter path", () => { + // score <= 5 keeps rows 1..5 in their relative order (a narrowing); + // clearing the filters afterwards is the widening twin. + const narrowQuery = { + filters: [ + { columnId: "score" as const, operator: "lte" as const, value: 5 }, + ], + sort: [{ columnId: "score" as const, direction: "asc" as const }], + rowGroups: [], + }; + const wideQuery = { + filters: [], + sort: [{ columnId: "score" as const, direction: "asc" as const }], + rowGroups: [], + }; + // score > 4 removes rows 1..4 — including row 4, the anchor row of the + // shared 130px-viewport geometry — while rows 5..10 survive. + const dropAnchorQuery = { + filters: [ + { columnId: "score" as const, operator: "gt" as const, value: 4 }, + ], + sort: [{ columnId: "score" as const, direction: "asc" as const }], + rowGroups: [], + }; + const survivorMeasurements = allMeasurements.slice(0, 5); + + test("a filter-only commit refilters existing heights without a replacement", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + for (const [rowId, height] of allMeasurements) { + controller.measure(data(rowId), height); + } + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + const refilterSpy = vi.spyOn( + controller.getState().rowHeights, + "refilter", + ); + + model.setQuery(narrowQuery); + + // Synchronous: ready again with no scheduler flush and no replacement. + const after = controller.getState(); + expect(after.status.kind).toBe("ready"); + expect(after.snapshot?.visibleRowCount).toBe(5); + expect(after.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount, + ); + expect(diagnostics.refilterPathCount).toBe( + before.refilterPathCount + 1, + ); + expect(diagnostics.refilterFallbackCount).toBe( + before.refilterFallbackCount, + ); + expect(diagnostics.reorderPathCount).toBe(before.reorderPathCount); + expect(refilterSpy).toHaveBeenCalledTimes(1); + expect(after.rowHeights).toBe(refilterSpy.mock.results[0]!.value); + + // Survivor measurements ride; the table equals a full replacement. + for (const [rowId, height] of survivorMeasurements) { + expect(after.rowHeights.hasMeasurement(data(rowId))).toBe(true); + expect( + after.rowHeights.getHeight( + model.getState().snapshot.indexOf(data(rowId)), + ), + ).toBe(height); + } + const oracle = createReplacementOracle(tenRows, allMeasurements); + oracle.model.setQuery(narrowQuery); + oracle.scheduler.flushAll(); + const reference = oracle.controller.getState(); + expect(reference.status.kind).toBe("ready"); + expect(after.rowHeights.rowCount).toBe(reference.rowHeights.rowCount); + const rankOffsets = ( + heights: (typeof after)["rowHeights"], + ): readonly number[] => + Array.from({ length: 5 }, (_, rank) => + heights.getOffsetForIndex(rank), + ); + expect(rankOffsets(after.rowHeights)).toEqual( + rankOffsets(reference.rowHeights), + ); + expect(after.totalHeight).toBe(reference.totalHeight); + }); + + test("a widening refilter ingests entrants under the estimate rule", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + // Only the survivors are measured, so the widening's entrants (rows + // 6..10) have NO retained measurement to restore and must take the + // estimate-or-default height. + for (const [rowId, height] of survivorMeasurements) { + controller.measure(data(rowId), height); + } + model.setQuery(narrowQuery); + expect(controller.getState().snapshot?.visibleRowCount).toBe(5); + const refilterSpy = vi.spyOn( + controller.getState().rowHeights, + "refilter", + ); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + + model.setQuery(wideQuery); + + const after = controller.getState(); + expect(after.status.kind).toBe("ready"); + expect(after.snapshot?.visibleRowCount).toBe(10); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount, + ); + expect(diagnostics.refilterPathCount).toBe( + before.refilterPathCount + 1, + ); + expect(refilterSpy).toHaveBeenCalledTimes(1); + const layout = heightIndexDiagnostics( + refilterSpy.mock.results[0]!.value, + ); + expect(layout.refilterEntriesReused).toBe(5); + expect(layout.refilterEntriesInserted).toBe(5); + + // Entrants estimate at the 44px default (short non-wrapping labels); + // survivors keep their measurements. + for (const row of tenRows) { + const rank = model.getState().snapshot.indexOf(data(row.id)); + expect(after.rowHeights.getHeight(rank)).toBe( + row.id <= 5 ? measuredHeightOf(row.id) : 44, + ); + } + const oracle = createReplacementOracle(tenRows, survivorMeasurements); + oracle.model.setQuery(narrowQuery); + oracle.scheduler.flushAll(); + oracle.model.setQuery(wideQuery); + oracle.scheduler.flushAll(); + const reference = oracle.controller.getState(); + expect(reference.status.kind).toBe("ready"); + const rankOffsets = ( + heights: (typeof after)["rowHeights"], + ): readonly number[] => + Array.from({ length: 10 }, (_, rank) => + heights.getOffsetForIndex(rank), + ); + expect(rankOffsets(after.rowHeights)).toEqual( + rankOffsets(reference.rowHeights), + ); + expect(after.totalHeight).toBe(reference.totalHeight); + }); + + test("an anchor row filtered out degrades like a full replacement", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + for (const [rowId, height] of allMeasurements) { + controller.measure(data(rowId), height); + } + // Anchor inside row 4 (rank 3, offsets 126..170 under the 41..50 + // measured heights), 4px below its top. `dropAnchorQuery` removes + // rows 1..4: the exact ref is GONE, and the replacement path's + // old-order neighbor search lands on row 5 — rank 0 in the filtered + // set — so the anchored viewport follows it to 0 + 4 = 4px. A path + // that skipped anchor restoration entirely would stay at the global + // 130. + controller.setViewport({ + scrollTop: 130, + viewportHeight: 88, + overscan: 0, + }); + expect(controller.getState().scrollTop).toBe(130); + + model.setQuery(dropAnchorQuery); + + const after = controller.getState(); + expect(after.status.kind).toBe("ready"); + expect(after.snapshot?.visibleRowCount).toBe(6); + expect(after.scrollTop).toBe(4); + expect( + getRowLayoutControllerDiagnosticsForTesting(controller) + .refilterPathCount, + ).toBe(1); + + const oracle = createReplacementOracle(tenRows, allMeasurements, { + scrollTop: 130, + viewportHeight: 88, + overscan: 0, + }); + oracle.model.setQuery(dropAnchorQuery); + oracle.scheduler.flushAll(); + expect(oracle.controller.getState().scrollTop).toBe(after.scrollTop); + }); + + test("a refilter reset with a misaligned revision falls back to replacement", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + controller.measure(data(2), 77); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + vi.spyOn(model, "changesSince").mockImplementation((revision) => ({ + kind: "reset" as const, + // One short of the committed revision: the range this reset claims + // to cover does not reach the snapshot the controller sees. + toRevision: revision, + reason: "refilter" as const, + })); + + model.setQuery(narrowQuery); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.visibleRowCount).toBe(5); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + expect(diagnostics.refilterPathCount).toBe(before.refilterPathCount); + expect(diagnostics.refilterFallbackCount).toBe( + before.refilterFallbackCount + 1, + ); + }); + + test("a refilter() throw falls back to replacement without an error publish", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + controller.measure(data(2), 77); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + const statuses: string[] = []; + controller.subscribe(() => { + statuses.push(controller.getState().status.kind); + }); + vi.spyOn( + controller.getState().rowHeights, + "refilter", + ).mockImplementation(() => { + throw new Error("injected refilter contract violation"); + }); + + model.setQuery(narrowQuery); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(statuses).not.toContain("error"); + expect(state.snapshot?.visibleRowCount).toBe(5); + expect(state.rowHeights.hasMeasurement(data(2))).toBe(true); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + expect(diagnostics.refilterPathCount).toBe(before.refilterPathCount); + expect(diagnostics.refilterFallbackCount).toBe( + before.refilterFallbackCount + 1, + ); + }); + + test('a "bulk-replace" reset still takes the replacement path', () => { + const oracle = createReplacementOracle(tenRows, [[2, 77]]); + const before = getRowLayoutControllerDiagnosticsForTesting( + oracle.controller, + ); + + oracle.model.setQuery(narrowQuery); + oracle.scheduler.flushAll(); + + const state = oracle.controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.visibleRowCount).toBe(5); + const diagnostics = getRowLayoutControllerDiagnosticsForTesting( + oracle.controller, + ); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + expect(diagnostics.refilterPathCount).toBe(before.refilterPathCount); + expect(diagnostics.refilterFallbackCount).toBe( + before.refilterFallbackCount, + ); + }); + + test("a refilter arriving mid-replacement restarts fail-closed", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + for (const [rowId, height] of allMeasurements) { + controller.measure(data(rowId), height); + } + // Retained state keeps the reset cooperative, so the refilter below + // really does arrive MID-replacement. + model.setRows(tenRows.map((row) => ({ ...row, label: "x" }))); + expect(controller.getState().status.kind).toBe("rebuilding"); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + + // This cycle's explicit scope decision: membership change + pending + // catch-up is exactly the complexity the reorder compose rule + // excluded, so a mid-replacement refilter restarts instead of + // composing. + model.setQuery(narrowQuery); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.visibleRowCount).toBe(5); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.refilterPathCount).toBe(before.refilterPathCount); + expect(diagnostics.reorderComposeCount).toBe( + before.reorderComposeCount, + ); + // The restart IS the observable: the interrupted replacement is + // abandoned and a fresh one runs against the filtered target. + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + }); + + test("a narrowing reuses survivors, retires leavers, re-measures none", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + for (const [rowId, height] of allMeasurements) { + controller.measure(data(rowId), height); + } + const refilterSpy = vi.spyOn( + controller.getState().rowHeights, + "refilter", + ); + + model.setQuery(narrowQuery); + + expect(refilterSpy).toHaveBeenCalledTimes(1); + const result = refilterSpy.mock.results[0]!.value as unknown; + const layout = heightIndexDiagnostics(result); + expect(layout.refilterEntriesReused).toBe(5); + expect(layout.refilterEntriesInserted).toBe(0); + expect(layout.refilterEntriesRetired).toBe(5); + // The published root IS the refilter result: nothing measured or + // re-ingested anything after the synchronous pass. + expect(controller.getState().rowHeights).toBe(result); + }); + }); }); }); diff --git a/packages/renderer-dom/src/__tests__/row-layout-dense.test.ts b/packages/renderer-dom/src/__tests__/row-layout-dense.test.ts new file mode 100644 index 000000000..d213fd6b9 --- /dev/null +++ b/packages/renderer-dom/src/__tests__/row-layout-dense.test.ts @@ -0,0 +1,553 @@ +import { describe, expect, test } from "vitest"; + +import { createColumnHelper, type PretableVisibleRowRef } from "@pretable/core"; +import { createLocalRowModel } from "@pretable-internal/row-model"; + +import { + createRowLayoutController, + getRowLayoutControllerDiagnosticsForTesting, + type RowLayoutScheduler, +} from "../row-layout-controller"; + +/** + * The dense-identity layout seam (Amendment I): a FLAT row-model snapshot + * supplies model slots (`ɵvisibleSlotRange` / `ɵslotCapacity` / + * `ɵslotOfRowId`), so the controller builds the height index in the DENSE + * lane — slot-keyed sources, slot-stamped operations, and slot-pooled frozen + * row refs — while a grouped snapshot falls back to today's string-identity + * shape wholesale. + */ + +type Row = { + id: number | string; + team: string; + score: number; + label: string; +}; + +const helper = createColumnHelper(); +const modelColumns = [ + helper.accessor("team", { type: "text" }), + helper.accessor("score", { type: "number", aggregate: "sum" }), + helper.accessor("label", { type: "text" }), +] as const; +const renderColumns = [ + { id: "label", header: "Label", wrap: true, widthPx: 90 }, + { id: "score", header: "Score", widthPx: 80 }, +] as const; + +const data = (rowId: Row["id"]): PretableVisibleRowRef => ({ + kind: "data", + rowId, +}); + +class ManualScheduler implements RowLayoutScheduler { + readonly tasks: Array<{ task: () => void; cancelled: boolean }> = []; + + schedule(task: () => void): () => void { + const entry = { task, cancelled: false }; + this.tasks.push(entry); + return () => { + entry.cancelled = true; + }; + } + + flushOne(): boolean { + const entry = this.tasks.shift(); + if (!entry) return false; + if (!entry.cancelled) entry.task(); + return true; + } + + flushAll(limit = 10_000): void { + let count = 0; + while (this.flushOne()) { + count += 1; + if (count > limit) throw new Error("Manual scheduler did not settle."); + } + } +} + +const tenRows: readonly Row[] = Array.from({ length: 10 }, (_, index) => ({ + id: `r${index}`, + team: index % 2 === 0 ? "A" : "B", + score: index, + label: `row ${index}`, +})); + +function createModel( + rows: readonly Row[], + options: { readonly grouped?: boolean } = {}, +) { + return createLocalRowModel({ + rows, + columns: modelColumns, + initialExpansion: { kind: "expanded" }, + query: { + filters: [], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: options.grouped ? [{ columnId: "team" }] : [], + }, + }); +} + +function createReadyController( + model: ReturnType, + scheduler = new ManualScheduler(), +) { + const controller = createRowLayoutController({ + model, + columns: renderColumns, + viewport: { scrollTop: 0, viewportHeight: 88, overscan: 1 }, + scheduler, + now: () => 0, + budgetMs: 5, + maxUnitsPerSlice: 256, + }); + scheduler.flushAll(); + expect(controller.getState().status.kind).toBe("ready"); + return { controller, scheduler }; +} + +/** + * The lane probe: on a DENSE generation, layout-core refuses any operation + * that arrives without a `denseKey` (the fallback contract). A string-lane + * generation accepts the same operation. `apply` is persistent and throws + * before producing anything, so probing never perturbs the published index. + */ +function isDenseIndex( + controller: ReturnType["controller"], + ref: PretableVisibleRowRef, +): boolean { + const rowHeights = controller.getState().rowHeights; + try { + rowHeights.apply([{ kind: "update", ref, index: 0 }]); + return false; + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch(/dense/i); + return true; + } +} + +/** + * Wraps a model so every published snapshot counts its per-row and bulk + * reads. Wrapper identity is memoized per underlying snapshot: the + * controller compares snapshots by reference across wakes. + */ +function instrumentModel(model: ReturnType) { + type ModelSnapshot = ReturnType< + ReturnType["getState"] + >["snapshot"]; + const counters = { + rowAt: 0, + rangeCalls: [] as Array, + slotRangeCalls: [] as Array, + }; + const wrapped = new WeakMap(); + const wrapSnapshot = (snapshot: ModelSnapshot): ModelSnapshot => { + const existing = wrapped.get(snapshot); + if (existing !== undefined) { + return existing; + } + // Spread rather than Proxy: published snapshots are frozen, and a Proxy + // over a frozen target may not report a different function for a + // non-configurable data property. + const slotRange = snapshot.ɵvisibleSlotRange?.bind(snapshot); + const instrumentedSnapshot = Object.freeze({ + ...snapshot, + rowAt(index: number) { + counters.rowAt += 1; + return snapshot.rowAt(index); + }, + range(start: number, end: number) { + counters.rangeCalls.push([start, end]); + return snapshot.range(start, end); + }, + ...(slotRange === undefined + ? {} + : { + ɵvisibleSlotRange(start: number, end: number) { + counters.slotRangeCalls.push([start, end]); + return slotRange(start, end); + }, + }), + }); + wrapped.set(snapshot, instrumentedSnapshot); + return instrumentedSnapshot; + }; + const instrumented = new Proxy(model, { + get(target, property, receiver) { + if (property === "getState") { + return () => { + const state = model.getState(); + return { ...state, snapshot: wrapSnapshot(state.snapshot) }; + }; + } + const value = Reflect.get(target, property, receiver) as unknown; + return typeof value === "function" + ? (value as (...args: unknown[]) => unknown).bind(target) + : value; + }, + }); + return { instrumented, counters }; +} + +describe("dense-identity layout seam", () => { + test("a flat snapshot builds a DENSE height index through bounded bulk range walks", () => { + // 300 rows: the build must cross a chunk boundary, so the walk shape — + // bounded bulk chunks, slots aligned, zero per-row descents — is + // actually exercised rather than collapsing into one tiny read. + const rows = Array.from({ length: 300 }, (_, index) => ({ + id: `r${index}`, + team: index % 2 === 0 ? "A" : "B", + score: index, + label: `row ${index}`, + })); + const model = createModel(rows); + const { instrumented, counters } = instrumentModel(model); + const { controller } = createReadyController( + instrumented as ReturnType, + ); + + // The replacement source materialized the visible set through chunked + // bulk `range` walks — bounded per call, never the whole dataset at + // once — with the slot reads aligned chunk-for-chunk, and the per-row + // `rowAt` rank descents are gone from the build path entirely. + const buildWalks = counters.rangeCalls.filter( + ([start, end]) => end - start > 12, + ); + expect(buildWalks).toEqual([ + [0, 256], + [256, 300], + ]); + expect( + counters.slotRangeCalls.filter(([start, end]) => end - start > 12), + ).toEqual([ + [0, 256], + [256, 300], + ]); + expect(counters.rowAt).toBe(0); + + // The published index is a dense generation: an operation without a + // `denseKey` is refused by layout-core's guard. + expect(isDenseIndex(controller, data("r0"))).toBe(true); + }); + + test("a grouped snapshot falls back to the string lane wholesale", () => { + const model = createModel(tenRows, { grouped: true }); + const { controller } = createReadyController(model); + const state = controller.getState(); + expect(state.snapshot?.ɵvisibleSlotRange?.(0, 1)).toBeUndefined(); + + // A group ref at index 0 (expanded grouping leads with a group row); the + // string lane accepts the un-keyed operation, proving no denseCapacity + // was declared. + const ref = state.window[0]!.ref; + expect(() => + state.rowHeights.apply([{ kind: "update", ref, index: 0 }]), + ).not.toThrow(); + }); + + test("incremental change operations are slot-stamped, so a dense index absorbs them without fallback", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + expect(isDenseIndex(controller, data("r0"))).toBe(true); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + + // An update (relabel), a remove (drop r3), and an insert (new r10), each + // an incremental "changes" sequence over the SAME dense index. Any + // missing denseKey would throw inside `apply` and restart a full + // replacement, which `replacementStartCount` would expose. The REMOVE is + // the deliberately hard one: the removed row's slot is only still bound + // in the PRE-change snapshot. + model.applyTransaction({ + update: tenRows.map((row) => ({ + id: row.id, + changes: { label: `${row.label}!` }, + })), + }); + expect(controller.getState().status.kind).toBe("ready"); + model.applyTransaction({ remove: ["r3"] }); + expect(controller.getState().status.kind).toBe("ready"); + model.applyTransaction({ + add: [{ id: "r10", team: "A", score: 10, label: "row 10" }], + }); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.visibleRowCount).toBe(10); + const diagnostics = getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount, + ); + // Still dense after the incremental ops. + expect(isDenseIndex(controller, data("r0"))).toBe(true); + }); + + test("a dense refilter round-trip keeps measurements with zero fallbacks", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + expect(isDenseIndex(controller, data("r0"))).toBe(true); + controller.measure(data("r0"), 61); + controller.measure(data("r1"), 62); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + + model.setQuery({ + filters: [{ columnId: "score", operator: "gt", value: 4 }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }); + expect(controller.getState().snapshot?.visibleRowCount).toBe(5); + model.setQuery({ + filters: [], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.visibleRowCount).toBe(10); + const diagnostics = getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.refilterPathCount).toBe(before.refilterPathCount + 2); + expect(diagnostics.refilterFallbackCount).toBe( + before.refilterFallbackCount, + ); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount, + ); + // The measured heights returned with their rows. + expect( + state.rowHeights.getHeight(state.snapshot!.indexOf(data("r0"))), + ).toBe(61); + expect( + state.rowHeights.getHeight(state.snapshot!.indexOf(data("r1"))), + ).toBe(62); + expect(isDenseIndex(controller, data("r0"))).toBe(true); + }); + + test("data-row refs are pooled by slot and reused across publications", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + const refOf = (rowId: Row["id"]) => + controller + .getState() + .window.find( + (row) => row.ref.kind === "data" && row.ref.rowId === rowId, + )?.ref; + const first = refOf("r0"); + expect(first).toBeDefined(); + + // Across a filter-only commit (dense refilter) … + model.setQuery({ + filters: [{ columnId: "score", operator: "lt", value: 5 }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }); + expect(controller.getState().status.kind).toBe("ready"); + expect(refOf("r0")).toBe(first); + + // … and across a FULL replacement (a column change rebuilds the index + // from a fresh source), the same frozen ref object is reused while the + // rowId still owns its slot. + controller.setColumns([ + { id: "label", wrap: true, widthPx: 140 }, + { id: "score", widthPx: 80 }, + ]); + scheduler.flushAll(); + expect(controller.getState().status.kind).toBe("ready"); + expect(refOf("r0")).toBe(first); + }); + + test("a staged measurement for a filtered-out row is retained through a dense restart", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + expect(isDenseIndex(controller, data("r0"))).toBe(true); + + // Retained state keeps the column-change reset cooperative (a bare mount + // base would complete it inline and nothing would ever be staged). + controller.measure(data("r0"), 50); + // Open a replacement (column change), stage a measurement while it is in + // flight, then filter the measured row OUT — a mid-replacement refilter + // fails closed into a restart whose staged replay must retain the + // now-absent row's measurement on the DENSE candidate (slot-keyed + // `retainMeasurement`). + controller.setColumns([ + { id: "label", wrap: true, widthPx: 140 }, + { id: "score", widthPx: 80 }, + ]); + controller.measure(data("r1"), 63); + model.setQuery({ + filters: [{ columnId: "score", operator: "gt", value: 4 }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }); + scheduler.flushAll(); + expect(controller.getState().status.kind).toBe("ready"); + expect(controller.getState().snapshot?.visibleRowCount).toBe(5); + + // Widen the filter back: r1 re-enters with its retained 63px, never + // re-measured. + model.setQuery({ + filters: [], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }); + scheduler.flushAll(); + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.rowHeights.hasMeasurement(data("r1"))).toBe(true); + expect( + state.rowHeights.getHeight(state.snapshot!.indexOf(data("r1"))), + ).toBe(63); + // The retention was honored ON the dense candidate (slot-keyed): the + // published index never dropped to the string lane, which is what an + // unstamped `retainMeasurement` would have forced. + expect(isDenseIndex(controller, data("r0"))).toBe(true); + }); + + test("a staged measurement for a permanently removed row drops the generation to the string lane, keeping retention", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + + // Same orchestration, but the measured row is REMOVED from the dataset + // entirely: its slot is released, so a dense candidate cannot retain the + // measurement — and must neither replay the refusal forever nor drop the + // retention. The amendment's escape hatch fires instead: this ONE + // generation falls back to the string lane, where retention is + // identity-keyed, and a later re-insert of the same rowId restores the + // measured height. + // + // Retained state keeps the column-change reset cooperative (a bare mount + // base would complete it inline and nothing would ever be staged). + controller.measure(data("r0"), 50); + controller.setColumns([ + { id: "label", wrap: true, widthPx: 140 }, + { id: "score", widthPx: 80 }, + ]); + controller.measure(data("r1"), 63); + model.setRows(tenRows.filter((row) => row.id !== "r1")); + scheduler.flushAll(); + const settled = controller.getState(); + expect(settled.status.kind).toBe("ready"); + expect(settled.snapshot?.visibleRowCount).toBe(9); + expect(settled.rowHeights.hasMeasurement(data("r1"))).toBe(true); + expect(isDenseIndex(controller, data("r0"))).toBe(false); + + model.applyTransaction({ + add: [{ id: "r1", team: "B", score: 1, label: "row 1 again" }], + }); + const reinserted = controller.getState(); + expect(reinserted.status.kind).toBe("ready"); + const rank = reinserted.snapshot!.indexOf(data("r1")); + expect(reinserted.rowHeights.getHeight(rank)).toBe(63); + expect(reinserted.rowHeights.hasMeasurement(data("r1"))).toBe(true); + + // The string lane lasts one generation: the next FULL replacement + // re-decides dense. + controller.setColumns([ + { id: "label", wrap: true, widthPx: 90 }, + { id: "score", widthPx: 80 }, + ]); + scheduler.flushAll(); + expect(controller.getState().status.kind).toBe("ready"); + expect(isDenseIndex(controller, data("r0"))).toBe(true); + }); + + test("a dense build whose source read throws drops the generation to the string lane, not a refusal loop", () => { + // The dense lane's bulk `range` chunks are authorized by the ɵ-seam + // probe, and that inference is conventional: a spread-based snapshot + // wrapper carries the ɵ members through while its OWN bounded-read guard + // still refuses a chunk-wide `range`. Such a wrapper must cost exactly + // one restart onto the string lane (whose per-row `rowAt` shape every + // structural wrapper supports) — never a dense→refuse→dense livelock, + // and never a dead grid. + const manyRows: readonly Row[] = Array.from( + { length: 300 }, + (_, index) => ({ + id: `r${index}`, + team: index % 2 === 0 ? "A" : "B", + score: index, + label: `row ${index}`, + }), + ); + const model = createModel(manyRows); + // Capped BELOW the controller's chunk size (`maxUnitsPerSlice`, 256 + // below), so the probe's slot read succeeds but the first bulk `range` + // chunk throws. `undefined` lifts the guard. + let rangeCap: number | undefined = 64; + type ModelSnapshot = ReturnType< + ReturnType["getState"] + >["snapshot"]; + const wrapped = new WeakMap(); + const wrapSnapshot = (snapshot: ModelSnapshot): ModelSnapshot => { + const existing = wrapped.get(snapshot); + if (existing !== undefined) return existing; + const guarded = Object.freeze({ + ...snapshot, + range(start: number, end: number) { + if (rangeCap !== undefined && end - start > rangeCap) { + throw new RangeError( + `bounded-read guard: range ${start}..${end} exceeds ${rangeCap}`, + ); + } + return snapshot.range(start, end); + }, + }); + wrapped.set(snapshot, guarded); + return guarded; + }; + const guardedModel = new Proxy(model, { + get(target, property, receiver) { + if (property === "getState") { + return () => { + const state = model.getState(); + return { ...state, snapshot: wrapSnapshot(state.snapshot) }; + }; + } + const value = Reflect.get(target, property, receiver) as unknown; + return typeof value === "function" + ? (value as (...args: unknown[]) => unknown).bind(target) + : value; + }, + }); + const scheduler = new ManualScheduler(); + const controller = createRowLayoutController({ + model: guardedModel, + columns: renderColumns, + viewport: { scrollTop: 0, viewportHeight: 88, overscan: 1 }, + scheduler, + now: () => 0, + budgetMs: 5, + maxUnitsPerSlice: 256, + }); + scheduler.flushAll(); + + // The mount SUCCEEDED — on the string lane, after exactly one restart + // (the dense attempt plus its string-lane replay; a livelock would trip + // the manual scheduler's settle limit long before this assertion). + const mounted = controller.getState(); + expect(mounted.status.kind).toBe("ready"); + expect(mounted.snapshot?.visibleRowCount).toBe(300); + expect(isDenseIndex(controller, data("r0"))).toBe(false); + expect( + getRowLayoutControllerDiagnosticsForTesting(controller) + .replacementStartCount, + ).toBe(2); + + // The escape hatch lasts ONE generation: with the guard lifted, the next + // full replacement re-decides dense. + rangeCap = undefined; + controller.setColumns([ + { id: "label", wrap: true, widthPx: 140 }, + { id: "score", widthPx: 80 }, + ]); + scheduler.flushAll(); + expect(controller.getState().status.kind).toBe("ready"); + expect(isDenseIndex(controller, data("r0"))).toBe(true); + expect( + getRowLayoutControllerDiagnosticsForTesting(controller) + .replacementStartCount, + ).toBe(3); + }); +}); diff --git a/packages/renderer-dom/src/row-layout-controller.ts b/packages/renderer-dom/src/row-layout-controller.ts index d4e3743b7..538412065 100644 --- a/packages/renderer-dom/src/row-layout-controller.ts +++ b/packages/renderer-dom/src/row-layout-controller.ts @@ -82,6 +82,18 @@ export interface RowLayoutControllerDiagnostics { * finding. */ readonly reorderComposeFallbackCount: number; + /** Filter-only commits absorbed by refiltering the height index in place. */ + readonly refilterPathCount: number; + /** + * Refilter resets that ended in a full replacement anyway — a misaligned + * revision, or a `refilter()` contract violation, which since Amendment I + * includes every DENSE-lane refusal (a missing/malformed/out-of-range + * `denseKey`): layout-core throws, the dispatch catches, and the count + * advances — making this the honest signal that the dense fast path + * actually ran. Expected 0 on the happy path; a nonzero count under a + * bench run is a finding. + */ + readonly refilterFallbackCount: number; readonly pendingCatchUpChangeSetCount: number; readonly pendingCatchUpOperationCount: number; readonly retainedCatchUpSnapshotCount: number; @@ -294,6 +306,38 @@ function rowRef( : Object.freeze({ kind: "group" as const, groupId: row.groupId }); } +/** + * The dense-lane op stamp (Amendment I): the model slot currently bound to a + * data ref, or `undefined` for group refs, grouped roots, and permanently + * removed rows. One HAMT get per call — k-sized paths only (op stamping and + * staged-measurement replay), never a per-visible-row walk; the bulk paths + * read `ɵvisibleSlotRange` instead. An `undefined` on a DENSE index is + * layout-core's problem to refuse (its lifecycle throw), and the throw lands + * in this controller's existing fallback-on-throw handling. + */ +function denseKeyFor< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +>( + snapshot: PretableRowModelSnapshot, + ref: PretableVisibleRowRef, +): number | undefined { + return ref.kind === "data" ? snapshot.ɵslotOfRowId?.(ref.rowId) : undefined; +} + +/** + * Layout-core deliberately keeps `RowHeightReplacementLifecycleError` off its + * barrel, so the one place this controller must DISTINGUISH a dense-contract + * refusal from a genuine failure matches it by name. + */ +function isDenseContractRefusal(error: unknown): boolean { + return ( + error instanceof Error && + error.name === "RowHeightReplacementLifecycleError" + ); +} + function identityOf( ref: PretableVisibleRowRef, ): string { @@ -397,6 +441,14 @@ interface ActiveReplacement< readonly token: object; readonly baseTarget: PretableRowModelSnapshot; readonly builder: RowHeightReplacementBuilder>; + /** + * Whether this replacement's source declared `denseCapacity` — i.e. the + * builder is ingesting through the DENSE lane's chunked bulk reads. Read by + * `runReplacementSlice`'s builder-phase catch, which routes a dense + * source-read throw to the string-lane escape hatch instead of the generic + * failure path. + */ + readonly dense: boolean; latestTarget: PretableRowModelSnapshot; capturedRevision: number; capturedWakeVersion: number; @@ -543,6 +595,29 @@ export function createRowLayoutController< } return Math.max(defaultRowHeight, height); }; + // Slot-indexed pool of frozen data-row refs (Amendment I / spec M5). A ref + // is a frozen value object compared everywhere via `identityOf`/`sameRef` — + // verified: no consumer compares refs by allocation identity — so a slot's + // ref is allocated once and reused across every source entry and window row + // while the model keeps that rowId bound to the slot. Group refs are not + // pooled (group rows carry no slot), and paths with no slot in hand (the + // k-sized anchor searches) keep allocating; only the n-sized walks pool. + // The pool never shrinks: it is proportional to slot capacity, and a + // rebound slot simply overwrites its cell. + const pooledDataRefs: Array< + Extract, { kind: "data" }> | undefined + > = []; + const pooledRowRef = ( + row: PretableVisibleRow, + slot: number | undefined, + ): PretableVisibleRowRef => { + if (row.kind !== "data" || slot === undefined) return rowRef(row); + const pooled = pooledDataRefs[slot]; + if (pooled !== undefined && pooled.rowId === row.rowId) return pooled; + const created = Object.freeze({ kind: "data" as const, rowId: row.rowId }); + pooledDataRefs[slot] = created; + return created; + }; const listeners = new Set<() => void>(); let disposed = false; let notifying = false; @@ -607,6 +682,8 @@ export function createRowLayoutController< let reorderFallbackCount = 0; let reorderComposeCount = 0; let reorderComposeFallbackCount = 0; + let refilterPathCount = 0; + let refilterFallbackCount = 0; let catchUpUnits = 0; let maxCatchUpUnitsPerSlice = 0; let deferredViewportWithoutAnchor = false; @@ -815,10 +892,17 @@ export function createRowLayoutController< "The row-model range did not match its published visible count.", ); } + // Aligned index-for-index with `rows`; `undefined` wholesale on a + // grouped root, which is exactly the string-lane case where the + // estimate operations below need no `denseKey` either. + const slots = snapshot.ɵvisibleSlotRange?.( + plan.range.start, + plan.range.end, + ); const estimates: RowHeightOperation>[] = []; for (let offset = 0; offset < rows.length; offset += 1) { const row = rows[offset]!; - const ref = rowRef(row); + const ref = pooledRowRef(row, slots?.[offset]); const index = plan.range.start + offset; const indexedRef = root.keyAt(index); if (indexedRef === undefined || !sameRef(indexedRef, ref)) { @@ -851,6 +935,10 @@ export function createRowLayoutController< // row never had. estimatedHeight: lastMeasuredHeights.get(identity) ?? estimate(row.row), + // On a dense root this update must carry the row's slot (the + // guard verifies it against the entry's stamped slot); on a + // string root the key is ignored. + denseKey: slots?.[offset], }); } } @@ -864,7 +952,7 @@ export function createRowLayoutController< const window = plan.rows.map((geometry, offset) => Object.freeze({ ...geometry, - ref: rowRef(rows[offset]!), + ref: pooledRowRef(rows[offset]!, slots?.[offset]), row: rows[offset]!, }), ); @@ -891,6 +979,14 @@ export function createRowLayoutController< readonly isCurrent: () => boolean; readonly commit: () => void; }, + // The status the publication carries. Every completing publish stamps + // READY; the one caller that overrides this is the mid-replacement + // viewport republish in `setViewport`, which repaints the STALE snapshot + // at a new scroll position while a rebuild toward a newer revision is + // still in flight — flipping to READY there would hide the in-progress + // rebuild from status consumers for its whole remaining duration, since + // nothing re-stamps "rebuilding" until the next `startReplacement`. + status: RowLayoutControllerState["status"] = READY, ): void => { projecting = true; try { @@ -917,7 +1013,7 @@ export function createRowLayoutController< window: prepared.window, totalHeight: prepared.totalHeight, leadingHeight: prepared.leadingHeight, - status: READY, + status, }); } finally { projecting = false; @@ -953,13 +1049,32 @@ export function createRowLayoutController< root: RowHeightIndex>, operation: PretableChangeOperation, revision: number, + // The snapshots the op's `denseKey` resolves against, BEFORE-first: a + // REMOVED row's slot is only still bound in the pre-change snapshot + // (release frees it), while an INSERTED row's binding exists only in the + // post-change one; updates and moves are bound in both, to the same slot. + // A row that exists in neither — inserted and removed inside one queued + // stretch — resolves to `undefined`, and a WRONG resolution (a reused + // slot) is refused by layout-core's dup/drift guards; either way the + // throw lands in this call site's existing fallback handling (the + // catch-up restart, or `synchronize`'s startReplacement catch). + slotsBefore: PretableRowModelSnapshot | undefined, + slotsAfter: PretableRowModelSnapshot, ): RowHeightIndex> => { + const denseKey = + operation.ref.kind === "data" + ? ((slotsBefore === undefined + ? undefined + : denseKeyFor(slotsBefore, operation.ref)) ?? + denseKeyFor(slotsAfter, operation.ref)) + : undefined; let heightOperation: RowHeightOperation>; if (operation.kind === "insert") { heightOperation = { kind: "insert", ref: operation.ref, index: operation.index, + denseKey, }; } else if (operation.kind === "remove") { // A removed row will never be looked up again, so its retained height is @@ -979,6 +1094,7 @@ export function createRowLayoutController< kind: "remove", ref: operation.ref, previousIndex: operation.previousIndex, + denseKey, }; } else if (operation.kind === "move") { heightOperation = { @@ -986,12 +1102,14 @@ export function createRowLayoutController< ref: operation.ref, previousIndex: operation.previousIndex, index: operation.index, + denseKey, }; } else { heightOperation = { kind: "update", ref: operation.ref, index: operation.index, + denseKey, }; const identity = identityOf(operation.ref); const staged = stagedMeasurements.get(identity); @@ -1154,13 +1272,36 @@ export function createRowLayoutController< const sliceStartedAt = now(); let builderUnitsThisSlice = 0; if (!replacement.builder.done) { - const progress = replacement.builder.advance({ - maxUnits: maxUnitsPerSlice, - deadline: ignoreDeadline - ? Number.MAX_VALUE - : sliceStartedAt + budgetMs, - now, - }); + let progress: ReturnType; + try { + progress = replacement.builder.advance({ + maxUnits: maxUnitsPerSlice, + deadline: ignoreDeadline + ? Number.MAX_VALUE + : sliceStartedAt + budgetMs, + now, + }); + } catch (error) { + if (active !== replacement || disposed) return; + if (!replacement.dense) throw error; + // A DENSE build's source reads are CHUNKED bulk walks + // (`replacementSourceOf`'s `range` / `ɵvisibleSlotRange` chunks), + // resolved lazily as the builder's `advance` ingests. The lane + // probe that authorized them reads only the slot seam, so its + // inference — "the ɵ members answer, therefore unbounded-ish bulk + // reads are safe" — is conventional, not structural: a spread-based + // snapshot wrapper carries the ɵ members through while its own + // bounded-read guard still refuses a chunk-wide `range`. Left to + // the generic error handling, that throw would either kill the + // grid or — via a refilter fallback — re-decide DENSE and refuse + // again. It is the amendment's escape-hatch case instead, the same + // one `retainMeasurement` takes for a permanently removed row: + // input the dense lane cannot read drops this ONE generation to + // the string lane, whose per-row `rowAt` shape every structural + // wrapper supports; the next full replacement re-decides dense. + startReplacement(replacement.latestTarget, true, false); + return; + } builderUnitsThisSlice = progress.unitsThisSlice; maxReplacementUnitsPerSlice = Math.max( maxReplacementUnitsPerSlice, @@ -1208,6 +1349,8 @@ export function createRowLayoutController< replacement.candidate!, operation, change.revision, + replacement.baseTarget, + replacement.latestTarget, ); replacement.pendingOperationIndex += 1; replacement.pendingOperationCount -= 1; @@ -1268,10 +1411,33 @@ export function createRowLayoutController< measurement.height, ); } else { - replacement.candidate = replacement.candidate!.retainMeasurement( - measurement.ref, - measurement.height, - ); + // A dense candidate keys its visible-check by slot, so the + // retention carries the row's CURRENT slot. A row that has been + // permanently REMOVED since it was measured has no slot left to + // offer (its release freed the binding, and a stale slot may + // already name another row), so layout-core REFUSES the + // retention — and that refusal must not escape into the slice's + // generic restart, which would replay this same staged entry + // against another dense candidate and refuse forever. It is the + // amendment's fallback case instead: input that cannot supply a + // dense key drops the WHOLE generation to the string lane, + // where retention is identity-keyed and a later re-insert of + // the same rowId still restores the measured height (pinned + // behavior). Any other throw — including the visible-row drift + // guard, which a rebuild against the live model self-heals — + // keeps its existing path. + try { + replacement.candidate = + replacement.candidate!.retainMeasurement( + measurement.ref, + measurement.height, + denseKeyFor(stagedTarget, measurement.ref), + ); + } catch (error) { + if (!isDenseContractRefusal(error)) throw error; + startReplacement(replacement.latestTarget, true, false); + return; + } } stagedMeasurements.set(stagedKey, { ...measurement, @@ -1417,40 +1583,156 @@ export function createRowLayoutController< }; /** - * The `{rowCount, entryAt}` source both re-ingest paths hand the height - * index: `startReplacement` feeds it to `beginReplacement`, and the sort-only - * permutation path feeds the SAME shape to `reorder`, so a snapshot that - * omits a visible row fails identically on either path. Reads - * `visibleRowCount` eagerly — construct inside a try. + * The `{rowCount, entryAt}` source every re-ingest path hands the height + * index: `startReplacement` feeds it to `beginReplacement`, and the + * sort-only/filter-only paths feed the SAME shape to `reorder`/`refilter`, + * so a snapshot that omits a visible row fails identically on any path. + * Reads `visibleRowCount` and the slot seam's lane decision eagerly — + * construct inside a try. + * + * On the DENSE lane the visible set is materialized through CHUNKED + * `range` walks — one `maxUnitsPerSlice`-sized bulk read per chunk, + * resolved lazily the first time `entryAt` lands in it — and `entryAt` + * indexes the cached chunk. The per-row `rowAt` alternative is an O(log n) + * rank descent EACH, which at 50k rows is the difference between a linear + * source and an O(n log n) one; chunking keeps that win while every + * snapshot read stays bounded per call and the cooperative build stays + * lazy — a 100k mount never materializes the dataset inside one + * synchronous construction. A short (lying) chunk surfaces as the same + * omitted-row error the per-row shape threw. + * + * The STRING lane keeps the per-row `rowAt` shape VERBATIM, on purpose: + * string-lane snapshots include structural wrappers (react's bounded-read + * guards among them) whose `range` legitimately refuses spans wider than + * their own window, while a snapshot that supplies the `ɵ` slot seam is by + * construction a real model snapshot, where the bulk walk is safe. Do not + * "optimize" the string lane onto the bulk walk — that is exactly the + * regression the react indexed suite's poisoned models exist to catch. + * + * DENSE LANE (Amendment I): when the snapshot supplies the internal slot + * seam — `ɵvisibleSlotRange` aligned with `range`, plus `ɵslotCapacity` — + * the source declares `denseCapacity` and stamps every entry's `denseKey`, + * which is what makes the built generation dense (slot-bitset membership, + * slot-indexed refilter/reorder). The lane is decided ONCE per source, off + * `ɵslotCapacity` plus a first-chunk slot read; Task 4's all-or-nothing + * contract makes that probe authoritative. A grouped root — or a + * structural snapshot wrapper that does not forward the `ɵ` seam — answers + * `undefined` and the source is today's string-identity shape verbatim; + * layout-core refuses a HALF-dense source (declared capacity, missing key) + * with a throw that the dispatch sites already route to the + * full-replacement fallback. + * + * `denseAllowed: false` forces the string-identity shape even on a flat + * root — the amendment's wholesale escape hatch, taken when input that + * cannot supply a slot must still be honored. Its takers, registered here + * so the set stays deliberate: + * + * - a staged measurement for a permanently REMOVED row, whose released + * slot no longer exists (`retainMeasurement`'s refusal in the catch-up + * replay); + * - a DENSE build whose source read throws (`runReplacementSlice`'s + * builder-phase catch) — a snapshot that carries the `ɵ` seam through a + * structural wrapper whose own guard refuses the chunk-wide `range`. + * + * The lane is re-decided at the next full replacement. */ const replacementSourceOf = ( target: PretableRowModelSnapshot, - ): RowHeightReplacementSource> => ({ - rowCount: target.visibleRowCount, - entryAt(index) { - const row = target.rowAt(index); + denseAllowed = true, + ): RowHeightReplacementSource> => { + const rowCount = target.visibleRowCount; + const chunkSize = maxUnitsPerSlice; + const rowChunks: Array< + readonly PretableVisibleRow[] | undefined + > = []; + const slotChunks: Array = []; + const capacity = denseAllowed ? target.ɵslotCapacity?.() : undefined; + // The lane probe reads ONLY the slot seam (never `range`): a snapshot + // that answers it is a real model snapshot, and only then is the bulk + // `range` walk below safe. Task 4's all-or-nothing contract makes the + // first-chunk probe authoritative for the whole root. + const firstSlots = + capacity !== undefined && rowCount > 0 + ? target.ɵvisibleSlotRange?.(0, Math.min(chunkSize, rowCount)) + : undefined; + if (firstSlots !== undefined) slotChunks[0] = firstSlots; + const dense = + capacity !== undefined && (rowCount === 0 || firstSlots !== undefined); + const rowAt = ( + index: number, + ): { + readonly row: PretableVisibleRow; + readonly slot: number | undefined; + } => { + const chunkIndex = Math.floor(index / chunkSize); + let chunk = rowChunks[chunkIndex]; + if (chunk === undefined) { + const start = chunkIndex * chunkSize; + const end = Math.min(rowCount, start + chunkSize); + chunk = target.range(start, end); + rowChunks[chunkIndex] = chunk; + if (dense && slotChunks[chunkIndex] === undefined) { + slotChunks[chunkIndex] = target.ɵvisibleSlotRange?.(start, end); + } + } + const row = chunk[index - chunkIndex * chunkSize]; if (row === undefined) { throw new RowLayoutControllerError( "layout-failed", `The row-model snapshot omitted visible row ${index}.`, ); } - return { key: rowRef(row) }; - }, - }); + return { + row, + slot: slotChunks[chunkIndex]?.[index - chunkIndex * chunkSize], + }; + }; + if (!dense) { + return { + rowCount, + entryAt(index) { + const row = target.rowAt(index); + if (row === undefined) { + throw new RowLayoutControllerError( + "layout-failed", + `The row-model snapshot omitted visible row ${index}.`, + ); + } + return { key: rowRef(row) }; + }, + }; + } + return { + rowCount, + denseCapacity: capacity, + entryAt(index) { + const { row, slot } = rowAt(index); + return { key: pooledRowRef(row, slot), denseKey: slot }; + }, + }; + }; const startReplacement = ( target: PretableRowModelSnapshot, shouldNotify: boolean, + // See `replacementSourceOf`: `false` forces this one generation onto the + // string lane. Restarts and later replacements default back to `true`, + // which is safe because the staged entry that demanded the string lane is + // consumed by the very replacement this flag builds — and a superseded + // string replacement simply re-trips the same escape hatch on replay. + denseAllowed = true, ): void => { cancelActive(); replacementStartCount += 1; const anchor = deferredViewportWithoutAnchor ? undefined : captureAnchor(); let builder: RowHeightReplacementBuilder>; let targetRevision: number; + let dense: boolean; try { targetRevision = target.revision; - builder = state.rowHeights.beginReplacement(replacementSourceOf(target)); + const source = replacementSourceOf(target, denseAllowed); + dense = source.denseCapacity !== undefined; + builder = state.rowHeights.beginReplacement(source); } catch (error) { clearStagedMeasurements(); rollbackDeferredViewport(); @@ -1465,6 +1747,7 @@ export function createRowLayoutController< token: {}, baseTarget: target, builder, + dense, latestTarget: target, capturedRevision: targetRevision, capturedWakeVersion: modelWakeVersion, @@ -1595,6 +1878,13 @@ export function createRowLayoutController< reorderComposeFallbackCount += 1; return false; } + // A "refilter" reset lands here deliberately and fails the changes + // check below into a restart: composing a MEMBERSHIP change into an + // active replacement (entrants/leavers over pending index-based + // catch-up) is exactly the complexity the reorder FINAL-retarget rule + // excluded, so mid-replacement refilters are fail-closed this cycle — + // an explicit scope decision, observable as `replacementStartCount` + // advancing while `refilterPathCount` does not. if ( sequence.kind !== "changes" || sequence.fromRevision !== replacement.capturedRevision || @@ -1627,11 +1917,18 @@ export function createRowLayoutController< const applyChanges = ( sequence: Extract, { kind: "changes" }>, + target: PretableRowModelSnapshot, ): RowHeightIndex> => { let root = state.rowHeights; for (const change of sequence.changes) { for (const operation of change.operations) { - root = applyOperation(root, operation, change.revision); + root = applyOperation( + root, + operation, + change.revision, + state.snapshot ?? undefined, + target, + ); } } return root; @@ -1645,13 +1942,66 @@ export function createRowLayoutController< * drift; the cooperative replacement path implements the same resolution * against its own staged candidate in `finishReplacement`. */ + /** + * The cooperative replacement's OLD-order neighbor search, run to + * completion synchronously: when the exact anchor ref no longer resolves — + * a membership change filtered the anchored row out — probe the old + * snapshot's neighbors at alternating distances (+1, -1, +2, ...) until one + * survives into `target`, exactly the order `runReplacementSlice` probes + * in. On a flat snapshot `nearestVisibleRef` is exact-or-undefined (no + * logical-neighbor resolution of its own), so without this search a + * filtered-out anchor would silently degrade to a global scroll while the + * replacement path anchors a neighbor — divergent UX for the same commit. + */ + const resolveSyncAnchorRef = ( + target: PretableRowModelSnapshot, + anchor: CapturedAnchor, + ): PretableVisibleRowRef | undefined => { + const exact = target.nearestVisibleRef(anchor.heightAnchor.ref); + if (exact !== undefined && target.indexOf(exact) >= 0) return exact; + let searchPrevious = false; + let searchDistance = 1; + for (;;) { + const candidateIndex = searchPrevious + ? anchor.oldIndex - searchDistance + : anchor.oldIndex + searchDistance; + searchPrevious = !searchPrevious; + if (!searchPrevious) searchDistance += 1; + if ( + candidateIndex >= 0 && + candidateIndex < anchor.oldSnapshot.visibleRowCount + ) { + const oldRow = anchor.oldSnapshot.rowAt(candidateIndex); + if (oldRow !== undefined) { + const resolved = target.nearestVisibleRef(rowRef(oldRow)); + if (resolved !== undefined && target.indexOf(resolved) >= 0) { + return resolved; + } + } + } + if ( + anchor.oldIndex - searchDistance < 0 && + anchor.oldIndex + searchDistance >= anchor.oldSnapshot.visibleRowCount + ) { + return undefined; + } + } + }; + const restoreAnchorRequest = ( target: PretableRowModelSnapshot, root: RowHeightIndex>, anchor: CapturedAnchor | undefined, + // The incremental journal path keeps exact-only resolution (its remove + // operations already re-anchor via the surviving exact ref or fall to a + // global scroll — pinned behavior); the synchronous reset paths opt into + // the replacement-mirroring neighbor search above. + searchOldNeighbors = false, ): ScrollRequest => { if (anchor !== undefined) { - const resolved = target.nearestVisibleRef(anchor.heightAnchor.ref); + const resolved = searchOldNeighbors + ? resolveSyncAnchorRef(target, anchor) + : target.nearestVisibleRef(anchor.heightAnchor.ref); if (resolved !== undefined) { const index = target.indexOf(resolved); if (index >= 0) { @@ -1711,48 +2061,74 @@ export function createRowLayoutController< let sequence: PretableChangeSequence; try { sequence = options.model.changesSince(state.observedRevision); - if (sequence.kind === "reset" && sequence.reason === "reorder") { - // A sort-only commit: the visible row SET and every height-relevant - // fact are unchanged, only the order moved, so the height index is - // permuted synchronously instead of re-ingested row by row. The - // reset carries no `fromRevision` — `changesSince` was called with - // `state.observedRevision`, so the range's start is pinned by the - // argument and only the target side needs to line up. + if ( + sequence.kind === "reset" && + (sequence.reason === "reorder" || sequence.reason === "refilter") + ) { + // A sort-only commit ("reorder": same row SET, new order) or a + // filter-only commit ("refilter": same relative order among + // survivors, changed membership): the height index absorbs either + // synchronously — `reorder` permutes existing entries, `refilter` + // reuses survivors, ingests entrants under the estimate rule, and + // retires leavers — instead of re-ingesting row by row through a + // cooperative replacement. The reset carries no `fromRevision` — + // `changesSince` was called with `state.observedRevision`, so the + // range's start is pinned by the argument and only the target + // side needs to line up. + // + // POLICY (measured, decided): ALL refilter resets route through + // `refilter()` regardless of direction. A narrowing runs + // 15-26ms@50k; a WIDENING runs 57-80ms because entrant ingest is + // irreducible — still taken, because one synchronous pass + // eliminates the multi-slice replacement interval (the window the + // blank-viewport defect lived in), and a direction threshold buys + // milliseconds at the price of two code paths. // // No staged/pending lifecycle: this path runs only when no // replacement is active (the `active` branch above owns everything - // else), the permutation is synchronous, and with no active - // replacement `measure` applies immediately, so the staged - // measurement queue is empty and stays untouched. + // else), the pass is synchronous, and with no active replacement + // `measure` applies immediately, so the staged measurement queue + // is empty and stays untouched. // - // ANY doubt — misaligned revision, a `reorder()` contract - // violation, a publish failure — falls back to the full - // replacement. The fallback IS the error handling; nothing here - // publishes an error state of its own. + // ANY doubt — misaligned revision, an index contract violation, a + // publish failure — falls back to the full replacement. The + // fallback IS the error handling; nothing here publishes an error + // state of its own. + const reason = sequence.reason; if (sequence.toRevision === target.revision) { try { const anchor = deferredViewportWithoutAnchor ? undefined : captureAnchor(); - const root = state.rowHeights.reorder( - replacementSourceOf(target), - ); + const source = replacementSourceOf(target); + const root = + reason === "reorder" + ? state.rowHeights.reorder(source) + : state.rowHeights.refilter(source); publishReady( target, root, - restoreAnchorRequest(target, root, anchor), + // The neighbor search matters only under "refilter": a + // membership change may have filtered the anchored row out, + // and the replacement path would re-anchor its nearest + // OLD-order neighbor. Under "reorder" the exact ref always + // survives and the search never engages. + restoreAnchorRequest(target, root, anchor, true), ); // Mirrors `finishReplacement`'s commit: a deferred viewport is // applied by the publish above (it reads the live `viewport`), // so the flag must not survive into the next capture. deferredViewportWithoutAnchor = false; - reorderPathCount += 1; + if (reason === "reorder") reorderPathCount += 1; + else refilterPathCount += 1; } catch { - reorderFallbackCount += 1; + if (reason === "reorder") reorderFallbackCount += 1; + else refilterFallbackCount += 1; startReplacement(target, true); } } else { - reorderFallbackCount += 1; + if (reason === "reorder") reorderFallbackCount += 1; + else refilterFallbackCount += 1; startReplacement(target, true); } continue; @@ -1769,7 +2145,7 @@ export function createRowLayoutController< } try { const previousAnchor = captureAnchor(); - const root = applyChanges(sequence); + const root = applyChanges(sequence, target); publishReady( target, root, @@ -1929,8 +2305,44 @@ export function createRowLayoutController< } viewport = normalized; if (active !== undefined) { + // The replacement's finish must honor this request as a GLOBAL + // scroll: the anchor was captured at the old position, and restoring + // it would snap the viewport back there. active.anchor = undefined; deferredViewportWithoutAnchor = true; + // Deferring alone is not enough: the last publication planned its + // window for the OLD scroll position, so if the new scrollTop lands + // outside it the grid is blank until `finishReplacement` — ~150ms+ of + // empty viewport on a 50k-row rebuild. Republish a window from the + // CURRENT (stale) snapshot and heights at the new scrollTop — the + // same stale-but-visible behavior the no-replacement branch below + // provides. The replacement lifecycle is untouched: no + // active/candidate/staged mutation, `observedRevision` stays the old + // snapshot's revision (the publish target IS the old snapshot), and + // the status keeps reporting the in-flight rebuild. + if (state.snapshot === null) return; + try { + publishReady( + state.snapshot, + state.rowHeights, + globalScroll(viewport.scrollTop), + undefined, + state.status, + ); + } catch (error) { + // Same surface as the branch below: the consumer learns the window + // could not be planned instead of silently staring at a stale (or + // blank) viewport. Harmless to the replacement — nothing in the + // slice/finish machinery consults `state.status`, and its own + // completing publish stamps READY over this transient error (or its + // own failure path rolls the deferred viewport back and publishes + // its error). + publishError( + "layout-failed", + "The requested row window could not be planned.", + error, + ); + } return; } deferredViewportWithoutAnchor = false; @@ -2066,6 +2478,8 @@ export function createRowLayoutController< reorderFallbackCount, reorderComposeCount, reorderComposeFallbackCount, + refilterPathCount, + refilterFallbackCount, pendingCatchUpChangeSetCount: active?.pendingChangeSetCount ?? 0, pendingCatchUpOperationCount: active?.pendingOperationCount ?? 0, retainedCatchUpSnapshotCount: retainedSnapshots.size, diff --git a/packages/row-model/src/__tests__/change-journal.test.ts b/packages/row-model/src/__tests__/change-journal.test.ts index ff4de5c62..0dc595762 100644 --- a/packages/row-model/src/__tests__/change-journal.test.ts +++ b/packages/row-model/src/__tests__/change-journal.test.ts @@ -549,6 +549,103 @@ describe("bounded revision change journal", () => { }); }); + test('a range of only "refilter" barriers resets with reason "refilter"', () => { + const journal = createChangeJournal(4); + journal.appendBarrier(0, 1, "refilter"); + + expect(journal.changesSince(0, 1)).toEqual({ + kind: "reset", + toRevision: 1, + reason: "refilter", + }); + + journal.appendBarrier(1, 2, "refilter"); + expect(journal.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "refilter", + }); + // A sub-range that is still all-refilter reports "refilter" too. + expect(journal.changesSince(1, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "refilter", + }); + }); + + test('a changes entry in the range demotes "refilter" to "bulk-replace" (both orders)', () => { + const changesFirst = createChangeJournal(4); + changesFirst.appendChanges(0, 1, [ + { kind: "insert", ref: data(1), index: 0 }, + ]); + changesFirst.appendBarrier(1, 2, "refilter"); + expect(changesFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + + const refilterFirst = createChangeJournal(4); + refilterFirst.appendBarrier(0, 1, "refilter"); + refilterFirst.appendChanges(1, 2, [ + { kind: "insert", ref: data(1), index: 0 }, + ]); + expect(refilterFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + // Resuming from PAST the refilter barrier replays the plain changes. + expect(refilterFirst.changesSince(1, 2)).toMatchObject({ + kind: "changes", + changes: [{ previousRevision: 1, revision: 2 }], + }); + }); + + test('a non-"refilter" barrier in the range wins over "refilter" (both orders)', () => { + const refilterFirst = createChangeJournal(4); + refilterFirst.appendBarrier(0, 1, "refilter"); + refilterFirst.appendBarrier(1, 2); + expect(refilterFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + + const barrierFirst = createChangeJournal(4); + barrierFirst.appendBarrier(0, 1); + barrierFirst.appendBarrier(1, 2, "refilter"); + expect(barrierFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + }); + + test('mixed "reorder" + "refilter" barriers degrade to "bulk-replace" (both orders)', () => { + // Each reason is a distinct promise (order-only vs membership-only). + // A range holding both delivers NEITHER promise as a whole, and no + // combined kind exists, so the aggregate degrades to the plain bulk + // reset rather than privileging either reason. + const reorderFirst = createChangeJournal(4); + reorderFirst.appendBarrier(0, 1, "reorder"); + reorderFirst.appendBarrier(1, 2, "refilter"); + expect(reorderFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + + const refilterFirst = createChangeJournal(4); + refilterFirst.appendBarrier(0, 1, "refilter"); + refilterFirst.appendBarrier(1, 2, "reorder"); + expect(refilterFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + }); + test("rejects malformed or non-contiguous append pairs without changing retained state", () => { const journal = createChangeJournal(1); journal.appendChanges(0, 1, [{ kind: "insert", ref: data(1), index: 0 }]); @@ -610,12 +707,13 @@ describe("bounded revision change journal", () => { reason: "bulk-replace", }); - // A FILTER change: a sort-only change would take the synchronous fast - // path and journal a "reorder" barrier instead (pinned in - // sort-fast-path.test.ts). + // A COMBINED filter+sort change: a sort-only change would take the + // synchronous fast path and journal a "reorder" barrier, and a + // filter-only change a "refilter" one (pinned in sort-fast-path.test.ts + // and filter-fast-path.test.ts respectively). const query = flat.setQuery({ filters: [{ columnId: "team", operator: "equals", value: "Z" }], - sort: [{ columnId: "score", direction: "asc" }], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); await query.finished; diff --git a/packages/row-model/src/__tests__/compile-filter-predicate.test.ts b/packages/row-model/src/__tests__/compile-filter-predicate.test.ts new file mode 100644 index 000000000..b1b27a572 --- /dev/null +++ b/packages/row-model/src/__tests__/compile-filter-predicate.test.ts @@ -0,0 +1,494 @@ +/** + * Exhaustive operator-semantics sweep for `compileFilterPredicate` — the + * compile step that specializes one `(value) => boolean` closure per runtime + * filter at plan construction. Every (column type, operator) pair in + * `FILTER_OPERATORS` appears at least once, and every expectation is pinned + * as a LITERAL derived from the pre-refactor `evaluateFilter` semantics — + * never generated by calling the code under test. The coverage test at the + * bottom fails if `FILTER_OPERATORS` grows a pair this sweep doesn't cover. + */ + +import { describe, expect, test } from "vitest"; + +import { FILTER_OPERATORS, compileFilterPredicate } from "../compiled-query"; + +interface SweepEntry { + readonly type: keyof typeof FILTER_OPERATORS; + readonly operator: string; + readonly operand?: unknown; + /** [cell value, expected verdict] — expectations are pinned literals. */ + readonly cases: readonly (readonly [unknown, boolean])[]; +} + +/** + * `isEmpty` semantics are type-independent (`isEmptyValue`): null, undefined, + * NaN, and whitespace-only strings are empty; everything else is not. + */ +const EMPTY_CASES: readonly (readonly [unknown, boolean])[] = [ + [null, true], + [undefined, true], + ["", true], + [" ", true], + [Number.NaN, true], + [0, false], + ["x", false], + [false, false], + [new Date(0), false], +]; + +const emptinessEntries: SweepEntry[] = ( + ["text", "number", "date", "enum", "boolean"] as const +).flatMap((type) => [ + { type, operator: "isEmpty", cases: EMPTY_CASES }, + { + type, + operator: "isNotEmpty", + cases: EMPTY_CASES.map(([value, expected]) => [value, !expected] as const), + }, +]); + +const MARCH_15_NOON_MS = Date.UTC(2024, 2, 15, 13, 45); + +const SWEEP: readonly SweepEntry[] = [ + ...emptinessEntries, + + // ── text ── cell and needle both String(...).toLocaleLowerCase(); a + // nullish cell folds to "". + { + type: "text", + operator: "contains", + operand: "Ab", + cases: [ + ["cab", true], + ["AB", true], + ["b", false], + ["", false], + [null, false], + [undefined, false], + [123, false], + ], + }, + { + type: "text", + operator: "contains", + operand: "", + // Every cell contains the empty needle — including a nullish cell. + cases: [ + [null, true], + ["x", true], + ["", true], + ], + }, + { + type: "text", + operator: "notContains", + operand: "Ab", + cases: [ + ["cab", false], + ["b", true], + [null, true], + ], + }, + { + type: "text", + operator: "equals", + operand: "Foo", + cases: [ + ["foo", true], + ["FOO", true], + [" foo", false], + ["foo ", false], + [null, false], + ["", false], + ], + }, + { + type: "text", + operator: "equals", + operand: "", + // A nullish cell folds to "" and therefore EQUALS the empty needle. + cases: [ + [null, true], + [undefined, true], + ["", true], + ["x", false], + ], + }, + { + type: "text", + operator: "notEquals", + operand: "foo", + cases: [ + ["bar", true], + ["FOO", false], + [null, true], + ["", true], + ], + }, + { + type: "text", + operator: "startsWith", + operand: "Ca", + cases: [ + ["Cab", true], + ["cAt", true], + ["aCa", false], + [null, false], + ], + }, + { + type: "text", + operator: "endsWith", + operand: "at", + cases: [ + ["Cat", true], + ["AT", true], + ["atx", false], + [null, false], + ], + }, + + // ── number ── a non-number or NaN cell fails EVERY operator, including + // notEquals. + { + type: "number", + operator: "equals", + operand: 5, + cases: [ + [5, true], + [4, false], + [Number.NaN, false], + [null, false], + ["5", false], + [undefined, false], + ], + }, + { + type: "number", + operator: "notEquals", + operand: 5, + cases: [ + [4, true], + [5, false], + [Number.NaN, false], + [null, false], + ["5", false], + ], + }, + { + type: "number", + operator: "gt", + operand: 5, + cases: [ + [6, true], + [5.000001, true], + [5, false], + [4, false], + [Number.NaN, false], + [Number.POSITIVE_INFINITY, true], + ], + }, + { + type: "number", + operator: "gte", + operand: 5, + cases: [ + [5, true], + [6, true], + [4.999, false], + [null, false], + ], + }, + { + type: "number", + operator: "lt", + operand: 5, + cases: [ + [4, true], + [5, false], + [Number.NEGATIVE_INFINITY, true], + [Number.NaN, false], + ], + }, + { + type: "number", + operator: "lte", + operand: 5, + cases: [ + [5, true], + [4, true], + [6, false], + [null, false], + ], + }, + { + type: "number", + operator: "between", + operand: [3, 7], + // Inclusive at BOTH bounds — the boundary cases the mutation test leans on. + cases: [ + [3, true], + [7, true], + [5, true], + [2.9999, false], + [7.0001, false], + [Number.NaN, false], + [null, false], + ["5", false], + ], + }, + { + type: "number", + operator: "between", + operand: [7, 3], + // Reversed range normalizes via min/max; still inclusive at both bounds. + cases: [ + [5, true], + [3, true], + [7, true], + [2, false], + [8, false], + ], + }, + + // ── date ── both sides collapse to a UTC calendar day via `toDayMs`; an + // unparsable cell fails every operator. + { + type: "date", + operator: "on", + operand: "2024-03-15", + cases: [ + ["2024-03-15", true], + ["2024-03-15T23:59:59Z", true], + [MARCH_15_NOON_MS, true], + [new Date(Date.UTC(2024, 2, 15, 5)), true], + ["2024-03-16", false], + // -05:00 pushes the instant into the NEXT UTC day. + ["2024-03-15T23:00:00-05:00", false], + ["garbage", false], + [null, false], + ["", false], + ], + }, + { + type: "date", + operator: "before", + operand: "2024-03-15", + cases: [ + ["2024-03-14", true], + ["2024-03-14T23:59:59Z", true], + ["2024-03-15", false], + ["2024-03-16", false], + [null, false], + ["junk", false], + ], + }, + { + type: "date", + operator: "after", + operand: "2024-03-15", + cases: [ + ["2024-03-16", true], + ["2024-03-15", false], + ["2024-03-15T23:59:59Z", false], + ["2024-03-14", false], + [null, false], + ], + }, + { + type: "date", + operator: "dateBetween", + operand: ["2024-03-10", "2024-03-20"], + // Inclusive at both DAY bounds; a time-of-day on the bound day still hits. + cases: [ + ["2024-03-10", true], + ["2024-03-20", true], + ["2024-03-20T23:59:59Z", true], + ["2024-03-15", true], + ["2024-03-09", false], + ["2024-03-21", false], + [null, false], + ["x", false], + ], + }, + { + type: "date", + operator: "dateBetween", + operand: ["2024-03-20", "2024-03-10"], + cases: [ + ["2024-03-15", true], + ["2024-03-10", true], + ["2024-03-09", false], + ], + }, + { + type: "date", + operator: "dateBetween", + operand: [new Date(Date.UTC(2024, 2, 10)), new Date(Date.UTC(2024, 2, 20))], + cases: [ + ["2024-03-15", true], + ["2024-03-21", false], + ], + }, + + // ── enum ── membership over String(...) coercion of BOTH sides; an empty + // selection matches everything regardless of direction. + { + type: "enum", + operator: "isAnyOf", + operand: ["a", "b"], + cases: [ + ["a", true], + ["b", true], + ["c", false], + [null, false], + [undefined, false], + ["", false], + ], + }, + { + type: "enum", + operator: "isAnyOf", + operand: [1, "2"], + cases: [ + [1, true], + ["1", true], + [2, true], + [3, false], + ], + }, + { + type: "enum", + operator: "isAnyOf", + operand: [null], + // String(null) === "null" on both sides. + cases: [ + [null, true], + ["null", true], + ["x", false], + ], + }, + { + type: "enum", + operator: "isAnyOf", + operand: [], + cases: [ + ["x", true], + [null, true], + ], + }, + { + type: "enum", + operator: "isNoneOf", + operand: ["a"], + cases: [ + ["a", false], + ["b", true], + [null, true], + ], + }, + { + type: "enum", + operator: "isNoneOf", + operand: [], + // Empty selection short-circuits to TRUE before direction applies. + cases: [["a", true]], + }, + + // ── boolean ── membership over `booleanValue` coercion of both sides. + { + type: "boolean", + operator: "isAnyOf", + operand: [true], + cases: [ + [true, true], + ["true", true], + [1, true], + ["1", true], + ["yes", true], // Boolean("yes") === true + [false, false], + [0, false], + ["false", false], + [null, false], + [Number.NaN, false], + ], + }, + { + type: "boolean", + operator: "isAnyOf", + operand: ["false"], + cases: [ + [false, true], + [0, true], + ["0", true], + ["", true], + [null, true], // Boolean(null) === false + [true, false], + ], + }, + { + type: "boolean", + operator: "isAnyOf", + operand: [], + cases: [ + [true, true], + [null, true], + ], + }, + { + type: "boolean", + operator: "isNoneOf", + operand: [true], + cases: [ + [false, true], + [null, true], + [true, false], + ["1", false], + ], + }, + { + type: "boolean", + operator: "isNoneOf", + operand: [], + cases: [[false, true]], + }, +]; + +describe("compileFilterPredicate", () => { + for (const entry of SWEEP) { + const operandLabel = + entry.operand === undefined ? "" : ` ${JSON.stringify(entry.operand)}`; + test(`${entry.type} ${entry.operator}${operandLabel}`, () => { + const predicate = compileFilterPredicate( + { columnId: "c", operator: entry.operator, value: entry.operand }, + { type: entry.type }, + ); + for (const [value, expected] of entry.cases) { + expect(predicate(value), `cell ${String(value)}`).toBe(expected); + } + }); + } + + test("a compiled predicate is a reusable closure", () => { + const predicate = compileFilterPredicate( + { columnId: "c", operator: "between", value: [3, 7] }, + { type: "number" }, + ); + expect(predicate(3)).toBe(true); + expect(predicate(3)).toBe(true); + expect(predicate(8)).toBe(false); + }); + + test("sweep covers every (type, operator) pair in FILTER_OPERATORS", () => { + const expected = new Set(); + for (const [type, operators] of Object.entries(FILTER_OPERATORS)) { + for (const operator of operators) expected.add(`${type}:${operator}`); + } + // 8 text + 9 number + 6 date + 4 enum + 4 boolean. + expect(expected.size).toBe(31); + const covered = new Set( + SWEEP.map((entry) => `${entry.type}:${entry.operator}`), + ); + expect([...covered].sort()).toEqual([...expected].sort()); + }); +}); diff --git a/packages/row-model/src/__tests__/compiled-query.test.ts b/packages/row-model/src/__tests__/compiled-query.test.ts index e9abbf11e..08278d585 100644 --- a/packages/row-model/src/__tests__/compiled-query.test.ts +++ b/packages/row-model/src/__tests__/compiled-query.test.ts @@ -6,6 +6,7 @@ import { CompiledQueryValidationError, compileQuery, createColumnHelper, + filterVerdict, sortKeysOf, type CompiledAggregateLeaf, type PretableAggregator, @@ -83,17 +84,23 @@ describe("compileQuery", () => { ignored: "never", }; - const metadata = plan.evaluate({ rowId: 7, row, sourceOrder: 3 }); + const metadata = plan.evaluate({ rowId: 7, row, sourceOrder: 3, slot: 3 }); expect(metadata).toMatchObject({ rowId: 7, row, sourceOrder: 3, - filterPasses: true, groupPath: [{ columnId: "sector", value: "Tech" }], }); + // The verdict is not metadata: it is asked of the plan, and recorded by + // the structure the row lands in. + expect( + filterVerdict(plan, { rowId: 7, row, sourceOrder: 3, slot: 3 }), + ).toBe(true); // Sort keys live in the plan's store, not on metadata. - expect(sortKeysOf(plan, metadata)).toEqual([ + expect( + sortKeysOf(plan, { ...metadata, slot: metadata.sourceOrder }), + ).toEqual([ { columnId: "quantity", value: 20 }, { columnId: "label", value: "item 2" }, ]); @@ -122,8 +129,8 @@ describe("compileQuery", () => { ignored: "z", }; - const first = plan.evaluate({ rowId: 1, row, sourceOrder: 0 }); - const second = plan.evaluate({ rowId: 1, row, sourceOrder: 0 }); + const first = plan.evaluate({ rowId: 1, row, sourceOrder: 0, slot: 0 }); + const second = plan.evaluate({ rowId: 1, row, sourceOrder: 0, slot: 0 }); expect(second).toBe(first); expect(calls.quantity).toHaveBeenCalledTimes(1); @@ -205,6 +212,7 @@ describe("compileQuery", () => { quantityPlan.evaluate({ rowId: id, sourceOrder, + slot: sourceOrder, row: { id, sector: null, quantity, label: "same", ignored: "" }, }); const numeric = [ @@ -216,7 +224,13 @@ describe("compileQuery", () => { expect( numeric - .sort((left, right) => compareRecordRows(quantityPlan, left, right)) + .sort((left, right) => + compareRecordRows( + quantityPlan, + { ...left, slot: left.sourceOrder }, + { ...right, slot: right.sourceOrder }, + ), + ) .map((row) => row.rowId), ).toEqual([3, 2, 1, 4]); @@ -232,12 +246,19 @@ describe("compileQuery", () => { labelPlan.evaluate({ rowId: sourceOrder, sourceOrder, + slot: sourceOrder, row: { id: sourceOrder, sector: null, quantity: 1, label, ignored: "" }, }), ); expect( labels - .sort((left, right) => compareRecordRows(labelPlan, left, right)) + .sort((left, right) => + compareRecordRows( + labelPlan, + { ...left, slot: left.sourceOrder }, + { ...right, slot: right.sourceOrder }, + ), + ) .map((row) => row.row.label), ).toEqual(["Item 2", "item 2", "item 10"]); }); @@ -263,27 +284,41 @@ describe("compileQuery", () => { const defined = defaultLast.evaluate({ rowId: 1, sourceOrder: 0, + slot: 0, row: { id: 1, sector: "Tech", quantity: 10, label: "", ignored: "" }, }); const missing = defaultLast.evaluate({ rowId: 2, sourceOrder: 1, + slot: 1, row: { id: 2, sector: null, quantity: null, label: "", ignored: "" }, }); const firstDefined = nullFirst.evaluate({ rowId: 1, sourceOrder: 0, + slot: 0, row: { id: 1, sector: "Tech", quantity: 10, label: "", ignored: "" }, }); const firstMissing = nullFirst.evaluate({ rowId: 2, sourceOrder: 1, + slot: 1, row: { id: 2, sector: null, quantity: null, label: "", ignored: "" }, }); - expect(compareRecordRows(defaultLast, defined, missing)).toBeLessThan(0); expect( - compareRecordRows(nullFirst, firstDefined, firstMissing), + compareRecordRows( + defaultLast, + { ...defined, slot: defined.sourceOrder }, + { ...missing, slot: missing.sourceOrder }, + ), + ).toBeLessThan(0); + expect( + compareRecordRows( + nullFirst, + { ...firstDefined, slot: firstDefined.sourceOrder }, + { ...firstMissing, slot: firstMissing.sourceOrder }, + ), ).toBeGreaterThan(0); expect( defaultLast.compareGroupKeys( @@ -301,7 +336,7 @@ describe("compileQuery", () => { ).toBeGreaterThan(0); }); - test("applies typed filters and emits both all and filtered aggregate leaves", () => { + test("applies typed filters and emits one aggregate leaf per aggregated column", () => { const { columns } = setup(); const plan = compileQuery({ derivations: columns, @@ -311,29 +346,37 @@ describe("compileQuery", () => { sort: [], }), }); - const hidden = plan.evaluate({ - rowId: 1, + const hiddenInput = { + rowId: 1 as const, sourceOrder: 0, + slot: 0, row: { id: 1, sector: null, quantity: 5, label: "a", ignored: "" }, - }); - const visible = plan.evaluate({ - rowId: 2, + }; + const visibleInput = { + rowId: 2 as const, sourceOrder: 1, + slot: 1, row: { id: 2, sector: null, quantity: 10, label: "b", ignored: "" }, - }); - - expect(hidden.filterPasses).toBe(false); - expect( - hidden.aggregateLeaves.every((leaf) => leaf.allLeaf !== undefined), - ).toBe(true); - expect( - hidden.aggregateLeaves.every((leaf) => leaf.filteredLeaf === undefined), - ).toBe(true); - expect( - visible.aggregateLeaves.every( - (leaf) => leaf.filteredLeaf === leaf.allLeaf, - ), - ).toBe(true); + }; + const hidden = plan.evaluate(hiddenInput); + const visible = plan.evaluate(visibleInput); + + expect(filterVerdict(plan, hiddenInput)).toBe(false); + expect(filterVerdict(plan, visibleInput)).toBe(true); + // A failing row's leaves are built exactly like a passing row's: which + // aggregate TREE a leaf joins is the consumer's decision, made from the + // verdict above, so the metadata carries no filtered variant of its own. + for (const metadata of [hidden, visible]) { + expect(metadata.aggregateLeaves.length).toBeGreaterThan(0); + expect(metadata.aggregateLeaves.every((leaf) => leaf.allLeaf.row)).toBe( + true, + ); + expect( + metadata.aggregateLeaves.every( + (leaf) => !Object.hasOwn(leaf, "filteredLeaf"), + ), + ).toBe(true); + } }); test("retains exact built-in and custom aggregate leaf inference", () => { @@ -344,6 +387,7 @@ describe("compileQuery", () => { }).evaluate({ rowId: 1, sourceOrder: 0, + slot: 0, row: { id: 1, sector: "Tech", quantity: 2, label: "x", ignored: "" }, }); @@ -466,7 +510,7 @@ describe("compileQuery", () => { label: "x", ignored: "", }; - const metadata = plan.evaluate({ rowId: 1, row, sourceOrder: 0 }); + const metadata = plan.evaluate({ rowId: 1, row, sourceOrder: 0, slot: 0 }); operand[0] = 100; aggregateOption.label = "mutated"; @@ -481,7 +525,6 @@ describe("compileQuery", () => { expect(Object.isFrozen(plan.query)).toBe(true); expect(Object.isFrozen(plan.query.filters)).toBe(true); expect(Object.isFrozen(plan.derivations)).toBe(true); - expect(metadata.filterPasses).toBe(true); expect(metadata.row).toBe(row); expect(metadata.aggregateLeaves[0].allLeaf.row).toBe(row); const label = metadata.aggregateLeaves.find( @@ -519,11 +562,12 @@ describe("compileQuery", () => { exposed.setUTCDate(8); expect( - plan.evaluate({ + filterVerdict(plan, { rowId: 1, sourceOrder: 0, + slot: 0, row: { id: 1, asOf: new Date("2026-08-06T18:00:00Z") }, - }).filterPasses, + }), ).toBe(true); expect(Object.prototype.toString.call(exposed)).toBe("[object Date]"); expect(exposed.constructor).toBe(Date); @@ -707,20 +751,36 @@ describe("compileQuery", () => { // The operand always coerces to `true`, so isAnyOf must match the true // row and exclude the false row, and isNoneOf must do the reverse. expect( - isAnyOf.evaluate({ rowId: 1, row: trueRow, sourceOrder: 0 }) - .filterPasses, + filterVerdict(isAnyOf, { + rowId: 1, + row: trueRow, + sourceOrder: 0, + slot: 0, + }), ).toBe(true); expect( - isAnyOf.evaluate({ rowId: 2, row: falseRow, sourceOrder: 0 }) - .filterPasses, + filterVerdict(isAnyOf, { + rowId: 2, + row: falseRow, + sourceOrder: 0, + slot: 0, + }), ).toBe(false); expect( - isNoneOf.evaluate({ rowId: 1, row: trueRow, sourceOrder: 0 }) - .filterPasses, + filterVerdict(isNoneOf, { + rowId: 1, + row: trueRow, + sourceOrder: 0, + slot: 0, + }), ).toBe(false); expect( - isNoneOf.evaluate({ rowId: 2, row: falseRow, sourceOrder: 0 }) - .filterPasses, + filterVerdict(isNoneOf, { + rowId: 2, + row: falseRow, + sourceOrder: 0, + slot: 0, + }), ).toBe(true); }, ); @@ -745,12 +805,20 @@ describe("compileQuery", () => { // Both states are in the operand set, so every row matches. expect( - plan.evaluate({ rowId: 1, row: { id: 1, active: true }, sourceOrder: 0 }) - .filterPasses, + filterVerdict(plan, { + rowId: 1, + row: { id: 1, active: true }, + sourceOrder: 0, + slot: 0, + }), ).toBe(true); expect( - plan.evaluate({ rowId: 2, row: { id: 2, active: false }, sourceOrder: 0 }) - .filterPasses, + filterVerdict(plan, { + rowId: 2, + row: { id: 2, active: false }, + sourceOrder: 0, + slot: 0, + }), ).toBe(true); }); @@ -773,18 +841,20 @@ describe("compileQuery", () => { }); expect( - plan.evaluate({ + filterVerdict(plan, { rowId: 1, row: { id: 1, active: false }, sourceOrder: 0, - }).filterPasses, + slot: 0, + }), ).toBe(true); expect( - plan.evaluate({ + filterVerdict(plan, { rowId: 2, row: { id: 2, active: true }, sourceOrder: 0, - }).filterPasses, + slot: 0, + }), ).toBe(false); }); @@ -827,16 +897,22 @@ describe("compileQuery", () => { const left = plan.evaluate({ rowId: 11, sourceOrder: 0, + slot: 0, row: { id: 11, value: 1 }, }); const right = plan.evaluate({ rowId: 22, sourceOrder: 1, + slot: 1, row: { id: 22, value: 2 }, }); let caught: unknown; try { - compareRecordRows(plan, left, right); + compareRecordRows( + plan, + { ...left, slot: left.sourceOrder }, + { ...right, slot: right.sourceOrder }, + ); } catch (error) { caught = error; } @@ -937,7 +1013,12 @@ describe("compileQuery", () => { ); const plan = compileQuery(input as never); - plan.evaluate({ rowId: 1, sourceOrder: 0, row: { id: 1, value: 3 } }); + plan.evaluate({ + rowId: 1, + sourceOrder: 0, + slot: 0, + row: { id: 1, value: 3 }, + }); expect(Object.fromEntries(reads)).toEqual({ "input.derivations": 1, @@ -1190,6 +1271,7 @@ describe("compileQuery", () => { const metadata = plan.evaluate({ rowId: index, sourceOrder: index, + slot: index, row: { id: index, value: index }, }); const current = metadata.aggregateLeaves[0].aggregate; @@ -1221,17 +1303,22 @@ describe("compileQuery", () => { const left = plan.evaluate({ rowId: 1, sourceOrder: 0, + slot: 0, row: { id: 1, value: 1 }, }); const right = plan.evaluate({ rowId: 2, sourceOrder: 1, + slot: 1, row: { id: 2, value: 2 }, }); + const leftInput = { ...left, slot: left.sourceOrder }; + const rightInput = { ...right, slot: right.sourceOrder }; + for (const invalid of ["invalid", Number.NaN]) { result = invalid; - expect(() => compareRecordRows(plan, left, right)).toThrowError( + expect(() => compareRecordRows(plan, leftInput, rightInput)).toThrowError( expect.objectContaining({ name: "CompiledQueryComparatorError", columnId: "value", @@ -1240,7 +1327,9 @@ describe("compileQuery", () => { ); } result = Number.POSITIVE_INFINITY; - expect(compareRecordRows(plan, left, right)).toBe(Number.POSITIVE_INFINITY); + expect(compareRecordRows(plan, leftInput, rightInput)).toBe( + Number.POSITIVE_INFINITY, + ); }); test("includes group values when a custom group comparator returns NaN", () => { @@ -1262,11 +1351,13 @@ describe("compileQuery", () => { const left = plan.evaluate({ rowId: 1, sourceOrder: 0, + slot: 0, row: { id: 1, group: "a" }, }); const right = plan.evaluate({ rowId: 2, sourceOrder: 1, + slot: 1, row: { id: 2, group: "b" }, }); let caught: unknown; diff --git a/packages/row-model/src/__tests__/external-filter-authority.test.ts b/packages/row-model/src/__tests__/external-filter-authority.test.ts index 6a14e59a7..9ac23a63e 100644 --- a/packages/row-model/src/__tests__/external-filter-authority.test.ts +++ b/packages/row-model/src/__tests__/external-filter-authority.test.ts @@ -7,6 +7,7 @@ import { ɵsetLocalRowModelFilterAuthority, type PretableQueryFor, } from "../index"; +import { filterVerdict } from "../compiled-query"; interface Holding { id: string; @@ -199,10 +200,13 @@ describe("compileQuery filter authority", () => { { columnId: "customer", operator: "contains", value: "Northwind" }, ]); expect( - rows.map( - (row, index) => - plan.evaluate({ row, rowId: row.id, sourceOrder: index }) - .filterPasses, + rows.map((row, index) => + filterVerdict(plan, { + row, + rowId: row.id, + sourceOrder: index, + slot: index, + }), ), ).toEqual([true, true, true, true]); }); diff --git a/packages/row-model/src/__tests__/filter-fast-path.test.ts b/packages/row-model/src/__tests__/filter-fast-path.test.ts new file mode 100644 index 000000000..430cc8f76 --- /dev/null +++ b/packages/row-model/src/__tests__/filter-fast-path.test.ts @@ -0,0 +1,1621 @@ +import { describe, expect, test, vi } from "vitest"; + +import { + compileQuery, + createColumnHelper, + createLocalRowModel, + PretableRowModelError, + PretableTransitionCancelledError, + type PretableQueryFor, +} from "../index"; +import { + adoptEvaluationCache, + compareRecordRows, + filterVerdict, + isFilterOnlyChange, + isSortOnlyChange, + sortKeysOf, + type CompiledQuery, +} from "../compiled-query"; +import { rowPassesFilter } from "../filter-membership"; +import type { CooperativeTransitionScheduler } from "../cooperative-transition"; +import { getLocalRowModelSlotInternalsForTesting } from "../create-local-row-model"; +import { createInstrumentedLocalRowModel } from "../diagnostics"; +import type { LocalRowModelInstrumentation } from "../diagnostics"; +import { rebuildRootForFilterOnlyChange } from "../filter-rebuild"; +import { rebuildRootForSortOnlyChange } from "../sort-rebuild"; +import type { RevisionRoot } from "../internal-types"; +import { compareOrderStatisticTreeIds } from "../persistent/order-statistic-tree"; +import { createPersistentMap } from "../persistent/persistent-map"; +import { buildRowStore } from "../row-store"; +import { createSlotAllocator } from "../slot-allocator"; +import type { PretableGroupId } from "../types"; +import { EMPTY_MEMBERSHIP } from "../membership-bitset"; +import { createVisibleIndex, membershipFromFlatTree } from "../visible-index"; + +interface Holding { + id: string; + team: string; + score: number; + note: string; +} + +const helper = createColumnHelper(); + +/** Checks a query literal against a column tuple, as the sibling suites do. */ +function queryFor( + value: PretableQueryFor, +): PretableQueryFor { + return value; +} + +function createColumns() { + return [ + helper.accessor("team", (row: Holding) => row.team, { type: "text" }), + helper.accessor("score", (row: Holding) => row.score, { + type: "number", + aggregate: "sum", + }), + helper.accessor("note", (row: Holding) => row.note, { type: "text" }), + ] as const; +} + +type FixtureColumns = ReturnType; + +function scoreQuery( + operator: "gte" | "gt" | "lte", + value: number, +): PretableQueryFor { + return queryFor({ + filters: [{ columnId: "score", operator, value }], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [], + }); +} + +const NO_FILTER_QUERY = queryFor({ + filters: [], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [], +}); + +/** + * Eight rows over one sort column (`note`, asc, ties by sourceOrder). Under + * the main change (score gte 40 -> score lte 60) the survivors are h1/h5/a8 + * and the four flipped-in rows land at the HEAD (h2, note "a"), in the + * MIDDLE between survivors (h4, note "c"), TIED with a survivor (z4, note + * "m" against a8), and at the TAIL (h6, note "zz") — the merge cannot pass + * by appending. The tie pair's id order OPPOSES its source order (z4 comes + * first in source, a8 sorts first by id), so sourceOrder tie resolution and + * an id-based one produce opposite orders; both controls are asserted. + */ +const ROOT_ROWS: readonly Holding[] = Object.freeze([ + { id: "h1", team: "Alpha", score: 50, note: "b" }, + { id: "h2", team: "Alpha", score: 30, note: "a" }, + { id: "z4", team: "Alpha", score: 15, note: "m" }, + { id: "h3", team: "Alpha", score: 90, note: "e" }, + { id: "h4", team: "Alpha", score: 35, note: "c" }, + { id: "h5", team: "Alpha", score: 60, note: "d" }, + { id: "a8", team: "Alpha", score: 45, note: "m" }, + { id: "h6", team: "Alpha", score: 10, note: "zz" }, +]); + +const OLD_VISIBLE_ORDER = ["h1", "h5", "h3", "a8"] as const; +const NEW_VISIBLE_ORDER = ["h2", "h1", "h4", "h5", "z4", "a8", "h6"] as const; +const SURVIVORS = ["h1", "h5", "a8"] as const; +const FLIPPED_IN = ["h2", "z4", "h4", "h6"] as const; +const FLIPPED_OUT = ["h3"] as const; + +function createRoot( + queryPlan: CompiledQuery, + rows: readonly Holding[], +): RevisionRoot { + const slots = createSlotAllocator(); + const store = buildRowStore({ + rows, + getRowId: (row) => row.id, + queryPlan, + slots, + }); + const defaultPolicy = Object.freeze({ kind: "expanded" as const }); + const expansion = Object.freeze({ + default: defaultPolicy, + overrides: createPersistentMap(), + state: Object.freeze({ default: defaultPolicy, overrideCount: 0 }), + }); + const visible = createVisibleIndex( + store.records, + queryPlan, + false, + expansion.overrides, + ); + return Object.freeze({ + revision: 0, + parentRevision: null, + rows: store.rows, + sourceOrder: store.sourceOrder, + recordsBySlot: store.recordsBySlot, + slotCapacity: slots.capacity, + // Same rule as the production initial-build site: flat roots index their + // membership per slot, grouped roots carry the sentinel. + visibleSlots: + queryPlan.query.rowGroups.length > 0 + ? EMPTY_MEMBERSHIP + : membershipFromFlatTree(visible.rows, slots.capacity), + visible, + queryPlan, + expansion, + cause: Object.freeze({ kind: "initial" as const }), + }); +} + +function rankedIds( + visible: RevisionRoot["visible"], +): readonly string[] { + const ids: string[] = []; + for (let index = 0; index < visible.rows.size; index += 1) { + ids.push(visible.rows.entryAt(index)!.record.rowId); + } + return ids; +} + +function testInstrumentation(): LocalRowModelInstrumentation { + return { + work: { + rowsEvaluated: 0, + hamtNodesCopied: 0, + orderNodesCopied: 0, + groupNodesCopied: 0, + aggregateMerges: 0, + transitionRows: 0, + snapshotOutputRowsRead: 0, + synchronousRebuilds: 0, + synchronousRebuildMs: 0, + filterRebuilds: 0, + filterRowsFlipped: 0, + filterMergeSortedInsertions: 0, + filterRebuildMs: 0, + bulkByIdDerived: 0, + bulkOrderVerificationsSkipped: 0, + evaluationCacheAdoptions: 0, + slotChunksTouched: 0, + sortKeyCarries: 0, + sortKeyEvaluations: 0, + schedulerSliceDurations: [], + }, + snapshotRoots: new WeakMap(), + retainedSnapshots: new Map(), + scheduledCallbacks: new Set(), + currentRevisionRoot: undefined, + model: undefined, + }; +} + +/** + * Cold oracle: an independently compiled twin plan (cold cache) evaluated + * from scratch, filtered and sorted with the same composite order the + * visible tree maintains. Returns the expected visible ids, the twin's + * verdict per row, and the twin's metadata per row so equivalence checks can + * reach aggregate values. + */ +function coldOracle( + columns: FixtureColumns, + query: PretableQueryFor, + rows: readonly Holding[], +) { + const twinPlan = compileQuery({ derivations: columns, query }); + const evaluated = rows.map((row, sourceOrder) => ({ + rowId: row.id, + input: { rowId: row.id, row, sourceOrder, slot: sourceOrder }, + metadata: twinPlan.evaluate({ + rowId: row.id, + row, + sourceOrder, + slot: sourceOrder, + }), + })); + const visibleIds = evaluated + .filter((entry) => filterVerdict(twinPlan, entry.input)) + .sort( + (left, right) => + compareRecordRows(twinPlan, left.input, right.input) || + compareOrderStatisticTreeIds(left.rowId, right.rowId), + ) + .map((entry) => entry.rowId); + return { + visibleIds, + metadataOf: new Map( + evaluated.map((entry) => [entry.rowId, entry.metadata]), + ), + passesOf: new Map( + evaluated.map((entry) => [ + entry.rowId, + filterVerdict(twinPlan, entry.input), + ]), + ), + }; +} + +/** + * Runs the rebuild for `previousQuery -> nextQuery` over `rows` and asserts + * full equivalence with the cold oracle: visible order (full walk), counts, + * every row's verdict READ AS MEMBERSHIP of the rebuilt root, and per-row + * aggregate leaf values. + */ +function expectEquivalence( + previousQuery: PretableQueryFor, + nextQuery: PretableQueryFor, + rows: readonly Holding[] = ROOT_ROWS, +) { + const columns = createColumns(); + const previousPlan = compileQuery({ + derivations: columns, + query: previousQuery, + }); + const nextPlan = compileQuery({ derivations: columns, query: nextQuery }); + expect(isFilterOnlyChange(previousPlan, nextPlan)).toBe(true); + const captured = createRoot(previousPlan, rows); + + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + const oracle = coldOracle(columns, nextQuery, rows); + expect(rankedIds(rebuilt.visible)).toEqual(oracle.visibleIds); + expect(rebuilt.visible.rows.size).toBe(oracle.visibleIds.length); + expect(rebuilt.rows.size).toBe(rows.length); + for (const row of rows) { + const record = rebuilt.rows.get(row.id)!; + // The verdict equivalence is now structural: the rebuilt root agrees with + // the cold model about who is a member. + expect(rowPassesFilter(rebuilt, row.id)).toBe(oracle.passesOf.get(row.id)); + const leaf = record.metadata.aggregateLeaves[0]!; + expect(leaf.allLeaf.value).toBe(row.score); + // Identity, for EVERY row: a filter-only change reconstructs no record. + expect(record).toBe(captured.rows.get(row.id)); + } + return { captured, rebuilt, nextPlan, previousPlan }; +} + +describe("rebuildRootForFilterOnlyChange", () => { + function createMainFixture() { + const columns = createColumns(); + const previousPlan = compileQuery({ + derivations: columns, + query: scoreQuery("gte", 40), + }); + const nextPlan = compileQuery({ + derivations: columns, + query: scoreQuery("lte", 60), + }); + const captured = createRoot(previousPlan, ROOT_ROWS); + // Fixture controls. The captured visible order is the hand-derived one; + // the flipped-in rows interleave with survivors rather than clustering: + // NEW order must differ from every append shape a broken merge produces. + expect(rankedIds(captured.visible)).toEqual([...OLD_VISIBLE_ORDER]); + expect([...NEW_VISIBLE_ORDER]).not.toEqual([...SURVIVORS, ...FLIPPED_IN]); + expect([...NEW_VISIBLE_ORDER]).not.toEqual([...FLIPPED_IN, ...SURVIVORS]); + // Tie control: z4 and a8 tie on the only sort key (note "m"). Their id + // order OPPOSES their source order, so sourceOrder resolution (z4 first, + // pinned by NEW_VISIBLE_ORDER) and id resolution are distinguishable. + expect(ROOT_ROWS.findIndex((row) => row.id === "z4")).toBeLessThan( + ROOT_ROWS.findIndex((row) => row.id === "a8"), + ); + expect(compareOrderStatisticTreeIds("a8", "z4")).toBeLessThan(0); + return { columns, previousPlan, nextPlan, captured }; + } + + test("disjoint flip in both directions matches the cold model", () => { + const { rebuilt } = expectEquivalence( + scoreQuery("gte", 40), + scoreQuery("lte", 60), + ); + // The oracle-derived order is the hand-derived merge fixture order. + expect(rankedIds(rebuilt.visible)).toEqual([...NEW_VISIBLE_ORDER]); + }); + + test("narrowing matches the cold model", () => { + expectEquivalence(scoreQuery("gte", 40), scoreQuery("gte", 50)); + }); + + test("widening matches the cold model", () => { + expectEquivalence(scoreQuery("gte", 40), scoreQuery("gte", 20)); + }); + + test("removing every filter matches the cold model", () => { + const { rebuilt } = expectEquivalence( + scoreQuery("gte", 40), + NO_FILTER_QUERY, + ); + expect(rebuilt.visible.rows.size).toBe(ROOT_ROWS.length); + }); + + test("filter-to-empty: every row flips out", () => { + const { rebuilt, captured } = expectEquivalence( + scoreQuery("gte", 40), + scoreQuery("gte", 1000), + ); + expect(rebuilt.visible.rows.size).toBe(0); + expect(captured.visible.rows.size).toBe(OLD_VISIBLE_ORDER.length); + }); + + test("empty-to-filter: rows flip into an empty visible set", () => { + const { rebuilt, captured } = expectEquivalence( + scoreQuery("gte", 1000), + scoreQuery("lte", 60), + ); + expect(captured.visible.rows.size).toBe(0); + expect(rankedIds(rebuilt.visible)).toEqual([...NEW_VISIBLE_ORDER]); + }); + + test("multi-filter: one of two filters changes, the other keeps failing rows out", () => { + const rows = Object.freeze([ + ...ROOT_ROWS, + // Passes each score filter it meets, always fails the team filter — + // its verdict is false on BOTH sides, so it must stay out AND carry. + { id: "h9", team: "Beta", score: 50, note: "aa" }, + ]); + const multi = ( + operator: "gte" | "lte", + value: number, + ): PretableQueryFor => + queryFor({ + filters: [ + { columnId: "team", operator: "equals", value: "Alpha" }, + { columnId: "score", operator, value }, + ], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [], + }); + const { captured, rebuilt } = expectEquivalence( + multi("gte", 40), + multi("lte", 60), + rows, + ); + expect(rebuilt.visible.rows.rankOf("h9")).toBeUndefined(); + // The unflipped failing row carries by identity. + expect(rebuilt.rows.get("h9")).toBe(captured.rows.get("h9")); + }); + + test("zero flips: new revision root, rows map and visible tree carried by identity", () => { + const columns = createColumns(); + const previousPlan = compileQuery({ + derivations: columns, + query: scoreQuery("gte", 40), + }); + // gt 39 differs as a FILTER (operator and value) but produces identical + // verdicts over integer scores: filtersChanged is true, flips are zero. + const nextPlan = compileQuery({ + derivations: columns, + query: scoreQuery("gt", 39), + }); + expect(isFilterOnlyChange(previousPlan, nextPlan)).toBe(true); + const captured = createRoot(previousPlan, ROOT_ROWS); + const instrumentation = testInstrumentation(); + + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 3, + now: () => 0, + instrumentation, + }); + + // Decided and pinned: a zero-flip change still publishes a NEW root at + // the requested revision under the NEW plan — only the persistent + // structures carry wholesale, including the visible tree OBJECT. + expect(rebuilt).not.toBe(captured); + expect(rebuilt.revision).toBe(3); + expect(rebuilt.parentRevision).toBe(2); + expect(rebuilt.queryPlan).toBe(nextPlan); + expect(rebuilt.cause).toEqual({ kind: "set-query" }); + expect(rebuilt.rows).toBe(captured.rows); + expect(rebuilt.visible.rows).toBe(captured.visible.rows); + expect(rankedIds(rebuilt.visible)).toEqual([...OLD_VISIBLE_ORDER]); + expect(instrumentation.work.filterRebuilds).toBe(1); + expect(instrumentation.work.filterRowsFlipped).toBe(0); + expect(instrumentation.work.filterMergeSortedInsertions).toBe(0); + }); + + test("EVERY record carries by identity, flipped ones included, and so does the rows map", () => { + const { nextPlan, captured } = createMainFixture(); + + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + // The headline of this cycle: five rows flip, and the rows HAMT is the + // captured object itself — no transient was ever opened. + expect(rebuilt.rows).toBe(captured.rows); + const flipped = [...FLIPPED_IN, ...FLIPPED_OUT]; + expect(flipped.length).toBeGreaterThan(0); + for (const row of ROOT_ROWS) { + expect(rebuilt.rows.get(row.id)).toBe(captured.rows.get(row.id)); + } + // Positive twin: the carry did NOT come at the cost of the answer — the + // membership really did change for exactly the flipped rows. + for (const id of FLIPPED_IN) { + expect(rowPassesFilter(captured, id)).toBe(false); + expect(rowPassesFilter(rebuilt, id)).toBe(true); + } + for (const id of FLIPPED_OUT) { + expect(rowPassesFilter(captured, id)).toBe(true); + expect(rowPassesFilter(rebuilt, id)).toBe(false); + } + for (const id of SURVIVORS) { + expect(rowPassesFilter(captured, id)).toBe(true); + expect(rowPassesFilter(rebuilt, id)).toBe(true); + } + // sourceOrder and expansion carry by reference from the captured root. + expect(rebuilt.sourceOrder).toBe(captured.sourceOrder); + expect(rebuilt.expansion).toBe(captured.expansion); + }); + + test("a flipped row's aggregate leaf and its dependency carry untouched", () => { + const { nextPlan, captured } = createMainFixture(); + + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + for (const id of [...FLIPPED_IN, ...FLIPPED_OUT]) { + const before = captured.rows.get(id)!.metadata.aggregateLeaves[0]!; + const after = rebuilt.rows.get(id)!.metadata.aggregateLeaves[0]!; + // Nothing about the leaf encodes the verdict, so nothing about it + // changes when the verdict flips. + expect(after).toBe(before); + expect(after.allLeaf).toBe(before.allLeaf); + expect(after.allLeaf.dependency).toBe(before.allLeaf.dependency); + } + }); + + test("still-passing rows reuse their tree ENTRY objects; flipped-in entries hold the new records", () => { + const { nextPlan, captured } = createMainFixture(); + const before = new Map( + [...captured.visible.rows.entries()].map((entry) => [ + entry.record.rowId, + entry, + ]), + ); + + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + for (const entry of rebuilt.visible.rows.entries()) { + const id = entry.record.rowId; + if (SURVIVORS.includes(id as never)) { + // A still-passing row is by definition unflipped: same entry object. + expect(entry).toBe(before.get(id)); + } else { + expect(before.has(id)).toBe(false); + expect(entry.record).toBe(rebuilt.rows.get(id)); + } + } + }); + + test("counters: flipped, merge insertions, carries, and wall time are exact", () => { + const { nextPlan, captured } = createMainFixture(); + const instrumentation = testInstrumentation(); + const ticks = [0, 7]; + let call = 0; + + rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => ticks[call++] ?? 7, + instrumentation, + }); + + // Hand-counted: h3 flips out; h2, z4, h4, h6 flip in. + expect(instrumentation.work.filterRebuilds).toBe(1); + expect(instrumentation.work.filterRowsFlipped).toBe( + FLIPPED_IN.length + FLIPPED_OUT.length, + ); + expect(instrumentation.work.filterMergeSortedInsertions).toBe( + FLIPPED_IN.length, + ); + expect(instrumentation.work.filterRebuildMs).toBe(7); + // ZERO sort-key work of either kind, which is the point of the + // adoption: the per-row fill this path used to run reported one carry + // per row (`ROOT_ROWS.length`) and zero accessor evaluations — a + // 100%-carry walk, i.e. a walk that produced value-identical copies of + // arrays the previous plan already held. The next plan now takes the + // whole store by reference instead, so there is nothing per-row left to + // count. `evaluationCacheAdoptions` is what pins the replacement, and it + // is exactly one per rebuild, never per row. + expect(instrumentation.work.sortKeyCarries).toBe(0); + expect(instrumentation.work.sortKeyEvaluations).toBe(0); + expect(instrumentation.work.evaluationCacheAdoptions).toBe(1); + expect(instrumentation.work.synchronousRebuilds).toBe(0); + }); + + test("the merge commit takes BOTH bulk-build proofs, exactly once", () => { + const { nextPlan, captured } = createMainFixture(); + const instrumentation = testInstrumentation(); + + rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + instrumentation, + }); + + // One visible tree is built, and it pays for neither the n−1 order + // verification nor the n-entry byId refill. NARROW flip: 1 leaver and 4 + // arrivals against 7 built entries, so the derivation is the cheap route + // and is taken. + expect(FLIPPED_OUT.length + FLIPPED_IN.length).toBeLessThan( + NEW_VISIBLE_ORDER.length, + ); + expect(instrumentation.work.bulkOrderVerificationsSkipped).toBe(1); + expect(instrumentation.work.bulkByIdDerived).toBe(1); + }); + + /** + * The routing pair. Both fixtures hand the builder an identical, always-on + * derivation offer; only the flip RATIO differs, and the builder alone + * decides. The wide case is the shape the S2 target bench measured, where + * an unconditional derivation ran 37,500 removes to replace 12,500 inserts + * — three times the work — and cost ~9ms of settle. + */ + function routeFixture(nextQuery: PretableQueryFor) { + const columns = createColumns(); + const previousPlan = compileQuery({ + derivations: columns, + query: NO_FILTER_QUERY, + }); + const nextPlan = compileQuery({ derivations: columns, query: nextQuery }); + const captured = createRoot(previousPlan, ROOT_ROWS); + expect(captured.visible.rows.size).toBe(ROOT_ROWS.length); + const instrumentation = testInstrumentation(); + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + instrumentation, + }); + return { captured, rebuilt, instrumentation, nextQuery, columns }; + } + + test("wide flip — more removals than survivors — REFILLS instead of deriving", () => { + // 8 visible, 5 flip out, 0 flip in: 5 removals against 3 built entries. + const { rebuilt, instrumentation } = routeFixture(scoreQuery("gte", 50)); + + expect(rankedIds(rebuilt.visible)).toEqual(["h1", "h5", "h3"]); + expect(instrumentation.work.filterRowsFlipped).toBe(5); + expect(instrumentation.work.filterMergeSortedInsertions).toBe(0); + expect(instrumentation.work.bulkByIdDerived).toBe(0); + // The free half of the proof is unaffected by the routing decision. + expect(instrumentation.work.bulkOrderVerificationsSkipped).toBe(1); + }); + + test("narrow flip — fewer removals than survivors — DERIVES", () => { + // 8 visible, 1 flips out (h3, score 90), 0 flip in: 1 against 7. + const { rebuilt, instrumentation } = routeFixture(scoreQuery("lte", 60)); + + expect(rankedIds(rebuilt.visible)).toEqual([ + "h2", + "h1", + "h4", + "h5", + "z4", + "a8", + "h6", + ]); + expect(instrumentation.work.filterRowsFlipped).toBe(1); + expect(instrumentation.work.bulkByIdDerived).toBe(1); + expect(instrumentation.work.bulkOrderVerificationsSkipped).toBe(1); + }); + + test("the two routes produce the same tree, key for key, on the same input", () => { + // Correctness twin for the pair above: whichever route ran, the result + // matches a cold model built directly under the next query. Routing is a + // cost decision and must be invisible in the output. + for (const nextQuery of [scoreQuery("gte", 50), scoreQuery("lte", 60)]) { + const { rebuilt, columns } = routeFixture(nextQuery); + const oracle = coldOracle(columns, nextQuery, ROOT_ROWS); + expect(rankedIds(rebuilt.visible)).toEqual(oracle.visibleIds); + const tree = rebuilt.visible.rows; + expect(tree.size).toBe(oracle.visibleIds.length); + for (const row of ROOT_ROWS) { + const visible = oracle.passesOf.get(row.id); + expect(rowPassesFilter(rebuilt, row.id)).toBe(visible); + if (visible) { + const entry = tree.get(row.id)!; + expect(entry.record.rowId).toBe(row.id); + // The map and the tree agree — the assertion the refill route gets + // for free and the derived route has to earn. + expect(tree.entryAt(tree.rankOf(row.id)!)).toBe(entry); + } else { + expect(tree.get(row.id)).toBeUndefined(); + } + } + } + }); + + test("zero flips takes NO bulk build at all, so neither proof is claimed", () => { + const columns = createColumns(); + const previousPlan = compileQuery({ + derivations: columns, + query: scoreQuery("gte", 40), + }); + const nextPlan = compileQuery({ + derivations: columns, + // Same membership, different plan identity: no row flips, so the + // visible tree carries whole and no builder runs. + query: scoreQuery("gt", 39), + }); + const captured = createRoot(previousPlan, ROOT_ROWS); + const instrumentation = testInstrumentation(); + + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + instrumentation, + }); + + expect(instrumentation.work.filterRowsFlipped).toBe(0); + expect(rebuilt.visible).toBe(captured.visible); + expect(instrumentation.work.bulkOrderVerificationsSkipped).toBe(0); + expect(instrumentation.work.bulkByIdDerived).toBe(0); + }); + + test("the derived byId agrees with the built tree at every visible id", () => { + const { nextPlan, captured } = createMainFixture(); + + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + // This is the derived map's whole correctness claim, and the survivors + // are the half a size check cannot see: `get` must return the SAME entry + // object the tree holds at that rank, not the captured tree's stale one. + const tree = rebuilt.visible.rows; + let survivorsChecked = 0; + for (let rank = 0; rank < tree.size; rank += 1) { + const entry = tree.entryAt(rank)!; + const id = entry.record.rowId; + expect(tree.get(id)).toBe(entry); + expect(tree.rankOf(id)).toBe(rank); + if (SURVIVORS.includes(id as never)) { + // Reused by identity — the precondition derived mode rides on. + expect(captured.visible.rows.get(id)).toBe(entry); + survivorsChecked += 1; + } + } + expect(survivorsChecked).toBe(SURVIVORS.length); + // The leavers are gone from the map, not merely from the tree. + for (const id of FLIPPED_OUT) { + expect(tree.get(id as never)).toBeUndefined(); + expect(tree.rankOf(id as never)).toBeUndefined(); + } + expect(tree.size).toBe(NEW_VISIBLE_ORDER.length); + }); + + test("throws TypeError when the plans are not a filter-only change", () => { + const { columns, captured } = createMainFixture(); + const sortAlsoChangedPlan = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [{ columnId: "score", operator: "lte", value: 60 }], + sort: [{ columnId: "note", direction: "desc" }], + rowGroups: [], + }), + }); + + expect(() => + rebuildRootForFilterOnlyChange({ + captured, + nextPlan: sortAlsoChangedPlan, + revision: 1, + now: () => 0, + }), + ).toThrowError( + new TypeError( + "Synchronous filter rebuild requires a filter-only plan change.", + ), + ); + }); + + test("throws TypeError for a grouped next plan", () => { + const columns = createColumns(); + const grouped = ( + operator: "gte" | "lte", + value: number, + ): PretableQueryFor => + queryFor({ + filters: [{ columnId: "score", operator, value }], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }); + const groupedPrevious = compileQuery({ + derivations: columns, + query: grouped("gte", 40), + }); + const groupedNext = compileQuery({ + derivations: columns, + query: grouped("lte", 60), + }); + const captured = createRoot(groupedPrevious, ROOT_ROWS); + + expect(() => + rebuildRootForFilterOnlyChange({ + captured, + nextPlan: groupedNext, + revision: 1, + now: () => 0, + }), + ).toThrowError( + new TypeError("Synchronous filter rebuild requires an ungrouped query."), + ); + }); + + test("a throwing filter-column accessor surfaces the accessor-failed shape and touches nothing", () => { + const boom = new Error("boom"); + // The captured root must already hold h5, so its evaluate must succeed; + // the accessor arms AFTER the capture and throws only on the rebuild's + // verdict read. h5 sits sixth in source order, so several rows succeed + // before the throw — partial work would be visible if state leaked. + // + // The throwing accessor belongs to the FIRST (and only) runtime filter, + // where the fast and slow paths are shape-identical. They deliberately + // diverge further right: the verdict seam evaluates filter values + // LAZILY with `every`-short-circuit, so a LATER filter's throwing + // accessor is skipped whenever an earlier filter already returned false + // — the eager slow path would have surfaced it. That case is + // intentional (the row's verdict is decidable without the read) and + // unreachable from this pin. + let armed = false; + const armedColumns = [ + helper.accessor("team", (row: Holding) => row.team, { type: "text" }), + helper.accessor( + "score", + (row: Holding): number => { + if (armed && row.id === "h5") throw boom; + return row.score; + }, + { type: "number", aggregate: "sum" }, + ), + helper.accessor("note", (row: Holding) => row.note, { type: "text" }), + ] as unknown as FixtureColumns; + const armedPrevious = compileQuery({ + derivations: armedColumns, + query: scoreQuery("gte", 40), + }); + const armedNext = compileQuery({ + derivations: armedColumns, + query: scoreQuery("lte", 60), + }); + const captured = createRoot(armedPrevious, ROOT_ROWS); + expect(rankedIds(captured.visible)).toEqual([...OLD_VISIBLE_ORDER]); + armed = true; + + let thrown: unknown; + try { + rebuildRootForFilterOnlyChange({ + captured, + nextPlan: armedNext, + revision: 1, + now: () => 0, + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(PretableRowModelError); + const error = thrown as PretableRowModelError; + expect(error.code).toBe("accessor-failed"); + expect(error.cause).toBe(boom); + expect(error.columnId).toBe("score"); + expect(error.rowId).toBe("h5"); + // State untouched: the captured root still publishes the OLD world. + expect(rankedIds(captured.visible)).toEqual([...OLD_VISIBLE_ORDER]); + for (const row of ROOT_ROWS) { + expect(rowPassesFilter(captured, row.id)).toBe(row.score >= 40); + } + }); +}); + +/** + * Minimal deterministic scheduler, duplicated from `sort-fast-path.test.ts` + * (test files here do not import from each other). + */ +class ManualScheduler implements CooperativeTransitionScheduler { + readonly entries: { readonly task: () => void; cancelled: boolean }[] = []; + + schedule(task: () => void): () => void { + const entry = { task, cancelled: false }; + this.entries.push(entry); + return () => { + entry.cancelled = true; + }; + } + + flushAll(limit = 1_000_000): void { + let count = 0; + for (;;) { + const entry = this.entries.shift(); + if (entry === undefined) return; + if (!entry.cancelled) entry.task(); + count += 1; + if (count > limit) throw new Error("Manual scheduler did not settle."); + } + } +} + +function snapshotIds(model: { + getState(): { snapshot: { range(a: number, b: number): readonly unknown[] } }; +}): readonly string[] { + return model + .getState() + .snapshot.range(0, Number.MAX_SAFE_INTEGER) + .flatMap((row) => + (row as { kind: string }).kind === "data" + ? [String((row as { rowId: unknown }).rowId)] + : [], + ); +} + +/** Cooperative vehicle: BOTH facets change, so neither fast path applies. */ +const COMBINED_CHANGE = queryFor({ + filters: [{ columnId: "score", operator: "lte", value: 60 }], + sort: [{ columnId: "note", direction: "desc" }], + rowGroups: [], +}); + +describe("setQuery filter-only fast path", () => { + /** + * Ticking clock + 1ms budget force the cooperative path to yield after + * every unit, so any scheduler entry is proof the cooperative machinery + * ran — and an empty queue is proof the fast path bypassed it. + */ + function createModelFixture(options?: { + readonly columns?: FixtureColumns; + readonly rows?: readonly Holding[]; + }) { + const scheduler = new ManualScheduler(); + let tick = 0; + const instrumented = createInstrumentedLocalRowModel({ + rows: options?.rows ?? ROOT_ROWS, + columns: options?.columns ?? createColumns(), + query: scoreQuery("gte", 40), + transitionScheduler: scheduler, + transitionClock: () => tick++, + transitionBudgetMs: 1, + }); + const model = instrumented.model; + expect(snapshotIds(model)).toEqual([...OLD_VISIBLE_ORDER]); + return { model, diagnostics: instrumented.diagnostics, scheduler }; + } + + test("resolves synchronously without any scheduler task", async () => { + const { model, diagnostics, scheduler } = createModelFixture(); + + const transition = model.setQuery(scoreQuery("lte", 60)); + + expect(scheduler.entries).toHaveLength(0); + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(snapshotIds(model)).toEqual([...NEW_VISIBLE_ORDER]); + expect(diagnostics.read().work.filterRebuilds).toBe(1); + expect(diagnostics.read().work.synchronousRebuilds).toBe(0); + await expect(transition.finished).resolves.toBe(1); + }); + + test("mutation twin: a combined sort+filter change takes the cooperative path", () => { + const { model, diagnostics, scheduler } = createModelFixture(); + + model.setQuery(COMBINED_CHANGE); + + expect( + scheduler.entries.length > 0 || + model.getState().status.kind === "rebuilding", + ).toBe(true); + expect(diagnostics.read().work.filterRebuilds).toBe(0); + }); + + test("supersedes an in-flight cooperative transition", async () => { + const { model, scheduler } = createModelFixture(); + const first = model.setQuery(COMBINED_CHANGE); + expect(model.getState().status.kind).toBe("rebuilding"); + + const second = model.setQuery(scoreQuery("lte", 60)); + + await expect(first.finished).rejects.toMatchObject({ + name: "PretableTransitionCancelledError", + reason: "superseded", + }); + await expect(first.finished).rejects.toBeInstanceOf( + PretableTransitionCancelledError, + ); + await expect(second.finished).resolves.toBe(1); + // The fast path rebuilt from the last COMMITTED root: OLD sort (note + // asc) + NEW filter. The abandoned note-desc sort must leave no trace. + expect(snapshotIds(model)).toEqual([...NEW_VISIBLE_ORDER]); + expect(model.getState().status).toEqual({ kind: "ready" }); + scheduler.flushAll(); + // Abandoned cooperative tasks must not resurrect the superseded query. + expect(snapshotIds(model)).toEqual([...NEW_VISIBLE_ORDER]); + }); + + test("notifies subscribers exactly once", () => { + const { model } = createModelFixture(); + let calls = 0; + model.subscribe(() => { + calls += 1; + }); + + model.setQuery(scoreQuery("lte", 60)); + + expect(calls).toBe(1); + }); + + test("snapshot.query and requestedQuery report the new filters", () => { + const { model } = createModelFixture(); + + const transition = model.setQuery(scoreQuery("lte", 60)); + + expect(transition.requestedQuery.filters).toEqual([ + { columnId: "score", operator: "lte", value: 60 }, + ]); + const snapshot = model.getState().snapshot; + expect(snapshot.query.filters).toEqual([ + { columnId: "score", operator: "lte", value: 60 }, + ]); + expect(snapshot.query.sort).toEqual([ + { columnId: "note", direction: "asc" }, + ]); + }); + + test('THE journal pin: the filter fast path journals a "refilter" reset, never "reorder"', () => { + // The highest-stakes assertion in this cycle: a "reorder" barrier tells + // renderers the row SET is unchanged and only permuted — after a filter + // change that is false, and acting on it would permute retained rows + // over a different membership and corrupt layout. "refilter" makes the + // opposite promise (membership changed, surviving order and identities + // did not), which is exactly what the filter fast path delivers. + const { model } = createModelFixture(); + const before = model.getState().snapshot.revision; + + model.setQuery(scoreQuery("lte", 60)); + + const sequence = model.changesSince(before); + expect(sequence).toEqual({ + kind: "reset", + toRevision: before + 1, + reason: "refilter", + }); + // The load-bearing half, kept explicit: whatever the reason evolves + // into, it must never be "reorder" for a membership change. + if (sequence.kind === "reset") { + expect(sequence.reason).not.toBe("reorder"); + } + }); + + test('mutation twin: a cooperative combined change journals "bulk-replace"', async () => { + const { model, scheduler } = createModelFixture(); + const before = model.getState().snapshot.revision; + + const transition = model.setQuery(COMBINED_CHANGE); + scheduler.flushAll(); + await expect(transition.finished).resolves.toBe(before + 1); + + expect(model.changesSince(before)).toEqual({ + kind: "reset", + toRevision: before + 1, + reason: "bulk-replace", + }); + }); + + test('setRows after a fast filter spans a mixed range: NOT "refilter"', () => { + const { model } = createModelFixture(); + const before = model.getState().snapshot.revision; + model.setQuery(scoreQuery("lte", 60)); + + const moved = ROOT_ROWS.map((row) => + row.id === "h3" ? { ...row, score: 20 } : row, + ); + model.setRows(moved); + + // The range [refilter barrier, setRows barrier] must NOT collapse to + // "refilter" — the setRows changed row content, not just membership. + expect(model.changesSince(before)).toEqual({ + kind: "reset", + toRevision: before + 2, + reason: "bulk-replace", + }); + // And the setRows commit alone is a plain barrier. + expect(model.changesSince(before + 1)).toEqual({ + kind: "reset", + toRevision: before + 2, + reason: "bulk-replace", + }); + }); + + test('positive twin: a sort-only setQuery on the same model still journals "reorder"', () => { + const { model } = createModelFixture(); + const before = model.getState().snapshot.revision; + + model.setQuery({ + filters: [{ columnId: "score", operator: "gte", value: 40 }], + sort: [{ columnId: "note", direction: "desc" }], + rowGroups: [], + }); + + expect(model.changesSince(before)).toEqual({ + kind: "reset", + toRevision: before + 1, + reason: "reorder", + }); + }); + + test('a fast filter then a fast sort spans a mixed range: "bulk-replace"', () => { + const { model } = createModelFixture(); + const before = model.getState().snapshot.revision; + + model.setQuery(scoreQuery("lte", 60)); + model.setQuery({ + filters: [{ columnId: "score", operator: "lte", value: 60 }], + sort: [{ columnId: "note", direction: "desc" }], + rowGroups: [], + }); + + // Neither promise holds over the whole range (membership changed AND + // order changed), so the aggregate degrades to the plain bulk reset — + // while each single-commit range keeps its own reason. + expect(model.changesSince(before)).toEqual({ + kind: "reset", + toRevision: before + 2, + reason: "bulk-replace", + }); + expect(model.changesSince(before + 1)).toEqual({ + kind: "reset", + toRevision: before + 2, + reason: "reorder", + }); + }); + + test("setRows immediately after a fast setQuery applies incrementally", () => { + const { model, diagnostics, scheduler } = createModelFixture(); + model.setQuery(scoreQuery("lte", 60)); + expect(diagnostics.read().work.filterRebuilds).toBe(1); + + // h3 (note "e", score 90) drops to 20: it now passes lte 60 and must + // insert between h5 ("d") and the "m" tie pair under the NEW plan. + const moved = ROOT_ROWS.map((row) => + row.id === "h3" ? { ...row, score: 20 } : row, + ); + model.setRows(moved); + + expect(snapshotIds(model)).toEqual([ + "h2", + "h1", + "h4", + "h5", + "h3", + "z4", + "a8", + "h6", + ]); + // Parity with normal incremental setRows: synchronous, no scheduler + // task, no additional whole-root rebuild. + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(scheduler.entries).toHaveLength(0); + expect(diagnostics.read().work.filterRebuilds).toBe(1); + }); + + test("order independence: slot reuse never leaks into the visible order", () => { + // Transaction history engineered so slot order ≠ source order ≠ visible + // order: build A,B,C,D (slots 0..3), remove B (slot 1 freed), add E (E + // takes B's slot, so it sits between A and C in SLOT order while sitting + // last in SOURCE order). A filter-only setQuery then flips A out and E + // in. This is the pin that fails if anyone later makes the rebuild's + // walk order-sensitive. + const rowA = { id: "A", team: "Alpha", score: 50, note: "d" }; + const rowB = { id: "B", team: "Alpha", score: 10, note: "x" }; + const rowC = { id: "C", team: "Alpha", score: 44, note: "b" }; + const rowD = { id: "D", team: "Alpha", score: 41, note: "a" }; + const rowE = { id: "E", team: "Alpha", score: 30, note: "c" }; + const instrumented = createInstrumentedLocalRowModel({ + rows: [rowA, rowB, rowC, rowD], + columns: createColumns(), + query: scoreQuery("gte", 40), + getRowId: (row: Holding) => row.id, + }); + const model = instrumented.model; + const internals = () => getLocalRowModelSlotInternalsForTesting(model); + const slotOf = (id: string) => internals().root.rows.get(id)!.slot; + const bSlot = slotOf("B"); + model.setRows([rowA, rowC, rowD]); + model.setRows([rowA, rowC, rowD, rowE]); + // Precondition, asserted so the pin cannot go vacuous: E really does + // reuse B's released slot, so E precedes C and D in slot order while + // following them in source order. + expect(slotOf("E")).toBe(bSlot); + expect(slotOf("E")).toBeLessThan(slotOf("C")); + + const transition = model.setQuery(scoreQuery("lte", 45)); + + // The fast path ran (the pin exercises the rebuild, not a fallback)… + expect(instrumented.diagnostics.read().work.filterRebuilds).toBe(1); + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(transition.id).toBeGreaterThan(0); + // …and the visible sequence is EXACTLY what a freshly-built model with + // the same final rows and query publishes — slot history invisible. + const fresh = createInstrumentedLocalRowModel({ + rows: [rowA, rowC, rowD, rowE], + columns: createColumns(), + query: scoreQuery("lte", 45), + getRowId: (row: Holding) => row.id, + }).model; + expect(snapshotIds(model)).toEqual(snapshotIds(fresh)); + expect(snapshotIds(model)).toEqual(["D", "C", "E"]); + }); + + test("equivalence with a cold model built directly under the next query", () => { + const { model: warm } = createModelFixture(); + warm.setQuery(scoreQuery("lte", 60)); + const cold = createInstrumentedLocalRowModel({ + rows: ROOT_ROWS, + columns: createColumns(), + query: scoreQuery("lte", 60), + }).model; + + const warmSnapshot = warm.getState().snapshot; + const coldSnapshot = cold.getState().snapshot; + expect(warmSnapshot.visibleRowCount).toBe(coldSnapshot.visibleRowCount); + for (let index = 0; index < warmSnapshot.visibleRowCount; index += 1) { + const warmRow = warmSnapshot.rowAt(index)!; + const coldRow = coldSnapshot.rowAt(index)!; + expect(warmRow.kind).toBe("data"); + expect(warmRow.kind === "data" && coldRow.kind === "data").toBe(true); + if (warmRow.kind === "data" && coldRow.kind === "data") { + expect(warmRow.rowId).toBe(coldRow.rowId); + expect(warmRow.row).toBe(coldRow.row); + } + } + expect(warmSnapshot.query).toEqual(coldSnapshot.query); + }); + + /** + * The throwing accessor belongs to the FIRST (and only) runtime filter, + * where the fast and slow paths are shape-identical (see the module-level + * failure test for the intentional lazy-evaluation divergence on LATER + * filters). It arms after mount so the initial build succeeds. + */ + function armedThrowingFixture(boom: Error) { + const armedRef = { current: false }; + const columns = [ + helper.accessor("team", (row: Holding) => row.team, { type: "text" }), + helper.accessor( + "score", + (row: Holding): number => { + if (armedRef.current && row.id === "h5") throw boom; + return row.score; + }, + { type: "number", aggregate: "sum" }, + ), + helper.accessor("note", (row: Holding) => row.note, { type: "text" }), + ] as unknown as FixtureColumns; + return { columns, armedRef }; + } + + function expectAccessorFailureShape( + model: ReturnType["model"], + transitionId: number, + boom: Error, + ): PretableRowModelError { + const status = model.getState().status; + expect(status.kind).toBe("error"); + if (status.kind !== "error") throw new Error("unreachable"); + expect(status.transitionId).toBe(transitionId); + expect(status.error).toBeInstanceOf(PretableRowModelError); + const error = status.error as PretableRowModelError; + expect(error.code).toBe("accessor-failed"); + expect(error.cause).toBe(boom); + return error; + } + + test("predicate accessor failure on the SLOW path pins the error shape", async () => { + const boom = new Error("boom"); + const { columns, armedRef } = armedThrowingFixture(boom); + const { model, scheduler } = createModelFixture({ columns }); + armedRef.current = true; + + // Filter AND sort change: not filter-only, so the cooperative path runs + // the throwing accessor. + const transition = model.setQuery(COMBINED_CHANGE); + scheduler.flushAll(); + + const error = expectAccessorFailureShape(model, transition.id, boom); + await expect(transition.finished).rejects.toBe(error); + // Root unchanged: the OLD committed order is still published. + expect(snapshotIds(model)).toEqual([...OLD_VISIBLE_ORDER]); + }); + + test("predicate accessor failure on the fast path matches the slow path's shape", async () => { + const boom = new Error("boom"); + const { columns, armedRef } = armedThrowingFixture(boom); + const { model, scheduler, diagnostics } = createModelFixture({ columns }); + armedRef.current = true; + + const transition = model.setQuery(scoreQuery("lte", 60)); + + // Must not throw synchronously, must not schedule cooperative work. + expect(scheduler.entries).toHaveLength(0); + const error = expectAccessorFailureShape(model, transition.id, boom); + await expect(transition.finished).rejects.toBe(error); + expect(snapshotIds(model)).toEqual([...OLD_VISIBLE_ORDER]); + expect(diagnostics.read().work.filterRebuilds).toBe(0); + + // A subsequent valid filter-only setQuery recovers to ready. + armedRef.current = false; + const recovery = model.setQuery(scoreQuery("lte", 60)); + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(snapshotIds(model)).toEqual([...NEW_VISIBLE_ORDER]); + await expect(recovery.finished).resolves.toBe(1); + }); +}); + +/** + * The adoption is a CACHE-SHARING change: after a filter-only rebuild the + * next plan reads the previous plan's evaluation cache by reference. These + * tests hold the two halves of that bargain — the shared fields really are + * valid under the new plan, and the one field that is NOT (the verdict memo) + * never answers for the adopter — plus every chain that composes adoption + * with another path. + */ +describe("evaluation-cache adoption", () => { + /** Columns whose accessors are spies, so "no re-read" is observable. */ + function spyColumns() { + const teamAccessor = vi.fn((row: Holding) => row.team); + const scoreAccessor = vi.fn((row: Holding) => row.score); + const noteAccessor = vi.fn((row: Holding) => row.note); + const columns = [ + helper.accessor("team", teamAccessor, { type: "text" }), + helper.accessor("score", scoreAccessor, { + type: "number", + aggregate: "sum", + }), + helper.accessor("note", noteAccessor, { type: "text" }), + ] as unknown as FixtureColumns; + return { columns, teamAccessor, scoreAccessor, noteAccessor }; + } + + function adoptedFixture( + previousQuery: PretableQueryFor = scoreQuery("gte", 40), + nextQuery: PretableQueryFor = scoreQuery("lte", 60), + ) { + const spies = spyColumns(); + const previousPlan = compileQuery({ + derivations: spies.columns, + query: previousQuery, + }); + const nextPlan = compileQuery({ + derivations: spies.columns, + query: nextQuery, + }); + const captured = createRoot(previousPlan, ROOT_ROWS); + const instrumentation = testInstrumentation(); + const rebuilt = rebuildRootForFilterOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + instrumentation, + }); + expect(instrumentation.work.evaluationCacheAdoptions).toBe(1); + return { ...spies, previousPlan, nextPlan, captured, rebuilt }; + } + + /** The input triple `evaluate` was originally called with, for one row. */ + function inputFor(rowId: string) { + const sourceOrder = ROOT_ROWS.findIndex((row) => row.id === rowId); + return { + rowId, + row: ROOT_ROWS[sourceOrder], + sourceOrder, + slot: sourceOrder, + }; + } + + test("an adopted metadata hit is CORRECT under the new plan and re-reads nothing", () => { + const fixture = adoptedFixture(); + // h1 survives the flip untouched — the case most likely to be served + // from the adopted entry rather than recomputed. + const input = inputFor("h1"); + // The oracle runs the same spy accessors, so build it BEFORE clearing. + const oracle = coldOracle(fixture.columns, scoreQuery("lte", 60), [ + ...ROOT_ROWS, + ]); + fixture.teamAccessor.mockClear(); + fixture.scoreAccessor.mockClear(); + fixture.noteAccessor.mockClear(); + + const metadata = fixture.nextPlan.evaluate(input); + + // Content, not identity: what the NEW plan promises for this row. + expect(metadata.rowId).toBe("h1"); + expect(metadata.row).toBe(input.row); + expect(metadata.sourceOrder).toBe(input.sourceOrder); + expect(metadata.groupPath).toEqual([]); + expect( + metadata.aggregateLeaves.map((leaf) => ({ + columnId: leaf.columnId, + value: leaf.allLeaf.value, + dependency: leaf.allLeaf.dependency, + })), + ).toEqual([ + { + columnId: "score", + value: 50, + dependency: { + sourceOrder: input.sourceOrder, + sortKeys: [{ columnId: "note", value: "b" }], + }, + }, + ]); + // …and against an independently compiled COLD twin of the new plan. + expect(metadata).toEqual(oracle.metadataOf.get("h1")); + // The whole point of the adoption: zero accessor work on the hit. + expect(fixture.teamAccessor).not.toHaveBeenCalled(); + expect(fixture.scoreAccessor).not.toHaveBeenCalled(); + expect(fixture.noteAccessor).not.toHaveBeenCalled(); + }); + + test("an adopted entry's VERDICT memo never answers for the adopting plan", () => { + // h3 passes `gte 40` (score 90) and fails `lte 60`: if the adopted memo + // leaked, the new plan would report the old verdict for it. + const fixture = adoptedFixture(); + const input = inputFor("h3"); + expect(filterVerdict(fixture.previousPlan, input)).toBe(true); + fixture.scoreAccessor.mockClear(); + + expect(filterVerdict(fixture.nextPlan, input)).toBe(false); + // Proof it was recomputed rather than remembered: the filter column's + // accessor ran. (An adopting plan pays exactly the pass it paid before + // the adoption existed — the memo was never available to it.) + expect(fixture.scoreAccessor).toHaveBeenCalledTimes(1); + // The previous plan keeps its own memo: sharing is symmetric-safe. + expect(filterVerdict(fixture.previousPlan, input)).toBe(true); + }); + + test("the adopted store hands back the previous plan's key arrays BY IDENTITY", () => { + const fixture = adoptedFixture(); + for (const row of ROOT_ROWS) { + const input = inputFor(row.id); + const previousKeys = sortKeysOf(fixture.previousPlan, input); + // Identity, not equality: the adoption's entire saving is that no new + // array is produced for any row. + expect(sortKeysOf(fixture.nextPlan, input)).toBe(previousKeys); + } + // And the entries the rebuild put in the tree carry those same arrays. + for (const entry of fixture.rebuilt.visible.rows.entries()) { + expect(entry.keys).toBe( + sortKeysOf(fixture.previousPlan, inputFor(entry.record.rowId)), + ); + } + }); + + test("chain: filter change, then a sort-only change over the ADOPTED cache", () => { + const fixture = adoptedFixture(); + const sortedPlan = compileQuery({ + derivations: fixture.columns, + query: queryFor({ + filters: [{ columnId: "score", operator: "lte", value: 60 }], + sort: [{ columnId: "note", direction: "desc" }], + rowGroups: [], + }), + }); + expect(isSortOnlyChange(fixture.rebuilt.queryPlan, sortedPlan)).toBe(true); + + const resorted = rebuildRootForSortOnlyChange({ + captured: fixture.rebuilt, + nextPlan: sortedPlan, + revision: 2, + now: () => 0, + }); + + // The sort path fills its OWN store from the adopted one, so the new + // plan's keys are fresh objects with the same values, and the order is + // the cold model's — note DESC, with the z4/a8 tie still broken by + // sourceOrder (a reversal of the asc order would swap them). + const oracle = coldOracle( + fixture.columns, + queryFor({ + filters: [{ columnId: "score", operator: "lte", value: 60 }], + sort: [{ columnId: "note", direction: "desc" }], + rowGroups: [], + }), + [...ROOT_ROWS], + ); + expect(rankedIds(resorted.visible)).toEqual(oracle.visibleIds); + expect(oracle.visibleIds).not.toEqual([...NEW_VISIBLE_ORDER].reverse()); + for (const row of ROOT_ROWS) { + const input = inputFor(row.id); + expect(sortKeysOf(sortedPlan, input)).toEqual([ + { columnId: "note", value: row.note }, + ]); + // Fresh arrays: a sort change is exactly the change that may NOT share. + expect(sortKeysOf(sortedPlan, input)).not.toBe( + sortKeysOf(fixture.nextPlan, input), + ); + } + }); + + test("chain: a filter change adopting an ALREADY-adopted cache", () => { + const fixture = adoptedFixture(); + const thirdPlan = compileQuery({ + derivations: fixture.columns, + query: scoreQuery("gte", 20), + }); + const instrumentation = testInstrumentation(); + + const third = rebuildRootForFilterOnlyChange({ + captured: fixture.rebuilt, + nextPlan: thirdPlan, + revision: 2, + now: () => 0, + instrumentation, + }); + + expect(instrumentation.work.evaluationCacheAdoptions).toBe(1); + expect(instrumentation.work.sortKeyCarries).toBe(0); + const oracle = coldOracle(fixture.columns, scoreQuery("gte", 20), [ + ...ROOT_ROWS, + ]); + expect(rankedIds(third.visible)).toEqual(oracle.visibleIds); + for (const row of ROOT_ROWS) { + expect(rowPassesFilter(third, row.id)).toBe(oracle.passesOf.get(row.id)); + // Still the FIRST plan's arrays: the map is the same object throughout. + expect(sortKeysOf(thirdPlan, inputFor(row.id))).toBe( + sortKeysOf(fixture.previousPlan, inputFor(row.id)), + ); + } + }); + + test("chain: filter change, then a same-reference mutation recompile", () => { + // The A2 rebuild-or-reseed invariant, run against an ADOPTED cache: the + // recompile is a plan swap whose fresh store is seeded from the plan + // that adopted, and the visible index must be rebuilt under the fresh + // plan so the mutated row re-ranks. + const mutable = Object.preventExtensions({ + id: "m1", + team: "Alpha", + score: 10, + note: "b", + }); + const other = { id: "m2", team: "Alpha", score: 20, note: "c" }; + const third = { id: "m3", team: "Alpha", score: 30, note: "a" }; + const model = createLocalRowModel({ + rows: [mutable, other, third], + columns: createColumns(), + query: scoreQuery("gte", 0), + getRowId: (row) => row.id, + }); + expect(snapshotIds(model)).toEqual(["m3", "m1", "m2"]); + + // Filter-only change first: this is the adoption. + model.setQuery(scoreQuery("gte", 15)); + expect(snapshotIds(model)).toEqual(["m3", "m2"]); + + // Now mutate IN PLACE on the sort column and hand back the same refs. + mutable.score = 99; + mutable.note = "zz"; + model.setRows([mutable, other, third]); + expect(snapshotIds(model)).toEqual(["m3", "m2", "m1"]); + + // Follow-up update of a CARRIED row: its previous record must resolve + // under the recompiled plan's own store, or the fail-loud miss throws. + model.setRows([mutable, { ...other, note: "zzz" }, third]); + expect(snapshotIds(model)).toEqual(["m3", "m1", "m2"]); + }); + + test("a row absent from the adopted cache evaluates fresh and correctly", () => { + const rows = ROOT_ROWS.map((row) => ({ ...row })); + const model = createLocalRowModel({ + rows, + columns: createColumns(), + query: scoreQuery("gte", 40), + getRowId: (row) => row.id, + }); + model.setQuery(scoreQuery("lte", 60)); + expect(snapshotIds(model)).toEqual([...NEW_VISIBLE_ORDER]); + + // `n1` was never seen by either plan, so the adopted map has no entry. + const arrival: Holding = { + id: "n1", + team: "Alpha", + score: 25, + note: "ba", + }; + model.setRows([...rows, arrival]); + + // Sorted by note: "ba" sits between "b" (h1) and "c" (h4), and 25 passes + // `lte 60`, so the newcomer is visible in its own rank. + expect(snapshotIds(model)).toEqual([ + "h2", + "h1", + "n1", + "h4", + "h5", + "z4", + "a8", + "h6", + ]); + }); + + test("a GROUPED model's filter change adopts nothing (it never takes the fast path)", () => { + const scheduler = new ManualScheduler(); + let tick = 0; + const groupedQuery = ( + operator: "gte" | "lte", + value: number, + ): PretableQueryFor => + queryFor({ + filters: [{ columnId: "score", operator, value }], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }); + const instrumented = createInstrumentedLocalRowModel({ + rows: ROOT_ROWS, + columns: createColumns(), + query: groupedQuery("gte", 40), + transitionScheduler: scheduler, + transitionClock: () => tick++, + transitionBudgetMs: 1, + }); + + instrumented.model.setQuery(groupedQuery("lte", 60)); + scheduler.flushAll(); + + expect(instrumented.diagnostics.read().work.filterRebuilds).toBe(0); + expect(instrumented.diagnostics.read().work.evaluationCacheAdoptions).toBe( + 0, + ); + expect(instrumented.model.getState().status).toEqual({ kind: "ready" }); + // The grouped result is still right, which is what "unaffected" means. + expect(snapshotIds(instrumented.model)).toEqual([ + "h2", + "h1", + "h4", + "h5", + "z4", + "a8", + "h6", + ]); + }); + + test("adoption requires compiled plans on BOTH sides", () => { + const fixture = adoptedFixture(); + const foreign = { evaluate: () => undefined } as never; + expect(() => adoptEvaluationCache(fixture.nextPlan, foreign)).toThrowError( + new TypeError("Evaluation-cache adoption requires compiled query plans."), + ); + expect(() => + adoptEvaluationCache(foreign, fixture.previousPlan), + ).toThrowError( + new TypeError("Evaluation-cache adoption requires compiled query plans."), + ); + }); +}); diff --git a/packages/row-model/src/__tests__/filter-membership.test.ts b/packages/row-model/src/__tests__/filter-membership.test.ts new file mode 100644 index 000000000..cecdae910 --- /dev/null +++ b/packages/row-model/src/__tests__/filter-membership.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, test } from "vitest"; + +import { + compileQuery, + createColumnHelper, + type PretableQueryFor, +} from "../index"; +import { filterVerdict, type CompiledQuery } from "../compiled-query"; +import { + rowPassesFilter, + rowPassesFilterInGroupIndex, +} from "../filter-membership"; +import { getGroupIndex } from "../group-index"; +import type { RevisionRoot } from "../internal-types"; +import { createPersistentMap } from "../persistent/persistent-map"; +import { buildRowStore } from "../row-store"; +import { createSlotAllocator } from "../slot-allocator"; +import type { PretableGroupId } from "../types"; +import { EMPTY_MEMBERSHIP } from "../membership-bitset"; +import { createVisibleIndex, membershipFromFlatTree } from "../visible-index"; + +interface Holding { + id: string; + team: string; + score: number; +} + +const helper = createColumnHelper(); + +function createColumns() { + return [ + helper.accessor("team", (row: Holding) => row.team, { type: "text" }), + helper.accessor("score", (row: Holding) => row.score, { + type: "number", + aggregate: "sum", + }), + ] as const; +} + +type FixtureColumns = ReturnType; + +/** + * Both populations are non-trivial in BOTH group branches: Alpha and Beta + * each hold a passing and a failing row, so a helper that answered from the + * group's existence, or from "any row in the root", would be caught. + */ +const ROWS: readonly Holding[] = Object.freeze([ + { id: "r1", team: "Alpha", score: 90 }, + { id: "r2", team: "Alpha", score: 10 }, + { id: "r3", team: "Beta", score: 70 }, + { id: "r4", team: "Beta", score: 20 }, +]); + +const PASSING = ["r1", "r3"] as const; +const FAILING = ["r2", "r4"] as const; + +function query(grouped: boolean): PretableQueryFor { + return { + filters: [{ columnId: "score", operator: "gte", value: 50 }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: grouped ? [{ columnId: "team", direction: "asc" }] : [], + } as PretableQueryFor; +} + +function createRoot( + queryPlan: CompiledQuery, + rows: readonly Holding[] = ROWS, +): RevisionRoot { + const slots = createSlotAllocator(); + const store = buildRowStore({ + rows, + getRowId: (row) => row.id, + queryPlan, + slots, + }); + const defaultPolicy = Object.freeze({ kind: "expanded" as const }); + const expansion = Object.freeze({ + default: defaultPolicy, + overrides: createPersistentMap(), + state: Object.freeze({ default: defaultPolicy, overrideCount: 0 }), + }); + const visible = createVisibleIndex( + store.records, + queryPlan, + false, + expansion.overrides, + ); + return Object.freeze({ + revision: 0, + parentRevision: null, + rows: store.rows, + sourceOrder: store.sourceOrder, + recordsBySlot: store.recordsBySlot, + slotCapacity: slots.capacity, + // Same rule as the production initial-build site: flat roots index their + // membership per slot, grouped roots carry the sentinel. + visibleSlots: + queryPlan.query.rowGroups.length > 0 + ? EMPTY_MEMBERSHIP + : membershipFromFlatTree(visible.rows, slots.capacity), + visible, + queryPlan, + expansion, + cause: Object.freeze({ kind: "initial" as const }), + }); +} + +describe("rowPassesFilter", () => { + test("a FLAT root answers from the visible tree", () => { + const plan = compileQuery({ + derivations: createColumns(), + query: query(false), + }); + const root = createRoot(plan); + // Control: this root's verdicts are genuinely mixed. + expect(root.visible.rows.size).toBe(PASSING.length); + + for (const id of PASSING) expect(rowPassesFilter(root, id)).toBe(true); + for (const id of FAILING) expect(rowPassesFilter(root, id)).toBe(false); + }); + + test("a GROUPED root answers from group-index leaf membership", () => { + const plan = compileQuery({ + derivations: createColumns(), + query: query(true), + }); + const root = createRoot(plan); + // Control: the grouped root's flat tree is empty, so a helper that only + // consulted `visible.rows` would answer false for EVERY row here. + expect(root.visible.rows.size).toBe(0); + expect(getGroupIndex(root.visible)).toBeDefined(); + + for (const id of PASSING) expect(rowPassesFilter(root, id)).toBe(true); + for (const id of FAILING) expect(rowPassesFilter(root, id)).toBe(false); + }); + + test("both shapes agree with the plan's own verdict for every row", () => { + for (const grouped of [false, true]) { + const plan = compileQuery({ + derivations: createColumns(), + query: query(grouped), + }); + const root = createRoot(plan); + for (const [sourceOrder, row] of ROWS.entries()) { + expect(rowPassesFilter(root, row.id)).toBe( + filterVerdict(plan, { + rowId: row.id, + row, + sourceOrder, + slot: sourceOrder, + }), + ); + } + } + }); + + test("an unknown row is not a member, in either shape", () => { + for (const grouped of [false, true]) { + const plan = compileQuery({ + derivations: createColumns(), + query: query(grouped), + }); + expect(rowPassesFilter(createRoot(plan), "never-seen")).toBe(false); + } + }); + + test("the grouped accessor answers directly from a group index", () => { + const plan = compileQuery({ + derivations: createColumns(), + query: query(true), + }); + const grouped = getGroupIndex(createRoot(plan).visible)!; + + for (const id of PASSING) + expect(rowPassesFilterInGroupIndex(grouped, id)).toBe(true); + for (const id of FAILING) { + // Present in the index (it has a parent) but not in the leaf tree. + expect(grouped.rowParents.get(id)).toBeDefined(); + expect(rowPassesFilterInGroupIndex(grouped, id)).toBe(false); + } + }); + + test("with no filters every row is a member, in either shape", () => { + for (const grouped of [false, true]) { + const plan = compileQuery({ + derivations: createColumns(), + query: { + filters: [], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: grouped ? [{ columnId: "team", direction: "asc" }] : [], + } as PretableQueryFor, + }); + const root = createRoot(plan); + for (const row of ROWS) expect(rowPassesFilter(root, row.id)).toBe(true); + } + }); +}); diff --git a/packages/row-model/src/__tests__/filter-verdicts.test.ts b/packages/row-model/src/__tests__/filter-verdicts.test.ts new file mode 100644 index 000000000..23fec8f6c --- /dev/null +++ b/packages/row-model/src/__tests__/filter-verdicts.test.ts @@ -0,0 +1,632 @@ +/** + * The membership-verdict migration's behavioural gates. `CompiledRowMetadata` + * no longer carries `filterPasses` and aggregate leaves no longer carry a + * filtered twin: a row's verdict is its MEMBERSHIP in the root's visible + * structure. These tests hold the consequences that a purely mechanical + * rewrite could get wrong — old-verdict resolution under a same-reference + * mutation, grouped aggregation under both populations, the distinct-value + * "filtered" population, and the zero-rebuild claim on the filter fast path. + */ + +import { describe, expect, test } from "vitest"; + +import { + createColumnHelper, + createLocalRowModel, + type PretableAggregator, + type PretableGroupId, + type PretableQueryFor, +} from "../index"; +import type { CooperativeTransitionScheduler } from "../cooperative-transition"; +import { createInstrumentedLocalRowModel } from "../diagnostics"; + +interface Holding { + id: string; + team: string; + score: number; +} + +const helper = createColumnHelper(); + +const trace: PretableAggregator = { + init: () => [], + accumulate: (accumulator, value) => [...accumulator, value], + merge: (left, right) => [...left, ...right], + finalize: (accumulator) => [...accumulator].sort((a, b) => a - b).join("|"), +}; + +function createColumns() { + return [ + helper.accessor("team", (row: Holding) => row.team, { type: "text" }), + helper.accessor("score", (row: Holding) => row.score, { + type: "number", + aggregate: "sum", + }), + helper.accessor("id", (row: Holding) => row.score, { + type: "number", + aggregate: trace, + }), + ] as const; +} + +type FixtureColumns = ReturnType; + +/** Both teams straddle every threshold used below, in both directions. */ +const ROWS: readonly Holding[] = Object.freeze([ + { id: "a1", team: "Alpha", score: 10 }, + { id: "a2", team: "Alpha", score: 50 }, + { id: "a3", team: "Alpha", score: 90 }, + { id: "b1", team: "Beta", score: 20 }, + { id: "b2", team: "Beta", score: 60 }, + { id: "b3", team: "Beta", score: 80 }, +]); + +function flatQuery(threshold: number): PretableQueryFor { + return { + filters: [{ columnId: "score", operator: "gte", value: threshold }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + } as PretableQueryFor; +} + +function groupedQuery(threshold: number): PretableQueryFor { + return { + filters: [{ columnId: "score", operator: "gte", value: threshold }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + } as PretableQueryFor; +} + +class ManualScheduler implements CooperativeTransitionScheduler { + readonly entries: { readonly task: () => void; cancelled: boolean }[] = []; + + schedule(task: () => void): () => void { + const entry = { task, cancelled: false }; + this.entries.push(entry); + return () => { + entry.cancelled = true; + }; + } + + flushAll(limit = 1_000_000): void { + let count = 0; + for (;;) { + const entry = this.entries.shift(); + if (entry === undefined) return; + if (!entry.cancelled) entry.task(); + count += 1; + if (count > limit) throw new Error("Manual scheduler did not settle."); + } + } +} + +function visibleDataIds(model: { + getState(): { snapshot: { range(a: number, b: number): readonly unknown[] } }; +}): readonly string[] { + return model + .getState() + .snapshot.range(0, Number.MAX_SAFE_INTEGER) + .flatMap((row) => + (row as { kind: string }).kind === "data" + ? [String((row as { rowId: unknown }).rowId)] + : [], + ); +} + +function groupSummaries(model: { + getState(): { snapshot: { range(a: number, b: number): readonly unknown[] } }; +}) { + return model + .getState() + .snapshot.range(0, Number.MAX_SAFE_INTEGER) + .flatMap((row) => { + const candidate = row as { + kind: string; + groupId: PretableGroupId; + childCount: number; + aggregates: Readonly>; + }; + return candidate.kind === "group" + ? [ + { + groupId: candidate.groupId, + childCount: candidate.childCount, + aggregates: { ...candidate.aggregates }, + }, + ] + : []; + }); +} + +describe("same-reference mutation that flips a verdict", () => { + /** + * The case a row-object-keyed verdict store could not serve: the row OBJECT + * is unchanged, so nothing keyed by it can hold two answers at once. The + * OLD verdict comes from the committed root's membership and the NEW one is + * computed under the drafting plan — two distinct authorities, and the + * visible set is only right if each site asked the right one. + */ + function mutatingFixture(from: number, to: number) { + // `preventExtensions` (not `freeze`): the model fingerprints extensible + // rows and rejects in-place edits to them, so this is how a real consumer + // reaches the same-reference-mutation path. + const rows: Holding[] = ROWS.map((row) => + Object.preventExtensions({ ...row }), + ); + const diagnostics: { readonly code: string }[] = []; + const model = createLocalRowModel({ + rows, + columns: createColumns(), + getRowId: (row) => row.id, + query: flatQuery(50), + onDiagnostic: (diagnostic) => diagnostics.push(diagnostic), + }); + expect(visibleDataIds(model)).toEqual(["a2", "b2", "b3", "a3"]); + const target = rows.find((row) => row.score === from)!; + // In place: same object, same id, same source order. + target.score = to; + return { model, rows, target, diagnostics }; + } + + /** Proof the fixture reached the path it claims to test. */ + function expectSameReferenceMutation( + diagnostics: readonly { readonly code: string }[], + ) { + expect(diagnostics.map((diagnostic) => diagnostic.code)).toContain( + "same-reference-row-mutation", + ); + } + + test("a row mutated INTO the filter is inserted at its sorted position", () => { + const { model, rows, target, diagnostics } = mutatingFixture(10, 55); + + model.setRows(rows); + + expectSameReferenceMutation(diagnostics); + expect(target.id).toBe("a1"); + expect(visibleDataIds(model)).toEqual(["a2", "a1", "b2", "b3", "a3"]); + }); + + test("a row mutated OUT of the filter is removed", () => { + const { model, rows, target, diagnostics } = mutatingFixture(90, 5); + + model.setRows(rows); + + expectSameReferenceMutation(diagnostics); + expect(target.id).toBe("a3"); + expect(visibleDataIds(model)).toEqual(["a2", "b2", "b3"]); + }); + + test("a mutation that does NOT flip the verdict leaves membership alone", () => { + const { model, rows, target, diagnostics } = mutatingFixture(90, 70); + + model.setRows(rows); + + expectSameReferenceMutation(diagnostics); + expect(target.id).toBe("a3"); + expect(visibleDataIds(model)).toEqual(["a2", "b2", "a3", "b3"]); + }); + + test("the grouped shape survives the same flip", () => { + // `preventExtensions` (not `freeze`): the model fingerprints extensible + // rows and rejects in-place edits to them, so this is how a real consumer + // reaches the same-reference-mutation path. + const rows: Holding[] = ROWS.map((row) => + Object.preventExtensions({ ...row }), + ); + const diagnostics: { readonly code: string }[] = []; + const model = createLocalRowModel({ + rows, + columns: createColumns(), + getRowId: (row) => row.id, + initialExpansion: { kind: "expanded" }, + query: groupedQuery(50), + onDiagnostic: (diagnostic) => diagnostics.push(diagnostic), + }); + expect(visibleDataIds(model)).toEqual(["a2", "a3", "b2", "b3"]); + rows.find((row) => row.id === "a1")!.score = 55; + + model.setRows(rows); + + expectSameReferenceMutation(diagnostics); + expect(visibleDataIds(model)).toEqual(["a2", "a1", "a3", "b2", "b3"]); + const alpha = groupSummaries(model).find((group) => + group.groupId.includes("Alpha"), + )!; + expect(alpha.childCount).toBe(3); + }); +}); + +describe("a rows replacement that flips a verdict", () => { + /** + * `replaceFlatRowsDraft` removes a row from the visible draft on its OLD + * verdict and re-inserts on its NEW one. Reading the new verdict for the + * removal leaves a flipped-out row stranded in the visible tree. + */ + function replaced(edit: (row: Holding) => Holding) { + const model = createLocalRowModel({ + rows: ROWS, + columns: createColumns(), + getRowId: (row) => row.id, + query: flatQuery(50), + }); + expect(visibleDataIds(model)).toEqual(["a2", "b2", "b3", "a3"]); + model.setRows(ROWS.map((row) => edit({ ...row }))); + return model; + } + + test("a row edited OUT of the filter leaves the visible set", () => { + const model = replaced((row) => + row.id === "b2" ? { ...row, score: 5 } : row, + ); + + expect(visibleDataIds(model)).toEqual(["a2", "b3", "a3"]); + }); + + test("a row edited INTO the filter joins at its sorted position", () => { + const model = replaced((row) => + row.id === "a1" ? { ...row, score: 55 } : row, + ); + + expect(visibleDataIds(model)).toEqual(["a2", "a1", "b2", "b3", "a3"]); + }); + + test("an unrelated edit leaves membership alone", () => { + const model = replaced((row) => + row.id === "b1" ? { ...row, team: "Gamma" } : row, + ); + + expect(visibleDataIds(model)).toEqual(["a2", "b2", "b3", "a3"]); + }); +}); + +describe("a transaction update that flips a verdict emits the right ops", () => { + /** + * The old-verdict sites in `applyFlatTransactionDraft` decide whether the + * row is REMOVED from the visible draft and whether a `remove`/`move` op + * carries a `previousIndex`. Resolving them against the drafting plan + * instead of the committed root's membership silently drops both. + */ + function transactionFixture() { + const model = createLocalRowModel({ + rows: ROWS, + columns: createColumns(), + getRowId: (row) => row.id, + query: flatQuery(50), + }); + expect(visibleDataIds(model)).toEqual(["a2", "b2", "b3", "a3"]); + return { model, before: model.getState().snapshot.revision }; + } + + function operationsSince( + model: ReturnType["model"], + before: number, + ) { + const sequence = model.changesSince(before); + if (sequence.kind !== "changes") + throw new Error(`Expected per-row changes, got ${sequence.kind}.`); + return sequence.changes.flatMap((change) => [...change.operations]); + } + + test("flipping OUT removes the row from its old index", () => { + const { model, before } = transactionFixture(); + + model.applyTransaction({ update: [{ id: "b2", changes: { score: 5 } }] }); + + expect(visibleDataIds(model)).toEqual(["a2", "b3", "a3"]); + expect(operationsSince(model, before)).toEqual([ + { kind: "remove", ref: { kind: "data", rowId: "b2" }, previousIndex: 1 }, + ]); + }); + + test("flipping IN inserts the row at its new index", () => { + const { model, before } = transactionFixture(); + + model.applyTransaction({ update: [{ id: "a1", changes: { score: 55 } }] }); + + expect(visibleDataIds(model)).toEqual(["a2", "a1", "b2", "b3", "a3"]); + expect(operationsSince(model, before)).toEqual([ + { kind: "insert", ref: { kind: "data", rowId: "a1" }, index: 1 }, + ]); + }); + + test("staying IN while moving emits move + update, not remove + insert", () => { + const { model, before } = transactionFixture(); + + model.applyTransaction({ update: [{ id: "a2", changes: { score: 99 } }] }); + + expect(visibleDataIds(model)).toEqual(["b2", "b3", "a3", "a2"]); + expect(operationsSince(model, before)).toEqual([ + { + kind: "move", + ref: { kind: "data", rowId: "a2" }, + previousIndex: 0, + index: 3, + }, + { + kind: "update", + ref: { kind: "data", rowId: "a2" }, + index: 3, + fields: ["row"], + }, + ]); + }); + + test("staying OUT emits nothing at all", () => { + const { model, before } = transactionFixture(); + + model.applyTransaction({ update: [{ id: "a1", changes: { score: 15 } }] }); + + expect(visibleDataIds(model)).toEqual(["a2", "b2", "b3", "a3"]); + expect(operationsSince(model, before)).toEqual([]); + }); +}); + +describe("grouped equivalence with a cold model", () => { + function warmGrouped(aggregateFilteredRows: boolean) { + const scheduler = new ManualScheduler(); + let tick = 0; + const model = createLocalRowModel({ + rows: ROWS, + columns: createColumns(), + getRowId: (row) => row.id, + aggregateFilteredRows, + initialExpansion: { kind: "expanded" }, + query: groupedQuery(0), + transitionScheduler: scheduler, + transitionClock: () => tick++, + transitionBudgetMs: 1, + }); + model.setQuery(groupedQuery(55)); + scheduler.flushAll(); + return model; + } + + function coldGrouped(aggregateFilteredRows: boolean) { + return createLocalRowModel({ + rows: ROWS, + columns: createColumns(), + getRowId: (row) => row.id, + aggregateFilteredRows, + initialExpansion: { kind: "expanded" }, + query: groupedQuery(55), + }); + } + + for (const aggregateFilteredRows of [false, true]) { + test(`the cooperative rebuild matches a cold build (aggregateFilteredRows: ${aggregateFilteredRows})`, () => { + const warm = warmGrouped(aggregateFilteredRows); + const cold = coldGrouped(aggregateFilteredRows); + + expect(visibleDataIds(warm)).toEqual(visibleDataIds(cold)); + expect(groupSummaries(warm)).toEqual(groupSummaries(cold)); + // Control: the rebuild really did move rows out of the population. + expect(visibleDataIds(cold)).toEqual(["a3", "b2", "b3"]); + }); + } + + test("the two populations disagree, so the aggregate assertions can fail", () => { + const filtered = groupSummaries(coldGrouped(false)).find((group) => + group.groupId.includes("Alpha"), + )!; + const all = groupSummaries(coldGrouped(true)).find((group) => + group.groupId.includes("Alpha"), + )!; + + // Filtered population: only a3 (90) survives `score >= 55`. + expect(filtered.aggregates).toEqual({ score: 90, id: "90" }); + // All population: every Alpha row counts, filtered out or not. + expect(all.aggregates).toEqual({ score: 150, id: "10|50|90" }); + expect(filtered.childCount).toBe(1); + expect(all.childCount).toBe(1); + }); + + /** + * The BULK group builder (`createGroupIndexBuildDraft`) is chosen only when + * every aggregate is a builtin, so the fixture above — which carries a + * custom aggregator — never reaches it. This one does, and it is the path + * that lost the per-leaf filtered wrapper. + */ + function builtinColumns() { + return [ + helper.accessor("team", (row: Holding) => row.team, { type: "text" }), + helper.accessor("score", (row: Holding) => row.score, { + type: "number", + aggregate: "sum", + }), + ] as const; + } + + for (const aggregateFilteredRows of [false, true]) { + test(`the BULK grouped builder aggregates the right population (aggregateFilteredRows: ${aggregateFilteredRows})`, () => { + const scheduler = new ManualScheduler(); + let tick = 0; + const columns = builtinColumns(); + const query = (threshold: number) => + ({ + filters: [{ columnId: "score", operator: "gte", value: threshold }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }) as PretableQueryFor>; + const warm = createLocalRowModel({ + rows: ROWS, + columns, + getRowId: (row) => row.id, + aggregateFilteredRows, + initialExpansion: { kind: "expanded" }, + query: query(0), + transitionScheduler: scheduler, + transitionClock: () => tick++, + transitionBudgetMs: 1, + }); + warm.setQuery(query(55)); + scheduler.flushAll(); + const cold = createLocalRowModel({ + rows: ROWS, + columns, + getRowId: (row) => row.id, + aggregateFilteredRows, + initialExpansion: { kind: "expanded" }, + query: query(55), + }); + + expect(groupSummaries(warm)).toEqual(groupSummaries(cold)); + expect( + groupSummaries(warm).find((group) => group.groupId.includes("Alpha"))! + .aggregates, + ).toEqual({ score: aggregateFilteredRows ? 150 : 90 }); + }); + } + + test("the cooperative rebuild reaches the same aggregates for BOTH populations", () => { + expect( + groupSummaries(warmGrouped(false)).find((group) => + group.groupId.includes("Alpha"), + )!.aggregates, + ).toEqual({ score: 90, id: "90" }); + expect( + groupSummaries(warmGrouped(true)).find((group) => + group.groupId.includes("Alpha"), + )!.aggregates, + ).toEqual({ score: 150, id: "10|50|90" }); + }); +}); + +describe("distinct values read the population from membership", () => { + async function distinct( + query: PretableQueryFor, + population: "all" | "filtered", + ) { + const scheduler = new ManualScheduler(); + let tick = 0; + const model = createLocalRowModel({ + rows: ROWS, + columns: createColumns(), + getRowId: (row) => row.id, + initialExpansion: { kind: "expanded" }, + query, + transitionScheduler: scheduler, + transitionClock: () => tick++, + transitionBudgetMs: 1, + }); + const pending = model.distinctValues("team", { limit: 10, population }); + scheduler.flushAll(); + const result = await pending.finished; + return result.values.map((value) => ({ + value: value.value, + count: value.count, + })); + } + + test("a FLAT root's filtered population counts only members", async () => { + expect(await distinct(flatQuery(55), "filtered")).toEqual([ + { value: "Alpha", count: 1 }, + { value: "Beta", count: 2 }, + ]); + expect(await distinct(flatQuery(55), "all")).toEqual([ + { value: "Alpha", count: 3 }, + { value: "Beta", count: 3 }, + ]); + }); + + test("a GROUPED root's filtered population counts only leaf members", async () => { + // The grouped root's flat visible tree is empty, so a membership read + // that only consulted it would report an empty filtered population here. + expect(await distinct(groupedQuery(55), "filtered")).toEqual([ + { value: "Alpha", count: 1 }, + { value: "Beta", count: 2 }, + ]); + expect(await distinct(groupedQuery(55), "all")).toEqual([ + { value: "Alpha", count: 3 }, + { value: "Beta", count: 3 }, + ]); + }); +}); + +describe("the filter fast path rebuilds no records", () => { + function fastPathFixture() { + const scheduler = new ManualScheduler(); + let tick = 0; + const instrumented = createInstrumentedLocalRowModel({ + rows: ROWS, + columns: createColumns(), + getRowId: (row: Holding) => row.id, + query: flatQuery(50), + transitionScheduler: scheduler, + transitionClock: () => tick++, + transitionBudgetMs: 1, + }); + return { ...instrumented, scheduler }; + } + + test("a filter-only change copies ZERO rows-map nodes, and still moves rows", () => { + const { model, diagnostics, scheduler } = fastPathFixture(); + const before = diagnostics.read().work.hamtNodesCopied; + + model.setQuery(flatQuery(15)); + + const work = diagnostics.read().work; + // The rows HAMT is the only persistent map on this path, and it is never + // opened: no record is reconstructed, so nothing is written to it. + expect(work.hamtNodesCopied - before).toBe(0); + expect(work.filterRebuilds).toBe(1); + // Positive twin: real work happened and the answer is right. + expect(work.filterRowsFlipped).toBeGreaterThan(0); + expect(visibleDataIds(model)).toEqual(["b1", "a2", "b2", "b3", "a3"]); + expect(scheduler.entries).toHaveLength(0); + }); + + test("mutation twin: a rows change on the same model DOES copy map nodes", () => { + const { model, diagnostics } = fastPathFixture(); + const before = diagnostics.read().work.hamtNodesCopied; + + model.applyTransaction({ update: [{ id: "a1", changes: { score: 99 } }] }); + + expect(diagnostics.read().work.hamtNodesCopied - before).toBeGreaterThan(0); + }); +}); + +describe("nearestVisibleRef", () => { + function snapshotFor(query: PretableQueryFor) { + return createLocalRowModel({ + rows: ROWS, + columns: createColumns(), + getRowId: (row) => row.id, + initialExpansion: { kind: "expanded" }, + query, + }).getState().snapshot; + } + + test("a FLAT root answers for members and refuses for non-members", () => { + const snapshot = snapshotFor(flatQuery(55)); + + expect(snapshot.nearestVisibleRef({ kind: "data", rowId: "a3" })).toEqual({ + kind: "data", + rowId: "a3", + }); + // "a1" scores 10: present in the rows map, absent from the visible tree — + // the same absence the verdict now reads. + expect( + snapshot.nearestVisibleRef({ kind: "data", rowId: "a1" }), + ).toBeUndefined(); + expect( + snapshot.nearestVisibleRef({ kind: "data", rowId: "never-seen" }), + ).toBeUndefined(); + }); + + test("a GROUPED root still falls back to the parent group, unchanged", () => { + const snapshot = snapshotFor(groupedQuery(55)); + + expect(snapshot.nearestVisibleRef({ kind: "data", rowId: "a3" })).toEqual({ + kind: "data", + rowId: "a3", + }); + expect(snapshot.nearestVisibleRef({ kind: "data", rowId: "a1" })).toEqual({ + kind: "group", + groupId: "__group__:team=s:Alpha", + }); + expect( + snapshot.nearestVisibleRef({ kind: "data", rowId: "never-seen" }), + ).toBeUndefined(); + }); +}); diff --git a/packages/row-model/src/__tests__/membership-bitset.test.ts b/packages/row-model/src/__tests__/membership-bitset.test.ts new file mode 100644 index 000000000..38c236c29 --- /dev/null +++ b/packages/row-model/src/__tests__/membership-bitset.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { + EMPTY_MEMBERSHIP, + cloneMembership, + createMembership, + clearMembershipBit, + setMembershipBit, + testMembershipBit, +} from "../membership-bitset"; + +describe("membership bitset", () => { + it("round-trips set/clear/test across word boundaries", () => { + const bits = createMembership(100); + for (const slot of [0, 31, 32, 63, 64, 99]) { + expect(testMembershipBit(bits, slot)).toBe(false); + setMembershipBit(bits, slot); + expect(testMembershipBit(bits, slot)).toBe(true); + } + clearMembershipBit(bits, 32); + expect(testMembershipBit(bits, 32)).toBe(false); + expect(testMembershipBit(bits, 31)).toBe(true); + expect(testMembershipBit(bits, 63)).toBe(true); + }); + + it("clone is independent of the original", () => { + const bits = createMembership(64); + setMembershipBit(bits, 10); + const copy = cloneMembership(bits, 64); + clearMembershipBit(copy, 10); + setMembershipBit(copy, 20); + expect(testMembershipBit(bits, 10)).toBe(true); + expect(testMembershipBit(bits, 20)).toBe(false); + }); + + it("clone can grow capacity, preserving low bits", () => { + const bits = createMembership(32); + setMembershipBit(bits, 31); + const grown = cloneMembership(bits, 200); + expect(testMembershipBit(grown, 31)).toBe(true); + setMembershipBit(grown, 199); + expect(testMembershipBit(grown, 199)).toBe(true); + }); + + it("reads beyond a bitset's words answer false (EMPTY sentinel contract)", () => { + expect(testMembershipBit(EMPTY_MEMBERSHIP, 0)).toBe(false); + expect(testMembershipBit(EMPTY_MEMBERSHIP, 12345)).toBe(false); + const bits = createMembership(32); + expect(testMembershipBit(bits, 500)).toBe(false); + }); + + it("cloning the EMPTY sentinel grows into a usable bitset, sentinel untouched", () => { + const grown = cloneMembership(EMPTY_MEMBERSHIP, 100); + expect(testMembershipBit(grown, 99)).toBe(false); + setMembershipBit(grown, 99); + expect(testMembershipBit(grown, 99)).toBe(true); + expect(EMPTY_MEMBERSHIP.length).toBe(0); + }); +}); diff --git a/packages/row-model/src/__tests__/order-statistic-tree.test.ts b/packages/row-model/src/__tests__/order-statistic-tree.test.ts index f5729c22c..f4a7faddb 100644 --- a/packages/row-model/src/__tests__/order-statistic-tree.test.ts +++ b/packages/row-model/src/__tests__/order-statistic-tree.test.ts @@ -6,10 +6,12 @@ import { createOrderStatisticTree, createOrderStatisticTreeFromSortedEntries, getOrderStatisticTreeDiagnosticsForTesting, + instrumentOrderStatisticTree, type OrderStatisticTree, type OrderStatisticTreeNodeDiagnostic, type TransientOrderStatisticTree, } from "../persistent/order-statistic-tree"; +import type { LocalRowModelInstrumentation } from "../diagnostics"; interface Item { readonly id: string | number; @@ -726,3 +728,410 @@ describe("createOrderStatisticTreeFromSortedEntries", () => { ); }); }); + +describe("createOrderStatisticTreeFromSortedEntries proofs", () => { + const compositeCompare = (left: Item, right: Item) => + left.score - right.score || compareOrderStatisticTreeIds(left.id, right.id); + + function sortedEntries(count: number, offset = 0): Item[] { + const entries = Array.from({ length: count }, (_, index) => + item( + index + offset, + adversarialOrder(index + offset) % 997, + ((index + offset) % 13) + 1, + ), + ); + entries.sort(compositeCompare); + return entries; + } + + /** + * The base/survivor/leaver/arrival split every derived-byId test needs: a + * base tree over `base`, and a target sequence that drops `leavers` (whose + * survivors are carried as the SAME OBJECTS the base holds) and merges in + * fresh `arrivals` drawn from a disjoint id range. + */ + function derivationFixture(size: number, leaveEvery: number) { + const base = sortedEntries(size); + const arrivals = sortedEntries(Math.max(1, size >> 3), 10_000); + const leavers = base.filter((_, index) => index % leaveEvery === 0); + const leaverIds = new Set(leavers.map((entry) => entry.id)); + const target = [ + ...base.filter((entry) => !leaverIds.has(entry.id)), + ...arrivals, + ].sort(compositeCompare); + return { + baseTree: createOrderStatisticTreeFromSortedEntries(createTree(), base), + base, + arrivals, + leaverIds, + target, + }; + } + + test("derived byId equals a refilled byId at every key", () => { + const { baseTree, base, arrivals, leaverIds, target } = derivationFixture( + 500, + 7, + ); + + const refilled = createOrderStatisticTreeFromSortedEntries( + createTree(), + target, + ); + const derived = createOrderStatisticTreeFromSortedEntries( + createTree(), + target, + { + derivedById: { + base: baseTree, + removedIds: leaverIds, + addedEntries: arrivals, + }, + }, + ); + + expect(derived.size).toBe(refilled.size); + // Every key of the union — survivors, arrivals, AND leavers, so a map + // that kept a leaver is caught by the same sweep that checks the rest. + for (const entry of [...base, ...arrivals]) { + expect(derived.get(entry.id)).toBe(refilled.get(entry.id)); + expect(derived.rankOf(entry.id)).toBe(refilled.rankOf(entry.id)); + } + for (const id of leaverIds) { + expect(derived.get(id)).toBeUndefined(); + expect(derived.rankOf(id)).toBeUndefined(); + } + // And the map agrees with the tree it was built alongside: each id maps + // to the very object the entries array holds at that rank. + for (let rank = 0; rank < target.length; rank += 1) { + const entry = derived.entryAt(rank)!; + expect(derived.get(entry.id)).toBe(entry); + expect(entry).toBe(target[rank]); + } + }); + + test("derived byId keeps working as a base for a later derivation", () => { + const first = derivationFixture(200, 5); + const derived = createOrderStatisticTreeFromSortedEntries( + createTree(), + first.target, + { + derivedById: { + base: first.baseTree, + removedIds: first.leaverIds, + addedEntries: first.arrivals, + }, + }, + ); + const secondLeavers = new Set( + first.target.filter((_, index) => index % 3 === 0).map((e) => e.id), + ); + const secondTarget = first.target.filter( + (entry) => !secondLeavers.has(entry.id), + ); + + const again = createOrderStatisticTreeFromSortedEntries( + createTree(), + secondTarget, + { + derivedById: { + base: derived, + removedIds: secondLeavers, + addedEntries: [], + }, + }, + ); + + const refilled = createOrderStatisticTreeFromSortedEntries( + createTree(), + secondTarget, + ); + expect(again.size).toBe(refilled.size); + for (const entry of first.target) { + expect(again.get(entry.id)).toBe(refilled.get(entry.id)); + } + }); + + test("derived byId rejects an edit set that disagrees with the built entries", () => { + const { baseTree, arrivals, leaverIds, target } = derivationFixture(64, 4); + + // Leavers left in the map — the exact slip an unchecked derivation would + // let through as a phantom `get`. + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), target, { + derivedById: { + base: baseTree, + removedIds: new Set(), + addedEntries: arrivals, + }, + }), + ).toThrow(TypeError); + // Arrivals missing from the map — the mirror slip. + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), target, { + derivedById: { + base: baseTree, + removedIds: leaverIds, + addedEntries: [], + }, + }), + ).toThrow(TypeError); + }); + + test("derived byId rejects a foreign base", () => { + const foreign = { size: 0, measure: 0 } as unknown as OrderStatisticTree< + string | number, + Item, + number + >; + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), [], { + derivedById: { + base: foreign, + removedIds: new Set(), + addedEntries: [], + }, + }), + ).toThrow(TypeError); + }); + + /** + * The hazard, pinned as behavior rather than as a guard: detecting a + * replaced survivor costs O(n) — the very scan derived mode exists to + * avoid — so the API does NOT reject it. It documents the identity + * precondition and the two in-package callers hold it or abstain + * (sort-rebuild reallocates every entry, so it takes `orderIsProven` only). + * This test exists so that anyone who wires derived byId into a + * reallocating caller sees exactly what they will get. + */ + test("derived byId goes stale when a survivor's entry object is REPLACED", () => { + const base = sortedEntries(32); + const baseTree = createOrderStatisticTreeFromSortedEntries( + createTree(), + base, + ); + const original = base[10]!; + // Same id, same order position, different object — exactly what a + // re-decorating rebuild produces for a row that never moved. + const replacement: Item = { ...original, label: "replaced" }; + const target = base.map((entry) => + entry === original ? replacement : entry, + ); + + const derived = createOrderStatisticTreeFromSortedEntries( + createTree(), + target, + { + derivedById: { + base: baseTree, + removedIds: new Set(), + addedEntries: [], + }, + }, + ); + + // The size check cannot see this: the key set is right, the value is not. + expect(derived.size).toBe(target.length); + expect(derived.entryAt(derived.rankOf(original.id)!)).toBe(replacement); + expect(derived.get(original.id)).toBe(original); + expect(derived.get(original.id)).not.toBe(replacement); + // The refill has no such split. + const refilled = createOrderStatisticTreeFromSortedEntries( + createTree(), + target, + ); + expect(refilled.get(original.id)).toBe(replacement); + }); + + test("a trusted-order build is observably identical to a verified one", () => { + for (const size of [0, 1, 2, 3, 7, 8, 9, 1_000]) { + const entries = sortedEntries(size); + const verified = createOrderStatisticTreeFromSortedEntries( + createTree(), + entries, + ); + const trusted = createOrderStatisticTreeFromSortedEntries( + createTree(), + entries, + { orderIsProven: true }, + ); + expect(trusted.size).toBe(verified.size); + expect(trusted.measure).toBe(verified.measure); + expect(ids(trusted)).toEqual(ids(verified)); + expect(getOrderStatisticTreeDiagnosticsForTesting(trusted).balanced).toBe( + true, + ); + for (const entry of entries) { + expect(trusted.rankOf(entry.id)).toBe(verified.rankOf(entry.id)); + expect(trusted.get(entry.id)).toBe(verified.get(entry.id)); + } + } + }); + + test("verification still fires for every caller that does not claim the proof", () => { + const misordered = [item("a", 1), item("b", 3), item("c", 2)]; + // Default: on. Explicit `false`: on. Only `true` opts out — a caller + // cannot skip the check by passing a proof object for the other field. + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), misordered), + ).toThrow(TypeError); + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), misordered, {}), + ).toThrow(TypeError); + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), misordered, { + orderIsProven: false, + }), + ).toThrow(TypeError); + // The opt-out is real, and the corruption it admits is silent — which is + // why it is package-internal and why the filter/sort callers carry a + // downstream order oracle. A misordered trusted build does NOT throw and + // does NOT produce the sorted order. + const trusted = createOrderStatisticTreeFromSortedEntries( + createTree(), + misordered, + { orderIsProven: true }, + ); + expect(ids(trusted)).toEqual(["a", "b", "c"]); + expect(ids(trusted)).not.toEqual( + ids( + createOrderStatisticTreeFromSortedEntries( + createTree(), + [...misordered].sort(compositeCompare), + ), + ), + ); + }); +}); + +describe("bulk-build byId routing", () => { + const compositeCompare = (left: Item, right: Item) => + left.score - right.score || compareOrderStatisticTreeIds(left.id, right.id); + + function instrumentation(): LocalRowModelInstrumentation { + return { + work: { + rowsEvaluated: 0, + hamtNodesCopied: 0, + orderNodesCopied: 0, + groupNodesCopied: 0, + aggregateMerges: 0, + transitionRows: 0, + synchronousRebuilds: 0, + synchronousRebuildMs: 0, + filterRebuilds: 0, + filterRowsFlipped: 0, + filterMergeSortedInsertions: 0, + filterRebuildMs: 0, + bulkByIdDerived: 0, + bulkOrderVerificationsSkipped: 0, + evaluationCacheAdoptions: 0, + slotChunksTouched: 0, + sortKeyCarries: 0, + sortKeyEvaluations: 0, + snapshotOutputRowsRead: 0, + schedulerSliceDurations: [], + }, + snapshotRoots: new WeakMap(), + retainedSnapshots: new Map(), + scheduledCallbacks: new Set(), + currentRevisionRoot: undefined, + model: undefined, + }; + } + + /** + * Builds a base of `size` entries and drops the first `leaverCount` of them + * in comparator order, adding nothing. The built entry count is then + * `size - leaverCount`, so `leaverCount` alone moves the input across the + * routing rule's boundary while everything else is held fixed. + */ + function buildWithLeavers(size: number, leaverCount: number) { + const base = Array.from({ length: size }, (_, id) => + item(id, adversarialOrder(id) % 997, (id % 13) + 1), + ).sort(compositeCompare); + const baseTree = createOrderStatisticTreeFromSortedEntries( + createTree(), + base, + ); + const leaverIds = new Set(base.slice(0, leaverCount).map((e) => e.id)); + const target = base.slice(leaverCount); + const work = instrumentation(); + const built = createOrderStatisticTreeFromSortedEntries( + instrumentOrderStatisticTree(createTree(), work), + target, + { + derivedById: { + base: baseTree, + removedIds: leaverIds, + addedEntries: [], + }, + }, + ); + return { built, target, base, leaverIds, work }; + } + + /** + * The rule, stated as an experiment: with the derivation offered + * unconditionally, only the ratio decides. `removals + additions` against + * the built entry count — algebraically `removals < survivors` — with the + * tie going to the REFILL, because at equal operation counts the refill + * also skips copying the base map's path nodes. + * + * The boundary is not decoration. At S2's 50,000-row target the filter + * leaves 12,500 survivors, so an unconditional derivation ran 37,500 + * removes against 12,500 inserts and cost ~9ms of settle. + */ + test("routes by operation count, and the boundary is exact", () => { + // 100 entries, 49 leavers: 49 removals, 51 built. Derives. + expect(buildWithLeavers(100, 49).work.work.bulkByIdDerived).toBe(1); + // 100 entries, 50 leavers: 50 removals, 50 built. TIE — refills. + expect(buildWithLeavers(100, 50).work.work.bulkByIdDerived).toBe(0); + // 100 entries, 51 leavers: 51 removals, 49 built. Refills. + expect(buildWithLeavers(100, 51).work.work.bulkByIdDerived).toBe(0); + }); + + test("both routes build identical trees on either side of the boundary", () => { + for (const leaverCount of [1, 49, 50, 51, 99]) { + const { built, target, base, leaverIds } = buildWithLeavers( + 100, + leaverCount, + ); + const refilled = createOrderStatisticTreeFromSortedEntries( + createTree(), + target, + ); + expect(built.size).toBe(refilled.size); + expect(built.measure).toBe(refilled.measure); + expect(ids(built)).toEqual(ids(refilled)); + for (const entry of base) { + expect(built.get(entry.id)).toBe(refilled.get(entry.id)); + expect(built.rankOf(entry.id)).toBe(refilled.rankOf(entry.id)); + } + for (const id of leaverIds) expect(built.get(id)).toBeUndefined(); + } + }); + + test("the order proof is unaffected by which byId route runs", () => { + for (const leaverCount of [1, 99]) { + const { target, base } = buildWithLeavers(100, leaverCount); + const work = instrumentation(); + createOrderStatisticTreeFromSortedEntries( + instrumentOrderStatisticTree(createTree(), work), + target, + { + orderIsProven: true, + derivedById: { + base: createOrderStatisticTreeFromSortedEntries(createTree(), base), + removedIds: new Set( + base.slice(0, leaverCount).map((entry) => entry.id), + ), + addedEntries: [], + }, + }, + ); + expect(work.work.bulkOrderVerificationsSkipped).toBe(1); + } + }); +}); diff --git a/packages/row-model/src/__tests__/properties.test.ts b/packages/row-model/src/__tests__/properties.test.ts index 2af7efdaf..76bd98f3b 100644 --- a/packages/row-model/src/__tests__/properties.test.ts +++ b/packages/row-model/src/__tests__/properties.test.ts @@ -625,15 +625,17 @@ describe("incremental row-model properties", () => { }; const sameJson = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); - // Mirrors the #457 fast path: a sort-only change on an ungrouped - // query commits synchronously, so its revision must be accounted - // BEFORE the concurrent mutations assert their previousRevision. + // Mirrors the #457 fast paths: a sort-only OR filter-only change + // (exactly ONE facet) on an ungrouped query commits synchronously, + // so its revision must be accounted BEFORE the concurrent + // mutations assert their previousRevision. Both facets changing + // stays cooperative. const commitsSynchronously = ( from: PropertyQuery, to: PropertyQuery, ) => - !sameJson(from.sort, to.sort) && - sameJson(from.filters, to.filters) && + !sameJson(from.sort, to.sort) !== + !sameJson(from.filters, to.filters) && sameJson(from.rowGroups, to.rowGroups) && to.rowGroups.length === 0; let committed: PropertyQuery = initialQuery; diff --git a/packages/row-model/src/__tests__/query-delta.test.ts b/packages/row-model/src/__tests__/query-delta.test.ts index b4340aeb3..75a746433 100644 --- a/packages/row-model/src/__tests__/query-delta.test.ts +++ b/packages/row-model/src/__tests__/query-delta.test.ts @@ -3,6 +3,7 @@ import { describe, expect, test } from "vitest"; import { compileQuery, createColumnHelper, + isFilterOnlyChange, isSortOnlyChange, type PretableQueryFor, } from "../index"; @@ -244,3 +245,241 @@ describe("isSortOnlyChange", () => { expect(isSortOnlyChange(real, foreign as never)).toBe(false); }); }); + +const NO_FILTER = queryFor({ + filters: [], + sort: [], + rowGroups: [], +}); + +const TECH_FILTER = queryFor({ + filters: [{ columnId: "sector", operator: "contains", value: "Tech" }], + sort: [], + rowGroups: [], +}); + +const ENERGY_FILTER = queryFor({ + filters: [{ columnId: "sector", operator: "contains", value: "Energy" }], + sort: [], + rowGroups: [], +}); + +describe("isFilterOnlyChange", () => { + test("true when only the filter value differs", () => { + const previous = compileQuery({ derivations: columns, query: TECH_FILTER }); + const next = compileQuery({ derivations: columns, query: ENERGY_FILTER }); + + expect(isFilterOnlyChange(previous, next)).toBe(true); + }); + + test.each([ + { + name: "operator change", + prevFilters: [ + { columnId: "sector", operator: "contains", value: "Tech" }, + ], + nextFilters: [{ columnId: "sector", operator: "equals", value: "Tech" }], + }, + { + name: "filter added", + prevFilters: [ + { columnId: "sector", operator: "contains", value: "Tech" }, + ], + nextFilters: [ + { columnId: "sector", operator: "contains", value: "Tech" }, + { columnId: "customer", operator: "contains", value: "Acme" }, + ], + }, + { + name: "filter removed", + prevFilters: [ + { columnId: "sector", operator: "contains", value: "Tech" }, + { columnId: "customer", operator: "contains", value: "Acme" }, + ], + nextFilters: [ + { columnId: "sector", operator: "contains", value: "Tech" }, + ], + }, + { + name: "all filters removed", + prevFilters: [ + { columnId: "sector", operator: "contains", value: "Tech" }, + ], + nextFilters: [], + }, + ] as const)("true for $name", ({ prevFilters, nextFilters }) => { + const previous = compileQuery({ + derivations: columns, + query: queryFor({ + filters: prevFilters, + sort: [], + rowGroups: [], + }), + }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ + filters: nextFilters, + sort: [], + rowGroups: [], + }), + }); + + expect(isFilterOnlyChange(previous, next)).toBe(true); + }); + + test("false when the filter is identical", () => { + // Two structurally-equal plans compiled independently (no `previous` + // passed to `compileQuery`), so they are distinct objects. + const previous = compileQuery({ + derivations: columns, + query: TECH_FILTER, + }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [{ columnId: "sector", operator: "contains", value: "Tech" }], + sort: [], + rowGroups: [], + }), + }); + + expect(previous).not.toBe(next); + expect(isFilterOnlyChange(previous, next)).toBe(false); + }); + + test("false when sort also changed", () => { + const previous = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [{ columnId: "sector", operator: "contains", value: "Tech" }], + sort: [{ columnId: "quantity", direction: "asc" }], + rowGroups: [], + }), + }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [ + { columnId: "sector", operator: "contains", value: "Energy" }, + ], + sort: [{ columnId: "quantity", direction: "desc" }], + rowGroups: [], + }), + }); + + expect(isFilterOnlyChange(previous, next)).toBe(false); + }); + + test("false when rowGroups also changed", () => { + const previous = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [{ columnId: "sector", operator: "contains", value: "Tech" }], + sort: [], + rowGroups: [{ columnId: "sector", direction: "asc" }], + }), + }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [ + { columnId: "sector", operator: "contains", value: "Energy" }, + ], + sort: [], + rowGroups: [{ columnId: "customer", direction: "asc" }], + }), + }); + + expect(isFilterOnlyChange(previous, next)).toBe(false); + }); + + test("false when derivations changed for an active column", () => { + const sectorA = (row: Holding) => row.sector; + const sectorB = (row: Holding) => row.sector; + const columnsA = [ + helper.accessor("sector", sectorA, { type: "text" }), + helper.accessor("customer", { type: "text" }), + helper.accessor("quantity", { type: "number", aggregate: "sum" }), + ] as const; + const columnsB = [ + helper.accessor("sector", sectorB, { type: "text" }), + helper.accessor("customer", { type: "text" }), + helper.accessor("quantity", { type: "number", aggregate: "sum" }), + ] as const; + + const previous = compileQuery({ + derivations: columnsA, + query: TECH_FILTER as unknown as PretableQueryFor, + }); + const next = compileQuery({ + derivations: columnsB, + query: ENERGY_FILTER as unknown as PretableQueryFor, + }); + + expect(isFilterOnlyChange(previous, next)).toBe(false); + }); + + test("false when filterAuthority differs between plans", () => { + const previous = compileQuery({ + derivations: columns, + query: TECH_FILTER, + }); + const next = compileQuery({ + derivations: columns, + query: ENERGY_FILTER, + filterAuthority: "external", + }); + + expect(isFilterOnlyChange(previous, next)).toBe(false); + }); + + test("false when sortAuthority differs between plans", () => { + const previous = compileQuery({ + derivations: columns, + query: TECH_FILTER, + }); + const next = compileQuery({ + derivations: columns, + query: ENERGY_FILTER, + sortAuthority: "external", + }); + + expect(isFilterOnlyChange(previous, next)).toBe(false); + }); + + test("false under external filter authority both sides with only a public filter change", () => { + const previous = compileQuery({ + derivations: columns, + query: TECH_FILTER, + filterAuthority: "external", + }); + const next = compileQuery({ + derivations: columns, + query: ENERGY_FILTER, + filterAuthority: "external", + }); + + // Runtime filters are [] for both under external authority, so there is + // no runtime-level change at all. + expect(isFilterOnlyChange(previous, next)).toBe(false); + }); + + test("false for foreign plan objects in either position", () => { + const real = compileQuery({ derivations: columns, query: TECH_FILTER }); + const foreign = { query: ENERGY_FILTER, derivations: columns }; + + expect(isFilterOnlyChange(foreign as never, real)).toBe(false); + expect(isFilterOnlyChange(real, foreign as never)).toBe(false); + }); + + test("false when no facet changed at all", () => { + const previous = compileQuery({ derivations: columns, query: NO_FILTER }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ filters: [], sort: [], rowGroups: [] }), + }); + + expect(isFilterOnlyChange(previous, next)).toBe(false); + }); +}); diff --git a/packages/row-model/src/__tests__/records-by-slot.test.ts b/packages/row-model/src/__tests__/records-by-slot.test.ts new file mode 100644 index 000000000..de1693b81 --- /dev/null +++ b/packages/row-model/src/__tests__/records-by-slot.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, test } from "vitest"; + +import { createColumnHelper, createLocalRowModel } from "../index"; +import { getLocalRowModelSlotInternalsForTesting } from "../create-local-row-model"; +import { createInstrumentedLocalRowModel } from "../diagnostics"; +import type { PretableRowId } from "../column-types"; +import type { RevisionRoot } from "../internal-types"; +import { forEachSlotEntry, slotVectorGet } from "../slot-vector"; + +interface Row { + id: string; + value: number; +} +const helper = createColumnHelper(); + +function createModel(rows: readonly Row[]) { + const columns = [helper.accessor("value", { type: "number" })] as const; + return createLocalRowModel({ + rows, + columns, + getRowId: (row: Row) => row.id, + }); +} + +function rootOf(model: object): RevisionRoot { + return getLocalRowModelSlotInternalsForTesting(model).root; +} + +/** + * The Task 5 invariant, verbatim from the `recordsBySlot` doc comment: + * `slotVectorGet(recordsBySlot, record.slot) === record` (IDENTITY) for every + * record in `rows`, at every committed root — plus live-entry count parity, + * so the vector holds nothing beyond the root's own rows (no stale binding + * survives a removal), and every slot stays inside the root's self-described + * capacity. + */ +function expectSlotInvariant( + root: RevisionRoot, +): void { + let recordCount = 0; + for (const [, record] of root.rows.entries()) { + recordCount += 1; + expect(slotVectorGet(root.recordsBySlot, record.slot)).toBe(record); + expect(record.slot).toBeLessThan(root.slotCapacity); + } + let liveEntries = 0; + forEachSlotEntry(root.recordsBySlot, (value, slot) => { + liveEntries += 1; + expect(value.slot).toBe(slot); + }); + expect(liveEntries).toBe(recordCount); +} + +const ROWS: readonly Row[] = Object.freeze([ + { id: "a", value: 1 }, + { id: "b", value: 2 }, + { id: "c", value: 3 }, +]); + +describe("recordsBySlot", () => { + test("invariant holds across the committed-revision script", async () => { + const model = createModel(ROWS); + + // 1. Initial build. + const initial = rootOf(model); + expect(initial.slotCapacity).toBe(3); + expectSlotInvariant(initial); + + // 2. Update transaction: the fresh record replaces the old binding. + expect( + model.applyTransaction({ update: [{ id: "b", changes: { value: 20 } }] }), + ).toMatchObject({ updated: 1 }); + expectSlotInvariant(rootOf(model)); + + // 3. Remove + add across two commits, reusing the released slot — with + // the held-snapshot pin: the root captured BEFORE the removal must + // keep binding the reused slot to the OLD record afterwards. + const held = rootOf(model); + const oldB = held.rows.get("b")!; + expect(model.applyTransaction({ remove: ["b"] })).toMatchObject({ + removed: 1, + }); + expectSlotInvariant(rootOf(model)); + expect( + model.applyTransaction({ add: [{ id: "d", value: 4 }] }), + ).toMatchObject({ added: 1 }); + const afterAdd = rootOf(model); + // Control: the add genuinely reused b's released slot — without this the + // held-snapshot pin below could pass vacuously. + expect(afterAdd.rows.get("d")!.slot).toBe(oldB.slot); + expectSlotInvariant(afterAdd); + expect(slotVectorGet(held.recordsBySlot, oldB.slot)).toBe(oldB); + expectSlotInvariant(held); + + // 4. setRows replacement. Rows before: a, c, d. Two retire (a, d), two + // ingest (e, f) — the transfer pool hands the retiring slots straight + // to the new rows, so a clear and a write land on the SAME slot in + // one commit and capacity does not grow. + expect( + model.setRows([ + { id: "c", value: 30 }, + { id: "e", value: 5 }, + { id: "f", value: 6 }, + ]), + ).toMatchObject({ updated: 1, added: 2, removed: 2 }); + const replaced = rootOf(model); + expect(replaced.slotCapacity).toBe(3); + expectSlotInvariant(replaced); + + // 5. Filter-only setQuery (synchronous fast path). `rows` is the FULL + // set — filtering must not disturb a single slot binding. + const filterTransition = model.setQuery({ + filters: [{ columnId: "value", operator: "gte", value: 6 }], + sort: [], + rowGroups: [], + }); + await filterTransition.finished; + expectSlotInvariant(rootOf(model)); + + // 6. Sort-only setQuery (synchronous fast path). + const sortTransition = model.setQuery({ + filters: [{ columnId: "value", operator: "gte", value: 6 }], + sort: [{ columnId: "value", direction: "desc" }], + rowGroups: [], + }); + await sortTransition.finished; + expectSlotInvariant(rootOf(model)); + + model.dispose(); + }); + + test("cooperative transition: the finished root carries the vector, including a mid-transition delta that grew the slot space", async () => { + const model = createModel(ROWS); + // Filter AND sort change together: no synchronous fast path, so this is + // the cooperative-transition `finish` construction site. + const transition = model.setQuery({ + filters: [{ columnId: "value", operator: "gte", value: 2 }], + sort: [{ columnId: "value", direction: "desc" }], + rowGroups: [], + }); + // A commit during the transition becomes a replayed delta whose target + // root allocated a slot BEYOND the captured root's capacity. + expect( + model.applyTransaction({ add: [{ id: "d", value: 9 }] }), + ).toMatchObject({ added: 1 }); + expectSlotInvariant(rootOf(model)); + await transition.finished; + const finished = rootOf(model); + expect(finished.rows.get("d")).toBeDefined(); + // The finished root's domain must come from the delta TARGET's + // self-described capacity, never the captured root's smaller one. + expect(finished.slotCapacity).toBe(4); + expectSlotInvariant(finished); + model.dispose(); + }); + + test("an effective transaction reports its COW chunk writes end to end", () => { + // End-to-end wiring pin: the success path's `slotVectorWithAll` result + // must actually land in the published work diagnostics — deleting the + // `slotChunksTouched` accumulation in `transaction-draft.ts` fails here. + const instrumented = createInstrumentedLocalRowModel({ + rows: ROWS, + columns: [helper.accessor("value", { type: "number" })] as const, + getRowId: (row: Row) => row.id, + }); + expect(instrumented.diagnostics.read().work.slotChunksTouched).toBe(0); + expect( + instrumented.model.applyTransaction({ + update: [{ id: "b", changes: { value: 20 } }], + }), + ).toMatchObject({ updated: 1 }); + expect( + instrumented.diagnostics.read().work.slotChunksTouched, + ).toBeGreaterThanOrEqual(1); + instrumented.model.dispose(); + }); +}); diff --git a/packages/row-model/src/__tests__/retention.test.ts b/packages/row-model/src/__tests__/retention.test.ts index 29fce84b8..8f3d37618 100644 --- a/packages/row-model/src/__tests__/retention.test.ts +++ b/packages/row-model/src/__tests__/retention.test.ts @@ -412,8 +412,10 @@ describe("instrumented local row-model retention", () => { for (let attempt = 0; attempt < 10; attempt += 1) { const superseded = instrumented.model.setQuery({ + // Filter AND sort change: either alone commits synchronously (#457 + // fast paths) and there would be nothing pending to supersede. filters: [{ columnId: "score", operator: "gte", value: attempt + 1 }], - sort: [], + sort: [{ columnId: "score", direction: "asc" }], rowGroups: [], }); const replacement = instrumented.model.setQuery({ diff --git a/packages/row-model/src/__tests__/slot-allocator.test.ts b/packages/row-model/src/__tests__/slot-allocator.test.ts new file mode 100644 index 000000000..44913b1ba --- /dev/null +++ b/packages/row-model/src/__tests__/slot-allocator.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { createSlotAllocator } from "../slot-allocator"; + +describe("slot allocator", () => { + it("allocates dense sequential slots from zero", () => { + const slots = createSlotAllocator(); + expect([slots.allocate(), slots.allocate(), slots.allocate()]).toEqual([ + 0, 1, 2, + ]); + expect(slots.capacity).toBe(3); + }); + + it("reuses released slots before growing", () => { + const slots = createSlotAllocator(); + slots.allocate(); + const b = slots.allocate(); + slots.allocate(); + slots.release(b); + expect(slots.allocate()).toBe(b); + expect(slots.capacity).toBe(3); + }); + + it("capacity is monotonic and counts the high-water mark", () => { + const slots = createSlotAllocator(); + for (let i = 0; i < 10; i += 1) slots.allocate(); + for (let i = 0; i < 10; i += 1) slots.release(i); + expect(slots.capacity).toBe(10); + for (let i = 0; i < 10; i += 1) slots.allocate(); + expect(slots.capacity).toBe(10); + }); + + it("throws on double release", () => { + const slots = createSlotAllocator(); + const a = slots.allocate(); + slots.release(a); + expect(() => slots.release(a)).toThrow(/released|live/i); + }); + + it("throws on releasing a never-allocated slot", () => { + const slots = createSlotAllocator(); + expect(() => slots.release(5)).toThrow(); + }); +}); diff --git a/packages/row-model/src/__tests__/slot-lifecycle.test.ts b/packages/row-model/src/__tests__/slot-lifecycle.test.ts new file mode 100644 index 000000000..e369b97ef --- /dev/null +++ b/packages/row-model/src/__tests__/slot-lifecycle.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, test } from "vitest"; + +import { + compileQuery, + createColumnHelper, + createLocalRowModel, +} from "../index"; +import { getLocalRowModelSlotInternalsForTesting } from "../create-local-row-model"; +import { buildRowStore } from "../row-store"; +import { createSlotAllocator } from "../slot-allocator"; + +interface Row { + id: string; + value: number; +} +const helper = createColumnHelper(); + +function createModel( + rows: readonly Row[], + options?: { readonly grouped?: boolean }, +) { + const columns = [helper.accessor("value", { type: "number" })] as const; + const model = createLocalRowModel({ + rows, + columns, + getRowId: (row: Row) => row.id, + ...(options?.grouped === true + ? { + query: { + filters: [], + sort: [], + rowGroups: [{ columnId: "value", direction: "asc" as const }], + }, + } + : {}), + }); + return { model, ...getLocalRowModelSlotInternalsForTesting(model) }; +} + +function slotOf( + internals: ReturnType, + rowId: string, +): number { + const record = internals.root.rows.get(rowId); + if (record === undefined) throw new Error(`Missing row ${rowId}.`); + return record.slot; +} + +const ROWS: readonly Row[] = Object.freeze([ + { id: "a", value: 1 }, + { id: "b", value: 2 }, + { id: "c", value: 3 }, +]); + +describe("slot lifecycle", () => { + test("slots are dense from zero at initial build", () => { + const { model, root, slots } = createModel(ROWS); + expect(slotOf({ root, slots }, "a")).toBe(0); + expect(slotOf({ root, slots }, "b")).toBe(1); + expect(slotOf({ root, slots }, "c")).toBe(2); + expect(slots.capacity).toBe(3); + model.dispose(); + }); + + test("update carries the slot", () => { + const { model, slots } = createModel(ROWS); + const before = getLocalRowModelSlotInternalsForTesting(model).root; + const previousA = before.rows.get("a"); + const previousB = before.rows.get("b"); + const previousC = before.rows.get("c"); + + expect( + model.applyTransaction({ update: [{ id: "b", changes: { value: 99 } }] }), + ).toMatchObject({ updated: 1 }); + + const after = getLocalRowModelSlotInternalsForTesting(model).root; + const nextB = after.rows.get("b"); + expect(nextB).not.toBe(previousB); + expect(nextB?.slot).toBe(previousB?.slot); + expect(nextB?.slot).toBe(1); + // Untouched rows keep record identity (and therefore their slots). + expect(after.rows.get("a")).toBe(previousA); + expect(after.rows.get("c")).toBe(previousC); + expect(slots.capacity).toBe(3); + model.dispose(); + }); + + test("remove releases; a later add reuses", () => { + const { model, slots } = createModel(ROWS); + expect(model.applyTransaction({ remove: ["b"] })).toMatchObject({ + removed: 1, + }); + expect( + model.applyTransaction({ add: [{ id: "d", value: 4 }] }), + ).toMatchObject({ added: 1 }); + + const after = getLocalRowModelSlotInternalsForTesting(model).root; + expect(after.rows.get("d")?.slot).toBe(1); + expect(slots.capacity).toBe(3); + model.dispose(); + }); + + test("remove releases; a later add reuses (grouped root)", () => { + const { model, slots } = createModel(ROWS, { grouped: true }); + expect(model.applyTransaction({ remove: ["b"] })).toMatchObject({ + removed: 1, + }); + expect( + model.applyTransaction({ add: [{ id: "d", value: 4 }] }), + ).toMatchObject({ added: 1 }); + + const after = getLocalRowModelSlotInternalsForTesting(model).root; + expect(after.rows.get("d")?.slot).toBe(1); + expect(slots.capacity).toBe(3); + model.dispose(); + }); + + test("set-rows replacement carries intersecting ids", () => { + const { model, slots } = createModel(ROWS); + expect( + model.setRows([ + { id: "b", value: 20 }, + { id: "e", value: 5 }, + ]), + ).toMatchObject({ updated: 1, added: 1, removed: 2 }); + + const after = getLocalRowModelSlotInternalsForTesting(model).root; + expect(after.rows.get("b")?.slot).toBe(1); + // E takes one of the slots the dropped rows (a=0, c=2) gave back. + expect([0, 2]).toContain(after.rows.get("e")?.slot); + expect(slots.capacity).toBe(3); + model.dispose(); + }); + + test("abandoned draft leaks nothing", () => { + const { model, slots } = createModel([{ id: "a", value: 1 }]); + expect(slots.capacity).toBe(1); + // Entirely ineffective: the only entry is an unknown remove id. + expect(model.applyTransaction({ remove: ["missing"] })).toMatchObject({ + removed: 0, + ignored: 1, + }); + expect( + model.applyTransaction({ add: [{ id: "b", value: 2 }] }), + ).toMatchObject({ added: 1 }); + + const after = getLocalRowModelSlotInternalsForTesting(model).root; + // No gap: capacity grew by exactly the rows actually added since build. + expect(after.rows.get("b")?.slot).toBe(1); + expect(slots.capacity).toBe(2); + model.dispose(); + }); + + test("buildRowStore direct: a rebuild carries surviving slots and releases dropped ones", () => { + // The only production call site (create-local-row-model.ts) never passes + // `previous`; this pins the carry/release branch directly so it isn't + // dead-but-untested. + const columns = [helper.accessor("value", { type: "number" })] as const; + type Columns = typeof columns; + const queryPlan = compileQuery({ + derivations: columns, + query: { filters: [], sort: [], rowGroups: [] }, + }); + const slots = createSlotAllocator(); + const getRowId = (row: Row) => row.id; + + const first = buildRowStore({ + rows: ROWS, // a, b, c + getRowId, + queryPlan, + slots, + }); + expect(slots.capacity).toBe(3); + const aSlot = first.rows.get("a")!.slot; + const bSlot = first.rows.get("b")!.slot; + const cSlot = first.rows.get("c")!.slot; + + const second = buildRowStore({ + rows: [ + { id: "b", value: 20 }, + { id: "d", value: 4 }, + ], + getRowId, + queryPlan, + previous: first.rows, + slots, + }); + + // B carried its original slot (carry branch). + expect(second.rows.get("b")!.slot).toBe(bSlot); + // D draws its slot from the allocator BEFORE this build releases the + // dropped rows' slots (the build loop allocates for new rows, then only + // afterward releases the previous rows that didn't survive) — so D gets + // a brand-new slot, not a reused one, and capacity grows by one. + expect([aSlot, bSlot, cSlot]).not.toContain(second.rows.get("d")!.slot); + expect(slots.capacity).toBe(4); + + // A and C's slots are nonetheless genuinely released (the release + // branch ran): the next allocation reuses one of them instead of + // drawing a fifth slot. + const reused = slots.allocate(); + expect([aSlot, cSlot]).toContain(reused); + expect(slots.capacity).toBe(4); + }); + + test("a throwing accessor mid-setRows releases its provisional slot; nothing leaks", () => { + // A second candidate row's metadata evaluation throws after an earlier + // candidate in the same draft has already drawn a fresh slot. The draft + // must release that fresh slot on the way out, or it is stranded forever + // (still marked live, never reachable again) even though the setRows + // call it belonged to never committed. + const columns = [ + helper.accessor( + "value", + (row: Row) => { + if (row.value === -1) throw new Error("poisoned accessor"); + return row.value; + }, + { type: "number" }, + ), + ] as const; + const model = createLocalRowModel({ + rows: ROWS, + columns, + getRowId: (row: Row) => row.id, + // Sorting by "value" makes the column active, so `evaluate` actually + // calls the accessor. An inactive column (referenced by no filter, + // sort, group, or aggregate) is never read, and the accessor would + // never run at all. + query: { + filters: [], + sort: [{ columnId: "value", direction: "asc" as const }], + rowGroups: [], + }, + }); + const { slots } = getLocalRowModelSlotInternalsForTesting(model); + expect(slots.capacity).toBe(3); + + // "ok" draws a fresh slot (3) before "poison" throws during evaluation. + // A throwing accessor surfaces by propagating out of the mutation call + // (setRows rethrows; it does not downgrade the failure into an issue). + expect(() => + model.setRows([ + ...ROWS, + { id: "ok", value: 4 }, + { id: "poison", value: -1 }, + ]), + ).toThrow(); + // The high-water mark reflects the provisional allocation; it never + // shrinks. What matters is whether that slot made it back onto the free + // list, which the next assertion checks indirectly. + expect(slots.capacity).toBe(4); + + // A genuinely new, successful add should reuse the released slot rather + // than drawing a fifth one. Net growth across the whole sequence is + // exactly one (3 -> 4), proving the poisoned draft leaked nothing. + expect( + model.applyTransaction({ add: [{ id: "e", value: 5 }] }), + ).toMatchObject({ added: 1 }); + const after = getLocalRowModelSlotInternalsForTesting(model).root; + expect(after.rows.get("e")?.slot).toBe(3); + expect(slots.capacity).toBe(4); + model.dispose(); + }); +}); diff --git a/packages/row-model/src/__tests__/slot-vector.test.ts b/packages/row-model/src/__tests__/slot-vector.test.ts new file mode 100644 index 000000000..67ba464ac --- /dev/null +++ b/packages/row-model/src/__tests__/slot-vector.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from "vitest"; +import { + SLOT_VECTOR_CHUNK, + emptySlotVector, + forEachSlotEntry, + slotVectorFromEntries, + slotVectorGet, + slotVectorWithAll, +} from "../slot-vector"; + +describe("slot vector", () => { + it("stores entries at their slots, holes read undefined", () => { + const vec = slotVectorFromEntries( + [ + [0, "a"], + [2, "c"], + [1500, "far"], + ], + 2000, + ); + expect(slotVectorGet(vec, 0)).toBe("a"); + expect(slotVectorGet(vec, 1)).toBeUndefined(); + expect(slotVectorGet(vec, 2)).toBe("c"); + expect(slotVectorGet(vec, 1500)).toBe("far"); + expect(slotVectorGet(vec, 1999)).toBeUndefined(); + }); + + it("withAll writes and clears land; result reports chunks copied", () => { + const base = slotVectorFromEntries( + [ + [0, "a"], + [1, "b"], + ], + 10, + ); + const { next, chunksTouched } = slotVectorWithAll( + base, + [ + [0, "A"], + [1, undefined], + [5, "f"], + ], + 10, + ); + expect(slotVectorGet(next, 0)).toBe("A"); + expect(slotVectorGet(next, 1)).toBeUndefined(); + expect(slotVectorGet(next, 5)).toBe("f"); + expect(chunksTouched).toBe(1); // all three slots share chunk 0 + }); + + it("old snapshots survive later writes, including slot overwrite (COW pin)", () => { + const v0 = slotVectorFromEntries( + [ + [5, "old-5"], + [1030, "old-1030"], + ], + 2048, + ); + const { next: v1 } = slotVectorWithAll( + v0, + [ + [5, "new-5"], + [1030, undefined], + ], + 2048, + ); + expect(slotVectorGet(v1, 5)).toBe("new-5"); + expect(slotVectorGet(v1, 1030)).toBeUndefined(); + // v0 is byte-identical to before: the snapshot-validity invariant that + // makes slot REUSE safe for held revisions. + expect(slotVectorGet(v0, 5)).toBe("old-5"); + expect(slotVectorGet(v0, 1030)).toBe("old-1030"); + }); + + it("a commit touching k slots in one chunk copies exactly one chunk", () => { + const entries: [number, string][] = []; + for (let s = 0; s < 4096; s += 1) entries.push([s, `v${s}`]); + const base = slotVectorFromEntries(entries, 4096); + const writes: [number, string][] = []; + for (let s = 100; s < 150; s += 1) writes.push([s, `w${s}`]); + const { next, chunksTouched } = slotVectorWithAll(base, writes, 4096); + expect(chunksTouched).toBe(1); + // untouched chunks are carried by reference, not copied + expect(next.chunks[1]).toBe(base.chunks[1]); + expect(next.chunks[0]).not.toBe(base.chunks[0]); + }); + + it("withAll can grow capacity for slots beyond the old table", () => { + const base = slotVectorFromEntries([[0, "a"]], 1); + const { next } = slotVectorWithAll(base, [[5000, "far"]], 5001); + expect(slotVectorGet(next, 5000)).toBe("far"); + expect(slotVectorGet(next, 0)).toBe("a"); + expect(slotVectorGet(base, 5000)).toBeUndefined(); + }); + + it("forEachSlotEntry skips holes and visits every live entry once", () => { + const vec = slotVectorFromEntries( + [ + [3, "c"], + [SLOT_VECTOR_CHUNK + 1, "x"], + ], + 3000, + ); + const seen: Array<[number, string]> = []; + forEachSlotEntry(vec, (value, slot) => seen.push([slot, value])); + expect(seen).toEqual([ + [3, "c"], + [SLOT_VECTOR_CHUNK + 1, "x"], + ]); + }); + + it("emptySlotVector reads undefined everywhere", () => { + expect(slotVectorGet(emptySlotVector(), 0)).toBeUndefined(); + }); + + it("slotVectorFromEntries throws on a slot beyond the chunk table", () => { + // capacity 10 rounds up to a single SLOT_VECTOR_CHUNK-sized table; slot + // SLOT_VECTOR_CHUNK falls outside it even though it's far past `capacity`. + expect(() => slotVectorFromEntries([[SLOT_VECTOR_CHUNK, "x"]], 10)).toThrow( + RangeError, + ); + expect(() => slotVectorFromEntries([[SLOT_VECTOR_CHUNK, "x"]], 10)).toThrow( + `Slot ${SLOT_VECTOR_CHUNK} is beyond capacity 10.`, + ); + }); + + it("slotVectorWithAll throws on a slot beyond the chunk table", () => { + const base = slotVectorFromEntries([[0, "a"]], 10); + expect(() => + slotVectorWithAll(base, [[SLOT_VECTOR_CHUNK, "x"]], 10), + ).toThrow(RangeError); + expect(() => + slotVectorWithAll(base, [[SLOT_VECTOR_CHUNK, "x"]], 10), + ).toThrow(`Slot ${SLOT_VECTOR_CHUNK} is beyond capacity 10.`); + }); + + it("entries at the exact chunk boundary land in adjacent chunks", () => { + const vec = slotVectorFromEntries( + [ + [SLOT_VECTOR_CHUNK - 1, "last-of-0"], + [SLOT_VECTOR_CHUNK, "first-of-1"], + ], + 2 * SLOT_VECTOR_CHUNK, + ); + expect(slotVectorGet(vec, SLOT_VECTOR_CHUNK - 1)).toBe("last-of-0"); + expect(slotVectorGet(vec, SLOT_VECTOR_CHUNK)).toBe("first-of-1"); + }); + + it("forEachSlotEntry on an empty vector never invokes the callback", () => { + const callback = () => { + throw new Error("should not be called"); + }; + expect(() => forEachSlotEntry(emptySlotVector(), callback)).not.toThrow(); + }); +}); diff --git a/packages/row-model/src/__tests__/snapshot-dense-reads.test.ts b/packages/row-model/src/__tests__/snapshot-dense-reads.test.ts new file mode 100644 index 000000000..6a9f2271f --- /dev/null +++ b/packages/row-model/src/__tests__/snapshot-dense-reads.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from "vitest"; + +import { + createColumnHelper, + createLocalRowModel, + type PretableQueryFor, +} from "../index"; +import { getLocalRowModelSlotInternalsForTesting } from "../create-local-row-model"; +import { createInstrumentedLocalRowModel } from "../diagnostics"; + +interface Row { + id: string; + value: number; +} + +const helper = createColumnHelper(); + +function createColumns() { + return [helper.accessor("value", { type: "number" })] as const; +} + +type Columns = ReturnType; + +function createModel( + rows: readonly Row[], + options?: { readonly grouped?: boolean }, +) { + const model = createLocalRowModel({ + rows, + columns: createColumns(), + getRowId: (row: Row) => row.id, + ...(options?.grouped === true + ? { + query: { + filters: [], + sort: [], + rowGroups: [{ columnId: "value", direction: "asc" as const }], + }, + } + : {}), + }); + return model; +} + +function valueAtLeast(value: number): PretableQueryFor { + return { + filters: [{ columnId: "value", operator: "gte", value }], + sort: [], + rowGroups: [], + }; +} + +const ROWS: readonly Row[] = Object.freeze([ + { id: "a", value: 1 }, + { id: "b", value: 2 }, + { id: "c", value: 3 }, + { id: "d", value: 4 }, + { id: "e", value: 5 }, +]); + +/** + * Asserts index-for-index alignment between `ɵvisibleSlotRange` and the + * slots of the records `rowAt` resolves, via the committed-root seam. This + * is what makes the range read non-vacuous: the expected slot comes from the + * root's row store, not from the read under test. + */ +function expectSlotAlignment(model: ReturnType): void { + const snapshot = model.getState().snapshot; + const { root } = getLocalRowModelSlotInternalsForTesting(model); + const slots = snapshot.ɵvisibleSlotRange?.(0, snapshot.visibleRowCount); + expect(slots).toBeDefined(); + expect(slots).toHaveLength(snapshot.visibleRowCount); + for (let index = 0; index < snapshot.visibleRowCount; index += 1) { + const row = snapshot.rowAt(index); + if (row?.kind !== "data") throw new Error(`Expected data row at ${index}`); + const record = root.rows.get(row.rowId); + if (record === undefined) throw new Error(`Missing record ${row.rowId}`); + expect(slots?.[index]).toBe(record.slot); + } +} + +describe("snapshot dense reads", () => { + test("flat: ɵvisibleSlotRange aligns with rowAt-resolved records across a filter change", async () => { + const model = createModel(ROWS); + expectSlotAlignment(model); + + const transition = model.setQuery(valueAtLeast(3)); + await transition.finished; + const narrowed = model.getState().snapshot; + expect(narrowed.visibleRowCount).toBe(3); + expectSlotAlignment(model); + model.dispose(); + }); + + test("flat: ɵslotCapacity reports the root's slot capacity", () => { + const model = createModel(ROWS); + const snapshot = model.getState().snapshot; + const { root } = getLocalRowModelSlotInternalsForTesting(model); + expect(typeof snapshot.ɵslotCapacity).toBe("function"); + expect(snapshot.ɵslotCapacity?.()).toBe(root.slotCapacity); + expect(snapshot.ɵslotCapacity?.()).toBe(ROWS.length); + model.dispose(); + }); + + test("flat: ɵslotOfRowId resolves the current binding; a missing id is undefined", () => { + const model = createModel(ROWS); + const snapshot = model.getState().snapshot; + const { root } = getLocalRowModelSlotInternalsForTesting(model); + expect(snapshot.ɵslotOfRowId?.("b")).toBe(root.rows.get("b")?.slot); + expect(snapshot.ɵslotOfRowId?.("missing")).toBeUndefined(); + model.dispose(); + }); + + test("grouped: all three reads are implemented and return undefined", () => { + const model = createModel(ROWS, { grouped: true }); + const snapshot = model.getState().snapshot; + expect(typeof snapshot.ɵvisibleSlotRange).toBe("function"); + expect(typeof snapshot.ɵslotOfRowId).toBe("function"); + expect(typeof snapshot.ɵslotCapacity).toBe("function"); + expect( + snapshot.ɵvisibleSlotRange?.(0, snapshot.visibleRowCount), + ).toBeUndefined(); + expect(snapshot.ɵslotOfRowId?.("a")).toBeUndefined(); + expect(snapshot.ɵslotCapacity?.()).toBeUndefined(); + model.dispose(); + }); + + test("slot reuse: a remove+add transaction rebinds the slot and the range reflects it", () => { + const model = createModel(ROWS); + expect(model.applyTransaction({ remove: ["b"] })).toMatchObject({ + removed: 1, + }); + expect( + model.applyTransaction({ add: [{ id: "f", value: 6 }] }), + ).toMatchObject({ added: 1 }); + + // Non-vacuous via the committed-root seam: prove the allocator actually + // reused b's slot for f before asserting the snapshot reads agree. + const { root } = getLocalRowModelSlotInternalsForTesting(model); + const reusedSlot = root.rows.get("f")?.slot; + expect(reusedSlot).toBe(1); + + const snapshot = model.getState().snapshot; + expect(snapshot.ɵslotOfRowId?.("f")).toBe(reusedSlot); + expect(snapshot.ɵslotOfRowId?.("b")).toBeUndefined(); + expectSlotAlignment(model); + const slots = snapshot.ɵvisibleSlotRange?.(0, snapshot.visibleRowCount); + const fIndex = [...Array(snapshot.visibleRowCount).keys()].find((index) => { + const row = snapshot.rowAt(index); + return row?.kind === "data" && row.rowId === "f"; + }); + expect(fIndex).toBeDefined(); + expect(slots?.[fIndex ?? -1]).toBe(reusedSlot); + model.dispose(); + }); + + test("instrumented wrapper passes the reads through and counts the range read", () => { + const instrumented = createInstrumentedLocalRowModel({ + rows: ROWS, + columns: createColumns(), + getRowId: (row: Row) => row.id, + }); + instrumented.diagnostics.resetWork(); + const snapshot = instrumented.model.getState().snapshot; + const slots = snapshot.ɵvisibleSlotRange?.(0, snapshot.visibleRowCount); + expect(slots).toHaveLength(ROWS.length); + expect(instrumented.diagnostics.read().work.snapshotOutputRowsRead).toBe( + ROWS.length, + ); + // The k-sized reads pass through without counting output rows. + expect(snapshot.ɵslotOfRowId?.("a")).toBe(0); + expect(snapshot.ɵslotCapacity?.()).toBe(ROWS.length); + expect(instrumented.diagnostics.read().work.snapshotOutputRowsRead).toBe( + ROWS.length, + ); + instrumented.model.dispose(); + }); +}); diff --git a/packages/row-model/src/__tests__/sort-fast-path.test.ts b/packages/row-model/src/__tests__/sort-fast-path.test.ts index b5ca10c5b..ae6786f98 100644 --- a/packages/row-model/src/__tests__/sort-fast-path.test.ts +++ b/packages/row-model/src/__tests__/sort-fast-path.test.ts @@ -10,9 +10,11 @@ import { } from "../index"; import { compareRecordRows, + filterVerdict, sortKeysOf, type CompiledQuery, } from "../compiled-query"; +import { rowPassesFilter } from "../filter-membership"; import type { CooperativeTransitionScheduler } from "../cooperative-transition"; import { createInstrumentedLocalRowModel } from "../diagnostics"; import type { LocalRowModelInstrumentation } from "../diagnostics"; @@ -20,9 +22,11 @@ import type { RevisionRoot } from "../internal-types"; import { compareOrderStatisticTreeIds } from "../persistent/order-statistic-tree"; import { createPersistentMap } from "../persistent/persistent-map"; import { buildRowStore } from "../row-store"; +import { createSlotAllocator } from "../slot-allocator"; import { rebuildRootForSortOnlyChange } from "../sort-rebuild"; import type { PretableGroupId } from "../types"; -import { createVisibleIndex } from "../visible-index"; +import { EMPTY_MEMBERSHIP } from "../membership-bitset"; +import { createVisibleIndex, membershipFromFlatTree } from "../visible-index"; interface Holding { id: string; @@ -114,10 +118,12 @@ function createRoot( queryPlan: CompiledQuery, rows: readonly Holding[], ): RevisionRoot { + const slots = createSlotAllocator(); const store = buildRowStore({ rows, getRowId: (row) => row.id, queryPlan, + slots, }); const defaultPolicy = Object.freeze({ kind: "expanded" as const }); const expansion = Object.freeze({ @@ -125,17 +131,26 @@ function createRoot( overrides: createPersistentMap(), state: Object.freeze({ default: defaultPolicy, overrideCount: 0 }), }); + const visible = createVisibleIndex( + store.records, + queryPlan, + false, + expansion.overrides, + ); return Object.freeze({ revision: 0, parentRevision: null, rows: store.rows, sourceOrder: store.sourceOrder, - visible: createVisibleIndex( - store.records, - queryPlan, - false, - expansion.overrides, - ), + recordsBySlot: store.recordsBySlot, + slotCapacity: slots.capacity, + // Same rule as the production initial-build site: flat roots index their + // membership per slot, grouped roots carry the sentinel. + visibleSlots: + queryPlan.query.rowGroups.length > 0 + ? EMPTY_MEMBERSHIP + : membershipFromFlatTree(visible.rows, slots.capacity), + visible, queryPlan, expansion, cause: Object.freeze({ kind: "initial" as const }), @@ -164,6 +179,14 @@ function testInstrumentation(): LocalRowModelInstrumentation { snapshotOutputRowsRead: 0, synchronousRebuilds: 0, synchronousRebuildMs: 0, + filterRebuilds: 0, + filterRowsFlipped: 0, + filterMergeSortedInsertions: 0, + filterRebuildMs: 0, + bulkByIdDerived: 0, + bulkOrderVerificationsSkipped: 0, + evaluationCacheAdoptions: 0, + slotChunksTouched: 0, sortKeyCarries: 0, sortKeyEvaluations: 0, schedulerSliceDurations: [], @@ -229,10 +252,15 @@ describe("rebuildRootForSortOnlyChange", () => { expect(twinPlan).not.toBe(nextPlan); const expected = ROOT_ROWS.map((row, sourceOrder) => ({ rowId: row.id, - input: { rowId: row.id, row, sourceOrder }, - metadata: twinPlan.evaluate({ rowId: row.id, row, sourceOrder }), + input: { rowId: row.id, row, sourceOrder, slot: sourceOrder }, + metadata: twinPlan.evaluate({ + rowId: row.id, + row, + sourceOrder, + slot: sourceOrder, + }), })) - .filter((entry) => entry.metadata.filterPasses) + .filter((entry) => filterVerdict(twinPlan, entry.input)) .sort( (left, right) => compareRecordRows(twinPlan, left.input, right.input) || @@ -256,7 +284,8 @@ describe("rebuildRootForSortOnlyChange", () => { expect(rebuilt.visible.rows.rankOf("h3")).toBeUndefined(); const record = rebuilt.rows.get("h3"); expect(record).toBeDefined(); - expect(record!.metadata.filterPasses).toBe(false); + // The rebuilt root's own membership is the verdict, and it says "out". + expect(rowPassesFilter(rebuilt, "h3")).toBe(false); // The NEW plan's store was filled for the filtered-out row too: sort keys // resolve under nextPlan as note, not score. expect(sortKeysOf(nextPlan, record!)).toEqual([ @@ -363,6 +392,36 @@ describe("rebuildRootForSortOnlyChange", () => { expect(instrumentation.work.synchronousRebuildMs).toBe(7); }); + test("the sort commit claims the order proof and NOT the derived byId", () => { + const { nextPlan, captured } = createRebuildFixture(); + const instrumentation = testInstrumentation(); + + const rebuilt = rebuildRootForSortOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + instrumentation, + }); + + expect(instrumentation.work.bulkOrderVerificationsSkipped).toBe(1); + // Deliberate abstention, not an oversight. A sort-only change keeps the + // entry SET but re-decorates every entry with the next plan's keys, so + // every "survivor" is a NEW object; a map derived from the captured + // tree would keep returning the previous plan's entries. The assertion + // below is the reason, measured: not one visible entry survives by + // identity, so there is nothing for a derivation to carry. + expect(instrumentation.work.bulkByIdDerived).toBe(0); + let reused = 0; + for (const entry of rebuilt.visible.rows.entries()) { + const id = entry.record.rowId; + expect(rebuilt.visible.rows.get(id)).toBe(entry); + if (captured.visible.rows.get(id) === entry) reused += 1; + } + expect(rebuilt.visible.rows.size).toBeGreaterThan(0); + expect(reused).toBe(0); + }); + test("throws TypeError when the plans are not a sort-only change", () => { const { fixture, captured } = createRebuildFixture(); const filterChangedPlan = compileQuery({ @@ -537,12 +596,15 @@ describe("setQuery sort-only fast path", () => { await expect(transition.finished).resolves.toBe(1); }); - test("mutation twin: a filter change takes the cooperative path", () => { + test("mutation twin: a combined sort+filter change takes the cooperative path", () => { const { model, diagnostics, scheduler } = createModelFixture(); + // Was a filter-only change until the filter fast path landed; BOTH + // facets must now change for the cooperative machinery to be the + // subject. model.setQuery({ filters: [{ columnId: "team", operator: "equals", value: "Beta" }], - sort: [{ columnId: "score", direction: "desc" }], + sort: [{ columnId: "score", direction: "asc" }], rowGroups: [], }); @@ -568,8 +630,10 @@ describe("setQuery sort-only fast path", () => { test("supersedes an in-flight cooperative transition", async () => { const { model, scheduler } = createModelFixture(); const first = model.setQuery({ + // Filter AND sort change: a filter-only change would now commit + // synchronously (filter fast path) and leave nothing to supersede. filters: [{ columnId: "team", operator: "equals", value: "Beta" }], - sort: [{ columnId: "score", direction: "desc" }], + sort: [{ columnId: "score", direction: "asc" }], rowGroups: [], }); expect(model.getState().status.kind).toBe("rebuilding"); @@ -662,13 +726,15 @@ describe("setQuery sort-only fast path", () => { }); }); - test('mutation twin: a cooperative filter setQuery journals "bulk-replace"', async () => { + test('mutation twin: a cooperative sort+filter setQuery journals "bulk-replace"', async () => { const { model, scheduler } = createModelFixture(); const before = model.getState().snapshot.revision; + // Was a filter-only change until the filter fast path landed; both + // facets change so the COOPERATIVE path stays this twin's subject. const transition = model.setQuery({ filters: [{ columnId: "team", operator: "equals", value: "Beta" }], - sort: [{ columnId: "score", direction: "desc" }], + sort: [{ columnId: "score", direction: "asc" }], rowGroups: [], }); scheduler.flushAll(); diff --git a/packages/row-model/src/__tests__/sort-key-store.test.ts b/packages/row-model/src/__tests__/sort-key-store.test.ts index ee5a0e18b..16254b9f0 100644 --- a/packages/row-model/src/__tests__/sort-key-store.test.ts +++ b/packages/row-model/src/__tests__/sort-key-store.test.ts @@ -11,6 +11,7 @@ import { compareRecordRows, compareWithSortKeys, fillSortKeysFromPrevious, + filterVerdict, sortKeysOf, } from "../compiled-query"; @@ -155,11 +156,13 @@ describe("compareRecordRows", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; const b = { rowId: "b", row: holding({ id: "b", score: 9 }), sourceOrder: 1, + slot: 1, }; plan.evaluate(a); plan.evaluate(b); @@ -186,6 +189,7 @@ describe("compareRecordRows", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; plan.evaluate(input); @@ -211,8 +215,13 @@ describe("compareRecordRows", () => { rowGroups: [], } as unknown as PretableQueryFor, }); - const leftInput = { rowId: left.id, row: left, sourceOrder: 0 }; - const rightInput = { rowId: right.id, row: right, sourceOrder: 1 }; + const leftInput = { rowId: left.id, row: left, sourceOrder: 0, slot: 0 }; + const rightInput = { + rowId: right.id, + row: right, + sourceOrder: 1, + slot: 1, + }; plan.evaluate(leftInput); plan.evaluate(rightInput); @@ -237,11 +246,13 @@ describe("compareRecordRows", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; const stranger = { rowId: "b", row: holding({ id: "b", score: 9 }), sourceOrder: 1, + slot: 1, }; plan.evaluate(known); @@ -263,6 +274,7 @@ describe("compareRecordRows", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; plan.evaluate(input); const foreign = { query: SCORE_ASC, derivations: fixture.columns }; @@ -287,8 +299,13 @@ describe("compareWithSortKeys", () => { rowGroups: [], } as unknown as PretableQueryFor, }); - const leftInput = { rowId: left.id, row: left, sourceOrder: 0 }; - const rightInput = { rowId: right.id, row: right, sourceOrder: 1 }; + const leftInput = { rowId: left.id, row: left, sourceOrder: 0, slot: 0 }; + const rightInput = { + rowId: right.id, + row: right, + sourceOrder: 1, + slot: 1, + }; plan.evaluate(leftInput); plan.evaluate(rightInput); const leftKeys = sortKeysOf(plan, leftInput); @@ -323,11 +340,13 @@ describe("compareWithSortKeys", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; const b = { rowId: "b", row: holding({ id: "b", score: 9 }), sourceOrder: 1, + slot: 1, }; plan.evaluate(a); plan.evaluate(b); @@ -356,6 +375,7 @@ describe("compareWithSortKeys", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; plan.evaluate(input); const keys = sortKeysOf(plan, input); @@ -514,6 +534,7 @@ describe("sortKeysOf", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; plan.evaluate(input); @@ -539,6 +560,7 @@ describe("sortKeysOf", () => { rowId: "ghost", row: holding({ id: "ghost", score: 9 }), sourceOrder: 0, + slot: 0, }; expect(() => sortKeysOf(plan, stranger)).toThrowError( @@ -556,6 +578,7 @@ describe("sortKeysOf", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; plan.evaluate(input); const foreign = { query: SCORE_ASC, derivations: fixture.columns }; @@ -581,6 +604,7 @@ describe("fillSortKeysFromPrevious", () => { rowId: "a", row: holding({ id: "a", score: 5, note: "steady" }), sourceOrder: 0, + slot: 0, }; previousPlan.evaluate(input); @@ -602,6 +626,7 @@ describe("fillSortKeysFromPrevious", () => { rowId: "b", row: holding({ id: "b", score: 9, note: "zzz" }), sourceOrder: 1, + slot: 1, }; fillSortKeysFromPrevious(nextPlan, previousPlan, other); expect(compareRecordRows(nextPlan, input, other)).toBeLessThan(0); @@ -621,6 +646,7 @@ describe("fillSortKeysFromPrevious", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; previousPlan.evaluate(input); const first = fillSortKeysFromPrevious(nextPlan, previousPlan, input); @@ -650,6 +676,7 @@ describe("fillSortKeysFromPrevious", () => { rowId: "a", row: holding({ id: "a", score: 5, note: "steady" }), sourceOrder: 0, + slot: 0, }; previousPlan.evaluate(input); // Keys-only state under nextPlan: filled, never evaluated. @@ -659,7 +686,7 @@ describe("fillSortKeysFromPrevious", () => { // Evaluate must NOT treat the keys-only state as a metadata cache hit — // it produces coherent metadata and refreshes the stored keys. const metadata = nextPlan.evaluate(input); - expect(metadata.filterPasses).toBe(true); + expect(filterVerdict(nextPlan, input)).toBe(true); expect(metadata.rowId).toBe("a"); const afterEvaluate = sortKeysOf(nextPlan, input); expect(afterEvaluate).toEqual([ @@ -692,6 +719,7 @@ describe("fillSortKeysFromPrevious", () => { rowId: "a", row: holding({ id: "a", score: 5, note: "steady" }), sourceOrder: 0, + slot: 0, }; // No previousPlan.evaluate: nothing to carry, every column re-runs. const keys = fillSortKeysFromPrevious(nextPlan, previousPlan, input); @@ -738,6 +766,7 @@ describe("fillSortKeysFromPrevious", () => { rowId: "r1", row: holding({ id: "r1", score: 5 }), sourceOrder: 0, + slot: 0, }; previousPlan.evaluate(input); @@ -766,6 +795,7 @@ describe("fillSortKeysFromPrevious", () => { rowId: "a", row: holding({ id: "a", score: 5 }), sourceOrder: 0, + slot: 0, }; plan.evaluate(input); const foreign = { query: SCORE_ASC, derivations: fixture.columns }; diff --git a/packages/row-model/src/__tests__/transitions.test.ts b/packages/row-model/src/__tests__/transitions.test.ts index 488e10b72..593611bc8 100644 --- a/packages/row-model/src/__tests__/transitions.test.ts +++ b/packages/row-model/src/__tests__/transitions.test.ts @@ -180,8 +180,11 @@ describe("cooperative query and derivation transitions", () => { try { model = createModel({ clock: tickingClock(), budgetMs: 1 }); const transition = model.setQuery({ + // Filter AND sort change together: either alone would commit + // synchronously (the #457 sort and filter fast paths) and postTask + // would never be consulted; the subject is the scheduler fallback. filters: [{ columnId: "score", operator: "gte", value: 2 }], - sort: [], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); const outcome = await Promise.race([ @@ -716,8 +719,10 @@ describe("cooperative query and derivation transitions", () => { expect(scheduler.entries).toHaveLength(0); const changed = model.setQuery({ + // Filter AND sort change: either alone commits synchronously (#457 + // fast paths); this handle must stay pending to be cancellable. filters: [{ columnId: "score", operator: "gte", value: 5 }], - sort: [], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); expect(changed.id).toBe(2); @@ -736,8 +741,11 @@ describe("cooperative query and derivation transitions", () => { budgetMs: 2, }); const transition = model.setQuery({ + // Filter AND sort change: either alone commits synchronously (#457 + // fast paths); cancellation needs scheduled cooperative work to + // release. filters: [{ columnId: "score", operator: "gte", value: 3 }], - sort: [], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); const rejection = expect(transition.finished).rejects.toEqual( @@ -770,8 +778,11 @@ describe("cooperative query and derivation transitions", () => { model.subscribe(listener); const first = model.setQuery({ + // Every transition in this test pairs the filter change with a sort + // change: either facet alone commits synchronously (#457 fast paths) + // and the hostile hooks need PENDING cooperative work to attack. filters: [{ columnId: "score", operator: "gte", value: 3 }], - sort: [], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); const firstStaleTask = scheduler.entries[0]?.task; @@ -792,15 +803,16 @@ describe("cooperative query and derivation transitions", () => { const superseded = model.setQuery({ filters: [{ columnId: "score", operator: "gte", value: 4 }], - sort: [], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); const supersededStaleTask = scheduler.entries.at(-1)?.task; const replacement = model.setQuery({ - // A filter change: a sort-only replacement would commit synchronously - // (#457) and this test needs a pending transition to dispose. + // Filter AND sort change against the committed plan: a sort-only or + // filter-only replacement would commit synchronously (#457 fast + // paths) and this test needs a pending transition to dispose. filters: [{ columnId: "score", operator: "gte", value: 5 }], - sort: [], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); await expect(superseded.finished).rejects.toMatchObject({ @@ -879,8 +891,10 @@ describe("cooperative query and derivation transitions", () => { const observed: unknown[] = []; model.subscribe(() => observed.push(model.getState().status)); const first = model.setQuery({ + // Filter AND sort change: either alone commits synchronously (#457 + // fast paths); cross-supersession needs `first` still rebuilding. filters: [{ columnId: "score", operator: "gte", value: 4 }], - sort: [], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); const stale = scheduler.entries[0]; @@ -934,8 +948,10 @@ describe("cooperative query and derivation transitions", () => { const listener = vi.fn(); model.subscribe(listener); const transition = model.setQuery({ + // Filter AND sort change: either alone commits synchronously (#457 + // fast paths); disposal needs an ACTIVE transition to reject. filters: [{ columnId: "score", operator: "gte", value: 1 }], - sort: [], + sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); const stale = scheduler.entries[0]; diff --git a/packages/row-model/src/__tests__/types.test.ts b/packages/row-model/src/__tests__/types.test.ts index 08a2d0f4a..59a86d883 100644 --- a/packages/row-model/src/__tests__/types.test.ts +++ b/packages/row-model/src/__tests__/types.test.ts @@ -586,8 +586,11 @@ function assertOperationalSignatures( void _from; } else { const _reason: - "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder" = - sequence.reason; + | "unknown-revision" + | "journal-evicted" + | "bulk-replace" + | "reorder" + | "refilter" = sequence.reason; void _reason; } diff --git a/packages/row-model/src/__tests__/visible-slots.test.ts b/packages/row-model/src/__tests__/visible-slots.test.ts new file mode 100644 index 000000000..245a4260f --- /dev/null +++ b/packages/row-model/src/__tests__/visible-slots.test.ts @@ -0,0 +1,236 @@ +import { describe, expect, test } from "vitest"; + +import { createColumnHelper, createLocalRowModel } from "../index"; +import { getLocalRowModelSlotInternalsForTesting } from "../create-local-row-model"; +import type { PretableRowId } from "../column-types"; +import type { RevisionRoot } from "../internal-types"; +import { EMPTY_MEMBERSHIP, testMembershipBit } from "../membership-bitset"; + +interface Row { + id: string; + value: number; +} +const helper = createColumnHelper(); + +function createModel( + rows: readonly Row[], + options?: { + readonly grouped?: boolean; + readonly filterGte?: number; + }, +) { + const columns = [helper.accessor("value", { type: "number" })] as const; + return createLocalRowModel({ + rows, + columns, + getRowId: (row: Row) => row.id, + query: { + filters: + options?.filterGte === undefined + ? [] + : [ + { + columnId: "value", + operator: "gte" as const, + value: options.filterGte, + }, + ], + sort: [], + rowGroups: + options?.grouped === true + ? [{ columnId: "value", direction: "asc" as const }] + : [], + }, + }); +} + +function rootOf(model: object): RevisionRoot { + return getLocalRowModelSlotInternalsForTesting(model).root; +} + +/** + * The Task 6 equivalence oracle, verbatim from the `visibleSlots` doc + * comment: for a FLAT root, a record's bit is set iff the record is a member + * of `visible.rows` — the bitset is an index of the same structural verdict, + * never a divergent copy. The set-bit count over the root's self-described + * capacity must equal the visible tree's size, so a stale set bit on a + * record-less (released) slot fails too, not just a wrong bit under a live + * record. + */ +function expectMembershipOracle( + root: RevisionRoot, +): void { + for (const [, record] of root.rows.entries()) { + expect(testMembershipBit(root.visibleSlots, record.slot)).toBe( + root.visible.rows.get(record.rowId) !== undefined, + ); + } + let setBits = 0; + for (let slot = 0; slot < root.slotCapacity; slot += 1) { + if (testMembershipBit(root.visibleSlots, slot)) setBits += 1; + } + expect(setBits).toBe(root.visible.rows.size); +} + +const ROWS: readonly Row[] = Object.freeze([ + { id: "a", value: 1 }, + { id: "b", value: 2 }, + { id: "c", value: 3 }, + { id: "d", value: 4 }, + { id: "e", value: 5 }, + { id: "f", value: 6 }, +]); + +describe("visibleSlots membership bitset", () => { + test("flat roots satisfy the equivalence oracle across the scripted sequence", async () => { + const model = createModel(ROWS, { filterGte: 3 }); + + // 1. Initial build with an active filter. The fixture must be able to + // disprove: some rows pass (c..f), some do not (a, b). + const initial = rootOf(model); + expect(initial.visible.rows.size).toBe(4); + expect( + testMembershipBit(initial.visibleSlots, initial.rows.get("d")!.slot), + ).toBe(true); + expect( + testMembershipBit(initial.visibleSlots, initial.rows.get("a")!.slot), + ).toBe(false); + expectMembershipOracle(initial); + + // 2. Transaction flipping rows across the filter boundary: `a` enters + // (1 -> 10), `d` leaves (4 -> 0). + expect( + model.applyTransaction({ + update: [ + { id: "a", changes: { value: 10 } }, + { id: "d", changes: { value: 0 } }, + ], + }), + ).toMatchObject({ updated: 2 }); + const flipped = rootOf(model); + expect( + testMembershipBit(flipped.visibleSlots, flipped.rows.get("a")!.slot), + ).toBe(true); + expect( + testMembershipBit(flipped.visibleSlots, flipped.rows.get("d")!.slot), + ).toBe(false); + expectMembershipOracle(flipped); + + // 3. Remove a VISIBLE row: its bit must clear with the removal, not + // linger on the released slot. + const removedSlot = flipped.rows.get("c")!.slot; + expect(model.applyTransaction({ remove: ["c"] })).toMatchObject({ + removed: 1, + }); + const afterRemove = rootOf(model); + expect(testMembershipBit(afterRemove.visibleSlots, removedSlot)).toBe( + false, + ); + expectMembershipOracle(afterRemove); + + // 4. Filter-only setQuery (synchronous rebuild): rows b (2) and e (5) + // flip in opposite directions under gte 6. + const filterTransition = model.setQuery({ + filters: [{ columnId: "value", operator: "gte", value: 6 }], + sort: [], + rowGroups: [], + }); + await filterTransition.finished; + const refiltered = rootOf(model); + expect(refiltered.visible.rows.size).toBe(2); + expectMembershipOracle(refiltered); + + // 5. Sort-only setQuery: the member SET is identical, so the committed + // root must CARRY the previous bitset by identity. + const sortTransition = model.setQuery({ + filters: [{ columnId: "value", operator: "gte", value: 6 }], + sort: [{ columnId: "value", direction: "desc" }], + rowGroups: [], + }); + await sortTransition.finished; + const resorted = rootOf(model); + expect(resorted.visibleSlots).toBe(refiltered.visibleSlots); + expectMembershipOracle(resorted); + + // 6. setRows replacement: retiring rows hand slots to new rows in the + // same commit, and the rebuilt bitset must match the rebuilt tree. + expect( + model.setRows([ + { id: "a", value: 1 }, + { id: "g", value: 9 }, + { id: "h", value: 2 }, + ]), + ).toMatchObject({ added: 2 }); + const replaced = rootOf(model); + expect(replaced.visible.rows.size).toBe(1); + expectMembershipOracle(replaced); + + // 7. Add transaction (fresh or reused slot) on both sides of the filter. + expect( + model.applyTransaction({ + add: [ + { id: "i", value: 100 }, + { id: "j", value: 0 }, + ], + }), + ).toMatchObject({ added: 2 }); + expectMembershipOracle(rootOf(model)); + + // 8. Filter AND sort change together: the cooperative transition's flat + // `finish` construction site. + const cooperative = model.setQuery({ + filters: [{ columnId: "value", operator: "gte", value: 2 }], + sort: [{ columnId: "value", direction: "asc" }], + rowGroups: [], + }); + await cooperative.finished; + expectMembershipOracle(rootOf(model)); + + model.dispose(); + }); + + test("grouped roots carry the EMPTY_MEMBERSHIP sentinel by identity", async () => { + const model = createModel(ROWS, { grouped: true, filterGte: 3 }); + expect(rootOf(model).visibleSlots).toBe(EMPTY_MEMBERSHIP); + + expect( + model.applyTransaction({ + update: [{ id: "a", changes: { value: 10 } }], + remove: ["c"], + add: [{ id: "g", value: 7 }], + }), + ).toMatchObject({ updated: 1, removed: 1, added: 1 }); + expect(rootOf(model).visibleSlots).toBe(EMPTY_MEMBERSHIP); + + expect( + model.setRows([ + { id: "a", value: 1 }, + { id: "b", value: 2 }, + ]), + ).toMatchObject({ removed: 4 }); + expect(rootOf(model).visibleSlots).toBe(EMPTY_MEMBERSHIP); + model.dispose(); + }); + + test("a flat model that regroups via setQuery lands on the sentinel, and back", async () => { + const model = createModel(ROWS, { filterGte: 3 }); + const toGrouped = model.setQuery({ + filters: [{ columnId: "value", operator: "gte", value: 3 }], + sort: [], + rowGroups: [{ columnId: "value", direction: "asc" }], + }); + await toGrouped.finished; + expect(rootOf(model).visibleSlots).toBe(EMPTY_MEMBERSHIP); + + const toFlat = model.setQuery({ + filters: [{ columnId: "value", operator: "gte", value: 3 }], + sort: [], + rowGroups: [], + }); + await toFlat.finished; + const flat = rootOf(model); + expect(flat.visibleSlots).not.toBe(EMPTY_MEMBERSHIP); + expectMembershipOracle(flat); + model.dispose(); + }); +}); diff --git a/packages/row-model/src/change-journal.ts b/packages/row-model/src/change-journal.ts index ea09b17fa..f67d51c52 100644 --- a/packages/row-model/src/change-journal.ts +++ b/packages/row-model/src/change-journal.ts @@ -283,25 +283,32 @@ export function createChangeJournal( ); if (start < 0) return reset(currentRevision, "unknown-revision"); const retained = entries.slice(start); - // "reorder" is a PROMISE (order moved, nothing else), so it only - // survives aggregation when every entry in the range is a reorder - // barrier. Any other entry — changes or a plain barrier — voids the - // promise and the whole range degrades to a plain bulk reset. - let allReorder = retained.length > 0; - let reorderExpected = fromRevision; - for (const entry of retained) { - if ( - entry.kind !== "barrier" || - entry.reason !== "reorder" || - entry.previousRevision !== reorderExpected - ) { - allReorder = false; - break; + // "reorder" (order moved, nothing else) and "refilter" (membership + // changed, surviving order and identities intact) are PROMISES, so a + // promised reason only survives aggregation when every entry in the + // range is a barrier with that SAME reason. Any other entry — changes, + // a plain barrier, or the OTHER promise (order + membership both + // changed delivers neither) — voids it and the whole range degrades + // to a plain bulk reset. + const firstReason = + retained[0]?.kind === "barrier" ? retained[0].reason : undefined; + if (firstReason === "reorder" || firstReason === "refilter") { + let allPromised = true; + let promisedExpected = fromRevision; + for (const entry of retained) { + if ( + entry.kind !== "barrier" || + entry.reason !== firstReason || + entry.previousRevision !== promisedExpected + ) { + allPromised = false; + break; + } + promisedExpected = entry.revision; + } + if (allPromised && promisedExpected === currentRevision) { + return reset(currentRevision, firstReason); } - reorderExpected = entry.revision; - } - if (allReorder && reorderExpected === currentRevision) { - return reset(currentRevision, "reorder"); } let expected = fromRevision; const changeSets: PretableChangeSet[] = []; @@ -312,7 +319,9 @@ export function createChangeJournal( if (entry.kind === "barrier") { return reset( currentRevision, - entry.reason === "reorder" ? "bulk-replace" : entry.reason, + entry.reason === "reorder" || entry.reason === "refilter" + ? "bulk-replace" + : entry.reason, ); } changeSets.push(entry.changeSet); diff --git a/packages/row-model/src/compiled-query.ts b/packages/row-model/src/compiled-query.ts index 38ede769a..c42e3559f 100644 --- a/packages/row-model/src/compiled-query.ts +++ b/packages/row-model/src/compiled-query.ts @@ -64,14 +64,6 @@ type CompiledAggregateLeafForDescriptor< TValue, CompiledAggregateDependency >; - readonly filteredLeaf: - | AggregateTreeLeaf< - TRowId, - TRow, - TValue, - CompiledAggregateDependency - > - | undefined; } : never; @@ -92,6 +84,14 @@ export interface CompiledRowInput< readonly rowId: TRowId; readonly row: TRow; readonly sourceOrder: number; + /** + * The row's dense handle slot (Amendment J §1). Every caller either holds + * the record — which already carries `.slot` — or is creating one and has + * just allocated the slot, so this is always available to stamp here. + * Unread today; threaded so slot-indexed storage can consume the handle + * directly instead of re-deriving it from the row id. + */ + readonly slot: number; } /** @@ -114,9 +114,12 @@ export interface CompiledRowMetadata< readonly rowId: TRowId; readonly row: TRow; readonly sourceOrder: number; - readonly filterPasses: boolean; readonly groupPath: readonly CompiledGroupKey[]; - /** `allLeaf` always exists; `filteredLeaf` exists only when filters pass. */ + /** + * One leaf per aggregated column. There is no filtered variant: filtered + * aggregation is membership in a separate aggregate TREE, decided by the + * row's verdict at insert time, so a per-leaf copy would only restate it. + */ readonly aggregateLeaves: readonly CompiledAggregateLeaf[]; } @@ -276,11 +279,29 @@ interface RuntimeQuery { * `sortKeys` reads are UNGUARDED: keys depend only on the row object's * values and this plan's sort columns — they embed no rowId/sourceOrder, so * re-evaluation under a changed sourceOrder overwrites harmlessly. + * + * `filterPasses` is written ONLY beside a `metadata` write and read ONLY + * under the same guard, so it means exactly "the verdict the cached metadata + * was built with" — a memo of `evaluate`, not a verdict store. Verdicts are + * never STORED anywhere: a committed root's verdict is its membership (see + * `./filter-membership`), and this field only spares a second accessor pass + * when a producer evaluates a row and then asks the plan what it decided. + * + * `verdictPlan` is the plan that wrote `filterPasses`. It is the ONE field + * that exists because a cache can be SHARED between plans + * (`adoptEvaluationCache`): every other field is a function of the row and + * of facets an adopting plan holds identical, but a verdict is a function of + * the FILTERS, which are exactly what changed. Tagging the writer keeps the + * memo plan-scoped at zero per-row cost — the tag is written inside a write + * that already happens, and an adopting plan simply misses the guard and + * runs the accessors it would have run anyway. */ interface CachedEvaluation { rowId: PretableRowId; sourceOrder: number; metadata: object | undefined; + filterPasses: boolean | undefined; + verdictPlan: object | undefined; sortKeys: readonly { readonly columnId: string; readonly value: unknown }[]; } @@ -302,7 +323,7 @@ const collator = new Intl.Collator(undefined, { sensitivity: "base", }); -const FILTER_OPERATORS = { +export const FILTER_OPERATORS = { text: new Set([ "contains", "notContains", @@ -1242,75 +1263,144 @@ function toDayMs(value: unknown): number { : isoDayMs(parts[1]); } -function evaluateFilter( - filter: RuntimeFilter, - column: RuntimeColumn, - value: unknown, -): boolean { - if (filter.operator === "isEmpty") return isEmptyValue(value); - if (filter.operator === "isNotEmpty") return !isEmptyValue(value); +type FilterPredicate = (value: unknown) => boolean; + +const alwaysTrue: FilterPredicate = () => true; +const alwaysFalse: FilterPredicate = () => false; + +function isNumberCell(value: unknown): value is number { + return typeof value === "number" && !Number.isNaN(value); +} + +function textCell(value: unknown): string { + return String(value ?? "").toLocaleLowerCase(); +} + +function compileNumberPredicate( + operator: string, + operand: unknown, +): FilterPredicate { + if (operator === "between") { + const range = operand as readonly unknown[]; + const a = range[0]; + const b = range[1]; + if (typeof a !== "number" || typeof b !== "number") return alwaysFalse; + const lower = Math.min(a, b); + const upper = Math.max(a, b); + return (value) => isNumberCell(value) && value >= lower && value <= upper; + } + if (typeof operand !== "number" || Number.isNaN(operand)) return alwaysFalse; + switch (operator) { + case "equals": + return (value) => isNumberCell(value) && value === operand; + case "notEquals": + return (value) => isNumberCell(value) && value !== operand; + case "gt": + return (value) => isNumberCell(value) && value > operand; + case "gte": + return (value) => isNumberCell(value) && value >= operand; + case "lt": + return (value) => isNumberCell(value) && value < operand; + default: + return (value) => isNumberCell(value) && value <= operand; + } +} + +function compileDatePredicate( + operator: string, + operand: unknown, +): FilterPredicate { + if (operator === "dateBetween") { + const range = operand as readonly unknown[]; + const a = toDayMs(range[0]); + const b = toDayMs(range[1]); + if (Number.isNaN(a) || Number.isNaN(b)) return alwaysFalse; + const lower = Math.min(a, b); + const upper = Math.max(a, b); + return (value) => { + const cell = toDayMs(value); + return cell >= lower && cell <= upper; + }; + } + const other = toDayMs(operand); + if (Number.isNaN(other)) return alwaysFalse; + // A NaN cell (unparsable date) fails every comparison below on its own — + // no explicit guard needed to preserve the "bad cell never passes" rule. + if (operator === "on") return (value) => toDayMs(value) === other; + if (operator === "before") return (value) => toDayMs(value) < other; + return (value) => toDayMs(value) > other; +} + +function compileSelectionPredicate( + operator: string, + operand: unknown, + coerce: (value: unknown) => unknown, +): FilterPredicate { + const entries = operand as readonly unknown[]; + // An empty selection matches EVERYTHING, regardless of direction. + if (entries.length === 0) return alwaysTrue; + const included = new Set(entries.map((entry) => coerce(entry))); + return operator === "isAnyOf" + ? (value) => included.has(coerce(value)) + : (value) => !included.has(coerce(value)); +} + +function compileTextPredicate( + operator: string, + operand: unknown, +): FilterPredicate { + const search = String(operand).toLocaleLowerCase(); + switch (operator) { + case "contains": + return (value) => textCell(value).includes(search); + case "notContains": + return (value) => !textCell(value).includes(search); + case "equals": + return (value) => textCell(value) === search; + case "notEquals": + return (value) => textCell(value) !== search; + case "startsWith": + return (value) => textCell(value).startsWith(search); + default: + return (value) => textCell(value).endsWith(search); + } +} + +/** + * The ONE home of filter-predicate semantics: resolves a validated runtime + * filter's column type + operator into a monomorphic `(value) => boolean` + * closure with operand normalization hoisted out of the row loop (between + * bounds destructured and min/maxed once, date operands collapsed to UTC + * day-ms once, text needles lowercased once, selection operands coerced into + * a Set once). Called once per filter at plan construction; filters reaching + * a plan have passed `validateFilter`, so the defensive `alwaysFalse` arms + * for malformed operands are unreachable there and exist only to preserve the + * legacy per-row semantics for direct callers. Predicates only ever read the + * CELL value — a throwing accessor throws at the value source + * (`#readColumnValue`), never here. + */ +export function compileFilterPredicate( + filter: { + readonly columnId: string; + readonly operator: string; + readonly value?: unknown; + }, + column: { readonly type: string }, +): (value: unknown) => boolean { + if (filter.operator === "isEmpty") return isEmptyValue; + if (filter.operator === "isNotEmpty") return (value) => !isEmptyValue(value); const operand = filter.value; switch (column.type) { - case "number": { - if (typeof value !== "number" || Number.isNaN(value)) return false; - if (filter.operator === "between") { - const range = operand as readonly unknown[]; - const a = range[0]; - const b = range[1]; - if (typeof a !== "number" || typeof b !== "number") return false; - return value >= Math.min(a, b) && value <= Math.max(a, b); - } - if (typeof operand !== "number" || Number.isNaN(operand)) return false; - if (filter.operator === "equals") return value === operand; - if (filter.operator === "notEquals") return value !== operand; - if (filter.operator === "gt") return value > operand; - if (filter.operator === "gte") return value >= operand; - if (filter.operator === "lt") return value < operand; - return value <= operand; - } - case "date": { - const cell = toDayMs(value); - if (Number.isNaN(cell)) return false; - if (filter.operator === "dateBetween") { - const range = operand as readonly unknown[]; - const a = toDayMs(range[0]); - const b = toDayMs(range[1]); - return ( - !Number.isNaN(a) && - !Number.isNaN(b) && - cell >= Math.min(a, b) && - cell <= Math.max(a, b) - ); - } - const other = toDayMs(operand); - if (Number.isNaN(other)) return false; - if (filter.operator === "on") return cell === other; - return filter.operator === "before" ? cell < other : cell > other; - } - case "enum": { - if ((operand as readonly unknown[]).length === 0) return true; - const included = (operand as readonly unknown[]) - .map(String) - .includes(String(value)); - return filter.operator === "isAnyOf" ? included : !included; - } - case "boolean": { - if ((operand as readonly unknown[]).length === 0) return true; - const included = (operand as readonly unknown[]) - .map(booleanValue) - .includes(booleanValue(value)); - return filter.operator === "isAnyOf" ? included : !included; - } - default: { - const cell = String(value ?? "").toLocaleLowerCase(); - const search = String(operand).toLocaleLowerCase(); - if (filter.operator === "contains") return cell.includes(search); - if (filter.operator === "notContains") return !cell.includes(search); - if (filter.operator === "equals") return cell === search; - if (filter.operator === "notEquals") return cell !== search; - if (filter.operator === "startsWith") return cell.startsWith(search); - return cell.endsWith(search); - } + case "number": + return compileNumberPredicate(filter.operator, operand); + case "date": + return compileDatePredicate(filter.operator, operand); + case "enum": + return compileSelectionPredicate(filter.operator, operand, String); + case "boolean": + return compileSelectionPredicate(filter.operator, operand, booleanValue); + default: + return compileTextPredicate(filter.operator, operand); } } @@ -1365,12 +1455,18 @@ class CompiledQueryPlan readonly #runtimeColumns: readonly RuntimeColumn[]; readonly #runtimeQuery: RuntimeQuery; readonly #byId: ReadonlyMap; + // Parallel to `#runtimeQuery.filters`: one compiled predicate per filter, + // built once at construction so no verdict ever re-normalizes operands or + // re-resolves columns per row. + readonly #compiledPredicates: readonly FilterPredicate[]; readonly #active: readonly RuntimeColumn[]; readonly #aggregateColumns: readonly RuntimeColumn[]; readonly #operation: "set-query" | "set-derivations"; readonly #filterAuthority: CompiledFilterAuthority; readonly #sortAuthority: CompiledSortAuthority; - readonly #evaluationCache = new WeakMap(); + // Not `readonly`: `adoptEvaluationCache` repoints it at a previous plan's + // map (by reference — no copy, no per-row work) on a filter-only change. + #evaluationCache = new WeakMap(); /* * The recompile cache compares against the PUBLIC query, not the runtime @@ -1433,6 +1529,9 @@ class CompiledQueryPlan this.#byId = new Map( this.#runtimeColumns.map((column) => [column.id, column]), ); + this.#compiledPredicates = this.#runtimeQuery.filters.map((filter) => + compileFilterPredicate(filter, this.#byId.get(filter.columnId)!), + ); const activeIds = new Set(); this.#runtimeQuery.filters.forEach((entry) => activeIds.add(entry.columnId), @@ -1472,28 +1571,14 @@ class CompiledQueryPlan const values = new Map(); for (const column of this.#active) { - try { - values.set(column.id, column.accessor(input.row as never)); - } catch (cause) { - throw new PretableRowModelError( - "accessor-failed", - `Column ${column.id} accessor failed.`, - { - operation: this.#operation, - rowId: input.rowId, - columnId: column.id, - cause, - }, - ); - } + values.set( + column.id, + this.#readColumnValue(column, input.row, input.rowId), + ); } - const filterPasses = this.#runtimeQuery.filters.every((filter) => - evaluateFilter( - filter, - this.#byId.get(filter.columnId)!, - values.get(filter.columnId), - ), + const filterPasses = this.#filterVerdict((columnId) => + values.get(columnId), ); const groupPath = Object.freeze( this.#runtimeQuery.rowGroups.map((entry) => @@ -1518,6 +1603,10 @@ class CompiledQueryPlan * builds the dependency, aggregate leaves, and the frozen metadata from a * per-column value source, then seeds the evaluation cache. `valueOf` must * cover every sorted and aggregated column of THIS plan. + * + * `filterPasses` reaches the cache entry and nothing else: the metadata it + * builds carries no verdict, because a verdict lives in the structure the + * row lands in, not on the row. */ #finalizeMetadata(input: { readonly rowId: TRowId; @@ -1540,26 +1629,23 @@ class CompiledQueryPlan sortKeys, }); const aggregateLeaves = Object.freeze( - this.#aggregateColumns.map((column) => { - const allLeaf = Object.freeze({ - id: input.rowId, - row: input.row, - value: input.valueOf(column.id), - dependency, - }); - return Object.freeze({ + this.#aggregateColumns.map((column) => + Object.freeze({ columnId: column.id, aggregate: column.aggregate, - allLeaf, - filteredLeaf: input.filterPasses ? allLeaf : undefined, - }); - }), + allLeaf: Object.freeze({ + id: input.rowId, + row: input.row, + value: input.valueOf(column.id), + dependency, + }), + }), + ), ) as unknown as readonly CompiledAggregateLeaf[]; const metadata = Object.freeze({ rowId: input.rowId, row: input.row, sourceOrder: input.sourceOrder, - filterPasses: input.filterPasses, groupPath: input.groupPath, aggregateLeaves, }) as CompiledRowMetadata, TRowId, TColumns>; @@ -1569,6 +1655,8 @@ class CompiledQueryPlan rowId: input.rowId, sourceOrder: input.sourceOrder, metadata, + filterPasses: input.filterPasses, + verdictPlan: this, sortKeys, }); } else { @@ -1577,11 +1665,99 @@ class CompiledQueryPlan existing.rowId = input.rowId; existing.sourceOrder = input.sourceOrder; existing.metadata = metadata; + existing.filterPasses = input.filterPasses; + existing.verdictPlan = this; existing.sortKeys = sortKeys; } return metadata; } + /* + * The one accessor-read site: every column value this plan reads for + * evaluation flows through here so the accessor-failed error shape cannot + * fork between `evaluate` and the verdict-only path. + */ + #readColumnValue( + column: RuntimeColumn, + row: object, + rowId: PretableRowId, + ): unknown { + try { + return column.accessor(row as never); + } catch (cause) { + throw new PretableRowModelError( + "accessor-failed", + `Column ${column.id} accessor failed.`, + { + operation: this.#operation, + rowId, + columnId: column.id, + cause, + }, + ); + } + } + + /* + * The one filter-predicate loop, parameterized over the value source the + * same way `#finalizeMetadata` is: `evaluate` supplies its collected value + * map, the verdict-only path supplies live accessor reads. Predicate + * semantics live in `compileFilterPredicate`, applied here through the + * construction-time `#compiledPredicates` array (parallel to + * `#runtimeQuery.filters`) — no `#byId` lookup and no operand + * re-normalization per row. + */ + #filterVerdict(valueOf: (columnId: string) => unknown): boolean { + const filters = this.#runtimeQuery.filters; + return this.#compiledPredicates.every((predicate, index) => + predicate(valueOf(filters[index].columnId)), + ); + } + + /** + * This plan's filter verdict for one row — accessor reads over the runtime + * filter columns only, no metadata construction, no cache writes. Error + * semantics match `evaluate`: a throwing accessor surfaces the same + * accessor-failed shape. + * + * A row this plan has already evaluated answers from the evaluation cache + * under `evaluate`'s own guard, so `evaluate` + this call costs ONE + * accessor pass, not two (the pinned per-row work budgets are exact). The + * memo is exactly as fresh as the metadata `evaluate` would hand back for + * the same input, and never answers for a DIFFERENT plan — old verdicts + * come from root membership, not from here. The `verdictPlan` arm is what + * makes that last clause true once a cache is SHARED: an adopted entry's + * memo belongs to the plan that wrote it, so this plan re-reads accessors + * rather than repeating a verdict its own filters never produced. + */ + static filterVerdict( + plan: unknown, + input: CompiledRowInput, TRowId>, + ): boolean { + if (!(plan instanceof CompiledQueryPlan)) { + throw new TypeError("Filter verdicts require a compiled query plan."); + } + const compiled = plan as CompiledQueryPlan; + const cached = compiled.#evaluationCache.get(input.row); + if ( + cached !== undefined && + cached.metadata !== undefined && + cached.filterPasses !== undefined && + cached.verdictPlan === compiled && + Object.is(cached.rowId, input.rowId) && + cached.sourceOrder === input.sourceOrder + ) { + return cached.filterPasses; + } + return compiled.#filterVerdict((columnId) => + compiled.#readColumnValue( + compiled.#byId.get(columnId)!, + input.row, + input.rowId, + ), + ); + } + /* * The single comparison loop behind `compareRecordRows`: per-ordering * `compareValues` over store-resolved keys, then the `sourceOrder` @@ -1696,9 +1872,10 @@ class CompiledQueryPlan /** * Fills `nextPlan`'s store for one row from `previousPlan`'s: values carry * by columnId where the sort columns overlap, accessors run only for - * newly-active sort columns. Precondition (caller-owned): - * `isSortOnlyChange(previousPlan, nextPlan)`, so carried values are the - * ones the next plan's accessors would produce. When instrumentation is + * newly-active sort columns. Precondition (caller-owned): the plan change + * preserves every carried sort column's accessor semantics, so carried + * values are the ones the next plan's accessors would produce — both + * `isSortOnlyChange` and `isFilterOnlyChange` qualify. When instrumentation is * supplied, one counter is bumped per (row, sort column) entry — carry vs * accessor — and an already-filled row counts nothing. */ @@ -1762,11 +1939,78 @@ class CompiledQueryPlan rowId: input.rowId, sourceOrder: input.sourceOrder, metadata: undefined, + filterPasses: undefined, + verdictPlan: undefined, sortKeys, }); return sortKeys; } + /** + * Points `nextPlan`'s evaluation cache at `previousPlan`'s — one reference + * assignment for the whole store, no copy and no per-row work. Replaces the + * per-row `fillSortKeysFromPrevious` walk on the filter fast path. + * + * Precondition (CALLER-OWNED, exactly like `fillSortKeysFromPrevious`): + * `isFilterOnlyChange(previousPlan, nextPlan)`. Only the plan-shape check + * is enforced here; passing a plan pair the classifier would reject + * silently corrupts `nextPlan`'s reads, so callers assert first. + * + * Why every cached field survives, field by field — this is the safety + * proof, and a filter-only delta is what each line spends: + * + * - `rowId` / `sourceOrder`: guard fields, not derived state. They record + * the input the entry was written for, and both `evaluate` and + * `filterVerdict` re-check them against the live input, so a drift + * demotes to a miss under either plan. + * - `row`: the WeakMap KEY. Adoption cannot change which row an entry + * describes. + * - `metadata.rowId` / `.row` / `.sourceOrder`: copies of the guarded + * input, so they are correct under any plan that hits the guard. + * - `metadata.groupPath`: one entry per `rowGroups` ordering, valued by + * that column's accessor. `isFilterOnlyChange` requires + * `!groupsChanged` (identical orderings) and `!derivationsChanged`, + * which compares the accessor IDENTITY of every grouped column in BOTH + * plans' queries. Same orderings + same accessors + same row object ⇒ + * the same path. (In practice the fast path also refuses grouped + * queries outright.) + * - `metadata.aggregateLeaves`: one entry per column with an `aggregate`, + * carrying the aggregate spec, the row, the accessor value, and a + * `dependency`. `derivationsEqualForPlan` compares column id, type and + * ORDER positionally, requires `semanticValueEqual` on every + * `aggregate`, and forces accessor identity for every aggregated + * column — so the leaf set, its order, its specs and its values are all + * identical. + * - the leaves' `dependency` (`{ sourceOrder, sortKeys }`): guarded + * `sourceOrder` plus the keys below. + * - `sortKeys` (on the entry and inside the dependency): one value per + * `sort` ordering. `isFilterOnlyChange` requires `!sortChanged`, and + * `derivationsEqualForPlan` pins both the accessor and the comparator of + * every sorted column. Identical orderings over identical accessors ⇒ + * value-identical keys, which is precisely why the per-row fill this + * replaces reported 100% carries and zero evaluations. + * - `filterPasses`: the ONE filter-dependent field, and the reason + * `verdictPlan` exists. The memo is only read when `verdictPlan` is the + * reading plan, so an adopted entry's verdict is invisible to the + * adopter and it runs its own filters instead. Nothing stale leaks; the + * adopter pays exactly the accessor pass it paid before this change. + * + * Sharing is symmetric-safe: the previous plan keeps reading the same map, + * and anything the next plan writes into it is either value-identical + * under the argument above or tagged with the writer (`verdictPlan`). + */ + static adoptEvaluationCache(nextPlan: unknown, previousPlan: unknown): void { + if ( + !(nextPlan instanceof CompiledQueryPlan) || + !(previousPlan instanceof CompiledQueryPlan) + ) { + throw new TypeError( + "Evaluation-cache adoption requires compiled query plans.", + ); + } + nextPlan.#evaluationCache = previousPlan.#evaluationCache; + } + compareGroupKeys( level: number, left: CompiledGroupKey, @@ -1892,6 +2136,25 @@ export function isSortOnlyChange( ); } +/** + * True only when the applied filters are the sole difference between the + * plans. + */ +export function isFilterOnlyChange( + previous: CompiledQuery, + next: CompiledQuery, +): boolean { + const delta = classifyQueryDelta(previous, next); + return ( + delta !== undefined && + delta.filtersChanged && + !delta.derivationsChanged && + !delta.groupsChanged && + !delta.sortChanged && + !delta.authorityChanged + ); +} + /** * Orders two evaluated row records under `plan` via the plan's own sort-key * store. Both rows must already be in the store (`evaluate` or @@ -1947,8 +2210,10 @@ export function sortKeysOf( /** * Fills `nextPlan`'s sort-key store for one row, carrying values from * `previousPlan`'s store where the sort columns overlap and running accessors - * only for newly-active sort columns. Idempotent per row. Valid ONLY when - * `isSortOnlyChange(previousPlan, nextPlan)` — the caller owns that check. + * only for newly-active sort columns. Idempotent per row. Valid ONLY under a + * plan change that preserves every carried sort column's accessor semantics + * (`isSortOnlyChange` and `isFilterOnlyChange` both qualify) — the caller + * owns that check. */ export function fillSortKeysFromPrevious< TColumns, @@ -1967,6 +2232,36 @@ export function fillSortKeysFromPrevious< ); } +/** + * Points `nextPlan` at `previousPlan`'s whole evaluation cache — sort keys + * AND metadata — by reference. One assignment replaces a per-row fill, which + * is why the filter fast path uses it instead of walking every row. + * + * Valid ONLY when `isFilterOnlyChange(previousPlan, nextPlan)` holds; the + * caller owns that check. The field-by-field argument for why every cached + * field survives such a change lives on + * `CompiledQueryPlan.adoptEvaluationCache`. + */ +export function adoptEvaluationCache( + nextPlan: CompiledQuery, + previousPlan: CompiledQuery, +): void { + CompiledQueryPlan.adoptEvaluationCache(nextPlan, previousPlan); +} + +/** + * Computes `plan`'s filter verdict for one row: each runtime filter's column + * accessor runs and its predicate is evaluated, with the same semantics and + * accessor-failed error shape as `evaluate` — the predicate loop is shared, + * not duplicated. No metadata is built and no cache entry is written. + */ +export function filterVerdict( + plan: CompiledQuery, + input: CompiledRowInput, TRowId>, +): boolean { + return CompiledQueryPlan.filterVerdict(plan, input); +} + export function compileQuery( input: CompileQueryInput, ): CompiledQuery { diff --git a/packages/row-model/src/cooperative-transition.ts b/packages/row-model/src/cooperative-transition.ts index 800b1e48b..1133726bc 100644 --- a/packages/row-model/src/cooperative-transition.ts +++ b/packages/row-model/src/cooperative-transition.ts @@ -1,4 +1,4 @@ -import type { CompiledQuery } from "./compiled-query"; +import { filterVerdict, type CompiledQuery } from "./compiled-query"; import type { PretableRowId } from "./column-types"; import type { LocalRowModelInstrumentation } from "./diagnostics"; import { @@ -22,9 +22,11 @@ import { } from "./persistent/persistent-map"; import type { TransientMap } from "./persistent/transient"; import { instrumentOrderStatisticTree } from "./persistent/order-statistic-tree"; +import { slotVectorFromEntries } from "./slot-vector"; import type { PretableGroupId } from "./types"; import { orderedRowEntry } from "./ordered-row-entry"; -import { createFlatVisibleTree } from "./visible-index"; +import { EMPTY_MEMBERSHIP } from "./membership-bitset"; +import { createFlatVisibleTree, membershipFromFlatTree } from "./visible-index"; export interface CooperativeTransitionScheduler { /** Queues one continuation and returns an idempotent cancellation hook. */ @@ -368,6 +370,21 @@ export function createCooperativeTransitionCandidate< sourceOrder: RevisionRoot["sourceOrder"]; expansion: RevisionRoot["expansion"]; flatRows: VisibleIndexRoot["rows"]; + /** + * Slot-indexed records, as a PLAIN MUTABLE array: nothing here is + * reachable outside the candidate until `finish` chunks it into the + * published root's immutable vector, so per-step writes are O(1) + * instead of a COW chunk copy per slice. + */ + recordsBySlot: Array | undefined>; + /** + * The slot-space size for the root `finish` will publish. Seeded from + * the CAPTURED root's self-described capacity and widened only by + * replayed delta targets' capacities — never read from the live + * allocator, so growth after capture cannot leak into this build's + * domain. + */ + slotCapacity: number; groups: GroupIndexRoot | undefined; groupBuilder: GroupIndexBuildDraft | undefined; groupSealRemaining: number; @@ -406,6 +423,8 @@ export function createCooperativeTransitionCandidate< createFlatVisibleTree(options.queryPlan), instrumentation, ), + recordsBySlot: [], + slotCapacity: options.captured.slotCapacity, groups: grouped && !useBulkGroupBuilder ? createGroupIndex( @@ -535,6 +554,7 @@ export function createCooperativeTransitionCandidate< const state = retained; if (state === undefined) return; state.rows = state.rows.delete(record.rowId); + state.recordsBySlot[record.slot] = undefined; if (state.groups === undefined) { state.flatRows = state.flatRows.remove(record.rowId); } else { @@ -560,15 +580,19 @@ export function createCooperativeTransitionCandidate< rowId: source.rowId, row: source.row as never, sourceOrder: source.sourceOrder, + slot: source.slot, }) as unknown as RowRecord["metadata"]; const record = Object.freeze({ ...source, metadata }); if (state.rowBuilder !== undefined) state.rowBuilder.set(record.rowId, record); else state.rows = state.rows.set(record.rowId, record); + state.recordsBySlot[record.slot] = record; if (state.groupBuilder !== undefined) { state.groupBuilder.insert(record); } else if (state.groups === undefined) { - if (metadata.filterPasses) { + // Computed here, used here: the flat tree this inserts into is where + // the verdict is recorded. + if (filterVerdict(state.queryPlan, record as never)) { state.flatRows = state.flatRows.insertOrReplace( orderedRowEntry(state.queryPlan, record), ); @@ -613,6 +637,13 @@ export function createCooperativeTransitionCandidate< if (state === undefined) return; resetOverrideReconciliation(state); state.deltas.push(delta); + // Capacity is monotone across commits, so the widest replayed target + // bounds every slot this candidate can ever bind (still a captured + // root's value — the live allocator is never consulted). + state.slotCapacity = Math.max( + state.slotCapacity, + delta.target.slotCapacity, + ); totalRows += delta.affectedRowIds.length * 2 + 1; }, step() { @@ -703,11 +734,27 @@ export function createCooperativeTransitionCandidate< } else { visible = attachGroupIndex(state.flatRows, state.groups); } + const slotEntries: Array< + readonly [number, RowRecord] + > = []; + for (let slot = 0; slot < state.recordsBySlot.length; slot += 1) { + const record = state.recordsBySlot[slot]; + if (record !== undefined) slotEntries.push([slot, record]); + } return Object.freeze({ revision, parentRevision: revision - 1, rows: state.rows, sourceOrder: state.sourceOrder, + recordsBySlot: slotVectorFromEntries(slotEntries, state.slotCapacity), + slotCapacity: state.slotCapacity, + // Flat transitions built their membership into `flatRows`; index it + // over the state's self-described capacity. Grouped transitions keep + // answering from the group index — sentinel. + visibleSlots: + state.groups === undefined + ? membershipFromFlatTree(state.flatRows, state.slotCapacity) + : EMPTY_MEMBERSHIP, visible, queryPlan: state.queryPlan, expansion: state.expansion, @@ -725,6 +772,7 @@ export function createCooperativeTransitionCandidate< state.groupSealRemaining = 0; state.deltas.fill(null); state.deltas.length = 0; + state.recordsBySlot.length = 0; state.overrideReconciliation = undefined; state.reconciledExpansion = undefined; retained = undefined; diff --git a/packages/row-model/src/create-local-row-model.ts b/packages/row-model/src/create-local-row-model.ts index 329d43ef0..c5ce11b9d 100644 --- a/packages/row-model/src/create-local-row-model.ts +++ b/packages/row-model/src/create-local-row-model.ts @@ -1,11 +1,13 @@ import { compileQuery, fillSortKeysFromPrevious, + isFilterOnlyChange, isSortOnlyChange, type CompiledFilterAuthority, type CompiledQuery, type CompiledSortAuthority, } from "./compiled-query"; +import { rebuildRootForFilterOnlyChange } from "./filter-rebuild"; import { rebuildRootForSortOnlyChange } from "./sort-rebuild"; import { createCooperativeTransitionCandidate, @@ -50,6 +52,7 @@ import { } from "./group-index"; import { createPersistentMap } from "./persistent/persistent-map"; import { buildRowStore } from "./row-store"; +import { createSlotAllocator, type SlotAllocator } from "./slot-allocator"; import type { PretableRowIntegrityDiagnostic, PretableRowIntegrityDiagnosticSink, @@ -69,7 +72,12 @@ import type { PretableVisibleRow, PretableVisibleRowRef, } from "./types"; -import { createFlatSnapshot, createVisibleIndex } from "./visible-index"; +import { EMPTY_MEMBERSHIP } from "./membership-bitset"; +import { + createFlatSnapshot, + createVisibleIndex, + membershipFromFlatTree, +} from "./visible-index"; import { applyFlatTransactionDraft, replaceFlatRowsDraft, @@ -97,6 +105,16 @@ function createSnapshot< instrumentation.work.snapshotOutputRowsRead += rows.length; return rows; }, + // The dense range read is a visible-row output walk like `range`; the + // k-sized reads (`ɵslotOfRowId`, `ɵslotCapacity`) pass through the + // spread above uncounted. + ɵvisibleSlotRange: (start: number, end: number) => { + const slots = snapshot.ɵvisibleSlotRange?.(start, end); + if (slots !== undefined) { + instrumentation.work.snapshotOutputRowsRead += slots.length; + } + return slots; + }, dataRowAt: (index: number) => count(snapshot.dataRowAt(index)), firstDataRow: () => count(snapshot.firstDataRow()), lastDataRow: () => count(snapshot.lastDataRow()), @@ -184,6 +202,13 @@ const modelSortAuthoritySetters = new WeakMap< object, (authority: CompiledSortAuthority) => void >(); +const modelSlotInternals = new WeakMap< + object, + () => { + readonly root: RevisionRoot; + readonly slots: SlotAllocator; + } +>(); /** * Re-declares who selected the loaded records, recompiling the plan when the @@ -251,6 +276,23 @@ export function getLocalRowModelActiveTransitionCandidateForTesting( return read(); } +/** + * Committed-root and slot-allocator seam for the slot-lifecycle tests; + * intentionally absent from the package barrel. + * + * @internal test-only + */ +export function getLocalRowModelSlotInternalsForTesting(model: object): { + readonly root: RevisionRoot; + readonly slots: SlotAllocator; +} { + const read = modelSlotInternals.get(model); + if (read === undefined) { + throw new TypeError("Diagnostics require a local Pretable row model."); + } + return read(); +} + export function getLocalRowModelChangeJournalDiagnosticsForTesting( model: object, ): ChangeJournalDiagnostics { @@ -595,24 +637,41 @@ export function createLocalRowModel< const aggregateFilteredRows = options.aggregateFilteredRows ?? false; const diagnosticSink = options.onDiagnostic as PretableRowIntegrityDiagnosticSink | undefined; + /** + * Per-model row-slot allocator — mutable instance state, like + * `nextSourceOrder`. Every record-creating path below threads it so a row's + * slot is assigned exactly once, at ingest, and released exactly once, on + * permanent removal. + */ + const slots = createSlotAllocator(); const initialStore = buildRowStore({ rows: options.rows, getRowId, queryPlan, + slots, instrumentation, }); const initialExpansion = createExpansionRoot(options.initialExpansion); + const initialVisible = createVisibleIndex( + initialStore.records, + queryPlan, + aggregateFilteredRows, + initialExpansion.overrides, + ); let root: RevisionRoot = Object.freeze({ revision: 0, parentRevision: null, rows: initialStore.rows, sourceOrder: initialStore.sourceOrder, - visible: createVisibleIndex( - initialStore.records, - queryPlan, - aggregateFilteredRows, - initialExpansion.overrides, - ), + recordsBySlot: initialStore.recordsBySlot, + slotCapacity: slots.capacity, + // Flat roots index their membership per slot; grouped roots carry the + // sentinel and keep answering from the group index. + visibleSlots: + queryPlan.query.rowGroups.length > 0 + ? EMPTY_MEMBERSHIP + : membershipFromFlatTree(initialVisible.rows, slots.capacity), + visible: initialVisible, queryPlan, expansion: initialExpansion, cause: Object.freeze({ kind: "initial" as const }), @@ -699,18 +758,22 @@ export function createLocalRowModel< committedRoot: RevisionRoot, previousRevision: number, revision: number, - // Only the sort-only fast path may pass "reorder": it is the one commit - // that provably changes order and nothing else. Every other publisher - // keeps the plain barrier default. - barrierReason: "bulk-replace" | "reorder" = "bulk-replace", + // Only the sort-only fast path may pass "reorder" (the one commit that + // provably changes order and nothing else) and only the filter-only + // fast path may pass "refilter" (membership changed, surviving order + // and identities intact). Every other publisher keeps the plain + // barrier default. + barrierReason: "bulk-replace" | "reorder" | "refilter" = "bulk-replace", ): void => { queryPlan = committedRoot.queryPlan; query = committedRoot.queryPlan.query; derivations = committedRoot.queryPlan.derivations; commit(committedRoot, READY); - // On a sort-only change this is structurally a no-op (distinct-value - // cache keys hash filter/column/population semantics, never sort); kept - // so both paths publish through one identical recipe. + // Every publisher goes through this one recipe. On a sort-only change + // the distinct publish is structurally a no-op (distinct-value cache + // keys hash filter/column/population semantics, never sort); on a + // filter-only change it is LOAD-BEARING — filters are part of those + // cache keys, so the new root must reach the manager here. distinctValues.publishTransitionRoot(committedRoot); changeJournal.appendBarrier(previousRevision, revision, barrierReason); }; @@ -981,6 +1044,7 @@ export function createLocalRowModel< getRowId, queryPlan: nextPlan, nextSourceOrder, + slots, instrumentation, }); const pendingDiagnostics = drafted.diagnostics; @@ -1014,6 +1078,7 @@ export function createLocalRowModel< queryPlan: nextPlan, nextSourceOrder, acceptSameReferenceMutation: true, + slots, instrumentation, }); if (drafted.effective) { @@ -1027,18 +1092,35 @@ export function createLocalRowModel< * rebuild is journal-invisible. */ const records: RowRecord[] = []; - for (const entry of drafted.sourceOrder.entries()) { + // `range(0, size)`, not `entries()` — a full walk into an + // array (see `iterateEntries`). + for (const entry of drafted.sourceOrder.range( + 0, + drafted.sourceOrder.size, + )) { const record = drafted.rows.get(entry.rowId); if (record !== undefined) records.push(record); } + const rebuiltVisible = createVisibleIndex( + records, + nextPlan, + aggregateFilteredRows, + previousRoot.expansion.overrides, + ); drafted = { ...drafted, - visible: createVisibleIndex( - records, - nextPlan, - aggregateFilteredRows, - previousRoot.expansion.overrides, - ), + visible: rebuiltVisible, + // The rebuilt index resolves the same membership (same rows, + // same query, fresh plan), but derive the bitset from the + // tree that ships rather than carrying the draft's — the two + // must never be allowed to drift. + visibleSlots: + nextPlan.query.rowGroups.length > 0 + ? EMPTY_MEMBERSHIP + : membershipFromFlatTree( + rebuiltVisible.rows, + slots.capacity, + ), }; } } @@ -1062,6 +1144,12 @@ export function createLocalRowModel< parentRevision: previousRevision, rows: drafted.rows, sourceOrder: drafted.sourceOrder, + recordsBySlot: drafted.recordsBySlot, + // Commit-time capacity: nothing between the draft and this + // commit allocates, so this is the capacity the vector above + // was built for. + slotCapacity: slots.capacity, + visibleSlots: drafted.visibleSlots, visible: drafted.visible, queryPlan: nextPlan, expansion: previousRoot.expansion, @@ -1109,6 +1197,7 @@ export function createLocalRowModel< getRowId, queryPlan, nextSourceOrder, + slots, instrumentation, }); const result = mutationResult( @@ -1126,6 +1215,10 @@ export function createLocalRowModel< parentRevision: previousRoot.revision, rows: drafted.rows, sourceOrder: drafted.sourceOrder, + recordsBySlot: drafted.recordsBySlot, + // Commit-time capacity (see the setRows commit above). + slotCapacity: slots.capacity, + visibleSlots: drafted.visibleSlots, visible: drafted.visible, queryPlan, expansion: previousRoot.expansion, @@ -1181,16 +1274,36 @@ export function createLocalRowModel< notify: superseded, }; } - if ( - isSortOnlyChange(queryPlan, nextPlan) && - nextPlan.query.rowGroups.length === 0 - ) { + // The two synchronous fast paths share one commit and error shape; + // they differ only in the rebuild and the barrier reason, and each + // reason is that path's exclusive promise. "reorder" is the sort + // path's — the one commit that provably changes order and nothing + // else. "refilter" is the filter path's — membership changed while + // surviving rows kept their relative order and identities. Neither + // may cross over: a "reorder" on a membership change would tell + // renderers to permute retained rows over a different row set and + // corrupt their layout, and vice versa. + const fastPath = + nextPlan.query.rowGroups.length > 0 + ? undefined + : isSortOnlyChange(queryPlan, nextPlan) + ? Object.freeze({ + rebuild: rebuildRootForSortOnlyChange, + barrierReason: "reorder" as const, + }) + : isFilterOnlyChange(queryPlan, nextPlan) + ? Object.freeze({ + rebuild: rebuildRootForFilterOnlyChange, + barrierReason: "refilter" as const, + }) + : undefined; + if (fastPath !== undefined) { cancelActiveTransition("superseded"); const previousRevision = root.revision; const revision = previousRevision + 1; let committedRoot: RevisionRoot; try { - committedRoot = rebuildRootForSortOnlyChange({ + committedRoot = fastPath.rebuild({ captured: root, nextPlan, revision, @@ -1228,7 +1341,7 @@ export function createLocalRowModel< committedRoot, previousRevision, revision, - "reorder", + fastPath.barrierReason, ); return { transition: Object.freeze({ @@ -1457,6 +1570,7 @@ export function createLocalRowModel< modelChangeJournals.set(model, changeJournal as ChangeJournal); modelRevisionCauses.set(model, () => root.cause); modelActiveTransitionCandidates.set(model, () => activeTransition?.candidate); + modelSlotInternals.set(model, () => ({ root: root as never, slots })); /* * Re-running `setQuery` with the query the model already holds is what makes * the flip take effect: the query is unchanged, so nothing reported moves, diff --git a/packages/row-model/src/diagnostics.ts b/packages/row-model/src/diagnostics.ts index c35f7cbf4..cbac32dfa 100644 --- a/packages/row-model/src/diagnostics.ts +++ b/packages/row-model/src/diagnostics.ts @@ -26,6 +26,41 @@ export interface LocalRowModelWorkDiagnostics { readonly synchronousRebuilds: number; /** Total wall time inside synchronous sort-only rebuilds. */ readonly synchronousRebuildMs: number; + /** Filter-only rebuilds taken synchronously, bypassing the cooperative path. */ + readonly filterRebuilds: number; + /** Rows whose filter verdict flipped (either direction) across those rebuilds. */ + readonly filterRowsFlipped: number; + /** Flipped-in rows merged into the surviving visible order — the ONLY rows sorted. */ + readonly filterMergeSortedInsertions: number; + /** + * Total wall time inside synchronous filter-only rebuilds. Its own field — + * not folded into `synchronousRebuildMs` — so sort and filter fast paths + * stay separately attributable in bench traces. + */ + readonly filterRebuildMs: number; + /** + * Bulk tree builds that derived `byId` from a base map (k edits) instead of + * refilling it from the built entries (n inserts). + */ + readonly bulkByIdDerived: number; + /** + * Bulk tree builds that skipped the n−1 strict-order verification on a + * caller-supplied proof. Every other build still pays for it. + */ + readonly bulkOrderVerificationsSkipped: number; + /** + * Plan changes that adopted the previous plan's evaluation cache wholesale + * (by reference, zero per-row work) instead of refilling it. Only a + * filter-only change qualifies, so this counts filter fast paths that took + * the cheap route — one per rebuild, never per row. + */ + readonly evaluationCacheAdoptions: number; + /** + * `recordsBySlot` chunks copied or allocated across transaction and + * set-rows commits — the COW maintenance cost of the slot vector, ~k/1024 + * plus table copies per commit rather than per-row. + */ + readonly slotChunksTouched: number; /** Sort-key entries carried from a previous plan's store, per (row, column). */ readonly sortKeyCarries: number; /** Sort-key entries produced by running an accessor, per (row, column). */ @@ -97,6 +132,14 @@ function newInstrumentation(): LocalRowModelInstrumentation { transitionRows: 0, synchronousRebuilds: 0, synchronousRebuildMs: 0, + filterRebuilds: 0, + filterRowsFlipped: 0, + filterMergeSortedInsertions: 0, + filterRebuildMs: 0, + bulkByIdDerived: 0, + bulkOrderVerificationsSkipped: 0, + evaluationCacheAdoptions: 0, + slotChunksTouched: 0, sortKeyCarries: 0, sortKeyEvaluations: 0, snapshotOutputRowsRead: 0, @@ -120,6 +163,14 @@ function resetWork(instrumentation: LocalRowModelInstrumentation): void { "transitionRows", "synchronousRebuilds", "synchronousRebuildMs", + "filterRebuilds", + "filterRowsFlipped", + "filterMergeSortedInsertions", + "filterRebuildMs", + "bulkByIdDerived", + "bulkOrderVerificationsSkipped", + "evaluationCacheAdoptions", + "slotChunksTouched", "sortKeyCarries", "sortKeyEvaluations", "snapshotOutputRowsRead", diff --git a/packages/row-model/src/distinct-values.ts b/packages/row-model/src/distinct-values.ts index 9b56c538a..33c8ba9a5 100644 --- a/packages/row-model/src/distinct-values.ts +++ b/packages/row-model/src/distinct-values.ts @@ -8,6 +8,7 @@ import { PretableRowModelError, type PretableRowModelOperation, } from "./errors"; +import { rowPassesFilter } from "./filter-membership"; import type { RevisionRoot, RowRecord } from "./internal-types"; import { createOrderStatisticTree, @@ -494,10 +495,12 @@ function readValue( column: RuntimeColumn, options: CapturedQueryOptions, operation: PretableRowModelOperation, + /** The row's verdict under the root being replayed; see the caller. */ + filterPasses: boolean, ): | { readonly description: ValueDescription; readonly value: unknown } | undefined { - if (options.population === "filtered" && !record.metadata.filterPasses) { + if (options.population === "filtered" && !filterPasses) { return undefined; } let value: unknown; @@ -593,7 +596,16 @@ function replayRecord< } const record = target.rows.get(rowId); if (record === undefined) return next; - const selected = readValue(record, column, options, operation); + // The "filtered" population is exactly the target root's visible + // membership, so the root the replay already holds answers it — no stored + // per-row verdict, and no accessor re-run for the "all" population. + const selected = readValue( + record, + column, + options, + operation, + options.population === "filtered" ? rowPassesFilter(target, rowId) : true, + ); if (selected === undefined) return next; try { next = insertValue(next, rowId, selected); diff --git a/packages/row-model/src/filter-membership.ts b/packages/row-model/src/filter-membership.ts new file mode 100644 index 000000000..314fed364 --- /dev/null +++ b/packages/row-model/src/filter-membership.ts @@ -0,0 +1,50 @@ +/** + * Filter verdicts are not stored. A committed root already carries the answer + * structurally: a row passes this root's filters exactly when it is a MEMBER + * of the root's visible structure. This module is the one place that knows + * which structure answers for which root shape. + * + * - **Flat root** (`query.rowGroups.length === 0`): `root.visible.rows` holds + * one entry per passing row and nothing else, so membership is a lookup. + * This is the same predicate `nearestVisibleRef` has always used. + * - **Grouped root**: `root.visible.rows` is deliberately EMPTY (a grouped + * visible index attaches the group index to an empty flat tree), so the + * answer lives in the group index: every inserted row — passing or not — + * gets a `rowParents` entry, and only passing rows are inserted into their + * leaf group's `leaves` tree. So `rowParents` locates the leaf group and + * `leaves` holds the verdict. + * + * Absence is the answer, not a fault: a row this root never saw, or one its + * filters reject, is simply not a member. Nothing here fails loud. + * + * These read a COMMITTED root, so they answer "did this row pass under the + * plan that built this root" — the OLD verdict at every site that compares + * against a new one. A NEW verdict is computed, never resolved: producers + * call `filterVerdict(plan, record)` and use it locally to decide where the + * row goes. + */ + +import type { PretableRowId } from "./column-types"; +import { getGroupIndex, rowPassesFilterInGroupIndex } from "./group-index"; +import type { RevisionRoot } from "./internal-types"; + +/** + * The grouped half of the seam, re-exported for callers that hold a group + * index rather than a root (a transaction rebuilds the index, so those + * callers must read the PREVIOUS one explicitly). Its body lives in + * `./group-index` because it reads that module's own invariant, and because + * importing it back from here would cycle. + */ +export { rowPassesFilterInGroupIndex }; + +/** Did `rowId` pass the filters of the plan that built `root`? */ +export function rowPassesFilter< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +>(root: RevisionRoot, rowId: TRowId): boolean { + const grouped = getGroupIndex(root.visible); + return grouped === undefined + ? root.visible.rows.get(rowId) !== undefined + : rowPassesFilterInGroupIndex(grouped, rowId); +} diff --git a/packages/row-model/src/filter-rebuild.ts b/packages/row-model/src/filter-rebuild.ts new file mode 100644 index 000000000..2f0f12250 --- /dev/null +++ b/packages/row-model/src/filter-rebuild.ts @@ -0,0 +1,227 @@ +/** + * Synchronous subset rebuild for a filter-only plan change on an ungrouped + * query. Runs to completion on the caller's stack, like `sort-rebuild` — a + * filter-only change is a membership change over an already-sorted set: + * values, sort keys, group paths, and relative order are all unchanged. No + * record is reconstructed — not even a flipped one, because no record holds a + * verdict any more — so the rows HAMT carries by identity exactly as + * sort-rebuild's does, and the only new structure is the visible tree: a + * linear merge of the surviving old order with the sorted flipped-in subset, + * with no comparator sort of the full set, ever. + */ + +import type { PretableRowId } from "./column-types"; +import { + adoptEvaluationCache, + compareWithSortKeys, + filterVerdict, + isFilterOnlyChange, + sortKeysOf, + type CompiledQuery, + type CompiledSortKey, +} from "./compiled-query"; +import type { LocalRowModelInstrumentation } from "./diagnostics"; +import type { OrderedRowEntry, RevisionRoot } from "./internal-types"; +import { + createMembership, + setMembershipBit, + testMembershipBit, +} from "./membership-bitset"; +import { + compareOrderStatisticTreeIds, + createOrderStatisticTreeFromSortedEntries, + instrumentOrderStatisticTree, +} from "./persistent/order-statistic-tree"; +import { forEachSlotEntry } from "./slot-vector"; +import { createFlatVisibleTree } from "./visible-index"; + +export function rebuildRootForFilterOnlyChange< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +>(options: { + readonly captured: RevisionRoot; + readonly nextPlan: CompiledQuery; + readonly revision: number; + readonly now: () => number; + readonly instrumentation?: LocalRowModelInstrumentation; +}): RevisionRoot { + const { captured, nextPlan, revision, now, instrumentation } = options; + if (!isFilterOnlyChange(captured.queryPlan, nextPlan)) { + throw new TypeError( + "Synchronous filter rebuild requires a filter-only plan change.", + ); + } + if (nextPlan.query.rowGroups.length > 0) { + throw new TypeError( + "Synchronous filter rebuild requires an ungrouped query.", + ); + } + const startedAt = now(); + // The next plan ADOPTS the captured plan's evaluation cache by reference: + // one assignment for the whole store instead of a per-row refill. A + // filter-only change leaves every cached field valid (the field-by-field + // argument lives on the seam), and the one filter-dependent field — the + // verdict memo — is tagged with the plan that wrote it, so the loop below + // still runs the NEW filters over every row. + adoptEvaluationCache(nextPlan, captured.queryPlan); + // One hole-skipping pass over ALL records via the slot vector: run the new + // plan's verdict, record it as a bit in the next root's membership bitset, + // and diff it against the captured root's bit. Every record carries by + // identity — flipped or not — so the pass collects nothing but the two flip + // sets, and only a flipped-IN row needs its keys resolved (survivors keep + // their existing entry objects, leavers need nothing). The bitset is sized + // by `captured.slotCapacity` — roots are self-describing, and reading the + // live allocator instead would let later growth leak into this snapshot's + // domain. This path throws on grouped plans above, so + // `captured.visibleSlots` is always the REAL flat bitset, never the + // grouped sentinel. + const nextVisibleSlots = createMembership(captured.slotCapacity); + const flippedIn: OrderedRowEntry[] = []; + const flippedOut = new Set(); + // Slot order, not source order — sound because nothing downstream reads + // this walk's order: flippedIn is comparator-sorted below, flippedOut is a + // set, and the merge consumes the OLD TREE's walk. recordsBySlot replaces + // the rows-HAMT get; visibleSlots replaces the old-verdict membership get. + forEachSlotEntry(captured.recordsBySlot, (previous) => { + const passes = filterVerdict(nextPlan, previous as never); + if (passes) setMembershipBit(nextVisibleSlots, previous.slot); + // The OLD verdict is the captured root's membership bit — the flip diff + // is a set difference between two structures, not a comparison of two + // stored flags. And since no record stores a verdict, a FLIPPED row needs + // no new record either: it carries by identity exactly like an unflipped + // one, and the flip is expressed entirely by where it sits in the new + // visible tree. + if (passes === testMembershipBit(captured.visibleSlots, previous.slot)) { + return; + } + if (passes) { + // Resolved from the adopted store — the same array the captured plan + // handed out, since a filter-only change leaves the keys untouched. + const keys = sortKeysOf( + nextPlan, + previous as never, + ) as readonly CompiledSortKey[]; + flippedIn.push(Object.freeze({ record: previous, keys })); + } else { + flippedOut.add(previous.rowId); + } + }); + + const flipped = flippedIn.length + flippedOut.size; + // Identity, unconditionally: a filter-only change reconstructs NO record, + // so the rows HAMT is carried whole and the transient is never opened. + const rows = captured.rows; + let visible = captured.visible; + // Zero flips carry the captured bitset by identity — same member set — + // and the freshly-computed (bit-identical) `nextVisibleSlots` is dropped. + let visibleSlots = captured.visibleSlots; + if (flipped === 0) { + // Zero flips (decided here, pinned by tests): still a NEW root at the + // requested revision under the next plan, with the rows map AND the + // visible tree object carried wholesale. Reusing the tree is sound even + // though its entries' keys resolved under the OLD plan and its comparator + // closure captured it: a filter-only change keeps sort columns and + // comparators identical, and the next plan now READS THE SAME STORE, so + // future inserts decorate entries with the very arrays the carried ones + // hold — ordering stays coherent. + } else { + // Both sequences below are strictly sorted by the same composite order + // the tree maintains (comparator, then id): the old tree's in-order walk + // by construction, the flipped-in subset by this k log k sort — the only + // sort in the rebuild, and it never sees an unflipped row. + const compareEntries = ( + left: OrderedRowEntry, + right: OrderedRowEntry, + ) => + compareWithSortKeys( + nextPlan, + left.record as never, + left.keys, + right.record as never, + right.keys, + ) || compareOrderStatisticTreeIds(left.record.rowId, right.record.rowId); + flippedIn.sort(compareEntries); + // Single linear merge: surviving entries keep their ENTRY objects (a + // still-passing row is by definition unflipped — record and keys are both + // unchanged), flipped-out rows are skipped, flipped-in entries interleave + // where the composite order puts them. + const merged: OrderedRowEntry[] = []; + let next = 0; + // `range(0, size)` rather than `entries()`: this walk always runs to + // completion, and the tree's non-generator walk is the cheaper way to + // get one — ~1ms against ~30ms at 50,000 rows (see `iterateEntries`). + for (const entry of captured.visible.rows.range( + 0, + captured.visible.rows.size, + )) { + if (flippedOut.has(entry.record.rowId)) continue; + while ( + next < flippedIn.length && + compareEntries(flippedIn[next], entry) < 0 + ) { + merged.push(flippedIn[next]); + next += 1; + } + merged.push(entry); + } + while (next < flippedIn.length) { + merged.push(flippedIn[next]); + next += 1; + } + visible = Object.freeze({ + rows: createOrderStatisticTreeFromSortedEntries( + instrumentOrderStatisticTree( + createFlatVisibleTree(nextPlan), + instrumentation, + ), + merged, + // Both proofs are earned by the merge directly above, and neither + // would be available to a caller that re-sorted the full set. + // Order: a merge of two strictly-increasing, id-disjoint sequences + // under one total order is strictly increasing — so the n−1 + // verification would re-derive what the loop just guaranteed. + // byId: the visible set changes by exactly `flippedOut` leaving and + // `flippedIn` arriving; every survivor is pushed into `merged` as the + // base tree's OWN entry object (unflipped ⇒ record and keys + // unchanged), which is the identity precondition derived mode + // requires. Cost drops from n inserts to `flipped` edits. + { + orderIsProven: true, + derivedById: { + base: captured.visible.rows, + removedIds: flippedOut, + addedEntries: flippedIn, + }, + }, + ), + }); + // The verdict pass above already set a bit for every member, so the new + // root takes its bitset directly — no second walk over the tree. + visibleSlots = nextVisibleSlots; + } + + const root: RevisionRoot = Object.freeze({ + revision, + parentRevision: revision - 1, + rows, + sourceOrder: captured.sourceOrder, + // A filter-only change reconstructs no record and touches no slot, so + // the slot vector and its domain carry by identity with the rows HAMT. + recordsBySlot: captured.recordsBySlot, + slotCapacity: captured.slotCapacity, + visibleSlots, + visible, + queryPlan: nextPlan, + expansion: captured.expansion, + cause: Object.freeze({ kind: "set-query" as const }), + }); + if (instrumentation !== undefined) { + instrumentation.work.filterRebuilds += 1; + instrumentation.work.evaluationCacheAdoptions += 1; + instrumentation.work.filterRowsFlipped += flipped; + instrumentation.work.filterMergeSortedInsertions += flippedIn.length; + instrumentation.work.filterRebuildMs += Math.max(0, now() - startedAt); + } + return root; +} diff --git a/packages/row-model/src/group-index.ts b/packages/row-model/src/group-index.ts index f2346ae9c..b914bc7de 100644 --- a/packages/row-model/src/group-index.ts +++ b/packages/row-model/src/group-index.ts @@ -1,5 +1,6 @@ import { compareWithSortKeys, + filterVerdict, type CompiledGroupKey, type CompiledQuery, } from "./compiled-query"; @@ -218,6 +219,24 @@ export type GroupedVisibleIndexRoot< readonly [groupedIndex]: GroupIndexRoot; }; +/** + * The grouped half of the filter-verdict seam (see `./filter-membership`, + * which owns the seam's documentation and re-exports this). It lives here + * because it reads this module's internal invariant: EVERY inserted row gets + * a `rowParents` entry, but only a PASSING row is inserted into its leaf + * group's `leaves` tree — so leaf membership is the stored verdict, and a row + * this index never saw is simply not a member. + */ +export function rowPassesFilterInGroupIndex< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +>(grouped: GroupIndexRoot, rowId: TRowId): boolean { + const parentGroupId = grouped.rowParents.get(rowId); + if (parentGroupId === undefined) return false; + return grouped.groups.get(parentGroupId)?.leaves.get(rowId) !== undefined; +} + export function getGroupIndex< TRow extends object, TRowId extends PretableRowId, @@ -903,8 +922,6 @@ type RuntimeAggregateLeaf = { | BuiltinAggregatorName | PretableAggregator; readonly allLeaf: AggregateTreeLeaf; - readonly filteredLeaf: - AggregateTreeLeaf | undefined; }; type AggregateLeafDependency = { @@ -970,6 +987,12 @@ function updateAggregateRoots< queryPlan: CompiledQuery, record: RowRecord, operation: "insert" | "remove", + /** + * The row's filter verdict, supplied by the caller. Filtered aggregation is + * membership in the `filtered` tree — there is no per-leaf flag — so this is + * the single bit that decides insert vs remove there. + */ + filterPasses: boolean, modelOperation: PretableRowModelOperation, instrumentation?: LocalRowModelInstrumentation, ): AggregateRoots { @@ -994,8 +1017,8 @@ function updateAggregateRoots< ); filtered.set( leaf.columnId, - operation === "insert" && leaf.filteredLeaf !== undefined - ? filteredTree.insertOrReplace(leaf.filteredLeaf) + operation === "insert" && filterPasses + ? filteredTree.insertOrReplace(leaf.allLeaf) : filteredTree.remove(record.rowId), ); } catch (cause) { @@ -1293,6 +1316,12 @@ function mutatePath< record: RowRecord, operation: "insert" | "remove", context: FinishContext, + /** + * On `"insert"` the row's verdict under the index's plan; on `"remove"` the + * verdict the row was INSERTED under (its membership in the previous + * index), so `filteredCount` unwinds exactly what it counted. + */ + filterPasses: boolean, ): void { const metadata = record.metadata; const path = metadata.groupPath; @@ -1336,7 +1365,7 @@ function mutatePath< if (leafLevel) { leaves = - operation === "insert" && metadata.filterPasses + operation === "insert" && filterPasses ? leaves.insertOrReplace(orderedRowEntry(context.queryPlan, record)) : leaves.remove(record.rowId); } else { @@ -1356,7 +1385,7 @@ function mutatePath< const filteredCount = (previous?.filteredCount ?? 0) + - (metadata.filterPasses ? (operation === "insert" ? 1 : -1) : 0); + (filterPasses ? (operation === "insert" ? 1 : -1) : 0); const allCount = (previous?.allCount ?? 0) + (operation === "insert" ? 1 : -1); if (allCount === 0) { @@ -1368,6 +1397,7 @@ function mutatePath< context.queryPlan, record, operation, + filterPasses, context.operation, context.instrumentation, ); @@ -1597,6 +1627,7 @@ export function createGroupIndexBuildDraft< const updateMutableAggregates = ( node: MutableBuildNode, record: RowRecord, + filterPasses: boolean, ): void => { for (const leaf of record.metadata .aggregateLeaves as unknown as readonly RuntimeAggregateLeaf[]) { @@ -1624,9 +1655,9 @@ export function createGroupIndexBuildDraft< ) as AnyTransientAggregateTree; node.aggregateRoots.filtered.set(leaf.columnId, filtered); } - if (leaf.filteredLeaf !== undefined) { + if (filterPasses) { const filteredSize = filtered.size; - filtered.insertOrReplace(leaf.filteredLeaf); + filtered.insertOrReplace(leaf.allLeaf); if (filtered.size > filteredSize) pendingUnits += 1; } } catch (cause) { @@ -1752,6 +1783,10 @@ export function createGroupIndexBuildDraft< throw new Error("Cannot insert after grouped index sealing started."); const path = record.metadata.groupPath; if (path.length === 0) return; + // Computed once per record and spent locally on the three membership + // decisions below (filteredCount, the filtered aggregate tree, and the + // leaf tree) — which together ARE this index's record of the verdict. + const filterPasses = filterVerdict(queryPlan, record as never); const pathKeys = path.map((entry) => encodeGroupValue(entry.value, { operation, @@ -1774,13 +1809,13 @@ export function createGroupIndexBuildDraft< children.set(key, current); } current.allCount += 1; - if (record.metadata.filterPasses) current.filteredCount += 1; - updateMutableAggregates(current, record); + if (filterPasses) current.filteredCount += 1; + updateMutableAggregates(current, record, filterPasses); current.triggerRowId = record.rowId; parentGroupId = current.groupId; children = current.childrenByKey; } - if (record.metadata.filterPasses) { + if (filterPasses) { current!.leaves.insertOrReplace(orderedRowEntry(queryPlan, record)); } rowParents.set(record.rowId, current!.groupId); @@ -1912,7 +1947,15 @@ export function createGroupIndex< operation, instrumentation, }; - for (const record of records) mutatePath(state, record, "insert", context); + for (const record of records) { + mutatePath( + state, + record, + "insert", + context, + filterVerdict(queryPlan, record as never), + ); + } // Apply retained overrides after all future/current groups have been materialized. let root = rootFromState(state, context); for (const [groupId, expanded] of overrides.entries()) { @@ -1948,8 +1991,36 @@ export function updateGroupIndex< operation, instrumentation, }; - for (const record of removals) mutatePath(state, record, "remove", context); - for (const record of insertions) mutatePath(state, record, "insert", context); + // A removal must unwind the verdict the row was counted under, which is + // exactly its membership in the PREVIOUS index — read before any mutation + // touches `state`. + // + // An insertion's verdict comes from the index's OWN plan, the same authority + // this function already trusts for leaf order (`orderedRowEntry` resolves + // keys from that plan's store) and for its comparators. That binds the + // verdict to the same standing precondition as the rest: records handed to + // an existing index must be coherent with the plan the index was built + // under. A same-reference mutation breaks that precondition for every + // derived value, not just this one, and its caller answers by rebuilding + // the index outright under the fresh plan. + for (const record of removals) { + mutatePath( + state, + record, + "remove", + context, + rowPassesFilterInGroupIndex(previous, record.rowId), + ); + } + for (const record of insertions) { + mutatePath( + state, + record, + "insert", + context, + filterVerdict(previous.queryPlan, record as never), + ); + } let root = rootFromState(state, context); if (overrides !== undefined) { for (const [groupId, expanded] of overrides.entries()) { diff --git a/packages/row-model/src/internal-types.ts b/packages/row-model/src/internal-types.ts index d8eec869e..00f6aafdd 100644 --- a/packages/row-model/src/internal-types.ts +++ b/packages/row-model/src/internal-types.ts @@ -6,7 +6,9 @@ import type { import type { PretableRowId } from "./column-types"; import type { OrderStatisticTree } from "./persistent/order-statistic-tree"; import type { PersistentMap } from "./persistent/persistent-map"; +import type { MembershipBitset } from "./membership-bitset"; import type { RowIntegrityRecord } from "./row-integrity"; +import type { SlotVector } from "./slot-vector"; import type { PretableDataRow, PretableExpansionDefault, @@ -27,6 +29,16 @@ export interface RowRecord< readonly rowId: TRowId; readonly row: TRow; readonly sourceOrder: number; + /** + * Dense integer handle, assigned at ingest, stable for the row's lifetime + * (updates carry it; only permanent removal releases it). This slot is + * what the slot-indexed structures the dense-handle arc adds next + * (`recordsBySlot`, `visibleSlots`) will be indexed by — the + * array-resident fast path that replaces string-keyed lookups on O(n) + * walks. Those structures don't exist yet; this field is laid down ahead + * of them. + */ + readonly slot: number; readonly metadata: CompiledRowMetadata; readonly publicRow: PretableDataRow; readonly integrity: RowIntegrityRecord; @@ -86,6 +98,35 @@ export interface RevisionRoot< SourceOrderKey, number >; + /** + * Slot-indexed view of `rows` — same records, array-resident. Per-revision + * immutable (chunked COW), which is what keeps THIS root's bindings valid + * when the allocator later reuses a slot. Invariant, test-pinned: + * slotVectorGet(recordsBySlot, record.slot) === record for every record in + * `rows`, at every committed root. + */ + readonly recordsBySlot: SlotVector>; + /** + * The slot-space size this root's slot-indexed structures were built for + * (the allocator's capacity at commit time). A root must be + * SELF-DESCRIBING: readers size bitsets and walks from this field, never + * from the live allocator — reading the allocator would let later growth + * leak into a held snapshot's domain. + */ + readonly slotCapacity: number; + /** + * Flat roots: one bit per slot, set iff the row is a member of + * `visible.rows` — the same structural verdict `filter-membership` + * resolves, indexed for O(1)/word-scan access (membership IS the verdict; + * this is never a stored copy that could diverge). Grouped roots carry + * `EMPTY_MEMBERSHIP` (their membership lives in the group index) and every + * reader must treat it per that module's contract. ("grouped" is + * equivalently `queryPlan.query.rowGroups.length > 0` or + * `getGroupIndex(visible) !== undefined` on a committed root; producers use + * whichever their structure source is.) Never mutated after the root + * commits. + */ + readonly visibleSlots: MembershipBitset; readonly visible: VisibleIndexRoot; readonly queryPlan: CompiledQuery; readonly expansion: ExpansionRoot; diff --git a/packages/row-model/src/membership-bitset.ts b/packages/row-model/src/membership-bitset.ts new file mode 100644 index 000000000..f1d62306d --- /dev/null +++ b/packages/row-model/src/membership-bitset.ts @@ -0,0 +1,53 @@ +/** + * Membership bitsets: one bit per SLOT (see `slot-allocator`). A committed + * root's verdict is its membership (the filter-membership invariant); the + * bitset is a faster INDEX of that same structural answer for flat roots, + * never a stored verdict. Grouped roots carry `EMPTY_MEMBERSHIP` and keep + * answering from the group index. + * + * Mutable while a producer is building the next revision's set; frozen by + * convention once a root captures it (no Object.freeze — typed arrays do not + * support it; discipline is "producers build fresh or clone, never write a + * captured root's bitset", the same convention every persistent structure + * here relies on). + * + * Whole-copy on change is the point: 50k rows is 6.25KB, negligible per + * commit (M0 measured ~1µs), so no COW machinery exists at this layer. + */ + +export type MembershipBitset = Uint32Array; + +/** Shared sentinel for roots whose membership lives elsewhere (grouped). */ +export const EMPTY_MEMBERSHIP: MembershipBitset = new Uint32Array(0); + +export function createMembership(capacity: number): MembershipBitset { + return new Uint32Array((capacity + 31) >>> 5); +} + +/** Clone, growing to `capacity` when it exceeds the source's words. */ +export function cloneMembership( + bits: MembershipBitset, + capacity: number, +): MembershipBitset { + const words = Math.max(bits.length, (capacity + 31) >>> 5); + const next = new Uint32Array(words); + next.set(bits); + return next; +} + +export function setMembershipBit(bits: MembershipBitset, slot: number): void { + bits[slot >>> 5]! |= 1 << (slot & 31); +} + +export function clearMembershipBit(bits: MembershipBitset, slot: number): void { + bits[slot >>> 5]! &= ~(1 << (slot & 31)); +} + +/** Out-of-range slots read as false — the EMPTY sentinel relies on this. */ +export function testMembershipBit( + bits: MembershipBitset, + slot: number, +): boolean { + const word = bits[slot >>> 5]; + return word === undefined ? false : ((word >>> (slot & 31)) & 1) === 1; +} diff --git a/packages/row-model/src/persistent/order-statistic-tree.ts b/packages/row-model/src/persistent/order-statistic-tree.ts index 9408d3348..f0aec9a64 100644 --- a/packages/row-model/src/persistent/order-statistic-tree.ts +++ b/packages/row-model/src/persistent/order-statistic-tree.ts @@ -35,6 +35,52 @@ export interface OrderStatisticTreeOptions< readonly measure: OrderStatisticTreeMeasure; } +/** + * A derivation of the bulk build's id→entry map from an existing tree's map: + * delete `removedIds`, set `addedEntries`, keep everything else. k edits on a + * transient instead of n inserts into a fresh one. + * + * PRECONDITION, unverifiable in O(k) and therefore the caller's to hold: every + * entry that is neither removed nor added must appear in `sorted` as the SAME + * OBJECT the base tree holds for it. A caller that reallocates surviving + * entries must not use this — the map would keep the old objects while the + * tree holds the new ones. + * + * Offering a derivation does not force one: the builder compares this edit + * count against a refill's and takes the cheaper route, so a caller may hand + * one over unconditionally. `removedIds` is a SET rather than an iterable + * precisely so that comparison is possible without draining it. + */ +export interface BulkBuildDerivedById< + TId extends OrderStatisticTreeId, + TEntry, + TMeasure, +> { + readonly base: OrderStatisticTree; + readonly removedIds: ReadonlySet; + readonly addedEntries: readonly TEntry[]; +} + +/** + * Package-internal claims a bulk-build caller can offer in place of work the + * builder would otherwise do. Never reachable from the package index, and + * never to be offered on the strength of "the input looks sorted" — each + * field is an unchecked assertion about how the caller built its input. + */ +export interface BulkBuildProof< + TId extends OrderStatisticTreeId, + TEntry, + TMeasure, +> { + /** + * Skips the n−1 strict-order verification. Only for callers whose input is + * strictly increasing under this tree's total order by construction. + */ + readonly orderIsProven?: boolean; + /** Derives the id→entry map instead of refilling it. */ + readonly derivedById?: BulkBuildDerivedById; +} + interface OrderStatisticTreeReads< TId extends OrderStatisticTreeId, TEntry, @@ -529,15 +575,45 @@ function rangeFromNode( return result; } +/** + * In-order walk over an EXPLICIT stack, and it must stay that way. + * + * The obvious shape — `yield* iterateEntries(node.left)` — makes every + * element bubble up through one generator frame per tree level on its way + * out, so walking n entries of a ~log2(n)-deep tree costs O(n log n) + * generator resumptions rather than O(n). Measured on a 50,000-entry tree: + * `yield*` delegation 30.04ms, this shape 1.77ms. The delegating version is + * shorter and 17× slower; do not "simplify" it back. + * + * Laziness is part of the contract — callers step this iterator across + * cooperative-transition slices — so materializing into an array here is not + * an option either. Callers that materialize anyway should use `range(0, + * size)`, which is faster still because it never suspends. + */ function* iterateEntries( node: TreeNode | null, ): IterableIterator { - if (node === null) return; - yield* iterateEntries(node.left); - yield node.entry; - yield* iterateEntries(node.right); + const pending: TreeNode[] = []; + let current = node; + for (;;) { + while (current !== null) { + pending.push(current); + current = current.left; + } + const visited = pending.pop(); + if (visited === undefined) return; + yield visited.entry; + current = visited.right; + } } +/** + * The transient walk, same explicit-stack shape and same reason (see + * `iterateEntries`), plus the liveness guard a draft owes its callers: the + * draft is checked on first resumption and again before every element, so a + * walk in flight when the draft freezes or is abandoned fails on its next + * step rather than yielding from a structure that has moved on. + */ function* iterateTransientEntries< TId extends OrderStatisticTreeId, TEntry, @@ -547,11 +623,19 @@ function* iterateTransientEntries< assertHealthy: () => void, ): IterableIterator { assertHealthy(); - if (node === null) return; - yield* iterateTransientEntries(node.left, assertHealthy); - assertHealthy(); - yield node.entry; - yield* iterateTransientEntries(node.right, assertHealthy); + const pending: TreeNode[] = []; + let current = node; + for (;;) { + while (current !== null) { + pending.push(current); + current = current.left; + } + const visited = pending.pop(); + if (visited === undefined) return; + assertHealthy(); + yield visited.entry; + current = visited.right; + } } class PersistentOrderStatisticTree< @@ -647,35 +731,58 @@ class PersistentOrderStatisticTree< } /** - * The strict-order check is unconditional: a misordered build silently - * corrupts every later rank and lookup, which is strictly worse than the - * O(n) cost of checking. Duplicates compare 0 and are rejected by the same - * check. + * The strict-order check is unconditional by default: a misordered build + * silently corrupts every later rank and lookup, which is strictly worse + * than the O(n) cost of checking. Duplicates compare 0 and are rejected by + * the same check. + * + * `proof.orderIsProven` is the only opt-out, and it exists for the two + * in-package callers whose input is strictly sorted BY CONSTRUCTION, not by + * assumption: + * + * - `filter-rebuild` merges the captured visible tree's in-order walk (the + * tree's own order, minus a skipped subset — still strictly increasing) + * with a subset it just sorted under the identical composite comparator + * (`compareWithSortKeys` then id). A merge of two strictly-increasing + * sequences under one total order is strictly increasing, and the two + * sequences are disjoint by id (a flipped-in row was not visible). + * - `sort-rebuild` hands over `Array.sort` output under that same composite + * comparator, id tiebreak included, so it is totally ordered and the ids + * are unique because they come from a HAMT keyed by id. + * + * Any caller that cannot make that argument from its own code — including + * every external caller, which is why the option is package-internal — must + * leave the check on. The escape hatch buys n−1 comparator calls per commit + * and nothing else; it is not worth taking on a hunch. */ [buildFromSortedEntries]( sorted: readonly TEntry[], + proof?: BulkBuildProof, ): OrderStatisticTree { const context = this.#context; const entryIds = sorted.map((entry) => context.getId(entry)); - for (let index = 1; index < sorted.length; index += 1) { - const comparison = compareEntries( - sorted[index - 1]!, - entryIds[index - 1]!, - sorted[index]!, - entryIds[index]!, - context, - ); - if (comparison >= 0) { - throw new TypeError( - "Bulk build input must be strictly sorted by the tree's total order.", + if (proof?.orderIsProven === true) { + if (context.instrumentation !== undefined) { + context.instrumentation.work.bulkOrderVerificationsSkipped += 1; + } + } else { + for (let index = 1; index < sorted.length; index += 1) { + const comparison = compareEntries( + sorted[index - 1]!, + entryIds[index - 1]!, + sorted[index]!, + entryIds[index]!, + context, ); + if (comparison >= 0) { + throw new TypeError( + "Bulk build input must be strictly sorted by the tree's total order.", + ); + } } } - const byId = createPersistentMap().asTransient(); - for (let index = 0; index < sorted.length; index += 1) { - byId.set(entryIds[index]!, sorted[index]!); - } + const byId = this.#buildById(sorted, entryIds, proof?.derivedById); const build = ( low: number, @@ -699,7 +806,7 @@ class PersistentOrderStatisticTree< return new PersistentOrderStatisticTree( build(0, sorted.length - 1), - byId.freeze(), + byId, context, ); } @@ -724,6 +831,96 @@ class PersistentOrderStatisticTree< return iterateEntries(this.#root); } + /** + * Builds the id→entry map by whichever of the two routes does fewer map + * operations, and the choice is made HERE because this is the only place + * that holds both counts. + * + * - Refill: `sorted.length` inserts into a fresh transient. + * - Derive: `removedIds.size` deletes plus `addedEntries.length` inserts on + * a transient over the base map. + * + * CONSTRAINT — a derivation offered is not a derivation taken. Deriving is + * only cheaper when its edit count is below the refill's insert count, and + * the difference is not academic: at S2's 50,000-row target the filter drops + * to 12,500 survivors, so an unconditional derivation ran **37,500 removes + * to replace 12,500 inserts — three times the work**. Measured, on that + * scenario, by single-variable A/B: 217.1ms settle with the derivation + * always on against 208.3ms with it always off, and 15.3ms of the + * interaction window's persistent-map time attributed to this method. The + * comparison below is algebraically `removals < survivors` (the added + * entries are inserted on either route and cancel), so a narrow flip + * derives and a wide one refills. + * + * The derivation is otherwise a CLAIM, and only one half of it is verified. + * The + * post-edit size check below is O(1) and catches the whole class of + * "wrong edit set" slips (leavers left in, an added entry missing, a + * duplicate id). What it cannot catch is a STALE survivor: derived mode + * never touches an id that is neither removed nor added, so it keeps + * whatever entry object the base map held for it. That is only correct + * when the caller REUSES survivors' entry objects by identity, which is + * exactly the precondition filter-rebuild's merge satisfies (a still-passing + * row is unflipped, so its record and keys are both unchanged) and exactly + * the one sort-rebuild does NOT: it allocates a fresh entry per row to + * carry the new plan's keys, so its survivors are new objects and derived + * mode would leave the map pointing at the previous plan's entries. That is + * why sort-rebuild takes `orderIsProven` and nothing else. + */ + #buildById( + sorted: readonly TEntry[], + entryIds: readonly TId[], + derived: BulkBuildDerivedById | undefined, + ): PersistentMap { + const context = this.#context; + if ( + derived !== undefined && + !(derived.base instanceof PersistentOrderStatisticTree) + ) { + // Checked before the routing decision, so an unusable base is a hard + // error rather than a silent fall-through to the refill. + throw new TypeError( + "Derived bulk-build maps require a base tree created by this module.", + ); + } + if ( + derived === undefined || + derived.removedIds.size + derived.addedEntries.length >= sorted.length + ) { + const draft = createPersistentMap().asTransient(); + for (let index = 0; index < sorted.length; index += 1) { + draft.set(entryIds[index]!, sorted[index]!); + } + return draft.freeze(); + } + const base = derived.base as PersistentOrderStatisticTree< + TId, + TEntry, + TMeasure + >; + // The base map is taken as-is rather than re-instrumented: the refill it + // replaces built into a FRESH, uninstrumented map, so re-instrumenting + // here would start charging visible-index byId churn to + // `hamtNodesCopied` — the counter the suite uses as the record-rebuild + // proxy, which must stay zero across a filter commit. The derivation's + // own cost is reported by `bulkByIdDerived` instead. + const draft = (base.#byId as PersistentMap).asTransient(); + for (const id of derived.removedIds) draft.delete(id); + for (const entry of derived.addedEntries) { + draft.set(context.getId(entry), entry); + } + const byId = draft.freeze(); + if (byId.size !== sorted.length) { + throw new TypeError( + "Derived bulk-build map size must equal the built entry count.", + ); + } + if (context.instrumentation !== undefined) { + context.instrumentation.work.bulkByIdDerived += 1; + } + return byId; + } + [attachInstrumentation]( instrumentation: LocalRowModelInstrumentation, beforeCombine?: () => void, @@ -1006,6 +1203,10 @@ export function compareOrderStatisticTreeIds( * Throws TypeError when adjacent entries compare `>= 0` — misordered input, * equal-compare entries with misordered IDs, and duplicate IDs alike — or * when `like` was not created by this module. + * + * `proof` lets an in-package caller trade a claim it can prove from its own + * construction for work the builder would otherwise redo; see + * {@link BulkBuildProof}. Omitting it keeps every check on. */ export function createOrderStatisticTreeFromSortedEntries< TId extends OrderStatisticTreeId, @@ -1014,11 +1215,12 @@ export function createOrderStatisticTreeFromSortedEntries< >( like: OrderStatisticTree, sorted: readonly TEntry[], + proof?: BulkBuildProof, ): OrderStatisticTree { if (!(like instanceof PersistentOrderStatisticTree)) { throw new TypeError("Bulk builds require a tree created by this module."); } - return like[buildFromSortedEntries](sorted); + return like[buildFromSortedEntries](sorted, proof); } export function instrumentOrderStatisticTree< diff --git a/packages/row-model/src/persistent/persistent-map.ts b/packages/row-model/src/persistent/persistent-map.ts index 3f557a0d0..da99e47bd 100644 --- a/packages/row-model/src/persistent/persistent-map.ts +++ b/packages/row-model/src/persistent/persistent-map.ts @@ -303,15 +303,33 @@ function nodePath( return path; } +/** + * Pre-order walk over an EXPLICIT stack, for the same reason the order- + * statistic tree's walk uses one: `yield*` delegation routes every element + * through one generator frame per trie level on its way out, so the cost of a + * full walk scales with depth as well as size. The order-statistic tree + * measured 30.04ms → 1.77ms at 50,000 entries on exactly this change; a + * 32-way trie is shallower, so the win is smaller, but the shape is the same + * mistake. Children are pushed in reverse so they pop in bitmap order — + * iteration order is unspecified but must stay stable run to run. + */ function* iterateEntries( node: Node | null, ): IterableIterator { if (node === null) return; - if (node.kind === "leaf") { - for (const [key, value] of node.entries) yield [key, value] as const; - return; + const pending: Node[] = [node]; + for (;;) { + const visited = pending.pop(); + if (visited === undefined) return; + if (visited.kind === "leaf") { + for (const [key, value] of visited.entries) yield [key, value] as const; + continue; + } + for (let index = visited.children.length - 1; index >= 0; index -= 1) { + const child = visited.children[index]; + if (child !== undefined) pending.push(child); + } } - for (const child of node.children) yield* iterateEntries(child); } class PersistentHashMap implements PersistentMap { diff --git a/packages/row-model/src/row-store.ts b/packages/row-model/src/row-store.ts index de1a8283e..b7c51c449 100644 --- a/packages/row-model/src/row-store.ts +++ b/packages/row-model/src/row-store.ts @@ -16,6 +16,8 @@ import { inspectRowIntegrity, type PretableRowIntegrityDiagnostic, } from "./row-integrity"; +import type { SlotAllocator } from "./slot-allocator"; +import { slotVectorFromEntries, type SlotVector } from "./slot-vector"; export interface BuildRowStoreInput< TRow extends object, @@ -26,6 +28,7 @@ export interface BuildRowStoreInput< readonly getRowId: (row: TRow) => TRowId; readonly queryPlan: CompiledQuery; readonly previous?: PersistentMap>; + readonly slots: SlotAllocator; readonly instrumentation?: LocalRowModelInstrumentation; } @@ -37,6 +40,8 @@ export interface BuiltRowStore< readonly rows: PersistentMap>; readonly sourceOrder: ReturnType>; readonly records: readonly RowRecord[]; + /** Slot-indexed view of `records`, sized to the allocator at build time. */ + readonly recordsBySlot: SlotVector>; readonly sameReferenceMutation: boolean; readonly sameReferenceMutationCount: number; readonly diagnostics: readonly PretableRowIntegrityDiagnostic[]; @@ -60,7 +65,10 @@ export function rebuildRowStoreForQuery< RowRecord >().asTransient(); const records: RowRecord[] = []; - for (const source of sourceOrder.entries()) { + // `range(0, size)` rather than `entries()`: a full walk into an array, and + // the tree's non-generator walk is the cheaper way to get one (see + // `iterateEntries`). The only exit below is a throw, not an early return. + for (const source of sourceOrder.range(0, sourceOrder.size)) { const previous = previousRows.get(source.rowId); if (previous === undefined) { throw new PretableRowModelError( @@ -73,7 +81,11 @@ export function rebuildRowStoreForQuery< rowId: previous.rowId, row: previous.row as never, sourceOrder: previous.sourceOrder, + // dead code, kept compiling: rebuildRowStoreForQuery has zero callers. + slot: previous.slot, }) as unknown as RowRecord["metadata"]; + // The spread carries `slot` (with everything else the query re-evaluation + // leaves untouched) — a query rebuild never changes row lifetimes. const record = Object.freeze({ ...previous, metadata }); draft.set(record.rowId, record); records.push(record); @@ -187,11 +199,24 @@ export function buildRowStore< const previous = input.previous?.get(rowId); if (input.instrumentation !== undefined) input.instrumentation.work.rowsEvaluated += 1; + // `evaluate` needs a slot up front (it's part of `CompiledRowInput` now, + // unread this task), but the REAL slot for a brand-new row is still + // resolved after evaluation succeeds, exactly as before this field + // existed: `input.slots.allocate()` is a real allocator mutation that + // bumps the high-water mark permanently (capacity never shrinks — see + // `slot-allocator.ts`), and drawing it before a throwing accessor runs + // would leak capacity a release can't undo. A carried row's slot has no + // such hazard (no allocator call), so it is used directly. `-1` is a + // placeholder for the fresh-slot case only: harmless because nothing + // reads `CompiledRowInput.slot` yet. const metadata = input.queryPlan.evaluate({ rowId, row: row as never, sourceOrder, + slot: previous !== undefined ? previous.slot : -1, }) as unknown as RowRecord["metadata"]; + const slot = + previous !== undefined ? previous.slot : input.slots.allocate(); const publicRow = previous !== undefined && Object.is(previous.row, row) && @@ -209,6 +234,7 @@ export function buildRowStore< rowId, row, sourceOrder, + slot, metadata, publicRow, integrity: inspections[sourceOrder]!.integrity, @@ -217,10 +243,19 @@ export function buildRowStore< sourceDraft.insertOrReplace(Object.freeze({ rowId, sourceOrder })); records.push(record); } + if (input.previous !== undefined) { + for (const [rowId, record] of input.previous.entries()) { + if (!seen.has(rowId)) input.slots.release(record.slot); + } + } return { rows: mapDraft.freeze(), sourceOrder: sourceDraft.freeze(), records: Object.freeze(records), + recordsBySlot: slotVectorFromEntries( + records.map((record) => [record.slot, record] as const), + input.slots.capacity, + ), sameReferenceMutation, sameReferenceMutationCount: inspections.filter( (inspection) => inspection.sameReferenceMutation, diff --git a/packages/row-model/src/slot-allocator.ts b/packages/row-model/src/slot-allocator.ts new file mode 100644 index 000000000..e1e6e4156 --- /dev/null +++ b/packages/row-model/src/slot-allocator.ts @@ -0,0 +1,51 @@ +/** + * Per-MODEL slot allocator: every row gets a small dense integer for its + * lifetime, assigned at ingest and released only on permanent removal. + * Mutable by design — this is instance state, not revision state; the + * revision-scoped structures (`slot-vector`, `membership-bitset`) are what + * keep old snapshots valid when a released slot is reused. + * + * Capacity is the high-water mark and never shrinks, so slot-indexed + * structures never renumber. Release is fail-loud (double release would hand + * one slot to two live rows, which corrupts every slot-indexed structure + * from that commit on). + */ + +export interface SlotAllocator { + readonly capacity: number; + allocate(): number; + release(slot: number): void; +} + +export function createSlotAllocator(): SlotAllocator { + const free: number[] = []; + let next = 0; + let live = new Uint8Array(1024); + const ensure = (slot: number) => { + if (slot < live.length) return; + const grown = new Uint8Array(Math.max(live.length * 2, slot + 1)); + grown.set(live); + live = grown; + }; + return { + get capacity() { + return next; + }, + allocate() { + const slot = free.length > 0 ? free.pop()! : next++; + ensure(slot); + live[slot] = 1; + return slot; + }, + release(slot) { + if (!Number.isInteger(slot) || slot < 0 || slot >= next) { + throw new RangeError(`Slot ${slot} was never allocated.`); + } + if (live[slot] !== 1) { + throw new RangeError(`Slot ${slot} is not live (double release).`); + } + live[slot] = 0; + free.push(slot); + }, + }; +} diff --git a/packages/row-model/src/slot-vector.ts b/packages/row-model/src/slot-vector.ts new file mode 100644 index 000000000..fd795dafb --- /dev/null +++ b/packages/row-model/src/slot-vector.ts @@ -0,0 +1,117 @@ +/** + * Immutable chunked slot-indexed vector: a chunk table over + * `SLOT_VECTOR_CHUNK`-element chunks, copy-on-write per commit. A commit + * touching k slots copies the table plus each touched chunk once — this is + * what keeps old snapshots valid under slot reuse: every revision holds its + * own table, so revision N still binds slot s to whatever row owned s at + * revision N no matter what later commits do (M0 priced maintenance at + * ~33–98µs per 100-write commit). + * + * Holes (`undefined`) are free slots. Iteration hole-skips, which is why no + * separate "live" bitset exists (recorded plan deviation from the spec). + */ + +export const SLOT_VECTOR_CHUNK = 1024; + +export interface SlotVector { + /** Sparse table: a missing/undefined chunk reads as all holes. */ + readonly chunks: ReadonlyArray | undefined>; +} + +const EMPTY: SlotVector = Object.freeze({ chunks: Object.freeze([]) }); + +export function emptySlotVector(): SlotVector { + return EMPTY; +} + +export function slotVectorFromEntries( + entries: Iterable, + capacity: number, +): SlotVector { + const tableSize = Math.ceil(capacity / SLOT_VECTOR_CHUNK); + const limit = tableSize * SLOT_VECTOR_CHUNK; + const chunks: Array | undefined> = new Array(tableSize); + for (const [slot, value] of entries) { + if (slot >= limit) { + throw new RangeError(`Slot ${slot} is beyond capacity ${capacity}.`); + } + const index = (slot / SLOT_VECTOR_CHUNK) | 0; + let chunk = chunks[index]; + if (chunk === undefined) { + chunk = new Array(SLOT_VECTOR_CHUNK); + chunks[index] = chunk; + } + chunk[slot % SLOT_VECTOR_CHUNK] = value; + } + return { chunks }; +} + +export function slotVectorGet( + vector: SlotVector, + slot: number, +): T | undefined { + const chunk = vector.chunks[(slot / SLOT_VECTOR_CHUNK) | 0]; + return chunk === undefined ? undefined : chunk[slot % SLOT_VECTOR_CHUNK]; +} + +/** + * One commit's writes (`undefined` value = clear the slot), COW: table copied + * once, each touched chunk copied once. `capacity` may exceed the old + * table's reach (allocator growth). + * + * `chunksTouched` counts every chunk this commit allocated or copied, + * including a brand-new chunk created for a hole beyond the old table — that + * counts as touched even though nothing was copied. + */ +export function slotVectorWithAll( + vector: SlotVector, + writes: ReadonlyArray, + capacity: number, +): { readonly next: SlotVector; readonly chunksTouched: number } { + const tableSize = Math.max( + vector.chunks.length, + Math.ceil(capacity / SLOT_VECTOR_CHUNK), + ); + const limit = tableSize * SLOT_VECTOR_CHUNK; + const chunks: Array< + Array | ReadonlyArray | undefined + > = new Array(tableSize); + for (let i = 0; i < vector.chunks.length; i += 1) + chunks[i] = vector.chunks[i]; + let chunksTouched = 0; + for (const [slot, value] of writes) { + if (slot >= limit) { + throw new RangeError(`Slot ${slot} is beyond capacity ${capacity}.`); + } + const index = (slot / SLOT_VECTOR_CHUNK) | 0; + // A chunk still equal to the base's (or absent) hasn't been copied for + // this commit yet; after copy/create it is a fresh object, so identity + // alone tells touched from untouched with no separate tracking set. + if (chunks[index] === undefined || chunks[index] === vector.chunks[index]) { + const existing = chunks[index]; + chunks[index] = + existing === undefined + ? new Array(SLOT_VECTOR_CHUNK) + : existing.slice(); + chunksTouched += 1; + } + (chunks[index] as Array)[slot % SLOT_VECTOR_CHUNK] = value; + } + return { next: { chunks }, chunksTouched }; +} + +/** Hole-skipping walk in slot order. */ +export function forEachSlotEntry( + vector: SlotVector, + callback: (value: T, slot: number) => void, +): void { + for (let index = 0; index < vector.chunks.length; index += 1) { + const chunk = vector.chunks[index]; + if (chunk === undefined) continue; + const base = index * SLOT_VECTOR_CHUNK; + for (let offset = 0; offset < chunk.length; offset += 1) { + const value = chunk[offset]; + if (value !== undefined) callback(value, base + offset); + } + } +} diff --git a/packages/row-model/src/sort-rebuild.ts b/packages/row-model/src/sort-rebuild.ts index df95315ed..53365a643 100644 --- a/packages/row-model/src/sort-rebuild.ts +++ b/packages/row-model/src/sort-rebuild.ts @@ -17,6 +17,7 @@ import { type CompiledSortKey, } from "./compiled-query"; import type { LocalRowModelInstrumentation } from "./diagnostics"; +import { rowPassesFilter } from "./filter-membership"; import type { OrderedRowEntry, RevisionRoot } from "./internal-types"; import { compareOrderStatisticTreeIds, @@ -52,7 +53,13 @@ export function rebuildRootForSortOnlyChange< // ~4x the decorated form. The pairs ARE the tree's entry type, so the // sorted array feeds the bulk constructor directly. const visible: OrderedRowEntry[] = []; - for (const source of captured.sourceOrder.entries()) { + // `range(0, size)` rather than `entries()`: this walk always runs to + // completion into an array, and the tree's non-generator walk is the + // cheaper way to get one (see `iterateEntries`). + for (const source of captured.sourceOrder.range( + 0, + captured.sourceOrder.size, + )) { const previous = captured.rows.get(source.rowId); if (previous === undefined) continue; // Seed the NEXT plan's store for every carried record — the one part of @@ -64,7 +71,9 @@ export function rebuildRootForSortOnlyChange< previous as never, instrumentation, ) as readonly CompiledSortKey[]; - if (previous.metadata.filterPasses) { + // A sort-only change cannot move a row across the filter, so the CAPTURED + // root's membership is this row's verdict under the next plan too. + if (rowPassesFilter(captured, source.rowId)) { visible.push(Object.freeze({ record: previous, keys })); } } @@ -92,6 +101,14 @@ export function rebuildRootForSortOnlyChange< instrumentation, ), visible, + // Order only. The `visible.sort` above is under the tree's own composite + // order (comparator, then id) over ids drawn from a HAMT, so it is + // strictly increasing by construction and the n−1 verification can only + // re-confirm it. Derived byId is deliberately NOT taken: a sort-only + // change keeps the same entry SET but allocates a fresh entry object per + // row to carry the next plan's keys, so every "survivor" is a new object + // and a derived map would keep pointing at the previous plan's entries. + { orderIsProven: true }, ); const root: RevisionRoot = Object.freeze({ revision, @@ -100,6 +117,13 @@ export function rebuildRootForSortOnlyChange< // rows HAMT all survive a sort-only change untouched. rows: captured.rows, sourceOrder: captured.sourceOrder, + // Same identity carry: a sort-only change touches no record and no slot. + recordsBySlot: captured.recordsBySlot, + slotCapacity: captured.slotCapacity, + // A sort-only change reorders the members but keeps the member SET + // identical (the loop above pushed exactly the captured membership), so + // the bitset carries by identity too. + visibleSlots: captured.visibleSlots, visible: Object.freeze({ rows: tree }), queryPlan: nextPlan, expansion: captured.expansion, diff --git a/packages/row-model/src/transaction-draft.ts b/packages/row-model/src/transaction-draft.ts index 4272d0a83..c1769cbaf 100644 --- a/packages/row-model/src/transaction-draft.ts +++ b/packages/row-model/src/transaction-draft.ts @@ -1,10 +1,15 @@ -import { sortKeysOf, type CompiledQuery } from "./compiled-query"; +import { + filterVerdict, + sortKeysOf, + type CompiledQuery, +} from "./compiled-query"; import { attachChangeOperationDiagnosticsForTesting, getChangeOperationDiagnosticsForTesting, } from "./change-journal"; import type { PretableRowId } from "./column-types"; import type { LocalRowModelInstrumentation } from "./diagnostics"; +import { rowPassesFilter } from "./filter-membership"; import { PretableRowIdentityChangeError, PretableRowModelError, @@ -33,7 +38,15 @@ import type { } from "./types"; import type { PretableGroupId } from "./types"; import { orderedRowEntry } from "./ordered-row-entry"; -import { createFlatVisibleTree } from "./visible-index"; +import { + clearMembershipBit, + cloneMembership, + setMembershipBit, + EMPTY_MEMBERSHIP, +} from "./membership-bitset"; +import type { SlotAllocator } from "./slot-allocator"; +import { slotVectorWithAll } from "./slot-vector"; +import { createFlatVisibleTree, membershipFromFlatTree } from "./visible-index"; interface TransactionDraftInput< TRow extends object, @@ -45,6 +58,7 @@ interface TransactionDraftInput< readonly getRowId: (row: TRow) => TRowId; readonly queryPlan: CompiledQuery; readonly nextSourceOrder: number; + readonly slots: SlotAllocator; readonly instrumentation?: LocalRowModelInstrumentation; } @@ -56,6 +70,20 @@ export interface TransactionDraftResult< readonly rows: RevisionRoot["rows"]; readonly sourceOrder: RevisionRoot["sourceOrder"]; readonly visible: RevisionRoot["visible"]; + /** + * Slot-indexed view of `rows` for the root this draft commits into + * (carried unchanged from the input root when the draft is ineffective). + * Built on the SUCCESS path only, so a throwing accessor still propagates + * with nothing published. + */ + readonly recordsBySlot: RevisionRoot["recordsBySlot"]; + /** + * Membership bitset for the root this draft commits into: flat drafts + * maintain the input root's bitset (clone-and-flip) or rebuild it from the + * final flat tree; grouped drafts return the `EMPTY_MEMBERSHIP` sentinel; + * ineffective drafts carry the input root's bitset unchanged. + */ + readonly visibleSlots: RevisionRoot["visibleSlots"]; readonly nextSourceOrder: number; readonly added: number; readonly updated: number; @@ -289,6 +317,7 @@ function createRecord< row: TRow, rowId: TRowId, sourceOrder: number, + slot: number, queryPlan: CompiledQuery, instrumentation: LocalRowModelInstrumentation | undefined, ): { @@ -300,6 +329,7 @@ function createRecord< rowId, row: row as never, sourceOrder, + slot, }) as unknown as RowRecord["metadata"]; const inspection = inspectRowIntegrity(row, rowId, undefined, false); return { @@ -307,6 +337,7 @@ function createRecord< rowId, row, sourceOrder, + slot, metadata, publicRow: Object.freeze({ kind: "data" as const, @@ -344,13 +375,21 @@ function sameFlatOrder< nextPlan: CompiledQuery, previous: RowRecord, next: RowRecord, + /** The committed root's membership verdict for this row (the OLD one). */ + previousPasses: boolean, + /** The drafting plan's verdict for `next`, computed by the caller. */ + nextPasses: boolean, ): boolean { // Each record's keys resolve from the plan that evaluated it: `previous` // from the committed root's plan, `next` from the drafting plan. Outside // the same-reference-mutation recompile these are one and the same object. + // The two VERDICTS likewise come from two different places, and must: the + // old one is structural (root membership), the new one is computed. A + // row-keyed verdict store could not tell them apart when the plan object is + // shared, which is exactly the same-reference-mutation case. return ( previous.sourceOrder === next.sourceOrder && - previous.metadata.filterPasses === next.metadata.filterPasses && + previousPasses === nextPasses && sameKeyValues( sortKeysOf(previousPlan, previous as never), sortKeysOf(nextPlan, next as never), @@ -367,9 +406,18 @@ function sameGroupIndexContribution< nextPlan: CompiledQuery, previous: RowRecord, next: RowRecord, + previousPasses: boolean, + nextPasses: boolean, ): boolean { if ( - !sameFlatOrder(previousPlan, nextPlan, previous, next) || + !sameFlatOrder( + previousPlan, + nextPlan, + previous, + next, + previousPasses, + nextPasses, + ) || !sameKeyValues(previous.metadata.groupPath, next.metadata.groupPath) ) { return false; @@ -384,7 +432,6 @@ function sameGroupIndexContribution< readonly sourceOrder: number; }; }; - readonly filteredLeaf: object | undefined; }[]; const nextLeaves = next.metadata.aggregateLeaves as typeof previousLeaves; return ( @@ -394,9 +441,12 @@ function sameGroupIndexContribution< if ( nextLeaf === undefined || previousLeaf.columnId !== nextLeaf.columnId || - previousLeaf.aggregate !== nextLeaf.aggregate || - (previousLeaf.filteredLeaf === undefined) !== - (nextLeaf.filteredLeaf === undefined) + previousLeaf.aggregate !== nextLeaf.aggregate + // A per-leaf filtered flag is deliberately NOT compared: whether a + // leaf belongs to the filtered aggregate tree is the row's filter + // verdict, and `sameFlatOrder` above already compared the old verdict + // against the new one for this very row. Comparing it again here only + // restated that check. ) { return false; } @@ -713,11 +763,9 @@ function rebaseSourceOrder< ...leaf.allLeaf.dependency, sourceOrder, }); - const allLeaf = Object.freeze({ ...leaf.allLeaf, dependency }); return Object.freeze({ ...leaf, - allLeaf, - filteredLeaf: leaf.filteredLeaf === undefined ? undefined : allLeaf, + allLeaf: Object.freeze({ ...leaf.allLeaf, dependency }), }); }); return Object.freeze({ @@ -735,6 +783,19 @@ export function applyFlatTransactionDraft< >( input: TransactionDraftInput, ): TransactionDraftResult { + /* + * Abandon rule: the draft allocates slots while preparing records, but this + * function can still throw after that (metadata evaluation runs user code); + * leaked allocations would pin free-list slots forever. Every allocation is + * recorded here; the tail `catch` releases them on that live invariant. The + * `effective: false` release just above it is defensive — every prepared + * candidate implies `effective`, so that branch cannot currently be taken + * with a non-empty `allocatedSlots`, but a future change to this function's + * effectiveness check must not silently reopen the leak. The success path + * releases the REMOVED rows' slots instead — a removed slot must never be + * released on a failure path, because the committed root still owns it. + */ + const allocatedSlots: number[] = []; try { const transaction = input.transaction as unknown; if (transaction === null || typeof transaction !== "object") { @@ -854,6 +915,8 @@ export function applyFlatTransactionDraft< readonly row: TRow; readonly sourceOrder: number; readonly kind: "add" | "update"; + /** An update carries the previous record's slot; an add allocates. */ + readonly previousSlot?: number; }[] = []; const prepared: RowRecord[] = []; const diagnostics: PretableRowIntegrityDiagnostic[] = []; @@ -883,6 +946,7 @@ export function applyFlatTransactionDraft< row: merged.row, sourceOrder: previous.sourceOrder, kind: "update", + previousSlot: previous.slot, }); } for (const [rowId, row] of addById) { @@ -930,24 +994,49 @@ export function applyFlatTransactionDraft< // All lists, IDs, partial values, and resulting identities are validated // before active derivation callbacks are allowed to run. + // Each prepared record's NEW verdict is computed once, here, and keyed by + // the record OBJECT (the same row id can appear twice in one transaction, + // and each occurrence carries its own record). It is never stored on the + // record: the structures this draft builds are where it lands. + const nextVerdicts = new Map, boolean>(); + const passesNext = (record: RowRecord): boolean => + nextVerdicts.get(record)!; + /** The committed root's membership — the OLD verdict for one row. */ + const passedPreviously = (rowId: TRowId): boolean => + rowPassesFilter(input.root, rowId); for (const candidate of pending) { + let slot: number; + if (candidate.previousSlot !== undefined) { + slot = candidate.previousSlot; + } else { + slot = input.slots.allocate(); + allocatedSlots.push(slot); + } const made = createRecord( candidate.row, candidate.rowId, candidate.sourceOrder, + slot, input.queryPlan, input.instrumentation, ); prepared.push(made.record); + nextVerdicts.set( + made.record, + filterVerdict(input.queryPlan, made.record as never), + ); if (made.diagnostic) diagnostics.push(made.diagnostic); } const effective = prepared.length > 0 || effectiveRemoves.length > 0; if (!effective) { + for (const slot of allocatedSlots) input.slots.release(slot); return { rows: input.root.rows, sourceOrder: input.root.sourceOrder, visible: input.root.visible, + recordsBySlot: input.root.recordsBySlot, + visibleSlots: input.root.visibleSlots, nextSourceOrder: input.nextSourceOrder, added: 0, updated: 0, @@ -973,20 +1062,19 @@ export function applyFlatTransactionDraft< const previousGroups = getGroupIndex(input.root.visible); const visibleNeedsChange = previousGroups === undefined && - (effectiveRemoves.some( - (rowId) => input.root.rows.get(rowId)?.metadata.filterPasses === true, - ) || + (effectiveRemoves.some((rowId) => passedPreviously(rowId)) || prepared.some((record) => { const previous = input.root.rows.get(record.rowId); return ( - (previous?.metadata.filterPasses === true || - record.metadata.filterPasses) && + (passedPreviously(record.rowId) || passesNext(record)) && (previous === undefined || !sameFlatOrder( input.root.queryPlan, input.queryPlan, previous, record, + passedPreviously(record.rowId), + passesNext(record), )) ); })); @@ -1018,6 +1106,11 @@ export function applyFlatTransactionDraft< } for (const record of prepared) { const previous = input.root.rows.get(record.rowId); + // Both verdicts, resolved from their own authorities: the old one from + // the committed root's membership (immutable while this loop mutates + // the drafts), the new one computed when the record was prepared. + const previouslyPassed = passedPreviously(record.rowId); + const passes = passesNext(record); rowDraft.set(record.rowId, record); if (previous === undefined) sourceDraft.insertOrReplace( @@ -1029,9 +1122,16 @@ export function applyFlatTransactionDraft< if (previousGroups === undefined) { if ( previous !== undefined && - sameFlatOrder(input.root.queryPlan, input.queryPlan, previous, record) + sameFlatOrder( + input.root.queryPlan, + input.queryPlan, + previous, + record, + previouslyPassed, + passes, + ) ) { - if (record.metadata.filterPasses) { + if (passes) { const index = visibleDraft?.rankOf(record.rowId) ?? input.root.visible.rows.rankOf(record.rowId); @@ -1048,18 +1148,16 @@ export function applyFlatTransactionDraft< } continue; } - const previousIndex = previous?.metadata.filterPasses + const previousIndex = previouslyPassed ? visibleDraft?.rankOf(record.rowId) : undefined; - if (previous?.metadata.filterPasses) visibleDraft?.remove(record.rowId); - if (record.metadata.filterPasses) { + if (previouslyPassed) visibleDraft?.remove(record.rowId); + if (passes) { visibleDraft?.insertOrReplace( orderedRowEntry(input.queryPlan, record), ); } - const index = record.metadata.filterPasses - ? visibleDraft?.rankOf(record.rowId) - : undefined; + const index = passes ? visibleDraft?.rankOf(record.rowId) : undefined; if (previousIndex !== undefined && index !== undefined) { operations.push( Object.freeze({ @@ -1096,14 +1194,24 @@ export function applyFlatTransactionDraft< } if ( previous !== undefined && - sameFlatOrder(input.root.queryPlan, input.queryPlan, previous, record) + sameFlatOrder( + input.root.queryPlan, + input.queryPlan, + previous, + record, + previouslyPassed, + passes, + ) ) continue; } const frozenRows = rowDraft.freeze(); const frozenFlatRows = visibleDraft?.freeze(); + const removedRecords = effectiveRemoves.map((rowId) => + input.root.rows.get(rowId)!, + ); const groupedRemovals = [ - ...effectiveRemoves.map((rowId) => input.root.rows.get(rowId)!), + ...removedRecords, ...prepared.flatMap((record) => { const previous = input.root.rows.get(record.rowId); return previous === undefined || @@ -1112,6 +1220,8 @@ export function applyFlatTransactionDraft< input.queryPlan, previous, record, + passedPreviously(record.rowId), + passesNext(record), ) ? [] : [previous]; @@ -1126,6 +1236,8 @@ export function applyFlatTransactionDraft< input.queryPlan, previous, record, + passedPreviously(record.rowId), + passesNext(record), ) ); }); @@ -1155,7 +1267,7 @@ export function applyFlatTransactionDraft< previous: input.root, nextVisible: visible, removals: [ - ...effectiveRemoves.map((rowId) => input.root.rows.get(rowId)!), + ...removedRecords, ...prepared.flatMap((record) => { const old = input.root.rows.get(record.rowId); return old === undefined ? [] : [old]; @@ -1163,10 +1275,62 @@ export function applyFlatTransactionDraft< ], insertions: prepared, }); + // Flat roots maintain the membership bitset incrementally: clone the + // committed root's set, clear every removal, then set/clear each prepared + // record by its NEW verdict — the clone-and-flip the visible tree just + // underwent, in bit form. When the flat visible tree was carried + // unchanged, the member SET is provably unchanged too (no removal passed + // previously, and every prepared record either failed the verdict on + // both sides or `sameFlatOrder` held — verdict equality included), so + // the bitset carries by identity. Grouped + // roots keep the sentinel: their membership lives in the group index. + let visibleSlots: RevisionRoot["visibleSlots"]; + if (previousGroups !== undefined) { + visibleSlots = EMPTY_MEMBERSHIP; + } else if (visibleDraft === undefined) { + visibleSlots = input.root.visibleSlots; + } else { + const nextVisibleSlots = cloneMembership( + input.root.visibleSlots, + input.slots.capacity, + ); + for (const record of removedRecords) + clearMembershipBit(nextVisibleSlots, record.slot); + for (const record of prepared) { + if (passesNext(record)) setMembershipBit(nextVisibleSlots, record.slot); + else clearMembershipBit(nextVisibleSlots, record.slot); + } + visibleSlots = nextVisibleSlots; + } + // Success path only (a throwing accessor never reaches here): removals + // clear their slots, prepared records write theirs. No prepared record + // can share a slot with a removal in one transaction — adds allocate + // before the removals release below — so the order is free, but clears + // are listed first anyway to match `replaceFlatRowsDraft`, where a later + // write to the same slot must win. + const slotWrites: Array< + readonly [number, RowRecord | undefined] + > = [ + ...removedRecords.map((record) => [record.slot, undefined] as const), + ...prepared.map((record) => [record.slot, record] as const), + ]; + const { next: recordsBySlot, chunksTouched } = slotVectorWithAll( + input.root.recordsBySlot, + slotWrites, + input.slots.capacity, + ); + if (input.instrumentation !== undefined) + input.instrumentation.work.slotChunksTouched += chunksTouched; + // The draft is committed as effective: removed rows are permanently gone, + // so their slots go back to the free list (see the abandon-rule comment + // above for why this must not happen any earlier). + for (const record of removedRecords) input.slots.release(record.slot); return { rows: frozenRows, sourceOrder: sourceDraft.freeze(), visible, + recordsBySlot, + visibleSlots, nextSourceOrder, added: addById.size, updated: prepared.length - addById.size, @@ -1183,6 +1347,8 @@ export function applyFlatTransactionDraft< effective: true, }; } catch (error) { + // Abandoned draft: give back what it allocated, keep every removed slot. + for (const slot of allocatedSlots) input.slots.release(slot); return remap(error); } } @@ -1199,6 +1365,7 @@ export function replaceFlatRowsDraft< readonly queryPlan: CompiledQuery; readonly nextSourceOrder: number; readonly acceptSameReferenceMutation?: boolean; + readonly slots: SlotAllocator; readonly instrumentation?: LocalRowModelInstrumentation; }): RowsReplacementDraftResult { let captured: readonly TRow[]; @@ -1244,8 +1411,22 @@ export function replaceFlatRowsDraft< readonly sourceOrder: number; readonly integrity: RowRecord["integrity"]; readonly cachedMetadata?: RowRecord["metadata"]; + /** A carried id keeps the previous record's slot; a new id draws one. */ + readonly previousSlot?: number; }[] = []; const changedRecords: RowRecord[] = []; + // The NEW verdict per changed record, computed once beside the record and + // spent on the drafts below. The OLD verdict never appears here: it is read + // from `input.root`'s membership, which matters most in the + // same-reference-mutation retry, where the row OBJECT is unchanged and the + // two plans are distinct compilations of the same query — nothing keyed by + // the row could separate the two answers, but the two ROOTS are separate + // objects. + const nextVerdicts = new Map, boolean>(); + const passesNext = (record: RowRecord): boolean => + nextVerdicts.get(record)!; + const passedPreviously = (rowId: TRowId): boolean => + rowPassesFilter(input.root, rowId); const diagnostics: PretableRowIntegrityDiagnostic[] = []; let sameReferenceMutation = false; let added = 0; @@ -1283,6 +1464,7 @@ export function replaceFlatRowsDraft< sameReference && !inspection.sameReferenceMutation ? previous?.metadata : undefined, + previousSlot: previous?.slot, }); if (previous === undefined) added += 1; else updated += 1; @@ -1298,6 +1480,8 @@ export function replaceFlatRowsDraft< rows: input.root.rows, sourceOrder: input.root.sourceOrder, visible: input.root.visible, + recordsBySlot: input.root.recordsBySlot, + visibleSlots: input.root.visibleSlots, nextSourceOrder: input.nextSourceOrder, added, updated, @@ -1313,58 +1497,266 @@ export function replaceFlatRowsDraft< }; } - for (const candidate of candidates) { - const { row, rowId, sourceOrder } = candidate; - let metadata: RowRecord["metadata"]; - try { - if (candidate.cachedMetadata === undefined) { - if (input.instrumentation !== undefined) - input.instrumentation.work.rowsEvaluated += 1; - metadata = input.queryPlan.evaluate({ - rowId, - row: row as never, - sourceOrder, - }) as unknown as RowRecord["metadata"]; - } else { - metadata = rebaseSourceOrder(candidate.cachedMetadata, sourceOrder); + /* + * Abandon rule (see `applyFlatTransactionDraft` for the rationale): fresh + * allocations are recorded and released if this draft is abandoned mid-way + * (metadata evaluation runs user code and can throw). Removed rows' slots + * are NEVER released on a failure path — the committed root still owns + * them. On success they are released at the very end, except the ones + * handed straight to new rows: a set-rows call both retires and ingests + * rows in one commit, and handing a retiring row's slot to a new row is + * release-then-reuse without an allocator round trip — which also keeps a + * mid-draft throw from ever leaving a still-live row's slot on the free + * list. + */ + const allocatedSlots: number[] = []; + const reusableRemovedSlots = removedRecords.map((record) => record.slot); + const takeSlot = (): number => { + const reused = reusableRemovedSlots.pop(); + if (reused !== undefined) return reused; + const slot = input.slots.allocate(); + allocatedSlots.push(slot); + return slot; + }; + try { + for (const candidate of candidates) { + const { row, rowId, sourceOrder } = candidate; + // `evaluate` needs a slot up front (it's part of `CompiledRowInput` + // now, unread this task), but the REAL slot for a brand-new row is + // still resolved after evaluation succeeds, exactly as before this + // field existed: `takeSlot()` calls `input.slots.allocate()`, a real + // allocator mutation that bumps the high-water mark permanently (see + // `slot-allocator.ts` — capacity never shrinks), and doing that before + // a throwing accessor runs would leak capacity that a release can't + // undo. A carried row's slot has no such hazard (no allocator call), + // so it is used directly. `-1` is a placeholder for the fresh-slot + // case only: harmless because nothing reads `CompiledRowInput.slot` + // yet. + const placeholderSlot = + candidate.previousSlot !== undefined ? candidate.previousSlot : -1; + let metadata: RowRecord["metadata"]; + try { + if (candidate.cachedMetadata === undefined) { + if (input.instrumentation !== undefined) + input.instrumentation.work.rowsEvaluated += 1; + metadata = input.queryPlan.evaluate({ + rowId, + row: row as never, + sourceOrder, + slot: placeholderSlot, + }) as unknown as RowRecord["metadata"]; + } else { + metadata = rebaseSourceOrder(candidate.cachedMetadata, sourceOrder); + } + } catch (error) { + if ( + error instanceof PretableRowModelError && + error.operation !== "set-rows" + ) { + throw new PretableRowModelError(error.code, error.message, { + operation: "set-rows", + rowId: error.rowId, + columnId: error.columnId, + cause: error.cause, + }); + } + throw error; } - } catch (error) { + const slot = + candidate.previousSlot !== undefined + ? candidate.previousSlot + : takeSlot(); + const publicRow = Object.freeze({ + kind: "data" as const, + rowId, + row, + sourceIndex: sourceOrder, + depth: 0, + }); + const record = Object.freeze({ + rowId, + row, + sourceOrder, + slot, + metadata, + publicRow, + integrity: candidate.integrity, + }); + changedRecords.push(record); + // A record whose metadata was CARRIED carries its verdict too, and the + // carried verdict is the previous root's membership: `cachedMetadata` is + // only offered for an unmutated same-reference row, whose filter-column + // values are by definition the ones the committed root already judged. + // Re-running the predicate here would be a second accessor pass over + // rows this path exists to avoid re-evaluating (a pinned budget). + nextVerdicts.set( + record, + candidate.cachedMetadata === undefined + ? filterVerdict(input.queryPlan, record as never) + : passedPreviously(rowId), + ); + } + if (!effective) { + return { + rows: input.root.rows, + sourceOrder: input.root.sourceOrder, + visible: input.root.visible, + recordsBySlot: input.root.recordsBySlot, + visibleSlots: input.root.visibleSlots, + nextSourceOrder: input.nextSourceOrder, + added, + updated, + removed, + unchanged, + ignored: 0, + issues: Object.freeze([]), + diagnostics: Object.freeze(diagnostics), + operations: Object.freeze([]), + affectedRowIds: Object.freeze([]), + effective: false, + sameReferenceMutation, + }; + } + + const rowDraft = instrumentPersistentMap( + input.root.rows, + input.instrumentation, + ).asTransient(); + const sourceDraft = instrumentOrderStatisticTree( + input.root.sourceOrder, + input.instrumentation, + ).asTransient(); + const orderChangedRecords = changedRecords.filter((record) => { + const previous = input.root.rows.get(record.rowId); + return ( + previous === undefined || + !sameFlatOrder( + input.root.queryPlan, + input.queryPlan, + previous, + record, + passedPreviously(record.rowId), + passesNext(record), + ) + ); + }); + const affectedVisibleIds = new Set( + orderChangedRecords + .filter( + (record) => passedPreviously(record.rowId) || passesNext(record), + ) + .map((record) => record.rowId), + ); + for (const record of removedRecords) { + // A removed row's verdict is the one it was drawn under: membership. + if (passedPreviously(record.rowId)) affectedVisibleIds.add(record.rowId); + } + let hasUnaffectedVisible = false; + for (const entry of input.root.visible.rows.entries()) { + if (!affectedVisibleIds.has(entry.record.rowId)) { + hasUnaffectedVisible = true; + break; + } + } + const visibleDraft = + affectedVisibleIds.size === 0 + ? undefined + : instrumentOrderStatisticTree( + hasUnaffectedVisible + ? input.root.visible.rows + : createFlatVisibleTree(input.queryPlan), + input.instrumentation, + ).asTransient(); + for (const record of removedRecords) { + rowDraft.delete(record.rowId); + sourceDraft.remove(record.rowId); + if (hasUnaffectedVisible) visibleDraft?.remove(record.rowId); + } + if (hasUnaffectedVisible) { + for (const record of orderChangedRecords) { + if (passedPreviously(record.rowId)) visibleDraft?.remove(record.rowId); + } + } + for (const record of changedRecords) { + rowDraft.set(record.rowId, record); + const previous = input.root.rows.get(record.rowId); if ( - error instanceof PretableRowModelError && - error.operation !== "set-rows" + previous === undefined || + previous.sourceOrder !== record.sourceOrder ) { - throw new PretableRowModelError(error.code, error.message, { - operation: "set-rows", - rowId: error.rowId, - columnId: error.columnId, - cause: error.cause, - }); + sourceDraft.insertOrReplace( + Object.freeze({ + rowId: record.rowId, + sourceOrder: record.sourceOrder, + }), + ); } - throw error; } - const publicRow = Object.freeze({ - kind: "data" as const, - rowId, - row, - sourceIndex: sourceOrder, - depth: 0, - }); - const record = Object.freeze({ - rowId, - row, - sourceOrder, - metadata, - publicRow, - integrity: candidate.integrity, - }); - changedRecords.push(record); - } - if (!effective) { + for (const record of orderChangedRecords) { + if (passesNext(record)) { + visibleDraft?.insertOrReplace(orderedRowEntry(input.queryPlan, record)); + } + } + const frozenRows = rowDraft.freeze(); + const frozenSource = sourceDraft.freeze(); + const previousGroups = getGroupIndex(input.root.visible); + const visible = + previousGroups === undefined + ? visibleDraft === undefined + ? input.root.visible + : Object.freeze({ rows: visibleDraft.freeze() }) + : attachGroupIndex( + input.root.visible.rows, + updateGroupIndex( + previousGroups, + [ + ...removedRecords, + ...changedRecords.flatMap((record) => { + const old = input.root.rows.get(record.rowId); + return old === undefined ? [] : [old]; + }), + ], + changedRecords, + input.root.expansion.overrides, + "set-rows", + input.instrumentation, + ), + ); + // Set-rows is O(n) already, so the flat bitset is simply rebuilt from the + // final flat tree (carried or frozen — either way that tree IS the + // membership this bitset indexes). Grouped roots keep the sentinel. + const visibleSlots = + previousGroups !== undefined + ? EMPTY_MEMBERSHIP + : membershipFromFlatTree(visible.rows, input.slots.capacity); + // Success path only. ORDER IS LOAD-BEARING: the transfer pool above can + // retire a record and ingest a new one ON THE SAME SLOT in this one + // commit, so every removal-clear must precede every record-write — a + // later write to the same slot wins inside `slotVectorWithAll`. Records + // that carried unchanged keep their bindings via the COW carry. + const slotWrites: Array< + readonly [number, RowRecord | undefined] + > = [ + ...removedRecords.map((record) => [record.slot, undefined] as const), + ...changedRecords.map((record) => [record.slot, record] as const), + ]; + const { next: recordsBySlot, chunksTouched } = slotVectorWithAll( + input.root.recordsBySlot, + slotWrites, + input.slots.capacity, + ); + if (input.instrumentation !== undefined) + input.instrumentation.work.slotChunksTouched += chunksTouched; + // Committed as effective: retired slots not handed to new rows go back to + // the free list (see the abandon-rule comment above). + for (const slot of reusableRemovedSlots) input.slots.release(slot); return { - rows: input.root.rows, - sourceOrder: input.root.sourceOrder, - visible: input.root.visible, - nextSourceOrder: input.nextSourceOrder, + rows: frozenRows, + sourceOrder: frozenSource, + visible, + recordsBySlot, + visibleSlots, + nextSourceOrder: Math.max(input.nextSourceOrder, captured.length), added, updated, removed, @@ -1373,126 +1765,16 @@ export function replaceFlatRowsDraft< issues: Object.freeze([]), diagnostics: Object.freeze(diagnostics), operations: Object.freeze([]), - affectedRowIds: Object.freeze([]), - effective: false, + affectedRowIds: Object.freeze([ + ...removedRecords.map((record) => record.rowId), + ...changedRecords.map((record) => record.rowId), + ]), + effective: true, sameReferenceMutation, }; + } catch (error) { + // Abandoned draft: give back what it allocated, keep every removed slot. + for (const slot of allocatedSlots) input.slots.release(slot); + throw error; } - - const rowDraft = instrumentPersistentMap( - input.root.rows, - input.instrumentation, - ).asTransient(); - const sourceDraft = instrumentOrderStatisticTree( - input.root.sourceOrder, - input.instrumentation, - ).asTransient(); - const orderChangedRecords = changedRecords.filter((record) => { - const previous = input.root.rows.get(record.rowId); - return ( - previous === undefined || - !sameFlatOrder(input.root.queryPlan, input.queryPlan, previous, record) - ); - }); - const affectedVisibleIds = new Set( - orderChangedRecords - .filter((record) => { - const previous = input.root.rows.get(record.rowId); - return ( - previous?.metadata.filterPasses === true || - record.metadata.filterPasses - ); - }) - .map((record) => record.rowId), - ); - for (const record of removedRecords) { - if (record.metadata.filterPasses) affectedVisibleIds.add(record.rowId); - } - let hasUnaffectedVisible = false; - for (const entry of input.root.visible.rows.entries()) { - if (!affectedVisibleIds.has(entry.record.rowId)) { - hasUnaffectedVisible = true; - break; - } - } - const visibleDraft = - affectedVisibleIds.size === 0 - ? undefined - : instrumentOrderStatisticTree( - hasUnaffectedVisible - ? input.root.visible.rows - : createFlatVisibleTree(input.queryPlan), - input.instrumentation, - ).asTransient(); - for (const record of removedRecords) { - rowDraft.delete(record.rowId); - sourceDraft.remove(record.rowId); - if (hasUnaffectedVisible) visibleDraft?.remove(record.rowId); - } - if (hasUnaffectedVisible) { - for (const record of orderChangedRecords) { - if (input.root.rows.get(record.rowId)?.metadata.filterPasses) { - visibleDraft?.remove(record.rowId); - } - } - } - for (const record of changedRecords) { - rowDraft.set(record.rowId, record); - const previous = input.root.rows.get(record.rowId); - if (previous === undefined || previous.sourceOrder !== record.sourceOrder) { - sourceDraft.insertOrReplace( - Object.freeze({ rowId: record.rowId, sourceOrder: record.sourceOrder }), - ); - } - } - for (const record of orderChangedRecords) { - if (record.metadata.filterPasses) { - visibleDraft?.insertOrReplace(orderedRowEntry(input.queryPlan, record)); - } - } - const frozenRows = rowDraft.freeze(); - const frozenSource = sourceDraft.freeze(); - const previousGroups = getGroupIndex(input.root.visible); - const visible = - previousGroups === undefined - ? visibleDraft === undefined - ? input.root.visible - : Object.freeze({ rows: visibleDraft.freeze() }) - : attachGroupIndex( - input.root.visible.rows, - updateGroupIndex( - previousGroups, - [ - ...removedRecords, - ...changedRecords.flatMap((record) => { - const old = input.root.rows.get(record.rowId); - return old === undefined ? [] : [old]; - }), - ], - changedRecords, - input.root.expansion.overrides, - "set-rows", - input.instrumentation, - ), - ); - return { - rows: frozenRows, - sourceOrder: frozenSource, - visible, - nextSourceOrder: Math.max(input.nextSourceOrder, captured.length), - added, - updated, - removed, - unchanged, - ignored: 0, - issues: Object.freeze([]), - diagnostics: Object.freeze(diagnostics), - operations: Object.freeze([]), - affectedRowIds: Object.freeze([ - ...removedRecords.map((record) => record.rowId), - ...changedRecords.map((record) => record.rowId), - ]), - effective: true, - sameReferenceMutation, - }; } diff --git a/packages/row-model/src/types.ts b/packages/row-model/src/types.ts index f48dfd94b..e8f5e5796 100644 --- a/packages/row-model/src/types.ts +++ b/packages/row-model/src/types.ts @@ -145,6 +145,42 @@ export interface PretableRowModelSnapshot< ): PretableVisibleRowRef | undefined; isGroupExpanded(groupId: PretableGroupId): boolean; + /** + * Slots of the visible rows in the half-open interval `[start, end)`, in + * visible order — the dense-identity seam for the renderer's layout + * source; aligns index-for-index with {@link range}. Flat roots only: on a + * flat root every visible entry is a data row bound to a slot. A grouped + * root returns `undefined` wholesale (group rows carry no slot) and the + * caller must fall back to string identities. + * + * A slot is the row's CURRENT model binding — it stays valid only while + * the model binds that slot to the same row (updates carry it; permanent + * removal releases it for reuse). Consumers own that currency. + * + * Optional at the type seam so structural snapshot wrappers stay valid; + * every model-produced snapshot implements it. + * + * @internal + */ + ɵvisibleSlotRange?(start: number, end: number): readonly number[] | undefined; + /** + * Slot currently bound to `rowId`, or `undefined` when the id is unknown + * or the root is grouped. One row-store lookup — for k-sized paths only + * (op stamping): never call this per visible row; use + * `ɵvisibleSlotRange` for the bulk walk. + * + * @internal + */ + ɵslotOfRowId?(rowId: TRowId): number | undefined; + /** + * The slot-space size this root's slot-indexed structures were built for + * (every bound slot is `< ɵslotCapacity()`), or `undefined` on a grouped + * root — same fallback contract as `ɵvisibleSlotRange`. + * + * @internal + */ + ɵslotCapacity?(): number | undefined; + readonly query: Readonly>; readonly expansion: Readonly; } @@ -293,12 +329,19 @@ export type PretableChangeSequence = readonly toRevision: number; /** * `"reorder"` asserts the visible row set and every row's content are - * unchanged — only the order moved (a sort-only commit). Every other - * reason makes no such promise; consumers that do not understand - * `"reorder"` may treat it exactly like `"bulk-replace"`. + * unchanged — only the order moved (a sort-only commit). `"refilter"` + * asserts the opposite: membership changed (rows entered or left) + * while surviving rows kept their relative order and identities (a + * filter-only commit). Every other reason makes no such promise; + * consumers that do not understand `"reorder"` or `"refilter"` may + * treat either exactly like `"bulk-replace"`. */ readonly reason: - "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder"; + | "unknown-revision" + | "journal-evicted" + | "bulk-replace" + | "reorder" + | "refilter"; }; /** @public */ diff --git a/packages/row-model/src/visible-index.ts b/packages/row-model/src/visible-index.ts index 6e987eee4..6ba94f07e 100644 --- a/packages/row-model/src/visible-index.ts +++ b/packages/row-model/src/visible-index.ts @@ -1,5 +1,9 @@ import type { PretableRowId } from "./column-types"; -import { compareWithSortKeys, type CompiledQuery } from "./compiled-query"; +import { + compareWithSortKeys, + filterVerdict, + type CompiledQuery, +} from "./compiled-query"; import type { PretableRowModelOperation } from "./errors"; import { orderedRowEntry } from "./ordered-row-entry"; import { @@ -23,6 +27,11 @@ import type { RowRecord, VisibleIndexRoot, } from "./internal-types"; +import { + createMembership, + setMembershipBit, + type MembershipBitset, +} from "./membership-bitset"; import { createOrderStatisticTree } from "./persistent/order-statistic-tree"; import type { PersistentMap } from "./persistent/persistent-map"; import type { @@ -43,7 +52,10 @@ export function createFlatVisibleIndex< queryPlan, ).asTransient(); for (const record of records) { - if (record.metadata.filterPasses) { + // The verdict is COMPUTED here and stays local: the tree this loop fills + // IS where the answer is recorded, so storing it on the record would only + // duplicate what membership already says. + if (filterVerdict(queryPlan, record as never)) { draft.insertOrReplace(orderedRowEntry(queryPlan, record)); } } @@ -78,6 +90,29 @@ export function createVisibleIndex< ); } +/** + * Membership bitset of a FLAT visible tree: one pass, `entry.record.slot`. + * `capacity` must be the owning root's self-described `slotCapacity` (or the + * value that will become it), never the live allocator's. + */ +export function membershipFromFlatTree< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +>( + rows: VisibleIndexRoot["rows"], + capacity: number, +): MembershipBitset { + const bits = createMembership(capacity); + // `range(0, size)` rather than `entries()`: a full walk, and the tree's + // materialized non-generator walk is the cheaper way to make one — ~1ms + // against ~30ms at 50,000 rows (see `iterateEntries`). + for (const entry of rows.range(0, rows.size)) { + setMembershipBit(bits, entry.record.slot); + } + return bits; +} + export function createFlatVisibleTree< TRow extends object, TRowId extends PretableRowId, @@ -167,6 +202,12 @@ export function createFlatSnapshot< nearestVisible(grouped, ref, policy), isGroupExpanded: (groupId: PretableGroupId) => isExpanded(grouped, groupId, policy), + // Dense reads answer `undefined` wholesale on grouped roots: group rows + // carry no slot, and the seam's contract is all-or-nothing — the caller + // falls back to string identities. + ɵvisibleSlotRange: () => undefined, + ɵslotOfRowId: () => undefined, + ɵslotCapacity: () => undefined, query: root.queryPlan.query, expansion: root.expansion.state, }); @@ -215,6 +256,16 @@ export function createFlatSnapshot< return dataRef(ref.rowId); }, isGroupExpanded: () => false, + // The tree's materialized `range` walk, slots only: on a flat root every + // entry is a data row, and a record's slot is carried across updates, so + // `entry.record.slot` IS the current binding — no row-store resolution + // per row (see `membershipFromFlatTree` for the same read). + ɵvisibleSlotRange: (start: number, end: number) => + Object.freeze( + visible.range(start, end).map((entry) => entry.record.slot), + ), + ɵslotOfRowId: (rowId: TRowId) => root.rows.get(rowId)?.slot, + ɵslotCapacity: () => root.slotCapacity, query: root.queryPlan.query, expansion: root.expansion.state, });