diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index 3050ff9f99a..9988139b36b 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -532,43 +532,77 @@ export interface StoreSearchResource { // the server's own result set rather than the reconciled one, so a locally // edited or created card can't be mistaken for a short page. readonly isPartial: boolean; + // Hand a running resource the result set a document fetched since it started + // carries. Optional: a store whose resources hold no state worth superseding + // implements no supersession. + reseed?(seed: StoreSearchSeed): void; + // The identity of the seeded result set the resource holds, and `undefined` + // once a search has re-derived that set for itself. Read it to decide whether + // a seed is worth handing over: a remembered "last seed applied" would go on + // claiming a set the resource has since replaced, and would then skip a + // document restoring the earlier one. + readonly appliedSeedIdentity?: string; } -export type GetSearchResourceFuncOpts = { +// A result set a producer already resolved, handed to a search resource in +// place of running the query. Generic in the row type so a `FileDef` search +// seeds with file-meta rows rather than being narrowed to `CardDef`. +export type StoreSearchSeed = { + cards: T[]; + // What this result set is, as against any other the same query could + // produce: two seeds sharing an identity assert the same thing, so a + // resource already holding one ignores the other. It has to cover + // everything the seed asserts and not just its rows — a page-clamped + // field whose match count moved holds the same row and a different + // answer. + identity?: string; + // The index generation this set was resolved at, and the realm whose counter + // that generation belongs to. Separate from the identity and doing a + // different job: the identity says whether two sets differ, these say which + // of them is newer. A generation counts writes within one realm, so it orders + // nothing without the realm that issued it. + // + // Keeping the generation out of the identity is deliberate — a realm + // generation moves on every write anywhere in the realm, so folding it in + // would make every set look different from every other and re-apply answers + // that had not changed. + generation?: number; + realm?: string; + searchURL?: string; + realms?: string[]; + queryErrors?: Array<{ + realm: string; + type: string; + message: string; + status?: number; + }>; + // IDs the parent doc named in `relationships.{field}.data`. Used + // by the SearchResource when `cards` is empty and the parent + // skipped query-backed expansion — the resource loads each ID by + // URL instead of running a live re-query. + cardURLs?: string[]; + // The result meta the seed was resolved under, chiefly `page.total` — + // the query's match count, which exceeds `cards.length` when the page + // ceiling clamped the expansion. Absent it, the resource takes the + // record count for the total and a truncated seed reads as complete. + meta?: QueryResultsMeta; + // The seed's match count is not knowable and must not be inferred from its + // rows — the producer resolved the field but deliberately reported no + // total, as a query-backed field does when one of its realms failed. + totalUnknown?: boolean; +}; + +export type GetSearchResourceFuncOpts = { isLive?: boolean; doWhileRefreshing?: (() => void) | undefined; dependencyTracking?: RuntimeDependencyTrackingContext; - seed?: { - cards: CardDef[]; - searchURL?: string; - realms?: string[]; - queryErrors?: Array<{ - realm: string; - type: string; - message: string; - status?: number; - }>; - // IDs the parent doc named in `relationships.{field}.data`. Used - // by the SearchResource when `cards` is empty and the parent - // skipped query-backed expansion — the resource loads each ID by - // URL instead of running a live re-query. - cardURLs?: string[]; - // The result meta the seed was resolved under, chiefly `page.total` — - // the query's match count, which exceeds `cards.length` when the page - // ceiling clamped the expansion. Absent it, the resource takes the - // record count for the total and a truncated seed reads as complete. - meta?: QueryResultsMeta; - // The seed's match count is not knowable and must not be inferred from its - // rows — the producer resolved the field but deliberately reported no - // total, as a query-backed field does when one of its realms failed. - totalUnknown?: boolean; - }; + seed?: StoreSearchSeed; }; export type GetSearchResourceFunc = ( parent: object, getQuery: () => Query | undefined, getRealms?: () => string[] | undefined, - opts?: GetSearchResourceFuncOpts, + opts?: GetSearchResourceFuncOpts, ) => StoreSearchResource; export interface CardStore { diff --git a/packages/base/query-field-support.ts b/packages/base/query-field-support.ts index 52a72b4de46..d0edfe5037d 100644 --- a/packages/base/query-field-support.ts +++ b/packages/base/query-field-support.ts @@ -62,6 +62,35 @@ interface QueryFieldState { message: string; status?: number; }>; + // Identity of the result set the owner's most recent document carried. Only + // an indexer-resolved umbrella gets one — a raw source document carries no + // authoritative answer, so it can never supersede one — and comparing it + // against the identity the search resource holds is what lets a document + // fetched after the resource started hand it a fresher result set. + seedIdentity?: string; + // The index generation the owner's document was serialized at, off + // `meta.generation`, and the realm that generation counts writes in, off + // `meta.realmURL`. What the resource compares against its own result set to + // order the two, so a document read before a search that has since completed + // does not overwrite it. + // + // A floor rather than the generation the field was resolved at: read-time + // resolution does not rewrite the owner's row, and a matching card changing + // never does either, since dependencies reached only through a query context + // are left out of that row's deps. It errs toward declining a document that + // is in fact fresh. + seedGeneration?: number; + seedRealm?: string; + // A document has been captured that no resource has been offered yet. Set by + // every capture and cleared the first time a read acts on it, so a running + // resource is offered a result set once per document fetched for the owner. + // + // This is what keeps the offer tied to a document arriving rather than to a + // field being read: a search re-derives the set for itself and reports no + // seeded identity afterwards, and without this gate the next read would hand + // the last document's answer straight back over the fresher one the search + // just produced. + seedHandoverPending?: boolean; searchResource?: StoreSearchResource; renderCycleBarrier?: Promise; // The sentinel `surfaceSearchResourceErrorState` planted on the most @@ -140,6 +169,38 @@ export function ensureQueryFieldSearchResource( log.debug( `ensureQueryFieldSearchResource: reusing existing resource from fieldState for field=${field.name}`, ); + // A document fetched after the resource started carries this field resolved + // as of that read, which supersedes what the resource holds: the resource's + // own refresh is driven by realm events for the realms its query targets, + // so it is behind for a write it never heard about — a subscription gap, or + // a query whose realms don't include the one the owner was written to. + // Handing that result set over costs nothing, because the document already + // paid for the resolution. + // + // Two gates, and both are needed. The outer one spends the document: the + // offer belongs to a document arriving, so a plain read never hands an + // answer back over a search that has since produced a fresher one. The + // inner one declines an offer the resource is already holding, asking the + // resource rather than remembering the last identity handed over — a + // memory would go on claiming a set a search had replaced, and would then + // turn away the document that corrects it. + if (fieldState.seedHandoverPending) { + fieldState.seedHandoverPending = false; + let seedIdentity = fieldState.seedIdentity; + if ( + seedIdentity && + searchResource.reseed && + seedIdentity !== searchResource.appliedSeedIdentity + ) { + let seed = queryFieldSeed(fieldState); + if (seed) { + log.info( + `ensureQueryFieldSearchResource: applying refreshed seed for field=${field.name}; count=${seed.cards.length}`, + ); + searchResource.reseed(seed); + } + } + } surfaceSearchResourceErrorState( fieldState, instance, @@ -149,8 +210,7 @@ export function ensureQueryFieldSearchResource( return searchResource; } - let seedRecords = fieldState?.seedRecords; - let seedSearchURL = fieldState?.seedSearchURL; + let seedRecords = fieldState.seedRecords; let args = () => { return resolveQueryAndRealm(store, instance, field, fieldDefinition); }; @@ -204,45 +264,12 @@ export function ensureQueryFieldSearchResource( { isLive, dependencyTracking: trackingContext, - seed: seedRecords - ? { - cards: seedRecords, - searchURL: seedSearchURL ?? undefined, - realms: fieldState?.seedRealms, - queryErrors: fieldState?.seedErrors, - cardURLs: fieldState?.seedCardURLs, - // What the resource is allowed to believe about the match count, - // in order of how much is known. A count the indexer reported - // passes through as the count. Where it reported none but recorded - // a realm failure, the rows in hand are labelled a floor — that - // says both that the count is unknown and why, which is what turns - // into the field's shortfall signal. Where it reported none and no - // realm failed, the count is simply unknowable and says so. - // - // The ordering matters because the fallback is inference: absent - // any of these the resource takes the total from the record count, - // and a set short by a realm nobody could count would read as the - // whole of it — a confident number over an incomplete set, which is - // the failure this field's status exists to report rather than - // reproduce. An ordinary seed reaches that inference legitimately, - // because there nothing was withheld. - ...(fieldState?.seedTotal != null - ? { meta: { page: { total: fieldState.seedTotal } } } - : fieldState?.seedErrors?.length - ? { - meta: { - page: { total: seedRecords.length }, - incomplete: true, - }, - } - : seedSearchURL != null - ? { totalUnknown: true } - : {}), - } - : undefined, + seed: queryFieldSeed(fieldState), }, ); fieldState.searchResource = searchResource; + // The document's result set went in with the resource, so it is spent. + fieldState.seedHandoverPending = false; trackQueryFieldLoads(store, field.name, fieldState); surfaceSearchResourceErrorState(fieldState, instance, field, searchResource); // Bridge `getRelationshipMembershipState(...).isLoading` to this freshly-created resource: @@ -689,6 +716,101 @@ export function captureQueryFieldSeedData( Number.isFinite(seedTotal) ? seedTotal : undefined; + fieldState.seedIdentity = seedIdentityFor(fieldState); + // The generation the row this document was serialized from was written at, + // and the realm that counter belongs to. Absent where the serialization did + // not come off the index — a freshly built resource that was never persisted + // — in which case the field's result set is ordered by identity alone. + let meta = resource.meta as + | { generation?: unknown; realmURL?: unknown } + | undefined; + fieldState.seedGeneration = + typeof meta?.generation === 'number' ? meta.generation : undefined; + fieldState.seedRealm = + typeof meta?.realmURL === 'string' ? meta.realmURL : undefined; + fieldState.seedHandoverPending = true; +} + +// The identity of an authoritative result set. An unauthoritative one has no +// identity, so it can never be mistaken for a fresher answer than the one a +// resource holds. +// +// It covers every part of the answer `queryFieldSeed` builds, not just the +// rows: a page-clamped field gains a match it cannot surface and reports the +// same row against a higher count, and a realm that stops answering leaves the +// rows it did contribute while turning the count into a floor. Identifying a +// result set by its rows alone would call both of those the answer already in +// hand and leave the field reporting a shortfall of none. +function seedIdentityFor(fieldState: QueryFieldState): string | undefined { + if (!fieldState.seedSearchURL) { + return undefined; + } + let ids = + fieldState.seedCardURLs ?? + (fieldState.seedRecords ?? []) + .map((card) => card.id) + .filter((id) => Boolean(id)); + let unreachableRealms = (fieldState.seedErrors ?? []) + .map((error) => error.realm) + .sort(); + return [ + fieldState.seedSearchURL, + ids.join(','), + fieldState.seedTotal ?? '', + unreachableRealms.join(','), + ].join('\n'); +} + +// The result set the owner's most recent document produced, in the shape the +// search resource consumes. `undefined` when the document resolved nothing for +// this field, which is the resource's signal to answer from a live query. +// +// Shared by resource creation and supersession so both describe the same set +// the same way: the count semantics below are what the field's shortfall signal +// reads, and a supersession that inferred a different count would report a +// shortfall the document never claimed. +function queryFieldSeed(fieldState: QueryFieldState) { + let seedRecords = fieldState.seedRecords; + if (!seedRecords) { + return undefined; + } + let seedSearchURL = fieldState.seedSearchURL; + return { + cards: seedRecords, + identity: fieldState.seedIdentity, + generation: fieldState.seedGeneration, + realm: fieldState.seedRealm, + searchURL: seedSearchURL ?? undefined, + realms: fieldState.seedRealms, + queryErrors: fieldState.seedErrors, + cardURLs: fieldState.seedCardURLs, + // What the resource is allowed to believe about the match count, in order + // of how much is known. A count the indexer reported passes through as the + // count. Where it reported none but recorded a realm failure, the rows in + // hand are labelled a floor — that says both that the count is unknown and + // why, which is what turns into the field's shortfall signal. Where it + // reported none and no realm failed, the count is simply unknowable and + // says so. + // + // The ordering matters because the fallback is inference: absent any of + // these the resource takes the total from the record count, and a set short + // by a realm nobody could count would read as the whole of it — a confident + // number over an incomplete set, which is the failure this field's status + // exists to report rather than reproduce. An ordinary seed reaches that + // inference legitimately, because there nothing was withheld. + ...(fieldState.seedTotal != null + ? { meta: { page: { total: fieldState.seedTotal } } } + : fieldState.seedErrors?.length + ? { + meta: { + page: { total: seedRecords.length }, + incomplete: true, + }, + } + : seedSearchURL != null + ? { totalUnknown: true } + : {}), + }; } function resolveQueryAndRealm( diff --git a/packages/host/app/lib/gc-card-store.ts b/packages/host/app/lib/gc-card-store.ts index 2a662ab95fe..300f63fd76c 100644 --- a/packages/host/app/lib/gc-card-store.ts +++ b/packages/host/app/lib/gc-card-store.ts @@ -98,6 +98,13 @@ type StoreHooks = { // so a hop that forwards the seed field by field rather than whole // would drop it and restore exactly the inference it prevents. totalUnknown?: boolean; + // Here for the same reason: the identity is what stops a result set + // being re-applied over one the resource already holds, and a hop + // that dropped it would put that back. The generation is what + // orders the two when they do differ. + identity?: string; + generation?: number; + realm?: string; } | undefined; }, @@ -1457,7 +1464,7 @@ export default class CardStoreWithGarbageCollection implements CardStore { parent: object, getQuery: () => Query | undefined, getRealms?: () => string[] | undefined, - opts?: GetSearchResourceFuncOpts, + opts?: GetSearchResourceFuncOpts, ) { if (!this.#storeHooks?.getSearchResource) { return { diff --git a/packages/host/app/resources/search.ts b/packages/host/app/resources/search.ts index 3c44b7e796a..54ec2ed9f3c 100644 --- a/packages/host/app/resources/search.ts +++ b/packages/host/app/resources/search.ts @@ -139,6 +139,17 @@ export interface Args { seed?: | { cards: T[]; + // What this result set is, as against any other the same query could + // produce. A resource already holding a seed with this identity + // ignores the seed rather than re-applying it. + identity?: string; + // The index generation the producer resolved this set at, and the + // realm whose counter it belongs to. Together they order this set + // against one the resource already holds — a generation counts + // writes within a single realm, so it means nothing without the + // realm that issued it. + generation?: number; + realm?: string; searchURL?: string; realms?: string[]; meta?: QueryResultsMeta; @@ -216,6 +227,39 @@ export class SearchResource< #cardInitiated = false; #getDefaultRealm: (() => string | undefined) | undefined; #seedApplied = false; + // Identity of the seeded result set this resource holds, cleared once a search + // completes and re-derives that set for itself. Deliberately untracked: the + // query-field getter reads it and then hands over a seed that writes it in the + // same render, and a tracked write there would trip the backtracking + // assertion. Reading it untracked is safe because that getter also consumes + // `instances`, so a search that cleared this has already dirtied what brought + // the reader here. + #appliedSeedIdentity: string | undefined; + // Per realm, the index generation this resource's result set is known to + // reflect, as a floor. It is what orders a document against a search: both + // name a generation, so the newer of the two wins rather than whichever + // arrived last. Each entry is raised and never lowered, so a set whose + // generation cannot be established keeps the floor already established + // rather than reverting to none — that refuses a fresh document at worst, + // where the other direction would let a stale one overwrite a newer answer. + // + // Keyed by realm because a generation counts writes within one realm and + // nothing else: `realm_generations` is per `realm_url`, so a document's + // generation and a search's are the same scale only when they came from the + // same realm. A query whose realms differ from its owner's would otherwise + // compare two unrelated sequences, and whichever ran ahead would decide every + // handover for that field — either declining all of them or none. + // + // Both sides are floors rather than exact resolution generations. A search + // is stamped with the generation of the event that woke it, and the index it + // then reads is at or after that. A document is stamped with the generation + // its owner's row was written at, which a query field's own membership never + // advances: dependencies reached only through a query context are left out of + // the row's deps, so a matching card changing never rewrites the owner. Both + // err toward declining a document that is in fact fresh. Making the document + // side exact means stamping the resolution's own generation on the umbrella + // relationship where the indexer resolves it. + #resultGenerations = new Map(); // No match count is known yet and one must not be inferred. Tracked, because // `totalMatchCount` is read during render and has to re-derive when a seed or // a live search supplies a real count. @@ -464,6 +508,10 @@ export class SearchResource< `modify: prepared realms for subscription=${this.realmsToSearch.join(',')}`, ); if (seed && !this.#seedApplied) { + // Recorded before the application starts, because the application itself + // reads it to confirm it is still the seed the resource reports holding. + this.#appliedSeedIdentity = seed.identity; + this.#raiseResultGeneration(seed.realm, seed.generation); this.trackStoreLoad(this.applySeed.perform(seed), 'seed'); this.#seedApplied = true; let hasQueryErrors = seed.queryErrors && seed.queryErrors.length > 0; @@ -549,8 +597,16 @@ export class SearchResource< if (!isIncrementalIndex && event.eventName !== 'prerender_html') { return; } + // The generation the indexing pass committed, and the realm whose + // counter it belongs to. A search answering this event reads the + // index at or after it, so together they stand as the floor for + // what the result set reflects of that realm. + let generation = + 'generation' in event && typeof event.generation === 'number' + ? event.generation + : undefined; this.trackStoreLoad( - this.search.perform(this.#previousQuery), + this.search.perform(this.#previousQuery, generation, realm), 'live-refresh', ); }), @@ -586,6 +642,102 @@ export class SearchResource< return canonicalQuerySignature(query, this.network.virtualNetwork); } + // Apply a result set to a resource that is already running. A document + // fetched for the owner carries this field resolved as of that read, which + // supersedes what the resource holds: the live subscription refreshes only on + // events for the realms the query targets, so a session that never received + // one — a dropped subscription, a tab resumed after a gap, or a write to a + // realm outside the query's own set — is behind until something hands it a + // fresher answer. A result set the producer resolved in full costs nothing to + // take: the query is suppressed the same way the initial seed suppresses it, + // because the document already paid for the resolution. Only a set a realm + // failure cut short sends the field back to a live query (below). + // + // A seed asserting what this resource already holds is dropped here rather + // than at the call site, so the comparison is against the resource's own + // state and not a caller's memory of it. Supersession needs an identity to + // compare: a seed carrying none cannot be told apart from any other and is + // always applied. + // + // Both `perform`s below write ember-concurrency's tracked task state, and + // this runs from the query-field getter during a render. A consumer that + // reads `getRelationshipMembershipState(...).isLoading` for a query field + // *before* reading the field itself in the same render would see that write + // land after its own read. + reseed(seed: NonNullable['named']['seed']>): void { + if (seed.identity && seed.identity === this.#appliedSeedIdentity) { + return; + } + // Older than what this resource holds for the realm the document came from, + // so it is not news — it is a read that was already in flight when a later + // one landed. Without this the handover has no ordering at all and the last + // arrival wins, which loses a completed search to a document resolved + // before it. The comparison needs a realm and a generation on both sides; + // absent any of them the identity check above decides alone. + let seedFloor = + seed.realm != null ? this.#resultGenerations.get(seed.realm) : undefined; + if ( + seed.generation != null && + seedFloor != null && + seed.generation < seedFloor + ) { + this.#log.info( + `decline seed from generation ${seed.generation}; holding generation ${seedFloor} for realm ${seed.realm}`, + ); + return; + } + // Before the application starts, for the reason the initial seed records it + // first: the application confirms against this that it has not been + // superseded. + this.#appliedSeedIdentity = seed.identity; + this.#raiseResultGeneration(seed.realm, seed.generation); + this.trackStoreLoad(this.applySeed.perform(seed), 'seed'); + let hasQueryErrors = seed.queryErrors && seed.queryErrors.length > 0; + if (seed.searchURL && !hasQueryErrors) { + let { query: seedQuery } = parseSearchURL(seed.searchURL); + this.#previousQueryString = this.querySignature(seedQuery); + } + if (seed.realms) { + this.#previousRealms = seed.realms; + } + // A producer that could not reach every realm resolved a set that is short + // by what the failure withheld, so its rows are a floor and not an answer. + // The initial seed reaches a live query for exactly this case by leaving + // the query signature unset and falling through to `modify`'s search; run + // that query directly here, because nothing brings a running resource back + // through that fall-through — its query and realms are unchanged, so the + // next `modify` skips, and the field would stay short until an unrelated + // event. + // + // Live only, matching the initial seed: a prerender treats the document's + // relationships as the authoritative cardinality and runs no query at all, + // and a query per field per loaded card is the cascade that rules out. + if (this.#isLive && hasQueryErrors && this.#previousQuery) { + this.trackStoreLoad(this.search.perform(this.#previousQuery), 'search'); + } + } + + get appliedSeedIdentity() { + return this.#appliedSeedIdentity; + } + + // Move one realm's floor up, never down. An unknown realm or generation + // leaves every floor alone: neither says anything about what the set + // reflects, and lowering a floor to `none` would reopen the ordering the + // floor exists to close. + #raiseResultGeneration( + realm: string | undefined, + generation: number | undefined, + ): void { + if (realm == null || generation == null) { + return; + } + let current = this.#resultGenerations.get(realm); + if (current == null || generation > current) { + this.#resultGenerations.set(realm, generation); + } + } + // Both routes to a result set count as loading. A seeded resource commonly // never performs a search at all — the seed's query signature suppresses it — // so reporting only the search task would leave a window where the resource @@ -912,6 +1064,19 @@ export class SearchResource< return type !== 'card-error'; }) as unknown as T[]; } + // Another seed has been handed over since this one, and it is the set the + // resource reports holding. This task runs unbounded — two documents + // arriving close together resolve their rows concurrently — so applying + // this one now would leave the resource holding the older answer under + // the newer identity, whichever of the two happens to finish last. The + // check is inert for a producer that supplies no identity, whose seeds + // are indistinguishable by design. + if (seed.identity !== this.#appliedSeedIdentity) { + this.#log.info( + `abandon seed superseded while resolving; searchURL=${seed.searchURL}`, + ); + return; + } // The row count stands in for the total only when the seed didn't report // one AND didn't say the count is unknowable. A producer that withheld // the count deliberately sets `totalUnknown`, and inferring it from the @@ -923,81 +1088,95 @@ export class SearchResource< }, ); - private search = restartableTask(async (query: Query) => { - this.#log.info( - `search task start; realms=${this.realmsToSearch.join(',')}; query=${JSON.stringify(query)}`, - ); - // we cannot use the `waitForPromise` test waiter helper as that will cast - // the Task instance to a promise which makes it uncancellable. When this is - // uncancellable it results in a flaky test. - let token = waiter.beginAsync(); - try { - let dependencyTrackingContext = this.dependencyTrackingContext( - 'search-resource:search', + private search = restartableTask( + async (query: Query, generation?: number, realm?: string) => { + this.#log.info( + `search task start; realms=${this.realmsToSearch.join(',')}; query=${JSON.stringify(query)}`, ); + // we cannot use the `waitForPromise` test waiter helper as that will cast + // the Task instance to a promise which makes it uncancellable. When this is + // uncancellable it results in a flaky test. + let token = waiter.beginAsync(); try { - // A card-`@context` search runs card-initiated — under the page, - // realms, and concurrency caps inside `store.search`. `realmsToSearch` - // has already resolved a no-realm card search to the current realm - // (see modify). Host-internal searches pass no flag and are unbounded. - let { instances, meta } = await this.runtimeStore.search( - query, - this.realmsToSearch, - { - includeMeta: true, - dependencyTrackingContext, - cardInitiated: this.#cardInitiated, - }, - ); - this.#log.info( - `search task complete; total instances=${instances.length}; refs=${instances - .map((r) => r.id) - .join(',')}`, + let dependencyTrackingContext = this.dependencyTrackingContext( + 'search-resource:search', ); - // A completed search reports its own count, which supersedes a seed - // that had none. - this.seedTotalUnknown = false; - this._meta = meta; - this._errors = undefined; - await this.updateInstances(instances, dependencyTrackingContext); - } catch (err) { - if (didCancel(err)) { - throw err; - } - this.#log.error(`search task failed`, err); - this._errors = [searchErrorEntry(err)]; - // Zero rows, and zero is not a count of anything: the search failed as - // a unit, so what the query matches was never computed. The zero is - // kept because the loading and rendering paths need the shape, and both - // markers beside it say it is not a count — a rollup reading it would - // otherwise settle on the one number no failure can justify, and it is - // the most believable wrong answer there is, an empty result set and an - // empty match set rendering identically. - // - // Both are needed, because they are read separately and answer - // different halves. `seedTotalUnknown` is what `totalMatchCount` - // consults, so it withholds the count. `incomplete` is what the field's - // probe consults for the shortfall, and without it the count comes back - // unknown while nothing reports rows missing — "how many is unknown, - // and you hold all of them", which is the contradiction the shortfall - // signal exists to prevent. - this.seedTotalUnknown = true; - this._meta = { page: { total: 0 }, incomplete: true }; - if (this._instances.length > 0) { - try { - await this.updateInstances([], dependencyTrackingContext); - } catch (cleanupErr) { - if (didCancel(cleanupErr)) { - throw cleanupErr; + try { + // A card-`@context` search runs card-initiated — under the page, + // realms, and concurrency caps inside `store.search`. `realmsToSearch` + // has already resolved a no-realm card search to the current realm + // (see modify). Host-internal searches pass no flag and are unbounded. + let { instances, meta } = await this.runtimeStore.search( + query, + this.realmsToSearch, + { + includeMeta: true, + dependencyTrackingContext, + cardInitiated: this.#cardInitiated, + }, + ); + this.#log.info( + `search task complete; total instances=${instances.length}; refs=${instances + .map((r) => r.id) + .join(',')}`, + ); + // A completed search reports its own count, which supersedes a seed + // that had none. + this.seedTotalUnknown = false; + // This set is the search's own, so no seed describes it any more. + // Keeping the last-applied identity here would go on claiming a set + // that has been replaced, and would then turn away a document + // restoring the very set the search moved off. What orders this set + // against a later document is the generation instead. + this.#appliedSeedIdentity = undefined; + this.#raiseResultGeneration(realm, generation); + this._meta = meta; + this._errors = undefined; + await this.updateInstances(instances, dependencyTrackingContext); + } catch (err) { + if (didCancel(err)) { + throw err; + } + this.#log.error(`search task failed`, err); + this._errors = [searchErrorEntry(err)]; + // Zero rows, and zero is not a count of anything: the search failed as + // a unit, so what the query matches was never computed. The zero is + // kept because the loading and rendering paths need the shape, and both + // markers beside it say it is not a count — a rollup reading it would + // otherwise settle on the one number no failure can justify, and it is + // the most believable wrong answer there is, an empty result set and an + // empty match set rendering identically. + // + // Both are needed, because they are read separately and answer + // different halves. `seedTotalUnknown` is what `totalMatchCount` + // consults, so it withholds the count. `incomplete` is what the field's + // probe consults for the shortfall, and without it the count comes back + // unknown while nothing reports rows missing — "how many is unknown, + // and you hold all of them", which is the contradiction the shortfall + // signal exists to prevent. + this.seedTotalUnknown = true; + this._meta = { page: { total: 0 }, incomplete: true }; + // The applied seed identity deliberately survives a failure, unlike a + // completed search above. A failed search computed nothing, so what a + // document last asserted about this field is still the last thing + // anyone asserted; clearing it here would let the next read reinstate + // those rows and, with them, clear the error this failure is reporting. + if (this._instances.length > 0) { + try { + await this.updateInstances([], dependencyTrackingContext); + } catch (cleanupErr) { + if (didCancel(cleanupErr)) { + throw cleanupErr; + } + this.#log.error(`search cleanup failed`, cleanupErr); } - this.#log.error(`search cleanup failed`, cleanupErr); } } + } finally { + waiter.endAsync(token); } - } finally { - waiter.endAsync(token); - } - }); + }, + ); } // WARNING! please don't import this directly into your component's module. @@ -1024,6 +1203,9 @@ export function getSearch( seed?: | { cards: T[]; + identity?: string; + generation?: number; + realm?: string; searchURL?: string; meta?: QueryResultsMeta; errors?: ErrorEntry[]; diff --git a/packages/host/app/services/store.ts b/packages/host/app/services/store.ts index 4b3cd99b53e..33e5076437f 100644 --- a/packages/host/app/services/store.ts +++ b/packages/host/app/services/store.ts @@ -1755,6 +1755,15 @@ export default class StoreService extends Service implements StoreInterface { getDefaultRealm?: () => string | undefined; seed?: { cards: T[]; + // Declared on this hop too, for the reason `totalUnknown` is below: + // the identity is what stops a seed being re-applied over a set a + // search has since re-derived, and a field-by-field forward that + // dropped it would restore that re-application silently. + identity?: string; + // What orders a handed-over set against one the resource already + // holds: the generation, and the realm whose counter it belongs to. + generation?: number; + realm?: string; searchURL?: string; meta?: QueryResultsMeta; errors?: ErrorEntry[]; diff --git a/packages/host/tests/integration/components/query-field-membership-status-test.gts b/packages/host/tests/integration/components/query-field-membership-status-test.gts index 146602d610b..483125680c7 100644 --- a/packages/host/tests/integration/components/query-field-membership-status-test.gts +++ b/packages/host/tests/integration/components/query-field-membership-status-test.gts @@ -88,6 +88,9 @@ module('Integration | query-field relationship status', function (hooks) { 'test-cards.gts': { Person, Host }, 'Person/one.json': new Person({ name: 'Anchor' }), 'Person/two.json': new Person({ name: 'Anchor' }), + // Deliberately outside the query. A live search can never return this + // card, so its presence in the field is proof a document put it there. + 'Person/three.json': new Person({ name: 'Different' }), 'Host/anchor.json': new Host({ cardTitle: 'Anchor' }), }, }); @@ -105,6 +108,46 @@ module('Integration | query-field relationship status', function (hooks) { return host; } + // The document the realm serves for the host card, which carries every + // query-backed field resolved as of that read. + async function fetchHostDoc(): Promise { + let response = await getService('network').authedFetch(HOST_URL, { + headers: { Accept: 'application/vnd.card+json' }, + }); + return await response.json(); + } + + // Add `Person/three` to the host document's `matches`, spelled the way the + // realm spells the members it already carries: the resolved instance in + // `included`, an id in the umbrella relationship's `data`, a per-member + // `matches.N` entry (which is what the deserializer reads), and a match total + // covering all three, so the document describes three matches rather than + // contradicting itself. + function spliceThirdMember(hostDoc: any): void { + let relationships = hostDoc.data.relationships; + let template = hostDoc.included?.find((resource: { id: string }) => + resource.id.endsWith('Person/one'), + ); + if (!template) { + throw new Error( + `expected the host document to carry Person/one as a resolved member of 'matches'; included ids were ${JSON.stringify( + (hostDoc.included ?? []).map((r: { id: string }) => r.id), + )}`, + ); + } + let spliced = JSON.parse(JSON.stringify(template)); + spliced.id = template.id.replace('Person/one', 'Person/three'); + hostDoc.included.push(spliced); + + let umbrella = relationships.matches; + umbrella.data.push({ type: 'card', id: spliced.id }); + umbrella.meta = { ...umbrella.meta, total: umbrella.data.length }; + relationships[`matches.${umbrella.data.length - 1}`] = { + links: { self: spliced.id }, + data: { type: 'card', id: spliced.id }, + }; + } + test('a singular query-backed field reports the one slot it surfaces, not the whole result set', async function (this: RenderingTestContext, assert) { let { getRelationshipMembershipState } = cardApi; let host = (await loadHost()) as CardDefType & { favorite: unknown }; @@ -262,6 +305,214 @@ module('Integration | query-field relationship status', function (hooks) { assert.false(status.isPartial); }); + test('a document fetched after the field resolved hands it the fresher result set', async function (this: RenderingTestContext, assert) { + let { getRelationshipMembershipState, updateFromSerialized } = cardApi; + let host = await loadHost(); + + assert.strictEqual( + getRelationshipMembershipState(host, 'matches').membership?.length, + 2, + 'the field resolves to the two cards the query matches', + ); + + assert.strictEqual( + getRelationshipMembershipState(host, 'matches').totalMatchCount, + 2, + 'and reports the count the indexer resolved it under', + ); + + // Start from the document the realm actually serves — the server resolves + // query fields at read time, so this carries `matches` already resolved — + // and splice in a third member. + let hostDoc = await fetchHostDoc(); + spliceThirdMember(hostDoc); + + await updateFromSerialized(host as any, hostDoc); + await settled(); + + // The count is what proves the document reached the resource. Membership + // cannot: deserializing the document deposits the spliced member in the + // store, and a live search's result set is reconciled against store + // residency, so the client-side merge adds a resident matching card to the + // displayed set whether or not anything superseded the result set. The + // match count comes off the resource's own meta, which only an applied + // result set moves. + let matches = getRelationshipMembershipState(host, 'matches'); + assert.strictEqual( + matches.totalMatchCount, + 3, + 'the newer document supersedes the result set the resource was holding', + ); + assert.strictEqual( + matches.membership?.length, + 3, + 'and the field surfaces all three members', + ); + assert.true( + matches.membership?.some( + (member) => + member.kind === 'present' && + member.reference.endsWith('Person/three'), + ), + 'including the one the document introduced', + ); + }); + + test('a document reporting a higher match count refreshes a page-clamped field', async function (this: RenderingTestContext, assert) { + let { getRelationshipMembershipState, updateFromSerialized } = cardApi; + let host = await loadHost(); + + let before = getRelationshipMembershipState(host, 'firstMatch'); + assert.strictEqual(before.totalMatchCount, 2, 'two matches are reported'); + + // `firstMatch` holds one row whatever its query matches, so a match count + // that moves leaves its rows and its query URL untouched — the count is the + // only thing that changed, and it is what says the rows fall short. + let hostDoc = await fetchHostDoc(); + hostDoc.data.relationships.firstMatch.meta = { total: 3 }; + + await updateFromSerialized(host as any, hostDoc); + await settled(); + + let after = getRelationshipMembershipState(host, 'firstMatch'); + assert.strictEqual( + after.membership?.length, + 1, + 'the field still holds the one row its page allowed', + ); + assert.strictEqual( + after.totalMatchCount, + 3, + 'against the count the newer document reports', + ); + assert.true(after.isPartial, 'so the shortfall is still reported'); + + // The field tracks whichever document arrived last rather than latching on + // the identities it has seen, so a count that moves back is applied too. + await updateFromSerialized(host as any, await fetchHostDoc()); + await settled(); + assert.strictEqual( + getRelationshipMembershipState(host, 'firstMatch').totalMatchCount, + 2, + 'and a document restoring the earlier count is applied in turn', + ); + }); + + test('a document resolved before the result set the field holds is declined', async function (this: RenderingTestContext, assert) { + let { getRelationshipMembershipState, updateFromSerialized } = cardApi; + let host = await loadHost(); + + // Derived from whatever the realm stamped, so the two documents order + // against each other and against the one the field already resolved from, + // whether or not this harness stamps a generation at all. + let hostDoc = await fetchHostDoc(); + let newerGeneration = (hostDoc.data.meta.generation ?? 0) + 10; + hostDoc.data.meta.generation = newerGeneration; + spliceThirdMember(hostDoc); + await updateFromSerialized(host as any, hostDoc); + await settled(); + assert.strictEqual( + getRelationshipMembershipState(host, 'matches').totalMatchCount, + 3, + 'the field holds the set that document resolved', + ); + + // A read that was already in flight when the newer one landed. It is not + // news, and taking it would walk the field backwards — the failure the + // handover has to avoid, because it has no other way to tell a document + // that arrived late from one that resolved late. + let staleDoc = await fetchHostDoc(); + staleDoc.data.meta.generation = newerGeneration - 1; + + await updateFromSerialized(host as any, staleDoc); + await settled(); + + assert.strictEqual( + getRelationshipMembershipState(host, 'matches').totalMatchCount, + 3, + 'the older document does not displace it', + ); + }); + + // Guards the gate rather than the handover: it holds trivially where nothing + // is handed over at all, and its job is to catch a gate that stops requiring + // an indexer-resolved umbrella. + test('a document that did not resolve the field cannot displace a result set', async function (this: RenderingTestContext, assert) { + let { getRelationshipMembershipState, updateFromSerialized } = cardApi; + let host = await loadHost(); + + // `links.search` is written only where the indexer resolved the field, so + // its absence marks a relationship this document is not authoritative + // about — a raw source file's own `data`, say. Stripping it while narrowing + // the field to a single member makes displacement unmistakable: a field + // that took this document's word for its membership would drop to one. + let hostDoc = await fetchHostDoc(); + let relationships = hostDoc.data.relationships; + delete relationships.matches.links.search; + relationships.matches.data = [relationships.matches.data[0]]; + relationships.matches.meta = { total: 1 }; + delete relationships['matches.1']; + + await updateFromSerialized(host as any, hostDoc); + await settled(); + + assert.strictEqual( + getRelationshipMembershipState(host, 'matches').membership?.length, + 2, + 'the field keeps the result set the indexer resolved', + ); + }); + + test('a document reporting an unreachable realm sends the field back to a live query', async function (this: RenderingTestContext, assert) { + let { getRelationshipMembershipState, updateFromSerialized } = cardApi; + let network = getService('network'); + let host = await loadHost(); + assert.strictEqual( + getRelationshipMembershipState(host, 'matches').membership?.length, + 2, + 'the field resolves from the document without querying', + ); + + let searchRequests: string[] = []; + let spy = async (request: Request) => { + if (new URL(request.url).pathname.endsWith('/_federated-search')) { + searchRequests.push(request.url); + } + // Fall through to the realm-server mock. + return null; + }; + network.virtualNetwork.mount(spy, { prepend: true }); + try { + // A realm that failed contributes its error and no rows, so what this + // document carries is a floor rather than an answer. Resolving from it + // and stopping there would leave the field short with nothing scheduled + // to correct it. + let hostDoc = await fetchHostDoc(); + let matches = hostDoc.data.relationships.matches; + matches.meta = { + ...matches.meta, + errors: [ + { + realm: 'http://unreachable-realm/test/', + type: 'realm-unreachable', + message: 'realm did not answer', + }, + ], + }; + + await updateFromSerialized(host as any, hostDoc); + await settled(); + + assert.strictEqual( + searchRequests.length, + 1, + 'the field runs the query the failed realm left unanswered', + ); + } finally { + network.virtualNetwork.unmount(spy); + } + }); + test('a declared linksToMany reports no match count', async function (this: RenderingTestContext, assert) { let { getRelationshipMembershipState } = cardApi; let host = await loadHost();