diff --git a/CHANGELOG.md b/CHANGELOG.md index 95bc6f304..04f2289e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ This file tracks product releases for Roomote (single monorepo version). Automated release entries are prepended by `pnpm run version`. +## 0.45.1 (2026-08-29) + +This patch restores complete Notion database discovery across Memory and the built-in Notion MCP. + +### Highlights + +- Find and ingest pages inside directly shared Notion databases even when Notion search omits them. + +### Patch changes + +- Discover pages inside directly shared Notion databases in Memory and let agents resolve the database through the Notion MCP even when Notion search omits its rows. + ## 0.45.0 (2026-08-27) This release adds secure hosted trial inference and self-run Brain model options, expands GLM 5.3 support, and improves reliability across Fast sessions, pull-request reviews, Memory, and chat. diff --git a/apps/api/src/handlers/mcp/notion/tools.ts b/apps/api/src/handlers/mcp/notion/tools.ts index 00a5cf188..2faf41b55 100644 --- a/apps/api/src/handlers/mcp/notion/tools.ts +++ b/apps/api/src/handlers/mcp/notion/tools.ts @@ -79,7 +79,7 @@ function registerSearchTool( { title: 'Search Notion', description: - 'Search pages and data sources explicitly shared with the deployment Notion integration.', + 'Search pages and data sources explicitly shared with the deployment Notion integration. Pages that live inside databases are often missing from search results: to find them, locate the data source (object_type "data_source") and list its rows with notion-query-data-sources.', inputSchema: { query: z.string().optional(), object_type: z.enum(['page', 'data_source']).optional(), @@ -118,10 +118,12 @@ function registerFetchTool( { title: 'Fetch Notion Content', description: - 'Fetch a page, data source, or block explicitly shared with the deployment Notion integration. Pages include enhanced Markdown content; blocks include one page of child blocks.', + 'Fetch a page, database, data source, or block explicitly shared with the deployment Notion integration. Pages include enhanced Markdown content; databases list their data sources (query rows with notion-query-data-sources); blocks include one page of child blocks.', inputSchema: { id: nonEmptyStringSchema, - object_type: z.enum(['page', 'data_source', 'block']).default('page'), + object_type: z + .enum(['page', 'database', 'data_source', 'block']) + .default('page'), include_transcript: z.boolean().optional(), ...paginationSchema, }, @@ -136,6 +138,13 @@ function registerFetchTool( page_size, }) => { const encodedId = encodeURIComponent(id); + if (objectType === 'database') { + const database = await notionApiRequestJson>({ + config, + path: `databases/${encodedId}`, + }); + return toMcpToolResult({ database }); + } if (objectType === 'data_source') { const dataSource = await notionApiRequestJson>({ config, diff --git a/apps/bullmq/src/scheduled-jobs/__tests__/brain-notion.test.ts b/apps/bullmq/src/scheduled-jobs/__tests__/brain-notion.test.ts index 91ad9a321..822ce94d0 100644 --- a/apps/bullmq/src/scheduled-jobs/__tests__/brain-notion.test.ts +++ b/apps/bullmq/src/scheduled-jobs/__tests__/brain-notion.test.ts @@ -360,7 +360,9 @@ describe('Notion traversal discovery', () => { mode: 'traverse' as const, lastSweepAt: '2026-08-27T00:00:00.000Z', scanStartedAt: '2026-08-27T00:00:00.000Z', - traverse: { afterItemId: '', pending: [] }, + // Most tests focus on the block/inventory walk; the shared data-source + // enumeration that runs first has its own tests below. + traverse: { afterItemId: '', pending: [], dataSourcesDone: true }, }; const pageObject = (id: string, title: string) => ({ object: 'page', @@ -521,6 +523,98 @@ describe('Notion traversal discovery', () => { expect(result.pages.map((page) => page.title)).toEqual(['Row page']); }); + it('discovers rows of directly-shared databases search never surfaced', async () => { + const dataSource = '99991111-0000-0000-0000-0000000000d5'; + const row = '99991111-0000-0000-0000-000000000101'; + const searchBodies: unknown[] = []; + mockNotionApiRequestJson.mockImplementation(async ({ path, body }) => { + if (path === 'search') { + searchBodies.push(body); + return { + results: [{ object: 'data_source', id: dataSource }], + has_more: false, + }; + } + if (path === `data_sources/${encodeURIComponent(dataSource)}/query`) { + return { + results: [pageObject(row, 'Shared database row')], + has_more: false, + }; + } + if (path === `pages/${encodeURIComponent(row)}`) { + return pageObject(row, 'Shared database row'); + } + if (path === `pages/${encodeURIComponent(row)}/markdown`) { + return { markdown: 'Row body' }; + } + if (path.startsWith('blocks/')) { + return { results: [], has_more: false }; + } + throw new Error(`unexpected path ${path}`); + }); + + const result = await collectNotionTraversal({ + config, + saved: { + ...savedTraverse, + traverse: { afterItemId: '', pending: [] }, + }, + limit: 10, + }); + + expect(searchBodies).toEqual([ + expect.objectContaining({ + filter: { property: 'object', value: 'data_source' }, + }), + ]); + expect(result.pages.map((page) => page.title)).toEqual([ + 'Shared database row', + ]); + expect(JSON.parse(result.stateUpdates![0]!.cursor as string)).toMatchObject( + { mode: 'idle' }, + ); + }); + + it('restarts data-source enumeration when its search cursor expires', async () => { + const searchCursors: unknown[] = []; + mockNotionApiRequestJson.mockImplementation(async ({ path, body }) => { + if (path === 'search') { + const cursor = (body as { start_cursor?: string }).start_cursor; + searchCursors.push(cursor ?? null); + if (cursor) { + throw new NotionApiError( + 'cursor expired', + 400, + 'validation_error', + null, + ); + } + return { results: [], has_more: false }; + } + throw new Error(`unexpected path ${path}`); + }); + + const result = await collectNotionTraversal({ + config, + saved: { + ...savedTraverse, + traverse: { + afterItemId: '', + pending: [], + dataSourceCursor: 'stale-cursor', + }, + }, + limit: 10, + }); + + // The stale cursor restarts the enumeration from the top instead of + // wedging the pass, and the cycle still completes. + expect(searchCursors).toEqual(['stale-cursor', null]); + expect(JSON.parse(result.stateUpdates![0]!.cursor as string)).toMatchObject( + { mode: 'idle' }, + ); + }); + it('descends into any block with children, not just a container allowlist', async () => { const parent = 'ffff6666-0000-0000-0000-000000000001'; const paragraph = 'ffff6666-0000-0000-0000-0000000000b1'; diff --git a/apps/bullmq/src/scheduled-jobs/brain-collectors/notion-pages.ts b/apps/bullmq/src/scheduled-jobs/brain-collectors/notion-pages.ts index 61af28228..496e4df41 100644 --- a/apps/bullmq/src/scheduled-jobs/brain-collectors/notion-pages.ts +++ b/apps/bullmq/src/scheduled-jobs/brain-collectors/notion-pages.ts @@ -79,9 +79,10 @@ const NOTION_MAX_SEARCH_REQUESTS_PER_PASS = 10; * directly-shared content is guaranteed; children reachable through a shared * parent may never appear (see * https://developers.notion.com/reference/search-optimizations-and-limitations). - * After each sweep+reconcile cycle the collector therefore walks the block - * tree and data sources of every inventoried page, discovering - * inheritance-shared pages the search index missed. + * After each sweep+reconcile cycle the collector therefore enumerates every + * data source the integration can see and walks the block tree of every + * inventoried page, discovering inheritance-shared pages — database rows + * above all — that the search index missed. */ const NOTION_MAX_TRAVERSAL_REQUESTS_PER_PASS = 24; const NOTION_TRAVERSAL_SEED_BATCH = 25; @@ -761,6 +762,23 @@ export function buildNotionSearchBody( }; } +function buildNotionDataSourceSearchBody( + cursor: string | null, +): Record { + return { + filter: { property: 'object', value: 'data_source' }, + page_size: NOTION_SEARCH_PAGE_SIZE, + ...(cursor ? { start_cursor: cursor } : {}), + }; +} + +function isNotionSearchDataSource( + value: unknown, +): value is { object: 'data_source'; id: string } { + const record = asObject(value); + return !!record && record.object === 'data_source' && !!asString(record.id); +} + async function fetchNotionPage( config: McpConnectionNotionConfig, page: NotionSearchPage, @@ -803,6 +821,10 @@ type NotionTraverseState = { afterItemId: string; /** Discovered containers awaiting expansion, bounded. */ pending: NotionTraverseNode[]; + /** Search cursor for the shared data-source enumeration, when paused. */ + dataSourceCursor?: string; + /** True once every directly-shared data source is enqueued this cycle. */ + dataSourcesDone?: boolean; }; type NotionScanCursor = { @@ -918,11 +940,13 @@ function isNotionBlock(value: unknown): value is NotionBlock { /** * Walk the inventory's block trees and data sources, emitting pages the - * search-based sweep never surfaced. The seed side iterates - * brain_collector_items by durable id cursor (no queue growth); only - * discovered containers carry over between passes, bounded by - * NOTION_TRAVERSAL_MAX_PENDING. A pass ends when its request budget or page - * limit is spent; completion flips the scan back to idle. + * search-based sweep never surfaced. Directly-shared data sources are + * enumerated first via search (their rows inherit access but are the classic + * search-index gap), then the seed side iterates brain_collector_items by + * durable id cursor (no queue growth); only discovered containers carry over + * between passes, bounded by NOTION_TRAVERSAL_MAX_PENDING. A pass ends when + * its request budget or page limit is spent; completion flips the scan back + * to idle. */ export async function collectNotionTraversal(input: { config: McpConnectionNotionConfig; @@ -936,6 +960,8 @@ export async function collectNotionTraversal(input: { pending: [], }; let afterItemId = state.afterItemId; + let dataSourceCursor = state.dataSourceCursor ?? null; + let dataSourcesDone = state.dataSourcesDone === true; const pending: NotionTraverseNode[] = [...state.pending]; const pages: CollectorPage[] = []; const itemUpdates: CollectorItemUpdate[] = []; @@ -1083,6 +1109,46 @@ export async function collectNotionTraversal(input: { const node = pending.shift(); if (!node) { + if (!dataSourcesDone) { + // Rows of a database shared directly with the integration inherit + // access but rarely reach the search index, and no inventoried page + // holds them as child_database blocks — search is the only way to + // find those data sources at all. + requests++; + let found: NotionSearchResponse; + try { + found = await notionCollectorRequest({ + config: input.config, + path: 'search', + method: 'POST', + body: buildNotionDataSourceSearchBody(dataSourceCursor), + }); + } catch (error) { + if ( + dataSourceCursor && + error instanceof NotionApiError && + error.status === 400 + ) { + // Notion pagination cursors expire; restart the enumeration. + dataSourceCursor = null; + continue; + } + throw error; + } + for (const raw of found.results ?? []) { + if (isNotionSearchDataSource(raw)) { + pushPending({ kind: 'data_source', id: raw.id }); + } + } + const nextCursor = + found.has_more && asString(found.next_cursor) + ? found.next_cursor!.trim() + : null; + dataSourceCursor = nextCursor; + dataSourcesDone = nextCursor === null; + continue; + } + const batch = await listBrainCollectorItemsAfter( db, NOTION_PAGES_COLLECTOR_ID, @@ -1259,6 +1325,8 @@ export async function collectNotionTraversal(input: { traverse: { afterItemId, pending: pending.slice(0, NOTION_TRAVERSAL_MAX_PENDING), + ...(dataSourceCursor ? { dataSourceCursor } : {}), + ...(dataSourcesDone ? { dataSourcesDone: true } : {}), }, }, ), diff --git a/apps/docs/integrations/notion.mdx b/apps/docs/integrations/notion.mdx index 1c7824081..09baeb90f 100644 --- a/apps/docs/integrations/notion.mdx +++ b/apps/docs/integrations/notion.mdx @@ -40,7 +40,10 @@ When Memory is enabled, Roomote also backfills the pages shared with this integration and keeps their Markdown snapshots current. Notion pages are stored under the `notion/` namespace. New and edited pages are picked up on regular Memory collector ticks, and a daily full sweep discovers older pages -that were newly shared without being edited. The same sweep replaces pages +that were newly shared without being edited. Because Notion's search index +does not reliably surface pages that live inside databases, the sweep also +enumerates every shared data source and walks page trees to capture database +rows and other inheritance-shared pages. The same sweep replaces pages that are no longer shared with unavailable tombstones, so their former content is no longer retained in Memory search results. diff --git a/package.json b/package.json index f3b9f26ee..66c0d4076 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "roomote", - "version": "0.45.0", + "version": "0.45.1", "license": "FCL-1.0-ALv2", "packageManager": "pnpm@10.29.3", "engines": {