diff --git a/.github/workflows/generate-builder-feeds.yml b/.github/workflows/generate-builder-feeds.yml new file mode 100644 index 0000000..ff22048 --- /dev/null +++ b/.github/workflows/generate-builder-feeds.yml @@ -0,0 +1,75 @@ +name: Generate builder feeds + +on: + schedule: + # Daily at 01:15 UTC + - cron: '15 1 * * *' + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: generate-builder-feeds + cancel-in-progress: false + +jobs: + generate: + name: generate · publish feeds + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Use the triggering ref so workflow_dispatch from a feature branch + # can publish with that branch's generator/sources. + ref: ${{ github.sha }} + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Generate public feeds + run: npm run generate:builder-feeds + + - name: Publish to feeds branch + run: | + set -euo pipefail + PUBLISH_DIR="${RUNNER_TEMP}/zero-tab-feeds" + rm -rf "$PUBLISH_DIR" + mkdir -p "$PUBLISH_DIR" + + if git ls-remote --exit-code --heads origin feeds >/dev/null 2>&1; then + git fetch origin feeds + git worktree add "$PUBLISH_DIR" origin/feeds + git -C "$PUBLISH_DIR" checkout -B feeds + else + git worktree add --detach "$PUBLISH_DIR" + git -C "$PUBLISH_DIR" checkout --orphan feeds + git -C "$PUBLISH_DIR" rm -rf . >/dev/null 2>&1 || true + fi + + cp builder-feeds/generated/feed-x.json "$PUBLISH_DIR/feed-x.json" + cp builder-feeds/generated/feed-blogs.json "$PUBLISH_DIR/feed-blogs.json" + cp builder-feeds/generated/feed-podcasts.json "$PUBLISH_DIR/feed-podcasts.json" + cp builder-feeds/generated/feed-videos.json "$PUBLISH_DIR/feed-videos.json" + cp builder-feeds/generated/generation-report.json "$PUBLISH_DIR/generation-report.json" + + git -C "$PUBLISH_DIR" config user.name "github-actions[bot]" + git -C "$PUBLISH_DIR" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git -C "$PUBLISH_DIR" add feed-x.json feed-blogs.json feed-podcasts.json feed-videos.json generation-report.json + + if git -C "$PUBLISH_DIR" diff --cached --quiet; then + echo "No feed changes to publish." + else + git -C "$PUBLISH_DIR" commit -m "Update builder feeds $(date -u +%Y-%m-%dT%H:%M:%SZ)" + git -C "$PUBLISH_DIR" push origin HEAD:feeds + fi diff --git a/.gitignore b/.gitignore index d344ec9..9a2994b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ dist/ # Personal config (landing page patterns, etc.) — never push to GitHub extension/config.local.js + +# Local feed generation output; published only on the feeds branch +builder-feeds/generated/ +tests/fixtures/out/ diff --git a/AGENTS.md b/AGENTS.md index d7ac442..2c18742 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ Before doing anything technical, tell the user what they're about to get: > - **Save for later** bookmark individual tabs to a checklist before closing them > - **Modular workspace cards** can be collapsed or hidden, with preferences stored locally > - **Daily horoscope** generates a private, deterministic reading from the local date and selected zodiac sign -> - **AI Builder daily brief** opens as an independent full-height drawer with public updates from builders, podcasts, and engineering blogs +> - **AI Builder daily brief** opens as an independent full-height drawer with public updates from builders, podcasts, engineering blogs, and conference videos > - **Local-first** tab and saved-item data never leaves the browser > > It's just a Chrome extension. Setup takes about 1 minute. @@ -102,5 +102,5 @@ Once the extension is loaded: - Run `npm run build` and load `dist/extension/` in Chrome. - Saved tabs are stored in `chrome.storage.local` (persists across sessions). - Tab management is fully local. Open-tab and saved-tab data is never uploaded. -- The optional AI Builder digest requests access only to `raw.githubusercontent.com`, fetches public feeds at most once per local day, and stores a compact cache locally. +- The optional AI Builder digest requests access only to `raw.githubusercontent.com`, fetches public feeds from this repository's `feeds` branch at most once per local day, and stores a compact cache locally. - To update: `cd zero-tab && git pull && npm install && npm run build`, then reload the extension in `chrome://extensions`. diff --git a/README.md b/README.md index 5ce46c1..48e389b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ No server and no account are required. Open-tab URLs, titles, and saved links st - Local, deterministic daily horoscope by zodiac sign - Localhost port labels - macOS-inspired light and dark themes -- Full-height AI Builder Daily Report drawer sourced from public Follow Builders feeds +- Full-height AI Builder Daily Report drawer sourced from public Zero Tab feeds - Optional on-device translation through Chrome's built-in Translator API ## Install locally @@ -46,7 +46,7 @@ No server and no account are required. Open-tab URLs, titles, and saved links st Core tab management makes no external requests. Tab URLs, titles, Saved for later items, read state, and preferences are stored locally using Chrome extension storage. -AI Builder Daily Report is disabled until the user enables it. When enabled, Zero Tab requests optional access to `raw.githubusercontent.com` and downloads three public Follow Builders JSON feeds at most once per local calendar day. No tab, browsing, saved-link, identifier, or API-key data is included in those requests. +AI Builder Daily Report is disabled until the user enables it. When enabled, Zero Tab requests optional access to `raw.githubusercontent.com` and downloads three public JSON feeds published by this repository's `feeds` branch at most once per local calendar day. No tab, browsing, saved-link, identifier, or API-key data is included in those requests. Translation uses Chrome's built-in on-device Translator API when available. The language model or language pack may be downloaded by Chrome, but report text is not sent to a third-party translation service by Zero Tab. @@ -87,6 +87,16 @@ The previous Vanilla implementation remains in `extension/` as a migration reference while the remaining publishing assets are moved to the new source tree. +## Builder feed generation + +Zero Tab publishes AI Builder JSON feeds from this repository's `feeds` branch. + +```bash +npm run generate:builder-feeds +``` + +Sources live in [`builder-feeds/sources.json`](builder-feeds/sources.json). GitHub Actions runs [`.github/workflows/generate-builder-feeds.yml`](.github/workflows/generate-builder-feeds.yml) daily and on `workflow_dispatch`, then commits `feed-x.json`, `feed-blogs.json`, `feed-podcasts.json`, and `feed-videos.json` to the `feeds` branch root. The extension reads those files from `raw.githubusercontent.com`. + ## Attribution and license Zero Tab began as a fork of Zara Zhang's MIT-licensed original project. diff --git a/builder-feeds/README.md b/builder-feeds/README.md new file mode 100644 index 0000000..0a57fe1 --- /dev/null +++ b/builder-feeds/README.md @@ -0,0 +1,9 @@ +# Builder feeds + +Curated source list and local generation output for the AI Builder Daily Report. + +- [`sources.json`](sources.json) — X handles, blogs (RSS and/or HTML), podcasts (RSS), conference videos (YouTube RSS) +- `generated/` — local output of `npm run generate:builder-feeds` (gitignored) +- Published artifacts live on the repository `feeds` branch root + +Generation is owned by Zero Tab. It does not call Follow Builders services or APIs. diff --git a/builder-feeds/sources.json b/builder-feeds/sources.json new file mode 100644 index 0000000..215cc3a --- /dev/null +++ b/builder-feeds/sources.json @@ -0,0 +1,70 @@ +{ + "version": 1, + "x": [ + { "name": "Boris Cherny", "handle": "bcherny" }, + { "name": "Anthropic", "handle": "AnthropicAI" }, + { "name": "Guillermo Rauch", "handle": "rauchg" }, + { "name": "Simon Willison", "handle": "simonw" }, + { "name": "Addy Osmani", "handle": "addyosmani" }, + { "name": "Swyx", "handle": "swyx" }, + { "name": "Andrej Karpathy", "handle": "karpathy" } + ], + "blogs": [ + { + "name": "Anthropic Engineering", + "url": "https://www.anthropic.com/engineering" + }, + { + "name": "Anthropic News", + "url": "https://www.anthropic.com/news" + }, + { + "name": "OpenAI Blog", + "url": "https://openai.com/blog", + "rssUrl": "https://openai.com/blog/rss.xml" + }, + { + "name": "Simon Willison", + "url": "https://simonwillison.net/", + "rssUrl": "https://simonwillison.net/atom/everything/" + }, + { + "name": "Julia Evans", + "url": "https://jvns.ca/", + "rssUrl": "https://jvns.ca/atom.xml" + }, + { + "name": "Latent Space", + "url": "https://www.latent.space/", + "rssUrl": "https://www.latent.space/feed" + }, + { + "name": "Cursor Blog", + "url": "https://cursor.com/blog" + } + ], + "podcasts": [ + { + "name": "Latent Space", + "rssUrl": "https://api.substack.com/feed/podcast/1084089.rss" + }, + { + "name": "Practical AI", + "rssUrl": "https://changelog.com/practicalai/feed" + } + ], + "videos": [ + { + "name": "AI Engineer", + "rssUrl": "https://www.youtube.com/feeds/videos.xml?channel_id=UCLKPca3kwwd-B59HNr-_lvA" + }, + { + "name": "Cursor Compile", + "rssUrl": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLuI2ZfvGpzwDCn0njJpjZ3ZiEpCqhN7BK" + }, + { + "name": "Figma Config", + "rssUrl": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLXDU_eVOJTx6erPKfFHtCNbyCmcCn4zrp" + } + ] +} diff --git a/docs/privacy.html b/docs/privacy.html index 4925fbc..d953d38 100644 --- a/docs/privacy.html +++ b/docs/privacy.html @@ -29,7 +29,7 @@

Local storage

Saved links, preferences, AI Builder Daily Report cache, read state, and translations are stored in chrome.storage.local. Zero Tab does not operate a server and does not synchronize this data to the publisher.

AI Builder Daily Report

-

This optional feature is disabled until the user enables it. When enabled, Zero Tab downloads public JSON feeds from https://raw.githubusercontent.com/. These requests do not contain tab data, saved links, browsing history, identifiers, credentials, or API keys.

+

This optional feature is disabled until the user enables it. When enabled, Zero Tab downloads public JSON feeds from https://raw.githubusercontent.com/beforeload/zero-tab/feeds/. These requests do not contain tab data, saved links, browsing history, identifiers, credentials, or API keys.

On-device translation

When supported and explicitly requested, Zero Tab uses Chrome's built-in Translator API. Chrome may download a language pack or model. Zero Tab does not send report text to a third-party translation API.

diff --git a/extension/app.js b/extension/app.js index dcb220c..0cfe70f 100644 --- a/extension/app.js +++ b/extension/app.js @@ -1313,6 +1313,7 @@ function formatDigestDate(dateStr) { function digestKindLabel(kind) { if (kind === 'blog') return 'Blog'; if (kind === 'podcast') return 'Podcast'; + if (kind === 'video') return 'Video'; return 'X'; } @@ -1372,7 +1373,7 @@ function renderDigestItemCard(item, state, stale, targetLanguage) { const safeExcerpt = escapeHtml(localized?.excerpt || item.excerpt); const safeSource = escapeHtml(item.source); const safeItemId = escapeHtml(item.id); - const kind = ['x', 'blog', 'podcast'].includes(item.kind) ? item.kind : 'x'; + const kind = ['x', 'blog', 'podcast', 'video'].includes(item.kind) ? item.kind : 'x'; const dateLabel = escapeHtml(formatDigestDate(item.publishedAt)); const isRead = state.readIds?.includes(item.id); diff --git a/extension/builder-digest.js b/extension/builder-digest.js index 28f4680..c773c57 100644 --- a/extension/builder-digest.js +++ b/extension/builder-digest.js @@ -8,12 +8,14 @@ const CACHE_KEY = 'builderDigestState'; const OPTIONAL_ORIGIN = 'https://raw.githubusercontent.com/'; const FEED_URLS = { - x: 'https://raw.githubusercontent.com/zarazhangrui/follow-builders/main/feed-x.json', - podcasts: 'https://raw.githubusercontent.com/zarazhangrui/follow-builders/main/feed-podcasts.json', - blogs: 'https://raw.githubusercontent.com/zarazhangrui/follow-builders/main/feed-blogs.json', + x: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-x.json', + podcasts: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-podcasts.json', + blogs: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-blogs.json', + videos: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-videos.json', }; const MAX_RESPONSE_CHARS = 1_500_000; const CACHE_RETENTION_MS = 48 * 60 * 60 * 1000; + const VIDEO_RETENTION_MS = 90 * 24 * 60 * 60 * 1000; const RETRY_COOLDOWN_MS = 15 * 60 * 1000; const KEYWORDS = /\b(launch|launched|release|released|ship|shipped|announce|model|agent|coding|code|api|open[\s-]?source|research|benchmark|security|product|tool|framework|developer|build|robot|autonom)/i; @@ -43,6 +45,53 @@ return `${text.slice(0, Math.max(0, maxLength - 1)).trimEnd()}…`; } + function cleanTweetText(value) { + let text = String(value || ''); + text = text.replace(/!\[[^\]]*\]\([^)]*\)/g, ' '); + text = text.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1'); + text = text.replace(/\[[^\]]*\]\([^)]*$/g, ' '); + text = text.replace(/\[[^\]]*\]\(/g, ' '); + text = text.replace(/^#{1,6}\s+/gm, ''); + text = text.replace(/^\*\s+/gm, ''); + text = text.replace(/\*\*|__/g, ''); + text = text.replace(/<[^>]+>/g, ' '); + return normalizeText(text); + } + + function isJunkTweetText(value) { + const raw = String(value || ''); + if (/pbs\.twimg\.com\/profile_images/i.test(raw)) return true; + if (/\buser avatar\b/i.test(raw)) return true; + if (/!\[[^\]]*\]\([^)]*profile_images/i.test(raw)) return true; + if (/is_blue_verified|entry_id|conversation_id_str|withheld_in_countries|"sort_index"/i.test(raw)) { + return true; + } + + const text = cleanTweetText(value); + if (!text || text.length < 20 || text.length > 400) return true; + if (/^(log in or sign up|sign up for x|create an account)\b/i.test(text)) return true; + if (/^joined (jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i.test(text)) return true; + if (/\bfollowing\b/i.test(text) && /\bfollowers?\b/i.test(text)) return true; + if (/^image\s+\d+\b/i.test(text)) return true; + if (/^(posts?|replies|highlights|media|likes|articles|subscriptions)\b/i.test(text)) return true; + if (/^(san francisco|singapore|new york|london|seattle|remote)\b/i.test(text) && text.length < 48) { + return true; + } + if (/^(?:[a-z0-9-]+\.)+[a-z]{2,}(?:\/\S*)?$/i.test(text)) return true; + if (/^@?[A-Za-z0-9_]{2,40}$/.test(text)) return true; + if ( + /^.{2,60}\s@\w{1,40}\s+(?:\[)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i.test( + text, + ) + ) { + return true; + } + if (/^[\p{L}\p{N}.''\-\s]{2,60}\s@\w{1,40}$/u.test(text)) return true; + if (/\[[^\]]*$/.test(text) || /\]\($/.test(text) || /\[[^\]]*\]\($/.test(text)) return true; + if (/[{}=]|\\u00|"type":/.test(text)) return true; + return false; + } + function safeHttpsUrl(value) { try { const url = new URL(value); @@ -79,26 +128,29 @@ const xFeed = feeds?.x; const podcastFeed = feeds?.podcasts; const blogFeed = feeds?.blogs; + const videoFeed = feeds?.videos; for (const builder of Array.isArray(xFeed?.x) ? xFeed.x : []) { for (const tweet of Array.isArray(builder?.tweets) ? builder.tweets : []) { const url = safeHttpsUrl(tweet?.url); const id = normalizeText(tweet?.id); - const excerpt = truncate(tweet?.text, 300); - if (!id || !url || !excerpt) continue; + const text = cleanTweetText(tweet?.text); + if (!id || !url || !text || isJunkTweetText(tweet?.text) || isJunkTweetText(text)) continue; const name = truncate(builder?.name || builder?.handle || 'AI Builder', 80); const handle = truncate(builder?.handle, 40); const publishedAt = new Date(timestamp(tweet?.createdAt, nowMs)).toISOString(); + const title = truncate(text, 180); + const excerpt = text.length > 180 ? truncate(text, 320) : ''; items.push({ id: `x:${id}`, kind: 'x', - source: name, - title: handle ? `${name} (@${handle})` : name, + source: handle ? `${name} (@${handle})` : name, + title, excerpt, url, publishedAt, - score: 40 + engagementScore(tweet) + keywordScore(excerpt) + recencyScore(publishedAt, nowMs), + score: 40 + engagementScore(tweet) + keywordScore(text) + recencyScore(publishedAt, nowMs), }); } } @@ -144,16 +196,41 @@ }); } + for (const video of Array.isArray(videoFeed?.videos) ? videoFeed.videos : []) { + const url = safeHttpsUrl(video?.url); + const title = truncate(video?.title, 180); + if (!url || !title) continue; + + const fallbackTime = timestamp(videoFeed?.generatedAt, nowMs); + const publishedAt = new Date(timestamp(video?.publishedAt, fallbackTime)).toISOString(); + const excerpt = truncate(video?.transcript, 340); + const guid = normalizeText(video?.guid) || url; + items.push({ + id: `video:${guid}`, + kind: 'video', + source: truncate(video?.name || 'Conference talk', 80), + title, + excerpt, + url, + publishedAt, + score: 64 + keywordScore(`${title} ${excerpt}`) + recencyScore(publishedAt, nowMs), + }); + } + return items.sort((a, b) => b.score - a.score || b.publishedAt.localeCompare(a.publishedAt)); } + function retentionMsFor(item) { + return item?.kind === 'video' ? VIDEO_RETENTION_MS : CACHE_RETENTION_MS; + } + function mergeItems(previous, incoming, now = new Date(), limit = Infinity) { - const cutoff = now.getTime() - CACHE_RETENTION_MS; + const nowMs = now.getTime(); const merged = new Map(); for (const item of [...(Array.isArray(previous) ? previous : []), ...(Array.isArray(incoming) ? incoming : [])]) { if (!item?.id || !safeHttpsUrl(item.url)) continue; - if (timestamp(item.publishedAt, now.getTime()) < cutoff) continue; + if (timestamp(item.publishedAt, nowMs) < nowMs - retentionMsFor(item)) continue; merged.set(item.id, item); } @@ -168,7 +245,7 @@ const selected = []; const selectedIds = new Set(); - for (const kind of ['x', 'blog', 'podcast']) { + for (const kind of ['x', 'blog', 'podcast', 'video']) { const item = sorted.find(candidate => candidate.kind === kind); if (item && !selectedIds.has(item.id)) { selected.push(item); diff --git a/extension/index.html b/extension/index.html index 6405e48..115232b 100644 --- a/extension/index.html +++ b/extension/index.html @@ -58,7 +58,7 @@

AI Builder Daily Report

-
Public updates from Follow Builders
+
Public updates from Zero Tab feeds
diff --git a/extension/style.css b/extension/style.css index 743f103..8d5319d 100644 --- a/extension/style.css +++ b/extension/style.css @@ -605,6 +605,11 @@ header { background: var(--orange-tint); } +.digest-kind[data-kind="video"] { + color: var(--green); + background: var(--green-tint); +} + .digest-freshness.is-stale { color: var(--orange); background: var(--orange-tint); diff --git a/package.json b/package.json index ec1d152..09b7da2 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,9 @@ "scripts": { "dev": "vite", "build": "tsc -b && vite build", - "test": "TZ=UTC node --test tests/*.test.js && TZ=UTC vitest run", - "typecheck": "tsc -b" + "test": "TZ=UTC node --test tests/*.test.js tests/*.test.mjs && TZ=UTC vitest run", + "typecheck": "tsc -b", + "generate:builder-feeds": "node scripts/generate-builder-feeds.mjs" }, "repository": { "type": "git", diff --git a/scripts/generate-builder-feeds.mjs b/scripts/generate-builder-feeds.mjs new file mode 100644 index 0000000..e06469d --- /dev/null +++ b/scripts/generate-builder-feeds.mjs @@ -0,0 +1,17 @@ +#!/usr/bin/env node +import { generateBuilderFeeds } from './lib/feed-generator.mjs'; + +const result = await generateBuilderFeeds(); +if (!result.hasData) { + console.error('Feed generation produced no items.'); + for (const error of result.errors) console.error(`- ${error}`); + process.exit(1); +} + +console.log( + `Generated feeds: x=${result.feedX.x.length} blogs=${result.feedBlogs.blogs.length} podcasts=${result.feedPodcasts.podcasts.length} videos=${result.feedVideos.videos.length}`, +); +if (result.errors.length) { + console.warn(`Completed with ${result.errors.length} source warning(s).`); + for (const error of result.errors) console.warn(`- ${error}`); +} diff --git a/scripts/lib/feed-generator.mjs b/scripts/lib/feed-generator.mjs new file mode 100644 index 0000000..bf7063c --- /dev/null +++ b/scripts/lib/feed-generator.mjs @@ -0,0 +1,774 @@ +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const DEFAULT_SOURCES = join(ROOT, 'builder-feeds', 'sources.json'); +const DEFAULT_OUT_DIR = join(ROOT, 'builder-feeds', 'generated'); +const USER_AGENT = + 'ZeroTabFeedBot/1.4 (+https://github.com/beforeload/zero-tab; public-feed-aggregator)'; +const FETCH_TIMEOUT_MS = 20_000; +const MAX_TWEETS_PER_HANDLE = 8; +const MAX_BLOG_ITEMS_PER_SOURCE = 6; +const MAX_PODCAST_ITEMS_PER_SOURCE = 4; +const MAX_VIDEO_ITEMS_PER_SOURCE = 6; + +export function normalizeText(value) { + return String(value || '') + .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +export function decodeEntities(value) { + return normalizeText(value) + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'/gi, "'") + .replace(/'/gi, "'") + .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code))) + .replace(/&#x([0-9a-f]+);/gi, (_, code) => + String.fromCharCode(Number.parseInt(code, 16)), + ); +} + +export function truncate(value, maxLength) { + const text = normalizeText(value); + if (text.length <= maxLength) return text; + return `${text.slice(0, Math.max(0, maxLength - 1)).trimEnd()}…`; +} + +export function safeHttpsUrl(value, base) { + if (!value || !String(value).trim()) return ''; + try { + const url = new URL(value, base); + return url.protocol === 'https:' ? url.href : ''; + } catch { + return ''; + } +} + +export function stripTags(value) { + const decoded = decodeEntities(String(value || '')); + return normalizeText( + decodeEntities( + decoded + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(/<[^>]+>/g, ' '), + ), + ); +} + +const SKIP_LINK_TITLE = + /^(跳到|跳轉到|跳转到)?\s*(主要)?(内容|內容|页脚|頁腳|导航|導航|選單|菜单|footer|main content|content|navigation|menu)\s*$/i; + +const NAV_TITLE = + /^(home|about|careers|pricing|docs|documentation|support|login|sign in|sign up|subscribe|privacy|terms|cookie|contact|research|news|blog|engineering|products?|company|api|claude|how to get support|如何获得支持|如何獲得支持)$/i; + +function isJunkTitle(title) { + const text = normalizeText(title); + if (!text || text.length < 8 || text.length > 200) return true; + if (SKIP_LINK_TITLE.test(text)) return true; + if (NAV_TITLE.test(text)) return true; + if (/^(skip to|jump to)\b/i.test(text)) return true; + // Concatenated card blobs usually contain multiple sentences/dates mashed together. + if ((text.match(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/g) || []).length >= 2) { + return true; + } + return false; +} + +function canonicalizeArticleUrl(value, baseUrl) { + const href = safeHttpsUrl(value, baseUrl); + if (!href) return ''; + try { + const url = new URL(href); + if (url.hash && /^#(main|main-content|content|footer|nav|navigation|top)$/i.test(url.hash)) { + return ''; + } + // Drop pure in-page skip targets that share the index path. + if (baseUrl) { + const base = new URL(baseUrl); + if (url.origin === base.origin && url.pathname.replace(/\/$/, '') === base.pathname.replace(/\/$/, '') && url.hash) { + return ''; + } + } + url.hash = ''; + // Keep YouTube watch IDs; stripping all search params breaks video links. + if (/^(www\.)?youtube\.com$/i.test(url.hostname) && url.pathname === '/watch') { + const videoId = url.searchParams.get('v'); + url.search = ''; + if (videoId) url.searchParams.set('v', videoId); + } else if (/^(www\.)?youtu\.be$/i.test(url.hostname)) { + url.search = ''; + } else { + url.search = ''; + } + return url.href.replace(/\/$/, ''); + } catch { + return ''; + } +} + +function looksLikeArticleUrl(url, baseUrl) { + try { + const parsed = new URL(url); + const path = parsed.pathname; + if (/\/(tag|tags|category|categories|author|authors|page|pages|search|login|signup)(\/|$)/i.test(path)) { + return false; + } + if (/\/(blog|news|engineering|posts|articles|research|changelog|index)\b/i.test(path)) { + // Index pages themselves are not articles. + if (/\/(blog|news|engineering|posts|articles|research|changelog|index)\/?$/i.test(path)) { + return false; + } + return true; + } + if (/\/\d{4}\/\d{2}\//.test(path)) return true; + if (/\/[a-z0-9-]{16,}\/?$/i.test(path)) return true; + if (baseUrl) { + const base = new URL(baseUrl); + if ( + parsed.origin === base.origin && + path.startsWith(base.pathname.replace(/\/$/, '') + '/') && + path.replace(/\/$/, '') !== base.pathname.replace(/\/$/, '') + ) { + return path.split('/').filter(Boolean).pop()?.length >= 8; + } + } + return false; + } catch { + return false; + } +} + +function extractHeadingTitle(anchorHtml) { + const heading = + anchorHtml.match(/]*>([\s\S]*?)<\/h[1-4]>/i) || + anchorHtml.match( + /<(?:span|div|p)[^>]*class=["'][^"']*(?:title|headline|card-title|post-title|featuredTitle)[^"']*["'][^>]*>([\s\S]*?)<\/(?:span|div|p)>/i, + ); + if (heading) return cleanArticleTitle(stripTags(heading[1])); + + // Prefer meaningful image alts (Cursor cards put the headline in alt). + const imageAlt = [...anchorHtml.matchAll(/]*\balt=["']([^"']{8,160})["'][^>]*>/gi)] + .map((match) => match[1].trim()) + .find(Boolean); + if (imageAlt) return cleanArticleTitle(imageAlt); + + return cleanArticleTitle(stripTags(anchorHtml)); +} + +function stripAuthorReadTimeCrumbs(title) { + let text = title; + for (let i = 0; i < 4; i += 1) { + const next = text + .replace(/\s+\d+\s*min(?:ute)?s?\s+read$/i, '') + // "Maxime Prades · 2m" / "Connor & Yuri · 6m" / "Chris, Rikki & Kevin · 7m" + .replace( + /\s+[A-Z][A-Za-z.]+(?:\s*(?:,|&|and)\s*|\s+)[A-Z][A-Za-z.]+(?:(?:\s*(?:,|&|and)\s*|\s+)[A-Z][A-Za-z.]+)*\s+·\s*\d+m$/u, + '', + ) + // "Connor & Yuri 6m" / "Chris, Rikki & Kevin 7m" + .replace( + /\s+[A-Z][A-Za-z.]+(?:\s*(?:,|&|and)\s*)[A-Z][A-Za-z.]+(?:(?:\s*(?:,|&|and)\s*)[A-Z][A-Za-z.]+)*\s+\d+m$/u, + '', + ) + // "Maxime Prades 2m" (exactly first + last before read-time) + .replace(/\s+[A-Z][a-z]+\s+[A-Z][a-z]+\s+\d+m$/u, ''); + if (next === text) break; + text = next; + } + return text; +} + +function cleanArticleTitle(title) { + let text = normalizeText(decodeEntities(title)); + text = text.replace( + /^(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},\s+\d{4}\s*[·•\-–|]?\s*(?:Research|Product|Company|company|product|Features|Announcements|News)?\s*/i, + '', + ); + text = text.replace( + /^(?:Featured|Announcements|Features|Product|News|Research|Company)\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},\s+\d{4}\s*/i, + '', + ); + text = text.replace( + /^(?:Featured|Announcements|Features|Product|News|Research|Company)\s+/i, + '', + ); + text = stripAuthorReadTimeCrumbs(text); + + // If a short headline is followed by a description sentence, keep the headline. + // Avoid bare "Cursor" here — titles like "… with Cursor for iOS" are valid. + const split = text.match( + /^(.{16,100}?)(?=\s+(?:We|The|How|This|A|An|Our|Built|I|Cursor is)\b)/, + ); + if (split?.[1] && !/[.!?]$/.test(split[1])) { + text = split[1]; + } else if (text.length > 110) { + const sentence = text.match(/^.{16,110}?(?:[.!?…]|$)/)?.[0]; + if (sentence) text = sentence; + } + return truncate(text, 120); +} + +function tagValue(block, tag) { + const cdata = block.match( + new RegExp(`<${tag}[^>]*>\\s*\\s*`, 'i'), + ); + if (cdata) return stripTags(cdata[1]); + const plain = block.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)`, 'i')); + return plain ? stripTags(plain[1]) : ''; +} + +function tagAttr(block, tag, attr) { + const match = block.match( + new RegExp(`<${tag}[^>]*\\s${attr}=["']([^"']+)["'][^>]*/?>`, 'i'), + ); + return match ? decodeEntities(match[1]) : ''; +} + +export function parseRssOrAtom(xml, { sourceName, baseUrl, limit = 6 } = {}) { + const text = String(xml || ''); + const items = []; + const itemBlocks = [...text.matchAll(//gi)].map((m) => m[0]); + const entryBlocks = [...text.matchAll(//gi)].map((m) => m[0]); + const blocks = itemBlocks.length ? itemBlocks : entryBlocks; + + for (const block of blocks) { + const title = truncate(tagValue(block, 'title'), 180); + const link = + canonicalizeArticleUrl(tagValue(block, 'link'), baseUrl) || + canonicalizeArticleUrl(tagAttr(block, 'link', 'href'), baseUrl) || + canonicalizeArticleUrl(tagValue(block, 'guid'), baseUrl) || + canonicalizeArticleUrl(tagValue(block, 'id'), baseUrl); + if (!title || !link || isJunkTitle(title)) continue; + + const publishedAt = + tagValue(block, 'pubDate') || + tagValue(block, 'published') || + tagValue(block, 'updated') || + tagValue(block, 'dc:date') || + null; + const description = truncate( + tagValue(block, 'description') || + tagValue(block, 'summary') || + tagValue(block, 'media:description') || + tagValue(block, 'content:encoded') || + tagValue(block, 'content') || + '', + 500, + ); + const guid = tagValue(block, 'guid') || tagValue(block, 'id') || link; + + items.push({ + name: truncate(sourceName || 'Source', 80), + title, + url: link, + guid, + description, + content: description, + transcript: description, + publishedAt: publishedAt ? new Date(Date.parse(publishedAt) || Date.now()).toISOString() : undefined, + }); + if (items.length >= limit) break; + } + + return items; +} + +export function parseBlogHtml(html, { sourceName, baseUrl, limit = 6 } = {}) { + // Strip site chrome only. Do NOT strip
— many blog cards (e.g. Cursor) + // wrap the title/media inside a card-level
. + const text = String(html || '') + .replace(//gi, ' ') + .replace(//gi, ' '); + const found = new Map(); + + // Document order; extractHeadingTitle already prefers h1–h4 / img[alt] inside the card. + const candidates = text.matchAll(/]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi); + + for (const match of candidates) { + const href = match[1]; + const inner = match[2] || ''; + const title = extractHeadingTitle(inner); + const url = canonicalizeArticleUrl(href, baseUrl); + if (!url || !title || isJunkTitle(title)) continue; + if (!looksLikeArticleUrl(url, baseUrl)) continue; + if (!found.has(url)) found.set(url, title); + if (found.size >= limit * 2) break; + } + + return [...found.entries()].slice(0, limit).map(([url, title]) => ({ + name: truncate(sourceName || 'Official blog', 80), + title, + url, + description: '', + content: '', + })); +} + +export function cleanTweetText(value) { + let text = String(value || ''); + text = text.replace(/!\[[^\]]*\]\([^)]*\)/g, ' '); + text = text.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1'); + // Drop broken/incomplete markdown links left by truncated scrapes: [Aug 15]( + text = text.replace(/\[[^\]]*\]\([^)]*$/g, ' '); + text = text.replace(/\[[^\]]*\]\(/g, ' '); + text = text.replace(/^#{1,6}\s+/gm, ''); + text = text.replace(/^\*\s+/gm, ''); + text = text.replace(/\*\*|__/g, ''); + text = stripTags(text); + return normalizeText(text); +} + +export function isJunkTweetText(value) { + const raw = String(value || ''); + // Check raw first — cleaning strips the evidence from avatar/profile chrome. + if (/pbs\.twimg\.com\/profile_images/i.test(raw)) return true; + if (/\buser avatar\b/i.test(raw)) return true; + if (/!\[[^\]]*\]\([^)]*profile_images/i.test(raw)) return true; + if (/is_blue_verified|entry_id|conversation_id_str|withheld_in_countries|"sort_index"/i.test(raw)) { + return true; + } + + const text = cleanTweetText(value); + if (!text || text.length < 20 || text.length > 400) return true; + if (/^(log in or sign up|sign up for x|create an account)\b/i.test(text)) return true; + if (/^joined (jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i.test(text)) return true; + if (/\bfollowing\b/i.test(text) && /\bfollowers?\b/i.test(text)) return true; + if (/^image\s+\d+\b/i.test(text)) return true; + if (/^(posts?|replies|highlights|media|likes|articles|subscriptions)\b/i.test(text)) return true; + if (/^(san francisco|singapore|new york|london|seattle|remote)\b/i.test(text) && text.length < 48) { + return true; + } + // Bare domain / vanity URL profile fields. + if (/^(?:[a-z0-9-]+\.)+[a-z]{2,}(?:\/\S*)?$/i.test(text)) return true; + if (/^@?[A-Za-z0-9_]{2,40}$/.test(text)) return true; + // "Simon Willison @simonw Aug 15" / "Name @handle [Aug 15](" profile chrome. + if ( + /^.{2,60}\s@\w{1,40}\s+(?:\[)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i.test( + text, + ) + ) { + return true; + } + // Author line with no tweet body left after cleaning: "Simon Willison @simonw" + if (/^[\p{L}\p{N}.''\-\s]{2,60}\s@\w{1,40}$/u.test(text)) return true; + // Leftover truncated markdown crumbs. + if (/\[[^\]]*$/.test(text) || /\]\($/.test(text) || /\[[^\]]*\]\($/.test(text)) return true; + // JSON/API debris that slipped through. + if (/[{}=]|\\u00|"type":/.test(text)) return true; + return false; +} + +function decodeJsonString(value) { + return String(value || '') + .replace(/\\n/g, ' ') + .replace(/\\"/g, '"') + .replace(/\\\//g, '/') + .replace(/\\u([0-9a-f]{4})/gi, (_, hex) => + String.fromCharCode(Number.parseInt(hex, 16)), + ); +} + +function extractTweetsFromStructuredJson(text, { handle, limit = MAX_TWEETS_PER_HANDLE } = {}) { + const tweets = []; + const seen = new Set(); + const expectedHandle = String(handle || '').replace(/^@/, '').toLowerCase(); + + const candidates = [ + ...text.matchAll(/"id_str"\s*:\s*"(\d+)"[\s\S]{0,1200}?"full_text"\s*:\s*"((?:\\.|[^"\\])*)"/g), + ...text.matchAll(/"full_text"\s*:\s*"((?:\\.|[^"\\])*)"[\s\S]{0,800}?"id_str"\s*:\s*"(\d+)"/g), + ...text.matchAll(/"id_str"\s*:\s*"(\d+)"[\s\S]{0,1200}?"text"\s*:\s*"((?:\\.|[^"\\])*)"/g), + ]; + + for (const match of candidates) { + const looksLikeFullTextFirst = match[0].trimStart().startsWith('"full_text"'); + const id = looksLikeFullTextFirst ? match[2] : match[1]; + const rawBody = looksLikeFullTextFirst ? match[1] : match[2]; + if (!id || seen.has(id)) continue; + const body = cleanTweetText(decodeJsonString(rawBody)); + if (!body || isJunkTweetText(rawBody) || isJunkTweetText(body)) continue; + + const around = text.slice(Math.max(0, match.index - 400), match.index + 1600); + const screen = + around.match(/"screen_name"\s*:\s*"([A-Za-z0-9_]+)"/)?.[1] || handle || ''; + if (expectedHandle && screen && screen.toLowerCase() !== expectedHandle) continue; + + seen.add(id); + const created = around.match(/"created_at"\s*:\s*"([^"]+)"/)?.[1] || null; + tweets.push({ + id, + text: truncate(body, 400), + createdAt: created + ? new Date(Date.parse(created) || Date.now()).toISOString() + : new Date().toISOString(), + url: `https://x.com/${screen || handle}/status/${id}`, + likes: Number(around.match(/"favorite_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + retweets: Number(around.match(/"retweet_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + replies: Number(around.match(/"reply_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + }); + if (tweets.length >= limit) break; + } + return tweets; +} + +function extractTweetBodyNearMatch(text, matchIndex, permalink, previousIndex = 0) { + // JSON timelines should be handled by extractTweetsFromStructuredJson. + if (/"is_blue_verified"|"entry_id"|"conversation_id_str"/.test(text)) return ''; + + const around = text.slice(Math.max(0, matchIndex - 900), matchIndex + 1400); + const textMatch = + around.match(/data-tweet-text=["']([^"']+)["']/i) || + around.match(/]*class=["'][^"']*tweet-text[^"']*["'][^>]*>([\s\S]*?)<\/p>/i) || + around.match(/"full_text"\s*:\s*"((?:\\.|[^"\\])*)"/); + + if (textMatch) { + const body = cleanTweetText(decodeJsonString(textMatch[1])); + if (body && !isJunkTweetText(body) && !isJunkTweetText(textMatch[1])) return body; + } + + // jina.ai markdown: only inspect text between the previous permalink and this one. + const before = text.slice(Math.max(previousIndex, matchIndex - 700), matchIndex); + const candidates = before + .split('\n') + .map((part) => cleanTweetText(part)) + .filter((part) => part.length >= 20) + .filter((part) => !/^https?:\/\//i.test(part)) + .filter((part) => !isJunkTweetText(part)) + .filter((part) => !part.includes(permalink)); + + if (!candidates.length) return ''; + return candidates.sort((a, b) => b.length - a.length)[0]; +} + +export function parseXSyndicationHtml(html, { name, handle } = {}) { + const text = String(html || ''); + const expectedHandle = String(handle || '').replace(/^@/, '').toLowerCase(); + + // Prefer structured JSON tweet objects when present (syndication/timeline payloads). + const fromJson = extractTweetsFromStructuredJson(text, { handle: expectedHandle }); + if (fromJson.length) { + return { + name: truncate(name || handle || 'AI Builder', 80), + handle: truncate(expectedHandle, 40), + tweets: fromJson, + }; + } + + const tweets = []; + const seen = new Set(); + + const permalinks = [ + ...text.matchAll( + /https?:\/\/(?:twitter\.com|x\.com)\/([A-Za-z0-9_]+)\/status\/(\d+)/gi, + ), + ]; + + let previousIndex = 0; + for (const match of permalinks) { + const tweetHandle = match[1]; + const id = match[2]; + const nextIndex = match.index + match[0].length; + if (expectedHandle && tweetHandle.toLowerCase() !== expectedHandle) { + previousIndex = nextIndex; + continue; + } + if (seen.has(id)) { + previousIndex = nextIndex; + continue; + } + seen.add(id); + + const around = text.slice(Math.max(0, match.index - 800), match.index + 1200); + const body = extractTweetBodyNearMatch(text, match.index, match[0], previousIndex); + previousIndex = nextIndex; + if (!body || isJunkTweetText(body)) continue; + + const created = + around.match(/datetime=["']([^"']+)["']/i)?.[1] || + around.match(/"created_at"\s*:\s*"([^"]+)"/)?.[1] || + null; + + tweets.push({ + id, + text: truncate(body, 400), + createdAt: created + ? new Date(Date.parse(created) || Date.now()).toISOString() + : new Date().toISOString(), + url: `https://x.com/${tweetHandle}/status/${id}`, + likes: Number(around.match(/"favorite_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + retweets: Number(around.match(/"retweet_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + replies: Number(around.match(/"reply_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + }); + if (tweets.length >= MAX_TWEETS_PER_HANDLE) break; + } + + if (!tweets.length) { + try { + const jsonMatch = text.match(/\{[\s\S]*"tweets"[\s\S]*\}/); + if (jsonMatch) { + const payload = JSON.parse(jsonMatch[0]); + for (const tweet of Array.isArray(payload.tweets) ? payload.tweets : []) { + const id = String(tweet.id_str || tweet.id || ''); + const body = cleanTweetText(tweet.full_text || tweet.text || ''); + const tweetHandle = tweet.user?.screen_name || handle; + if (!id || !body || !tweetHandle || isJunkTweetText(body)) continue; + tweets.push({ + id, + text: truncate(body, 400), + createdAt: new Date(Date.parse(tweet.created_at) || Date.now()).toISOString(), + url: `https://x.com/${tweetHandle}/status/${id}`, + likes: Number(tweet.favorite_count || 0) || undefined, + retweets: Number(tweet.retweet_count || 0) || undefined, + replies: Number(tweet.reply_count || 0) || undefined, + }); + if (tweets.length >= MAX_TWEETS_PER_HANDLE) break; + } + } + } catch { + // Ignore malformed JSON blobs. + } + } + + return { + name: truncate(name || handle || 'AI Builder', 80), + handle: truncate(String(handle || '').replace(/^@/, ''), 40), + tweets, + }; +} + +export async function fetchText(url, { fetchImpl = fetch, timeoutMs = FETCH_TIMEOUT_MS } = {}) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(url, { + signal: controller.signal, + headers: { + 'user-agent': USER_AGENT, + accept: 'text/html,application/xhtml+xml,application/xml,application/json;q=0.9,*/*;q=0.8', + }, + redirect: 'follow', + }); + if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`); + return await response.text(); + } finally { + clearTimeout(timer); + } +} + +export async function collectXFeed(sources, options = {}) { + const builders = []; + const errors = []; + for (const source of sources || []) { + const handle = String(source.handle || '').replace(/^@/, '').trim(); + if (!handle) continue; + const urls = [ + `https://cdn.syndication.twimg.com/timeline/profile?screen_name=${encodeURIComponent(handle)}`, + `https://syndication.twitter.com/srv/timeline-profile/screen-name/${encodeURIComponent(handle)}`, + `https://r.jina.ai/https://x.com/${encodeURIComponent(handle)}`, + ]; + let parsed = null; + let lastError = null; + for (const url of urls) { + try { + const html = await fetchText(url, { + ...options, + timeoutMs: options.timeoutMs || 45_000, + }); + parsed = parseXSyndicationHtml(html, { name: source.name, handle }); + if (parsed.tweets.length) break; + } catch (error) { + lastError = error; + } + } + if (parsed?.tweets?.length) builders.push(parsed); + else errors.push(`x:@${handle}: ${lastError?.message || 'no public tweets parsed'}`); + } + return { builders, errors }; +} + +export async function collectBlogFeed(sources, options = {}) { + const blogs = []; + const errors = []; + for (const source of sources || []) { + try { + if (source.rssUrl) { + const xml = await fetchText(source.rssUrl, options); + const items = parseRssOrAtom(xml, { + sourceName: source.name, + baseUrl: source.rssUrl, + limit: MAX_BLOG_ITEMS_PER_SOURCE, + }); + for (const item of items) { + blogs.push({ + name: item.name, + title: item.title, + url: item.url, + description: item.description, + content: item.content, + publishedAt: item.publishedAt, + }); + } + if (items.length) continue; + } + + if (!source.url) { + errors.push(`blog:${source.name || 'unknown'}: missing url/rssUrl`); + continue; + } + const html = await fetchText(source.url, options); + const items = parseBlogHtml(html, { + sourceName: source.name, + baseUrl: source.url, + limit: MAX_BLOG_ITEMS_PER_SOURCE, + }); + if (!items.length) { + errors.push(`blog:${source.name}: no articles parsed from HTML`); + continue; + } + blogs.push(...items); + } catch (error) { + errors.push(`blog:${source.name || source.url}: ${error.message}`); + } + } + return { blogs, errors }; +} + +async function collectRssListFeed(sources, { kind, limit, options = {} } = {}) { + const items = []; + const errors = []; + for (const source of sources || []) { + try { + if (!source.rssUrl) { + errors.push(`${kind}:${source.name || 'unknown'}: missing rssUrl`); + continue; + } + const xml = await fetchText(source.rssUrl, options); + const parsed = parseRssOrAtom(xml, { + sourceName: source.name, + baseUrl: source.rssUrl, + limit, + }); + if (!parsed.length) { + errors.push(`${kind}:${source.name}: empty RSS`); + continue; + } + for (const item of parsed) { + items.push({ + name: item.name, + title: item.title, + url: item.url, + guid: item.guid, + transcript: item.transcript, + publishedAt: item.publishedAt, + }); + } + } catch (error) { + errors.push(`${kind}:${source.name || source.rssUrl}: ${error.message}`); + } + } + return { items, errors }; +} + +export async function collectPodcastFeed(sources, options = {}) { + const result = await collectRssListFeed(sources, { + kind: 'podcast', + limit: MAX_PODCAST_ITEMS_PER_SOURCE, + options, + }); + return { podcasts: result.items, errors: result.errors }; +} + +export async function collectVideoFeed(sources, options = {}) { + const result = await collectRssListFeed(sources, { + kind: 'video', + limit: MAX_VIDEO_ITEMS_PER_SOURCE, + options, + }); + return { videos: result.items, errors: result.errors }; +} + +export function atomicWriteJson(filePath, value) { + mkdirSync(dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.${process.pid}.tmp`; + writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); + renameSync(tempPath, filePath); +} + +export async function generateBuilderFeeds({ + sourcesPath = DEFAULT_SOURCES, + outDir = DEFAULT_OUT_DIR, + now = new Date(), + fetchImpl, +} = {}) { + const sources = JSON.parse(readFileSync(sourcesPath, 'utf8')); + const generatedAt = now.toISOString(); + const options = { fetchImpl }; + + const [xResult, blogResult, podcastResult, videoResult] = await Promise.all([ + collectXFeed(sources.x, options), + collectBlogFeed(sources.blogs, options), + collectPodcastFeed(sources.podcasts, options), + collectVideoFeed(sources.videos, options), + ]); + + const feedX = { generatedAt, x: xResult.builders }; + const feedBlogs = { generatedAt, blogs: blogResult.blogs }; + const feedPodcasts = { generatedAt, podcasts: podcastResult.podcasts }; + const feedVideos = { generatedAt, videos: videoResult.videos }; + const errors = [ + ...xResult.errors, + ...blogResult.errors, + ...podcastResult.errors, + ...videoResult.errors, + ]; + + const hasData = + feedX.x.some((builder) => builder.tweets?.length) || + feedBlogs.blogs.length > 0 || + feedPodcasts.podcasts.length > 0 || + feedVideos.videos.length > 0; + + mkdirSync(outDir, { recursive: true }); + atomicWriteJson(join(outDir, 'feed-x.json'), feedX); + atomicWriteJson(join(outDir, 'feed-blogs.json'), feedBlogs); + atomicWriteJson(join(outDir, 'feed-podcasts.json'), feedPodcasts); + atomicWriteJson(join(outDir, 'feed-videos.json'), feedVideos); + atomicWriteJson(join(outDir, 'generation-report.json'), { + generatedAt, + hasData, + counts: { + xBuilders: feedX.x.length, + xTweets: feedX.x.reduce((sum, builder) => sum + (builder.tweets?.length || 0), 0), + blogs: feedBlogs.blogs.length, + podcasts: feedPodcasts.podcasts.length, + videos: feedVideos.videos.length, + }, + errors, + }); + + return { hasData, errors, feedX, feedBlogs, feedPodcasts, feedVideos }; +} + +const isMain = process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1]); + +if (isMain) { + const result = await generateBuilderFeeds(); + if (!result.hasData) { + console.error('Feed generation produced no items.'); + for (const error of result.errors) console.error(`- ${error}`); + process.exit(1); + } + console.log( + `Generated feeds: x=${result.feedX.x.length} blogs=${result.feedBlogs.blogs.length} podcasts=${result.feedPodcasts.podcasts.length} videos=${result.feedVideos.videos.length}`, + ); + if (result.errors.length) { + console.warn(`Completed with ${result.errors.length} source warning(s).`); + for (const error of result.errors) console.warn(`- ${error}`); + } +} diff --git a/src/components/BuilderDigestDrawer.tsx b/src/components/BuilderDigestDrawer.tsx index e83f251..9c09a0a 100644 --- a/src/components/BuilderDigestDrawer.tsx +++ b/src/components/BuilderDigestDrawer.tsx @@ -237,7 +237,7 @@ export function BuilderDigestDrawer({ open, onClose, onToast }: Props) {
Follow the people actually building AI products

- Downloads public Follow Builders updates from GitHub at most + Downloads public Zero Tab builder feeds from GitHub at most once per day. Your tabs and saved links never leave Chrome.

diff --git a/store/PUBLISHING_CHECKLIST.md b/store/PUBLISHING_CHECKLIST.md index 92942a8..09113b9 100644 --- a/store/PUBLISHING_CHECKLIST.md +++ b/store/PUBLISHING_CHECKLIST.md @@ -71,8 +71,8 @@ Generate package: - [ ] Submit for review - [ ] Monitor the publisher email for review questions -## Known publication decision +## Builder feed publication -The first release includes AI Builder Daily Report. Its listing and review notes position it as part of one developer-focused new-tab workspace. The Follow Builders data source currently lacks a repository-level license; obtain written permission for public feed display before final publication. +AI Builder Daily Report reads public JSON from `beforeload/zero-tab` on the `feeds` branch. Generate those files with the scheduled GitHub Action (or `workflow_dispatch`) before submitting listing updates that mention the report. See [`SUBMISSION_GUIDE.md`](SUBMISSION_GUIDE.md) for the full submission walkthrough. diff --git a/store/SUBMISSION_GUIDE.md b/store/SUBMISSION_GUIDE.md index cb78dee..2bcda5b 100644 --- a/store/SUBMISSION_GUIDE.md +++ b/store/SUBMISSION_GUIDE.md @@ -90,7 +90,7 @@ Paste the contents of [`store/test-instructions.md`](test-instructions.md) into | `tabs` | Read, focus, and close user-selected tabs | | `storage` | Saved for later, preferences, report cache | | `favicon` | Show local favicons without third-party requests | -| `https://raw.githubusercontent.com/*` (optional) | Fetch public Follow Builders JSON feeds after user opt-in | +| `https://raw.githubusercontent.com/*` (optional) | Fetch public Zero Tab JSON feeds after user opt-in | ## 8. Distribution settings @@ -100,9 +100,9 @@ Recommended first release: - Regions: all supported countries - Publishing: manual review, then publish when approved -## 9. Known blocker before final approval +## 9. Feed readiness before final approval -AI Builder Daily Report displays public content from the Follow Builders repository. That repository does not currently include a repository-level license for public feed display. Before the final public release, obtain written permission from the feed owner or remove/replace the feed source. +AI Builder Daily Report reads public JSON from this repository's `feeds` branch. Before relying on the report in production listings, run `.github/workflows/generate-builder-feeds.yml` at least once and confirm the three feed files are present on `feeds`. ## 10. After approval diff --git a/store/listing-en.md b/store/listing-en.md index d9a00db..4a5225c 100644 --- a/store/listing-en.md +++ b/store/listing-en.md @@ -32,7 +32,7 @@ Move a tab into a local Saved for later checklist before closing it. Completed i ### Optional AI Builder Daily Report -Enable a chronological feed of public updates from selected AI builders, engineering blogs, and podcasts. The report is disabled by default and requests access only to public JSON files on GitHub when enabled. +Enable a chronological feed of public updates from selected AI builders, engineering blogs, podcasts, and conference videos. The report is disabled by default and requests access only to public JSON files on GitHub when enabled. Cards can be marked as read, opened at their original source, and translated with Chrome's on-device Translator API when supported. diff --git a/store/privacy-policy.md b/store/privacy-policy.md index 4424f93..513d6f8 100644 --- a/store/privacy-policy.md +++ b/store/privacy-policy.md @@ -26,7 +26,7 @@ Zero Tab does not operate a server and does not synchronize this local data to t AI Builder Daily Report is optional and disabled until the user enables it. -When enabled, Zero Tab requests access only to `https://raw.githubusercontent.com/` and downloads public JSON feeds published by the Follow Builders project. Requests do not include open-tab data, saved links, browsing history, identifiers, credentials, or API keys. As with any HTTPS request, the remote host may receive standard network metadata such as the user's IP address and user agent under its own privacy policy. +When enabled, Zero Tab requests access only to `https://raw.githubusercontent.com/` and downloads public JSON feeds published by the Zero Tab repository on its `feeds` branch. Those feeds are generated by a scheduled GitHub Action that aggregates public page and RSS summaries. Requests do not include open-tab data, saved links, browsing history, identifiers, credentials, or API keys. As with any HTTPS request, the remote host may receive standard network metadata such as the user's IP address and user agent under its own privacy policy. Feed content is cached locally and filtered in the browser. Original report text is not sent by Zero Tab to a cloud AI service. diff --git a/store/review-notes.md b/store/review-notes.md index 640fd0b..2429a62 100644 --- a/store/review-notes.md +++ b/store/review-notes.md @@ -27,9 +27,10 @@ Required to display Chrome's local favicon representation without sending tab ho Requested only after the user clicks **Enable AI Builder Daily Report**. Used only to fetch these public data files: -- `zarazhangrui/follow-builders/main/feed-x.json` -- `zarazhangrui/follow-builders/main/feed-podcasts.json` -- `zarazhangrui/follow-builders/main/feed-blogs.json` +- `beforeload/zero-tab/feeds/feed-x.json` +- `beforeload/zero-tab/feeds/feed-podcasts.json` +- `beforeload/zero-tab/feeds/feed-videos.json` +- `beforeload/zero-tab/feeds/feed-blogs.json` The files contain data, not executable logic. All parsing, ranking, rendering, and interaction logic is packaged in the extension. @@ -37,7 +38,7 @@ No tab data, saved links, identifiers, credentials, or API keys are transmitted ## Remote code -Zero Tab does not download or execute remotely hosted code. The optional GitHub resources are JSON content feeds only. +Zero Tab does not download or execute remotely hosted code. The optional GitHub resources are JSON content feeds only. Feed generation runs in GitHub Actions and aggregates public web/RSS summaries into static JSON; the extension does not scrape third-party sites itself. ## Built-in AI diff --git a/tests/builder-digest.test.js b/tests/builder-digest.test.js index 4265c08..bbbd347 100644 --- a/tests/builder-digest.test.js +++ b/tests/builder-digest.test.js @@ -42,14 +42,77 @@ test('normalizes and ranks all supported feed types', () => { transcript: 'Today we discuss product iteration and shipping reliable AI tools.', }], }, + videos: { + generatedAt: NOW.toISOString(), + videos: [{ + name: 'Cursor Compile', + title: 'Opening Keynote, Michael Truell | Compile 26', + guid: 'yt:video:compile1', + url: 'https://www.youtube.com/watch?v=compile1', + transcript: 'Cursor Compile conference keynote about shipping agent coding tools.', + publishedAt: '2026-06-20T17:00:00.000Z', + }], + }, }, NOW); - assert.equal(items.length, 3); - assert.deepEqual(new Set(items.map(item => item.kind)), new Set(['x', 'blog', 'podcast'])); + assert.equal(items.length, 4); + assert.deepEqual(new Set(items.map(item => item.kind)), new Set(['x', 'blog', 'podcast', 'video'])); + const xItem = items.find(item => item.kind === 'x'); + assert.equal(xItem.title, 'We launched a new open-source agent framework today.'); + assert.equal(xItem.source, 'Builder (@builder)'); + assert.equal(xItem.excerpt, ''); assert.ok(items.every(item => item.url.startsWith('https://'))); assert.ok(items.every(item => Number.isFinite(item.score))); }); +test('drops junk X markdown profile chrome from feed normalization', () => { + const items = digest.normalizeFeeds({ + x: { + generatedAt: NOW.toISOString(), + x: [{ + name: 'Swyx', + handle: 'swyx', + tweets: [ + { + id: 'good', + text: 'Shipping agents that actually stay useful for a week', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/swyx/status/good', + }, + { + id: 'bad-avatar', + text: '* [![Image 8: user avatar](https://pbs.twimg.com/profile_images/x.jpg)](https://x.com/swyx) [swyx](https://x.com/swyx) [Aug 15](', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/swyx/status/bad-avatar', + }, + { + id: 'bad-date-chrome', + text: 'Simon Willison @simonw [Aug 15](', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/simonw/status/bad-date-chrome', + }, + { + id: 'bad-json', + text: ',"is_blue_verified":true}}}},{"type":"tweet","entry_id":"tweet-1"', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/swyx/status/bad-json', + }, + { + id: 'bad-login', + text: '## Log in or sign up for X', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/swyx/status/bad-login', + }, + ], + }], + }, + }, NOW); + + assert.equal(items.length, 1); + assert.equal(items[0].id, 'x:good'); + assert.equal(items[0].title, 'Shipping agents that actually stay useful for a week'); +}); + test('rejects unsafe links and control characters', () => { assert.equal(digest.safeHttpsUrl('javascript:alert(1)'), ''); assert.equal(digest.safeHttpsUrl('http://example.com'), ''); @@ -63,10 +126,11 @@ test('keeps a balanced top selection when sources exist', () => { { id: 'x:2', kind: 'x', score: 90 }, { id: 'blog:1', kind: 'blog', score: 50 }, { id: 'podcast:1', kind: 'podcast', score: 40 }, + { id: 'video:1', kind: 'video', score: 35 }, ]; - const selected = digest.selectTopItems(items, 3); - assert.deepEqual(new Set(selected.map(item => item.kind)), new Set(['x', 'blog', 'podcast'])); + const selected = digest.selectTopItems(items, 4); + assert.deepEqual(new Set(selected.map(item => item.kind)), new Set(['x', 'blog', 'podcast', 'video'])); }); test('orders report items by published time descending', () => { @@ -104,6 +168,36 @@ test('merges duplicate items and drops entries older than 48 hours', () => { assert.equal(merged[0].score, 30); }); +test('keeps conference videos for 90 days while dropping older tweets', () => { + const recentTweet = { + id: 'x:recent', + kind: 'x', + url: 'https://x.com/example/status/recent', + publishedAt: '2026-07-25T07:00:00.000Z', + score: 10, + }; + const oldTweet = { + id: 'x:old', + kind: 'x', + url: 'https://x.com/example/status/old', + publishedAt: '2026-07-20T07:00:00.000Z', + score: 100, + }; + const conferenceVideo = { + id: 'video:compile', + kind: 'video', + url: 'https://www.youtube.com/watch?v=compile1', + publishedAt: '2026-06-20T17:00:00.000Z', + score: 64, + }; + + const merged = digest.mergeItems([recentTweet, oldTweet, conferenceVideo], [], NOW); + assert.deepEqual( + merged.map((item) => item.id).sort(), + ['video:compile', 'x:recent'], + ); +}); + test('detects local-day cache hits and stale feeds', () => { // Use explicit UTC instants so the local-day check is stable across CI timezones. assert.equal( @@ -137,11 +231,31 @@ test('returns partial feed results when one source fails', async () => { }; const result = await digest.fetchFeeds({ fetchImpl, timeoutMs: 100 }); - assert.deepEqual(Object.keys(result.feeds).sort(), ['blogs', 'x']); + assert.deepEqual(Object.keys(result.feeds).sort(), ['blogs', 'videos', 'x']); assert.equal(result.errors.length, 1); assert.match(result.errors[0], /podcasts/); }); +test('points AI Builder feeds at this repository feeds branch', () => { + assert.equal(digest.OPTIONAL_ORIGIN, 'https://raw.githubusercontent.com/'); + assert.equal( + digest.FEED_URLS.x, + 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-x.json', + ); + assert.equal( + digest.FEED_URLS.blogs, + 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-blogs.json', + ); + assert.equal( + digest.FEED_URLS.podcasts, + 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-podcasts.json', + ); + assert.equal( + digest.FEED_URLS.videos, + 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-videos.json', + ); +}); + test('requests the declared raw GitHub origin without a wildcard path', async () => { const storage = {}; let requestedOrigins; @@ -205,6 +319,8 @@ test('enables optional permission, caches a refresh, and skips a second same-day } : url.includes('feed-podcasts') ? { generatedAt: NOW.toISOString(), podcasts: [] } + : url.includes('feed-videos') + ? { generatedAt: NOW.toISOString(), videos: [] } : { generatedAt: NOW.toISOString(), blogs: [] }; return { ok: true, text: async () => JSON.stringify(payload) }; }; @@ -215,7 +331,7 @@ test('enables optional permission, caches a refresh, and skips a second same-day const first = await digest.refresh({ force: true, now: NOW, fetchImpl }); assert.equal(first.items.length, 1); - assert.equal(fetchCount, 3); + assert.equal(fetchCount, 4); await digest.refresh({ now: new Date('2026-07-25T12:00:00.000Z'), @@ -223,7 +339,7 @@ test('enables optional permission, caches a refresh, and skips a second same-day throw new Error('same-day refresh should use cache'); }, }); - assert.equal(fetchCount, 3); + assert.equal(fetchCount, 4); } finally { delete global.chrome; } @@ -271,6 +387,8 @@ test('serializes concurrent refreshes and lets the second caller reuse the new c ? { generatedAt: NOW.toISOString(), x: [] } : url.includes('feed-podcasts') ? { generatedAt: NOW.toISOString(), podcasts: [] } + : url.includes('feed-videos') + ? { generatedAt: NOW.toISOString(), videos: [] } : { generatedAt: NOW.toISOString(), blogs: [] }; return { ok: true, text: async () => JSON.stringify(payload) }; }; @@ -280,7 +398,7 @@ test('serializes concurrent refreshes and lets the second caller reuse the new c digest.refresh({ force: true, now: NOW, fetchImpl }), digest.refresh({ now: NOW, fetchImpl }), ]); - assert.equal(fetchCount, 3); + assert.equal(fetchCount, 4); } finally { delete global.chrome; if (originalNavigator) { diff --git a/tests/feed-generator.test.mjs b/tests/feed-generator.test.mjs new file mode 100644 index 0000000..c3c9535 --- /dev/null +++ b/tests/feed-generator.test.mjs @@ -0,0 +1,139 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { + generateBuilderFeeds, + parseBlogHtml, + parseRssOrAtom, + parseXSyndicationHtml, +} from '../scripts/lib/feed-generator.mjs'; + +const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures'); + +describe('feed generator parsers', () => { + it('parses RSS blog entries', () => { + const xml = readFileSync(join(fixtures, 'blog.rss.xml'), 'utf8'); + const items = parseRssOrAtom(xml, { + sourceName: 'Example Blog', + baseUrl: 'https://example.com/feed.xml', + }); + assert.equal(items.length, 2); + assert.equal(items[0].title, 'Shipping smaller context windows'); + assert.equal(items[0].url, 'https://example.com/posts/context-windows'); + assert.match(items[0].description, /prompts/i); + assert.equal(items[0].description.includes('<'), false); + assert.equal(items[0].description.includes('<'), false); + assert.equal(items[0].description.includes(' { + const xml = readFileSync(join(fixtures, 'podcast.atom.xml'), 'utf8'); + const items = parseRssOrAtom(xml, { + sourceName: 'Builders Podcast', + baseUrl: 'https://example.com/podcast/atom.xml', + limit: 1, + }); + assert.equal(items.length, 1); + assert.equal(items[0].title, 'Evaluating coding agents'); + assert.equal(items[0].url, 'https://example.com/episodes/agents'); + }); + + it('parses blog HTML article links and ignores skip/nav junk', () => { + const html = readFileSync(join(fixtures, 'blog.html'), 'utf8'); + const items = parseBlogHtml(html, { + sourceName: 'Cursor Blog', + baseUrl: 'https://cursor.com/blog', + }); + assert.equal(items.length, 3); + assert.deepEqual( + items.map((item) => [item.url, item.title]), + [ + ['https://cursor.com/blog/agent-harness', 'Building a reliable agent harness'], + ['https://cursor.com/blog/tab-workstation', 'Designing a personal tab workstation'], + ['https://cursor.com/blog/ios-mobile-app', 'Build from anywhere with Cursor for iOS'], + ], + ); + assert.equal( + items.every((item) => !/跳到|Skip to|如何获得支持/i.test(item.title)), + true, + ); + }); + + it('parses YouTube Atom video entries and keeps watch IDs', () => { + const xml = readFileSync(join(fixtures, 'youtube.atom.xml'), 'utf8'); + const items = parseRssOrAtom(xml, { + sourceName: 'Cursor Compile', + baseUrl: 'https://www.youtube.com/feeds/videos.xml?playlist_id=example', + limit: 1, + }); + assert.equal(items.length, 1); + assert.equal(items[0].title, 'Opening Keynote, Michael Truell | Compile 26'); + assert.equal(items[0].url, 'https://www.youtube.com/watch?v=abc123Compile'); + assert.match(items[0].description, /Compile keynote/i); + }); + + it('parses X syndication markup into tweets', () => { + const html = readFileSync(join(fixtures, 'x-syndication.html'), 'utf8'); + const builder = parseXSyndicationHtml(html, { + name: 'Simon Willison', + handle: 'simonw', + }); + assert.equal(builder.handle, 'simonw'); + assert.equal(builder.tweets.length, 1); + assert.equal(builder.tweets[0].id, '1234567890'); + assert.match(builder.tweets[0].text, /Translator API/i); + assert.equal(builder.tweets[0].url, 'https://x.com/simonw/status/1234567890'); + }); + + it('parses jina markdown timelines and ignores profile chrome', () => { + const markdown = readFileSync(join(fixtures, 'x-jina.md'), 'utf8'); + const builder = parseXSyndicationHtml(markdown, { + name: 'Swyx', + handle: 'swyx', + }); + assert.equal(builder.tweets.length, 1); + assert.equal(builder.tweets[0].id, '2088381680478540096'); + assert.equal(builder.tweets[0].text, 'Shipping agents that actually stay useful for a week'); + assert.equal(builder.tweets.every((tweet) => !/avatar|Log in|Joined|Following/i.test(tweet.text)), true); + }); + + it('writes feed JSON through a mocked fetch layer', async () => { + const fixturesByUrl = { + 'https://cdn.syndication.twimg.com/timeline/profile?screen_name=simonw': + readFileSync(join(fixtures, 'x-syndication.html'), 'utf8'), + 'https://example.com/feed.xml': readFileSync(join(fixtures, 'blog.rss.xml'), 'utf8'), + 'https://example.com/podcast.xml': readFileSync(join(fixtures, 'podcast.atom.xml'), 'utf8'), + 'https://example.com/youtube.xml': readFileSync(join(fixtures, 'youtube.atom.xml'), 'utf8'), + }; + const fetchImpl = async (url) => { + const body = fixturesByUrl[url]; + if (!body) throw new Error(`unexpected url ${url}`); + return { + ok: true, + status: 200, + text: async () => body, + }; + }; + + const sourcesPath = join(fixtures, 'sources.json'); + const outDir = join(fixtures, 'out'); + const result = await generateBuilderFeeds({ + sourcesPath, + outDir, + now: new Date('2026-08-11T12:00:00.000Z'), + fetchImpl, + }); + + assert.equal(result.hasData, true); + assert.equal(result.feedX.x[0].tweets.length, 1); + assert.ok(result.feedBlogs.blogs.length >= 1); + assert.ok(result.feedPodcasts.podcasts.length >= 1); + assert.ok(result.feedVideos.videos.length >= 1); + assert.equal(result.feedVideos.videos[0].url, 'https://www.youtube.com/watch?v=abc123Compile'); + + const writtenX = JSON.parse(readFileSync(join(outDir, 'feed-x.json'), 'utf8')); + assert.equal(writtenX.generatedAt, '2026-08-11T12:00:00.000Z'); + }); +}); diff --git a/tests/fixtures/blog.html b/tests/fixtures/blog.html new file mode 100644 index 0000000..7f6038a --- /dev/null +++ b/tests/fixtures/blog.html @@ -0,0 +1,35 @@ + + + + Skip to main content + 跳到页脚 + +
+ +
+ Building a reliable agent harness +

How we keep agents productive without expanding blast radius.

+ Cursor Team · 3m +
+ + +

Designing a personal tab workstation

+ Announcements +

Extra card body that should not become the title.

+ + +
+ Build from anywhere with Cursor for iOS +

Cursor is available as a native iOS app on your phone.

+ Chris, Rikki & Kevin · 7m +
+ + 跳到主要内容 + 如何获得支持 +
+
Footer
+ + diff --git a/tests/fixtures/blog.rss.xml b/tests/fixtures/blog.rss.xml new file mode 100644 index 0000000..c5ed018 --- /dev/null +++ b/tests/fixtures/blog.rss.xml @@ -0,0 +1,20 @@ + + + + Example Blog + + Shipping smaller context windows + https://example.com/posts/context-windows + https://example.com/posts/context-windows + Mon, 10 Aug 2026 09:00:00 GMT + + + + Local-first browser tools + https://example.com/posts/local-first + https://example.com/posts/local-first + Sun, 09 Aug 2026 11:00:00 GMT + Why tab metadata should stay on the device. + + + diff --git a/tests/fixtures/podcast.atom.xml b/tests/fixtures/podcast.atom.xml new file mode 100644 index 0000000..c0eda70 --- /dev/null +++ b/tests/fixtures/podcast.atom.xml @@ -0,0 +1,11 @@ + + + Builders Podcast + + Evaluating coding agents + episode-agents + + 2026-08-09T19:40:00Z + Rollout guardrails and evaluation harnesses. + + diff --git a/tests/fixtures/sources.json b/tests/fixtures/sources.json new file mode 100644 index 0000000..ca491ac --- /dev/null +++ b/tests/fixtures/sources.json @@ -0,0 +1,23 @@ +{ + "version": 1, + "x": [{ "name": "Simon Willison", "handle": "simonw" }], + "blogs": [ + { + "name": "Example Blog", + "url": "https://example.com/", + "rssUrl": "https://example.com/feed.xml" + } + ], + "podcasts": [ + { + "name": "Builders Podcast", + "rssUrl": "https://example.com/podcast.xml" + } + ], + "videos": [ + { + "name": "Cursor Compile", + "rssUrl": "https://example.com/youtube.xml" + } + ] +} diff --git a/tests/fixtures/x-jina.md b/tests/fixtures/x-jina.md new file mode 100644 index 0000000..dfad676 --- /dev/null +++ b/tests/fixtures/x-jina.md @@ -0,0 +1,15 @@ +Title: swyx (swyx) + +Joined November 2007 +san francisco / singapore + +[664 Following](https://x.com/swyx/following)[100K Followers](https://x.com/swyx/verified_followers) + +## Log in or sign up for X + +* [![Image 8: user avatar](https://pbs.twimg.com/profile_images/2073162797354217472/hNny55eF_normal.jpg)](https://x.com/swyx) [swyx](https://x.com/swyx) +Shipping agents that actually stay useful for a week +https://x.com/swyx/status/2088381680478540096 + +* [![Image 13: user avatar](https://pbs.twimg.com/profile_images/2073162797354217472/hNny55eF_normal.jpg)](https://x.com/swyx) [swyx](https://x.com/swyx) +https://x.com/swyx/status/2088358628000768263 diff --git a/tests/fixtures/x-syndication.html b/tests/fixtures/x-syndication.html new file mode 100644 index 0000000..2faa880 --- /dev/null +++ b/tests/fixtures/x-syndication.html @@ -0,0 +1,12 @@ + + + +
+ +

+ Chrome on-device Translator API notes for local coding agents +

+ +
+ + diff --git a/tests/fixtures/youtube.atom.xml b/tests/fixtures/youtube.atom.xml new file mode 100644 index 0000000..f9ee6d2 --- /dev/null +++ b/tests/fixtures/youtube.atom.xml @@ -0,0 +1,16 @@ + + + Cursor Compile + + yt:video:abc123Compile + abc123Compile + Opening Keynote, Michael Truell | Compile 26 + + 2026-06-20T17:00:00+00:00 + + Cursor's first Compile keynote. + + +