From 12c6f42fa90b3bf14867225dd4a6c8a25e0f467e Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 12 Aug 2026 14:29:05 +0800 Subject: [PATCH 1/6] Add self-hosted AI Builder feed generation Generate X, blog, and podcast JSON via GitHub Actions and publish them on the feeds branch so Zero Tab no longer depends on Follow Builders. Co-authored-by: Cursor --- .github/workflows/generate-builder-feeds.yml | 72 +++ .gitignore | 4 + AGENTS.md | 2 +- README.md | 14 +- builder-feeds/README.md | 9 + builder-feeds/sources.json | 56 +++ docs/privacy.html | 2 +- extension/builder-digest.js | 6 +- extension/index.html | 2 +- package.json | 5 +- scripts/generate-builder-feeds.mjs | 17 + scripts/lib/feed-generator.mjs | 459 +++++++++++++++++++ src/components/BuilderDigestDrawer.tsx | 2 +- store/PUBLISHING_CHECKLIST.md | 4 +- store/SUBMISSION_GUIDE.md | 6 +- store/privacy-policy.md | 2 +- store/review-notes.md | 8 +- tests/builder-digest.test.js | 16 + tests/feed-generator.test.mjs | 98 ++++ tests/fixtures/blog.html | 10 + tests/fixtures/blog.rss.xml | 20 + tests/fixtures/podcast.atom.xml | 11 + tests/fixtures/sources.json | 17 + tests/fixtures/x-syndication.html | 12 + 24 files changed, 833 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/generate-builder-feeds.yml create mode 100644 builder-feeds/README.md create mode 100644 builder-feeds/sources.json create mode 100644 scripts/generate-builder-feeds.mjs create mode 100644 scripts/lib/feed-generator.mjs create mode 100644 tests/feed-generator.test.mjs create mode 100644 tests/fixtures/blog.html create mode 100644 tests/fixtures/blog.rss.xml create mode 100644 tests/fixtures/podcast.atom.xml create mode 100644 tests/fixtures/sources.json create mode 100644 tests/fixtures/x-syndication.html diff --git a/.github/workflows/generate-builder-feeds.yml b/.github/workflows/generate-builder-feeds.yml new file mode 100644 index 0000000..3a0f5b4 --- /dev/null +++ b/.github/workflows/generate-builder-feeds.yml @@ -0,0 +1,72 @@ +name: Generate builder feeds + +on: + schedule: + # Daily at 01:15 UTC + - cron: '15 1 * * *' + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: generate-builder-feeds + cancel-in-progress: false + +jobs: + generate: + name: generate · publish feeds + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Generate public feeds + run: npm run generate:builder-feeds + + - name: Publish to feeds branch + run: | + set -euo pipefail + PUBLISH_DIR="${RUNNER_TEMP}/zero-tab-feeds" + rm -rf "$PUBLISH_DIR" + mkdir -p "$PUBLISH_DIR" + + if git ls-remote --exit-code --heads origin feeds >/dev/null 2>&1; then + git fetch origin feeds + git worktree add "$PUBLISH_DIR" origin/feeds + git -C "$PUBLISH_DIR" checkout -B feeds + else + git worktree add --detach "$PUBLISH_DIR" + git -C "$PUBLISH_DIR" checkout --orphan feeds + git -C "$PUBLISH_DIR" rm -rf . >/dev/null 2>&1 || true + fi + + cp builder-feeds/generated/feed-x.json "$PUBLISH_DIR/feed-x.json" + cp builder-feeds/generated/feed-blogs.json "$PUBLISH_DIR/feed-blogs.json" + cp builder-feeds/generated/feed-podcasts.json "$PUBLISH_DIR/feed-podcasts.json" + cp builder-feeds/generated/generation-report.json "$PUBLISH_DIR/generation-report.json" + + git -C "$PUBLISH_DIR" config user.name "github-actions[bot]" + git -C "$PUBLISH_DIR" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git -C "$PUBLISH_DIR" add feed-x.json feed-blogs.json feed-podcasts.json generation-report.json + + if git -C "$PUBLISH_DIR" diff --cached --quiet; then + echo "No feed changes to publish." + else + git -C "$PUBLISH_DIR" commit -m "Update builder feeds $(date -u +%Y-%m-%dT%H:%M:%SZ)" + git -C "$PUBLISH_DIR" push origin HEAD:feeds + fi diff --git a/.gitignore b/.gitignore index d344ec9..9a2994b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ dist/ # Personal config (landing page patterns, etc.) — never push to GitHub extension/config.local.js + +# Local feed generation output; published only on the feeds branch +builder-feeds/generated/ +tests/fixtures/out/ diff --git a/AGENTS.md b/AGENTS.md index d7ac442..ccbc478 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,5 +102,5 @@ Once the extension is loaded: - Run `npm run build` and load `dist/extension/` in Chrome. - Saved tabs are stored in `chrome.storage.local` (persists across sessions). - Tab management is fully local. Open-tab and saved-tab data is never uploaded. -- The optional AI Builder digest requests access only to `raw.githubusercontent.com`, fetches public feeds at most once per local day, and stores a compact cache locally. +- The optional AI Builder digest requests access only to `raw.githubusercontent.com`, fetches public feeds from this repository's `feeds` branch at most once per local day, and stores a compact cache locally. - To update: `cd zero-tab && git pull && npm install && npm run build`, then reload the extension in `chrome://extensions`. diff --git a/README.md b/README.md index 5ce46c1..b8162d4 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ No server and no account are required. Open-tab URLs, titles, and saved links st - Local, deterministic daily horoscope by zodiac sign - Localhost port labels - macOS-inspired light and dark themes -- Full-height AI Builder Daily Report drawer sourced from public Follow Builders feeds +- Full-height AI Builder Daily Report drawer sourced from public Zero Tab feeds - Optional on-device translation through Chrome's built-in Translator API ## Install locally @@ -46,7 +46,7 @@ No server and no account are required. Open-tab URLs, titles, and saved links st Core tab management makes no external requests. Tab URLs, titles, Saved for later items, read state, and preferences are stored locally using Chrome extension storage. -AI Builder Daily Report is disabled until the user enables it. When enabled, Zero Tab requests optional access to `raw.githubusercontent.com` and downloads three public Follow Builders JSON feeds at most once per local calendar day. No tab, browsing, saved-link, identifier, or API-key data is included in those requests. +AI Builder Daily Report is disabled until the user enables it. When enabled, Zero Tab requests optional access to `raw.githubusercontent.com` and downloads three public JSON feeds published by this repository's `feeds` branch at most once per local calendar day. No tab, browsing, saved-link, identifier, or API-key data is included in those requests. Translation uses Chrome's built-in on-device Translator API when available. The language model or language pack may be downloaded by Chrome, but report text is not sent to a third-party translation service by Zero Tab. @@ -87,6 +87,16 @@ The previous Vanilla implementation remains in `extension/` as a migration reference while the remaining publishing assets are moved to the new source tree. +## Builder feed generation + +Zero Tab publishes AI Builder JSON feeds from this repository's `feeds` branch. + +```bash +npm run generate:builder-feeds +``` + +Sources live in [`builder-feeds/sources.json`](builder-feeds/sources.json). GitHub Actions runs [`.github/workflows/generate-builder-feeds.yml`](.github/workflows/generate-builder-feeds.yml) daily and on `workflow_dispatch`, then commits `feed-x.json`, `feed-blogs.json`, and `feed-podcasts.json` to the `feeds` branch root. The extension reads those files from `raw.githubusercontent.com`. + ## Attribution and license Zero Tab began as a fork of Zara Zhang's MIT-licensed original project. diff --git a/builder-feeds/README.md b/builder-feeds/README.md new file mode 100644 index 0000000..d127c76 --- /dev/null +++ b/builder-feeds/README.md @@ -0,0 +1,9 @@ +# Builder feeds + +Curated source list and local generation output for the AI Builder Daily Report. + +- [`sources.json`](sources.json) — X handles, blogs (RSS and/or HTML), podcasts (RSS) +- `generated/` — local output of `npm run generate:builder-feeds` (gitignored) +- Published artifacts live on the repository `feeds` branch root + +Generation is owned by Zero Tab. It does not call Follow Builders services or APIs. diff --git a/builder-feeds/sources.json b/builder-feeds/sources.json new file mode 100644 index 0000000..53d9b1b --- /dev/null +++ b/builder-feeds/sources.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "x": [ + { "name": "Boris Cherny", "handle": "boris_cherny" }, + { "name": "Anthropic", "handle": "AnthropicAI" }, + { "name": "Guillermo Rauch", "handle": "rauchg" }, + { "name": "Simon Willison", "handle": "simonw" }, + { "name": "Addy Osmani", "handle": "addyosmani" }, + { "name": "Swyx", "handle": "swyx" }, + { "name": "Andrej Karpathy", "handle": "karpathy" } + ], + "blogs": [ + { + "name": "Anthropic Engineering", + "url": "https://www.anthropic.com/engineering" + }, + { + "name": "Anthropic News", + "url": "https://www.anthropic.com/news" + }, + { + "name": "OpenAI Blog", + "url": "https://openai.com/blog", + "rssUrl": "https://openai.com/blog/rss.xml" + }, + { + "name": "Simon Willison", + "url": "https://simonwillison.net/", + "rssUrl": "https://simonwillison.net/atom/everything/" + }, + { + "name": "Julia Evans", + "url": "https://jvns.ca/", + "rssUrl": "https://jvns.ca/atom.xml" + }, + { + "name": "Latent Space", + "url": "https://www.latent.space/", + "rssUrl": "https://www.latent.space/feed" + }, + { + "name": "Cursor Blog", + "url": "https://cursor.com/blog" + } + ], + "podcasts": [ + { + "name": "Latent Space", + "rssUrl": "https://api.substack.com/feed/podcast/1084089.rss" + }, + { + "name": "Practical AI", + "rssUrl": "https://changelog.com/practicalai/feed" + } + ] +} diff --git a/docs/privacy.html b/docs/privacy.html index 4925fbc..d953d38 100644 --- a/docs/privacy.html +++ b/docs/privacy.html @@ -29,7 +29,7 @@

Local storage

Saved links, preferences, AI Builder Daily Report cache, read state, and translations are stored in chrome.storage.local. Zero Tab does not operate a server and does not synchronize this data to the publisher.

AI Builder Daily Report

-

This optional feature is disabled until the user enables it. When enabled, Zero Tab downloads public JSON feeds from https://raw.githubusercontent.com/. These requests do not contain tab data, saved links, browsing history, identifiers, credentials, or API keys.

+

This optional feature is disabled until the user enables it. When enabled, Zero Tab downloads public JSON feeds from https://raw.githubusercontent.com/beforeload/zero-tab/feeds/. These requests do not contain tab data, saved links, browsing history, identifiers, credentials, or API keys.

On-device translation

When supported and explicitly requested, Zero Tab uses Chrome's built-in Translator API. Chrome may download a language pack or model. Zero Tab does not send report text to a third-party translation API.

diff --git a/extension/builder-digest.js b/extension/builder-digest.js index 28f4680..b43ad44 100644 --- a/extension/builder-digest.js +++ b/extension/builder-digest.js @@ -8,9 +8,9 @@ const CACHE_KEY = 'builderDigestState'; const OPTIONAL_ORIGIN = 'https://raw.githubusercontent.com/'; const FEED_URLS = { - x: 'https://raw.githubusercontent.com/zarazhangrui/follow-builders/main/feed-x.json', - podcasts: 'https://raw.githubusercontent.com/zarazhangrui/follow-builders/main/feed-podcasts.json', - blogs: 'https://raw.githubusercontent.com/zarazhangrui/follow-builders/main/feed-blogs.json', + x: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-x.json', + podcasts: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-podcasts.json', + blogs: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-blogs.json', }; const MAX_RESPONSE_CHARS = 1_500_000; const CACHE_RETENTION_MS = 48 * 60 * 60 * 1000; diff --git a/extension/index.html b/extension/index.html index 6405e48..115232b 100644 --- a/extension/index.html +++ b/extension/index.html @@ -58,7 +58,7 @@

AI Builder Daily Report

-
Public updates from Follow Builders
+
Public updates from Zero Tab feeds
diff --git a/package.json b/package.json index ec1d152..09b7da2 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,9 @@ "scripts": { "dev": "vite", "build": "tsc -b && vite build", - "test": "TZ=UTC node --test tests/*.test.js && TZ=UTC vitest run", - "typecheck": "tsc -b" + "test": "TZ=UTC node --test tests/*.test.js tests/*.test.mjs && TZ=UTC vitest run", + "typecheck": "tsc -b", + "generate:builder-feeds": "node scripts/generate-builder-feeds.mjs" }, "repository": { "type": "git", diff --git a/scripts/generate-builder-feeds.mjs b/scripts/generate-builder-feeds.mjs new file mode 100644 index 0000000..4fb8175 --- /dev/null +++ b/scripts/generate-builder-feeds.mjs @@ -0,0 +1,17 @@ +#!/usr/bin/env node +import { generateBuilderFeeds } from './lib/feed-generator.mjs'; + +const result = await generateBuilderFeeds(); +if (!result.hasData) { + console.error('Feed generation produced no items.'); + for (const error of result.errors) console.error(`- ${error}`); + process.exit(1); +} + +console.log( + `Generated feeds: x=${result.feedX.x.length} blogs=${result.feedBlogs.blogs.length} podcasts=${result.feedPodcasts.podcasts.length}`, +); +if (result.errors.length) { + console.warn(`Completed with ${result.errors.length} source warning(s).`); + for (const error of result.errors) console.warn(`- ${error}`); +} diff --git a/scripts/lib/feed-generator.mjs b/scripts/lib/feed-generator.mjs new file mode 100644 index 0000000..99184aa --- /dev/null +++ b/scripts/lib/feed-generator.mjs @@ -0,0 +1,459 @@ +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const DEFAULT_SOURCES = join(ROOT, 'builder-feeds', 'sources.json'); +const DEFAULT_OUT_DIR = join(ROOT, 'builder-feeds', 'generated'); +const USER_AGENT = + 'ZeroTabFeedBot/1.4 (+https://github.com/beforeload/zero-tab; public-feed-aggregator)'; +const FETCH_TIMEOUT_MS = 20_000; +const MAX_TWEETS_PER_HANDLE = 8; +const MAX_BLOG_ITEMS_PER_SOURCE = 6; +const MAX_PODCAST_ITEMS_PER_SOURCE = 4; + +export function normalizeText(value) { + return String(value || '') + .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +export function decodeEntities(value) { + return normalizeText(value) + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'/gi, "'") + .replace(/'/gi, "'") + .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code))) + .replace(/&#x([0-9a-f]+);/gi, (_, code) => + String.fromCharCode(Number.parseInt(code, 16)), + ); +} + +export function truncate(value, maxLength) { + const text = normalizeText(value); + if (text.length <= maxLength) return text; + return `${text.slice(0, Math.max(0, maxLength - 1)).trimEnd()}…`; +} + +export function safeHttpsUrl(value, base) { + if (!value || !String(value).trim()) return ''; + try { + const url = new URL(value, base); + return url.protocol === 'https:' ? url.href : ''; + } catch { + return ''; + } +} + +export function stripTags(value) { + return decodeEntities( + String(value || '') + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(/<[^>]+>/g, ' '), + ); +} + +function tagValue(block, tag) { + const cdata = block.match( + new RegExp(`<${tag}[^>]*>\\s*\\s*`, 'i'), + ); + if (cdata) return decodeEntities(cdata[1]); + const plain = block.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)`, 'i')); + return plain ? stripTags(plain[1]) : ''; +} + +function tagAttr(block, tag, attr) { + const match = block.match( + new RegExp(`<${tag}[^>]*\\s${attr}=["']([^"']+)["'][^>]*/?>`, 'i'), + ); + return match ? decodeEntities(match[1]) : ''; +} + +export function parseRssOrAtom(xml, { sourceName, baseUrl, limit = 6 } = {}) { + const text = String(xml || ''); + const items = []; + const itemBlocks = [...text.matchAll(//gi)].map((m) => m[0]); + const entryBlocks = [...text.matchAll(//gi)].map((m) => m[0]); + const blocks = itemBlocks.length ? itemBlocks : entryBlocks; + + for (const block of blocks) { + const title = truncate(tagValue(block, 'title'), 180); + const link = + safeHttpsUrl(tagValue(block, 'link'), baseUrl) || + safeHttpsUrl(tagAttr(block, 'link', 'href'), baseUrl) || + safeHttpsUrl(tagValue(block, 'guid'), baseUrl) || + safeHttpsUrl(tagValue(block, 'id'), baseUrl); + if (!title || !link) continue; + + const publishedAt = + tagValue(block, 'pubDate') || + tagValue(block, 'published') || + tagValue(block, 'updated') || + tagValue(block, 'dc:date') || + null; + const description = + tagValue(block, 'description') || + tagValue(block, 'summary') || + tagValue(block, 'content:encoded') || + tagValue(block, 'content') || + ''; + const guid = tagValue(block, 'guid') || tagValue(block, 'id') || link; + + items.push({ + name: truncate(sourceName || 'Source', 80), + title, + url: link, + guid, + description: truncate(description, 500), + content: truncate(description, 500), + transcript: truncate(description, 500), + publishedAt: publishedAt ? new Date(Date.parse(publishedAt) || Date.now()).toISOString() : undefined, + }); + if (items.length >= limit) break; + } + + return items; +} + +export function parseBlogHtml(html, { sourceName, baseUrl, limit = 6 } = {}) { + const text = String(html || ''); + const found = new Map(); + + const patterns = [ + /]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, + ]; + + for (const pattern of patterns) { + for (const match of text.matchAll(pattern)) { + const href = match[1]; + const title = truncate(stripTags(match[2]), 180); + const url = safeHttpsUrl(href, baseUrl); + if (!url || !title || title.length < 12) continue; + if (!/\/(blog|news|engineering|posts|articles|research|changelog)\b/i.test(url) && + !/blog|news|engineering|post|article/i.test(href)) { + // Keep homepage-relative article-looking paths with dates or long slugs. + if (!/\/\d{4}\/\d{2}\//.test(url) && !/\/[a-z0-9-]{16,}\/?$/i.test(url)) continue; + } + if (/#(respond|comments)|\/tag\/|\/category\/|\/author\//i.test(url)) continue; + if (!found.has(url)) found.set(url, title); + if (found.size >= limit * 3) break; + } + } + + return [...found.entries()].slice(0, limit).map(([url, title]) => ({ + name: truncate(sourceName || 'Official blog', 80), + title, + url, + description: '', + content: '', + })); +} + +export function parseXSyndicationHtml(html, { name, handle } = {}) { + const text = String(html || ''); + const tweets = []; + const seen = new Set(); + const expectedHandle = String(handle || '').replace(/^@/, '').toLowerCase(); + + const permalinks = [ + ...text.matchAll( + /https?:\/\/(?:twitter\.com|x\.com)\/([A-Za-z0-9_]+)\/status\/(\d+)/gi, + ), + ]; + + for (const match of permalinks) { + const tweetHandle = match[1]; + const id = match[2]; + if (expectedHandle && tweetHandle.toLowerCase() !== expectedHandle) continue; + if (seen.has(id)) continue; + seen.add(id); + + const around = text.slice(Math.max(0, match.index - 800), match.index + 1200); + const textMatch = + around.match(/data-tweet-text=["']([^"']+)["']/i) || + around.match(/]*class=["'][^"']*tweet-text[^"']*["'][^>]*>([\s\S]*?)<\/p>/i) || + around.match(/"full_text"\s*:\s*"((?:\\.|[^"\\])*)"/) || + around.match(/"text"\s*:\s*"((?:\\.|[^"\\])*)"/) || + around.match(/\n([^\n]{20,280})\n/); + let body = ''; + if (textMatch) { + body = textMatch[1] + .replace(/\\n/g, ' ') + .replace(/\\"/g, '"') + .replace(/\\u([0-9a-f]{4})/gi, (_, hex) => + String.fromCharCode(Number.parseInt(hex, 16)), + ); + body = stripTags(body); + } + if (!body) { + // jina.ai markdown often has the tweet body on the previous lines. + const before = text.slice(Math.max(0, match.index - 400), match.index); + const line = before + .split('\n') + .map((part) => normalizeText(part)) + .filter((part) => part.length >= 24 && !/^https?:\/\//i.test(part) && !/^@/.test(part)) + .at(-1); + body = line || ''; + } + if (!body) continue; + + const created = + around.match(/datetime=["']([^"']+)["']/i)?.[1] || + around.match(/"created_at"\s*:\s*"([^"]+)"/)?.[1] || + null; + + tweets.push({ + id, + text: truncate(body, 400), + createdAt: created + ? new Date(Date.parse(created) || Date.now()).toISOString() + : new Date().toISOString(), + url: `https://x.com/${tweetHandle}/status/${id}`, + likes: Number(around.match(/"favorite_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + retweets: Number(around.match(/"retweet_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + replies: Number(around.match(/"reply_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + }); + if (tweets.length >= MAX_TWEETS_PER_HANDLE) break; + } + + if (!tweets.length) { + try { + const jsonMatch = text.match(/\{[\s\S]*"tweets"[\s\S]*\}/); + if (jsonMatch) { + const payload = JSON.parse(jsonMatch[0]); + for (const tweet of Array.isArray(payload.tweets) ? payload.tweets : []) { + const id = String(tweet.id_str || tweet.id || ''); + const body = normalizeText(tweet.full_text || tweet.text || ''); + const tweetHandle = tweet.user?.screen_name || handle; + if (!id || !body || !tweetHandle) continue; + tweets.push({ + id, + text: truncate(body, 400), + createdAt: new Date(Date.parse(tweet.created_at) || Date.now()).toISOString(), + url: `https://x.com/${tweetHandle}/status/${id}`, + likes: Number(tweet.favorite_count || 0) || undefined, + retweets: Number(tweet.retweet_count || 0) || undefined, + replies: Number(tweet.reply_count || 0) || undefined, + }); + if (tweets.length >= MAX_TWEETS_PER_HANDLE) break; + } + } + } catch { + // Ignore malformed JSON blobs. + } + } + + return { + name: truncate(name || handle || 'AI Builder', 80), + handle: truncate(String(handle || '').replace(/^@/, ''), 40), + tweets, + }; +} + +export async function fetchText(url, { fetchImpl = fetch, timeoutMs = FETCH_TIMEOUT_MS } = {}) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(url, { + signal: controller.signal, + headers: { + 'user-agent': USER_AGENT, + accept: 'text/html,application/xhtml+xml,application/xml,application/json;q=0.9,*/*;q=0.8', + }, + redirect: 'follow', + }); + if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`); + return await response.text(); + } finally { + clearTimeout(timer); + } +} + +export async function collectXFeed(sources, options = {}) { + const builders = []; + const errors = []; + for (const source of sources || []) { + const handle = String(source.handle || '').replace(/^@/, '').trim(); + if (!handle) continue; + const urls = [ + `https://cdn.syndication.twimg.com/timeline/profile?screen_name=${encodeURIComponent(handle)}`, + `https://syndication.twitter.com/srv/timeline-profile/screen-name/${encodeURIComponent(handle)}`, + `https://r.jina.ai/https://x.com/${encodeURIComponent(handle)}`, + ]; + let parsed = null; + let lastError = null; + for (const url of urls) { + try { + const html = await fetchText(url, { + ...options, + timeoutMs: options.timeoutMs || 45_000, + }); + parsed = parseXSyndicationHtml(html, { name: source.name, handle }); + if (parsed.tweets.length) break; + } catch (error) { + lastError = error; + } + } + if (parsed?.tweets?.length) builders.push(parsed); + else errors.push(`x:@${handle}: ${lastError?.message || 'no public tweets parsed'}`); + } + return { builders, errors }; +} + +export async function collectBlogFeed(sources, options = {}) { + const blogs = []; + const errors = []; + for (const source of sources || []) { + try { + if (source.rssUrl) { + const xml = await fetchText(source.rssUrl, options); + const items = parseRssOrAtom(xml, { + sourceName: source.name, + baseUrl: source.rssUrl, + limit: MAX_BLOG_ITEMS_PER_SOURCE, + }); + for (const item of items) { + blogs.push({ + name: item.name, + title: item.title, + url: item.url, + description: item.description, + content: item.content, + publishedAt: item.publishedAt, + }); + } + if (items.length) continue; + } + + if (!source.url) { + errors.push(`blog:${source.name || 'unknown'}: missing url/rssUrl`); + continue; + } + const html = await fetchText(source.url, options); + const items = parseBlogHtml(html, { + sourceName: source.name, + baseUrl: source.url, + limit: MAX_BLOG_ITEMS_PER_SOURCE, + }); + if (!items.length) { + errors.push(`blog:${source.name}: no articles parsed from HTML`); + continue; + } + blogs.push(...items); + } catch (error) { + errors.push(`blog:${source.name || source.url}: ${error.message}`); + } + } + return { blogs, errors }; +} + +export async function collectPodcastFeed(sources, options = {}) { + const podcasts = []; + const errors = []; + for (const source of sources || []) { + try { + if (!source.rssUrl) { + errors.push(`podcast:${source.name || 'unknown'}: missing rssUrl`); + continue; + } + const xml = await fetchText(source.rssUrl, options); + const items = parseRssOrAtom(xml, { + sourceName: source.name, + baseUrl: source.rssUrl, + limit: MAX_PODCAST_ITEMS_PER_SOURCE, + }); + if (!items.length) { + errors.push(`podcast:${source.name}: empty RSS`); + continue; + } + for (const item of items) { + podcasts.push({ + name: item.name, + title: item.title, + url: item.url, + guid: item.guid, + transcript: item.transcript, + publishedAt: item.publishedAt, + }); + } + } catch (error) { + errors.push(`podcast:${source.name || source.rssUrl}: ${error.message}`); + } + } + return { podcasts, errors }; +} + +export function atomicWriteJson(filePath, value) { + mkdirSync(dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.${process.pid}.tmp`; + writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); + renameSync(tempPath, filePath); +} + +export async function generateBuilderFeeds({ + sourcesPath = DEFAULT_SOURCES, + outDir = DEFAULT_OUT_DIR, + now = new Date(), + fetchImpl, +} = {}) { + const sources = JSON.parse(readFileSync(sourcesPath, 'utf8')); + const generatedAt = now.toISOString(); + const options = { fetchImpl }; + + const [xResult, blogResult, podcastResult] = await Promise.all([ + collectXFeed(sources.x, options), + collectBlogFeed(sources.blogs, options), + collectPodcastFeed(sources.podcasts, options), + ]); + + const feedX = { generatedAt, x: xResult.builders }; + const feedBlogs = { generatedAt, blogs: blogResult.blogs }; + const feedPodcasts = { generatedAt, podcasts: podcastResult.podcasts }; + const errors = [...xResult.errors, ...blogResult.errors, ...podcastResult.errors]; + + const hasData = + feedX.x.some((builder) => builder.tweets?.length) || + feedBlogs.blogs.length > 0 || + feedPodcasts.podcasts.length > 0; + + mkdirSync(outDir, { recursive: true }); + atomicWriteJson(join(outDir, 'feed-x.json'), feedX); + atomicWriteJson(join(outDir, 'feed-blogs.json'), feedBlogs); + atomicWriteJson(join(outDir, 'feed-podcasts.json'), feedPodcasts); + atomicWriteJson(join(outDir, 'generation-report.json'), { + generatedAt, + hasData, + counts: { + xBuilders: feedX.x.length, + xTweets: feedX.x.reduce((sum, builder) => sum + (builder.tweets?.length || 0), 0), + blogs: feedBlogs.blogs.length, + podcasts: feedPodcasts.podcasts.length, + }, + errors, + }); + + return { hasData, errors, feedX, feedBlogs, feedPodcasts }; +} + +const isMain = process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1]); + +if (isMain) { + const result = await generateBuilderFeeds(); + if (!result.hasData) { + console.error('Feed generation produced no items.'); + for (const error of result.errors) console.error(`- ${error}`); + process.exit(1); + } + console.log( + `Generated feeds: x=${result.feedX.x.length} blogs=${result.feedBlogs.blogs.length} podcasts=${result.feedPodcasts.podcasts.length}`, + ); + if (result.errors.length) { + console.warn(`Completed with ${result.errors.length} source warning(s).`); + for (const error of result.errors) console.warn(`- ${error}`); + } +} diff --git a/src/components/BuilderDigestDrawer.tsx b/src/components/BuilderDigestDrawer.tsx index e83f251..9c09a0a 100644 --- a/src/components/BuilderDigestDrawer.tsx +++ b/src/components/BuilderDigestDrawer.tsx @@ -237,7 +237,7 @@ export function BuilderDigestDrawer({ open, onClose, onToast }: Props) {
Follow the people actually building AI products

- Downloads public Follow Builders updates from GitHub at most + Downloads public Zero Tab builder feeds from GitHub at most once per day. Your tabs and saved links never leave Chrome.

diff --git a/store/PUBLISHING_CHECKLIST.md b/store/PUBLISHING_CHECKLIST.md index 92942a8..09113b9 100644 --- a/store/PUBLISHING_CHECKLIST.md +++ b/store/PUBLISHING_CHECKLIST.md @@ -71,8 +71,8 @@ Generate package: - [ ] Submit for review - [ ] Monitor the publisher email for review questions -## Known publication decision +## Builder feed publication -The first release includes AI Builder Daily Report. Its listing and review notes position it as part of one developer-focused new-tab workspace. The Follow Builders data source currently lacks a repository-level license; obtain written permission for public feed display before final publication. +AI Builder Daily Report reads public JSON from `beforeload/zero-tab` on the `feeds` branch. Generate those files with the scheduled GitHub Action (or `workflow_dispatch`) before submitting listing updates that mention the report. See [`SUBMISSION_GUIDE.md`](SUBMISSION_GUIDE.md) for the full submission walkthrough. diff --git a/store/SUBMISSION_GUIDE.md b/store/SUBMISSION_GUIDE.md index cb78dee..2bcda5b 100644 --- a/store/SUBMISSION_GUIDE.md +++ b/store/SUBMISSION_GUIDE.md @@ -90,7 +90,7 @@ Paste the contents of [`store/test-instructions.md`](test-instructions.md) into | `tabs` | Read, focus, and close user-selected tabs | | `storage` | Saved for later, preferences, report cache | | `favicon` | Show local favicons without third-party requests | -| `https://raw.githubusercontent.com/*` (optional) | Fetch public Follow Builders JSON feeds after user opt-in | +| `https://raw.githubusercontent.com/*` (optional) | Fetch public Zero Tab JSON feeds after user opt-in | ## 8. Distribution settings @@ -100,9 +100,9 @@ Recommended first release: - Regions: all supported countries - Publishing: manual review, then publish when approved -## 9. Known blocker before final approval +## 9. Feed readiness before final approval -AI Builder Daily Report displays public content from the Follow Builders repository. That repository does not currently include a repository-level license for public feed display. Before the final public release, obtain written permission from the feed owner or remove/replace the feed source. +AI Builder Daily Report reads public JSON from this repository's `feeds` branch. Before relying on the report in production listings, run `.github/workflows/generate-builder-feeds.yml` at least once and confirm the three feed files are present on `feeds`. ## 10. After approval diff --git a/store/privacy-policy.md b/store/privacy-policy.md index 4424f93..513d6f8 100644 --- a/store/privacy-policy.md +++ b/store/privacy-policy.md @@ -26,7 +26,7 @@ Zero Tab does not operate a server and does not synchronize this local data to t AI Builder Daily Report is optional and disabled until the user enables it. -When enabled, Zero Tab requests access only to `https://raw.githubusercontent.com/` and downloads public JSON feeds published by the Follow Builders project. Requests do not include open-tab data, saved links, browsing history, identifiers, credentials, or API keys. As with any HTTPS request, the remote host may receive standard network metadata such as the user's IP address and user agent under its own privacy policy. +When enabled, Zero Tab requests access only to `https://raw.githubusercontent.com/` and downloads public JSON feeds published by the Zero Tab repository on its `feeds` branch. Those feeds are generated by a scheduled GitHub Action that aggregates public page and RSS summaries. Requests do not include open-tab data, saved links, browsing history, identifiers, credentials, or API keys. As with any HTTPS request, the remote host may receive standard network metadata such as the user's IP address and user agent under its own privacy policy. Feed content is cached locally and filtered in the browser. Original report text is not sent by Zero Tab to a cloud AI service. diff --git a/store/review-notes.md b/store/review-notes.md index 640fd0b..04cdf37 100644 --- a/store/review-notes.md +++ b/store/review-notes.md @@ -27,9 +27,9 @@ Required to display Chrome's local favicon representation without sending tab ho Requested only after the user clicks **Enable AI Builder Daily Report**. Used only to fetch these public data files: -- `zarazhangrui/follow-builders/main/feed-x.json` -- `zarazhangrui/follow-builders/main/feed-podcasts.json` -- `zarazhangrui/follow-builders/main/feed-blogs.json` +- `beforeload/zero-tab/feeds/feed-x.json` +- `beforeload/zero-tab/feeds/feed-podcasts.json` +- `beforeload/zero-tab/feeds/feed-blogs.json` The files contain data, not executable logic. All parsing, ranking, rendering, and interaction logic is packaged in the extension. @@ -37,7 +37,7 @@ No tab data, saved links, identifiers, credentials, or API keys are transmitted ## Remote code -Zero Tab does not download or execute remotely hosted code. The optional GitHub resources are JSON content feeds only. +Zero Tab does not download or execute remotely hosted code. The optional GitHub resources are JSON content feeds only. Feed generation runs in GitHub Actions and aggregates public web/RSS summaries into static JSON; the extension does not scrape third-party sites itself. ## Built-in AI diff --git a/tests/builder-digest.test.js b/tests/builder-digest.test.js index 4265c08..d3a2f5d 100644 --- a/tests/builder-digest.test.js +++ b/tests/builder-digest.test.js @@ -142,6 +142,22 @@ test('returns partial feed results when one source fails', async () => { assert.match(result.errors[0], /podcasts/); }); +test('points AI Builder feeds at this repository feeds branch', () => { + assert.equal(digest.OPTIONAL_ORIGIN, 'https://raw.githubusercontent.com/'); + assert.equal( + digest.FEED_URLS.x, + 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-x.json', + ); + assert.equal( + digest.FEED_URLS.blogs, + 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-blogs.json', + ); + assert.equal( + digest.FEED_URLS.podcasts, + 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-podcasts.json', + ); +}); + test('requests the declared raw GitHub origin without a wildcard path', async () => { const storage = {}; let requestedOrigins; diff --git a/tests/feed-generator.test.mjs b/tests/feed-generator.test.mjs new file mode 100644 index 0000000..c60ef9e --- /dev/null +++ b/tests/feed-generator.test.mjs @@ -0,0 +1,98 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { + generateBuilderFeeds, + parseBlogHtml, + parseRssOrAtom, + parseXSyndicationHtml, +} from '../scripts/lib/feed-generator.mjs'; + +const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures'); + +describe('feed generator parsers', () => { + it('parses RSS blog entries', () => { + const xml = readFileSync(join(fixtures, 'blog.rss.xml'), 'utf8'); + const items = parseRssOrAtom(xml, { + sourceName: 'Example Blog', + baseUrl: 'https://example.com/feed.xml', + }); + assert.equal(items.length, 2); + assert.equal(items[0].title, 'Shipping smaller context windows'); + assert.equal(items[0].url, 'https://example.com/posts/context-windows'); + assert.match(items[0].description, /prompts/i); + }); + + it('parses Atom podcast entries', () => { + const xml = readFileSync(join(fixtures, 'podcast.atom.xml'), 'utf8'); + const items = parseRssOrAtom(xml, { + sourceName: 'Builders Podcast', + baseUrl: 'https://example.com/podcast/atom.xml', + limit: 1, + }); + assert.equal(items.length, 1); + assert.equal(items[0].title, 'Evaluating coding agents'); + assert.equal(items[0].url, 'https://example.com/episodes/agents'); + }); + + it('parses blog HTML article links', () => { + const html = readFileSync(join(fixtures, 'blog.html'), 'utf8'); + const items = parseBlogHtml(html, { + sourceName: 'Cursor Blog', + baseUrl: 'https://cursor.com/blog', + }); + assert.ok(items.length >= 1); + assert.equal(items[0].url, 'https://cursor.com/blog/agent-harness'); + assert.match(items[0].title, /agent harness/i); + }); + + it('parses X syndication markup into tweets', () => { + const html = readFileSync(join(fixtures, 'x-syndication.html'), 'utf8'); + const builder = parseXSyndicationHtml(html, { + name: 'Simon Willison', + handle: 'simonw', + }); + assert.equal(builder.handle, 'simonw'); + assert.equal(builder.tweets.length, 1); + assert.equal(builder.tweets[0].id, '1234567890'); + assert.match(builder.tweets[0].text, /Translator API/i); + assert.equal(builder.tweets[0].url, 'https://x.com/simonw/status/1234567890'); + }); + + it('writes feed JSON through a mocked fetch layer', async () => { + const fixturesByUrl = { + 'https://cdn.syndication.twimg.com/timeline/profile?screen_name=simonw': + readFileSync(join(fixtures, 'x-syndication.html'), 'utf8'), + 'https://example.com/feed.xml': readFileSync(join(fixtures, 'blog.rss.xml'), 'utf8'), + 'https://example.com/podcast.xml': readFileSync(join(fixtures, 'podcast.atom.xml'), 'utf8'), + }; + const fetchImpl = async (url) => { + const body = fixturesByUrl[url]; + if (!body) throw new Error(`unexpected url ${url}`); + return { + ok: true, + status: 200, + text: async () => body, + }; + }; + + const sourcesPath = join(fixtures, 'sources.json'); + const outDir = join(fixtures, 'out'); + const result = await generateBuilderFeeds({ + sourcesPath, + outDir, + now: new Date('2026-08-11T12:00:00.000Z'), + fetchImpl, + }); + + assert.equal(result.hasData, true); + assert.equal(result.feedX.x[0].tweets.length, 1); + assert.ok(result.feedBlogs.blogs.length >= 1); + assert.ok(result.feedPodcasts.podcasts.length >= 1); + + const writtenX = JSON.parse(readFileSync(join(outDir, 'feed-x.json'), 'utf8')); + assert.equal(writtenX.generatedAt, '2026-08-11T12:00:00.000Z'); + }); +}); diff --git a/tests/fixtures/blog.html b/tests/fixtures/blog.html new file mode 100644 index 0000000..3c55612 --- /dev/null +++ b/tests/fixtures/blog.html @@ -0,0 +1,10 @@ + + + +
+ Building a reliable agent harness + Designing a personal tab workstation + Pricing +
+ + diff --git a/tests/fixtures/blog.rss.xml b/tests/fixtures/blog.rss.xml new file mode 100644 index 0000000..5c88789 --- /dev/null +++ b/tests/fixtures/blog.rss.xml @@ -0,0 +1,20 @@ + + + + Example Blog + + Shipping smaller context windows + https://example.com/posts/context-windows + https://example.com/posts/context-windows + Mon, 10 Aug 2026 09:00:00 GMT + Practical notes on trimming prompts and keeping the edit loop fast. + + + Local-first browser tools + https://example.com/posts/local-first + https://example.com/posts/local-first + Sun, 09 Aug 2026 11:00:00 GMT + Why tab metadata should stay on the device. + + + diff --git a/tests/fixtures/podcast.atom.xml b/tests/fixtures/podcast.atom.xml new file mode 100644 index 0000000..c0eda70 --- /dev/null +++ b/tests/fixtures/podcast.atom.xml @@ -0,0 +1,11 @@ + + + Builders Podcast + + Evaluating coding agents + episode-agents + + 2026-08-09T19:40:00Z + Rollout guardrails and evaluation harnesses. + + diff --git a/tests/fixtures/sources.json b/tests/fixtures/sources.json new file mode 100644 index 0000000..0f6da74 --- /dev/null +++ b/tests/fixtures/sources.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "x": [{ "name": "Simon Willison", "handle": "simonw" }], + "blogs": [ + { + "name": "Example Blog", + "url": "https://example.com/", + "rssUrl": "https://example.com/feed.xml" + } + ], + "podcasts": [ + { + "name": "Builders Podcast", + "rssUrl": "https://example.com/podcast.xml" + } + ] +} diff --git a/tests/fixtures/x-syndication.html b/tests/fixtures/x-syndication.html new file mode 100644 index 0000000..2faa880 --- /dev/null +++ b/tests/fixtures/x-syndication.html @@ -0,0 +1,12 @@ + + + +
+ +

+ Chrome on-device Translator API notes for local coding agents +

+ +
+ + From bc7006868dcdb2de8311f74da6863753324db2fc Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 14 Aug 2026 12:18:07 +0800 Subject: [PATCH 2/6] Fix builder feed scrape quality for blogs and RSS. Filter skip/nav junk, strip entity-escaped HTML in summaries, and clean Cursor card titles so the AI Brief shows real article headlines. Co-authored-by: Cursor --- builder-feeds/sources.json | 2 +- scripts/lib/feed-generator.mjs | 224 +++++++++++++++++++++++++++------ tests/feed-generator.test.mjs | 21 +++- tests/fixtures/blog.html | 31 ++++- tests/fixtures/blog.rss.xml | 2 +- 5 files changed, 232 insertions(+), 48 deletions(-) diff --git a/builder-feeds/sources.json b/builder-feeds/sources.json index 53d9b1b..ef0118d 100644 --- a/builder-feeds/sources.json +++ b/builder-feeds/sources.json @@ -1,7 +1,7 @@ { "version": 1, "x": [ - { "name": "Boris Cherny", "handle": "boris_cherny" }, + { "name": "Boris Cherny", "handle": "bcherny" }, { "name": "Anthropic", "handle": "AnthropicAI" }, { "name": "Guillermo Rauch", "handle": "rauchg" }, { "name": "Simon Willison", "handle": "simonw" }, diff --git a/scripts/lib/feed-generator.mjs b/scripts/lib/feed-generator.mjs index 99184aa..5f9919e 100644 --- a/scripts/lib/feed-generator.mjs +++ b/scripts/lib/feed-generator.mjs @@ -50,19 +50,166 @@ export function safeHttpsUrl(value, base) { } export function stripTags(value) { - return decodeEntities( - String(value || '') - .replace(//gi, ' ') - .replace(//gi, ' ') - .replace(/<[^>]+>/g, ' '), + const decoded = decodeEntities(String(value || '')); + return normalizeText( + decodeEntities( + decoded + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(/<[^>]+>/g, ' '), + ), ); } +const SKIP_LINK_TITLE = + /^(跳到|跳轉到|跳转到)?\s*(主要)?(内容|內容|页脚|頁腳|导航|導航|選單|菜单|footer|main content|content|navigation|menu)\s*$/i; + +const NAV_TITLE = + /^(home|about|careers|pricing|docs|documentation|support|login|sign in|sign up|subscribe|privacy|terms|cookie|contact|research|news|blog|engineering|products?|company|api|claude|how to get support|如何获得支持|如何獲得支持)$/i; + +function isJunkTitle(title) { + const text = normalizeText(title); + if (!text || text.length < 16 || text.length > 160) return true; + if (SKIP_LINK_TITLE.test(text)) return true; + if (NAV_TITLE.test(text)) return true; + if (/^(skip to|jump to)\b/i.test(text)) return true; + // Concatenated card blobs usually contain multiple sentences/dates mashed together. + if ((text.match(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/g) || []).length >= 2) { + return true; + } + return false; +} + +function canonicalizeArticleUrl(value, baseUrl) { + const href = safeHttpsUrl(value, baseUrl); + if (!href) return ''; + try { + const url = new URL(href); + if (url.hash && /^#(main|main-content|content|footer|nav|navigation|top)$/i.test(url.hash)) { + return ''; + } + // Drop pure in-page skip targets that share the index path. + if (baseUrl) { + const base = new URL(baseUrl); + if (url.origin === base.origin && url.pathname.replace(/\/$/, '') === base.pathname.replace(/\/$/, '') && url.hash) { + return ''; + } + } + url.hash = ''; + url.search = ''; + return url.href.replace(/\/$/, ''); + } catch { + return ''; + } +} + +function looksLikeArticleUrl(url, baseUrl) { + try { + const parsed = new URL(url); + const path = parsed.pathname; + if (/\/(tag|tags|category|categories|author|authors|page|pages|search|login|signup)(\/|$)/i.test(path)) { + return false; + } + if (/\/(blog|news|engineering|posts|articles|research|changelog|index)\b/i.test(path)) { + // Index pages themselves are not articles. + if (/\/(blog|news|engineering|posts|articles|research|changelog|index)\/?$/i.test(path)) { + return false; + } + return true; + } + if (/\/\d{4}\/\d{2}\//.test(path)) return true; + if (/\/[a-z0-9-]{16,}\/?$/i.test(path)) return true; + if (baseUrl) { + const base = new URL(baseUrl); + if ( + parsed.origin === base.origin && + path.startsWith(base.pathname.replace(/\/$/, '') + '/') && + path.replace(/\/$/, '') !== base.pathname.replace(/\/$/, '') + ) { + return path.split('/').filter(Boolean).pop()?.length >= 8; + } + } + return false; + } catch { + return false; + } +} + +function extractHeadingTitle(anchorHtml) { + const heading = + anchorHtml.match(/]*>([\s\S]*?)<\/h[1-4]>/i) || + anchorHtml.match( + /<(?:span|div|p)[^>]*class=["'][^"']*(?:title|headline|card-title|post-title|featuredTitle)[^"']*["'][^>]*>([\s\S]*?)<\/(?:span|div|p)>/i, + ); + if (heading) return cleanArticleTitle(stripTags(heading[1])); + + // Prefer meaningful image alts (Cursor cards put the headline in alt). + const imageAlt = [...anchorHtml.matchAll(/]*\balt=["']([^"']{8,160})["'][^>]*>/gi)] + .map((match) => match[1].trim()) + .find(Boolean); + if (imageAlt) return cleanArticleTitle(imageAlt); + + return cleanArticleTitle(stripTags(anchorHtml)); +} + +function stripAuthorReadTimeCrumbs(title) { + let text = title; + for (let i = 0; i < 4; i += 1) { + const next = text + .replace(/\s+\d+\s*min(?:ute)?s?\s+read$/i, '') + // "Maxime Prades · 2m" / "Connor & Yuri · 6m" / "Chris, Rikki & Kevin · 7m" + .replace( + /\s+[A-Z][A-Za-z.]+(?:\s*(?:,|&|and)\s*|\s+)[A-Z][A-Za-z.]+(?:(?:\s*(?:,|&|and)\s*|\s+)[A-Z][A-Za-z.]+)*\s+·\s*\d+m$/u, + '', + ) + // "Connor & Yuri 6m" / "Chris, Rikki & Kevin 7m" + .replace( + /\s+[A-Z][A-Za-z.]+(?:\s*(?:,|&|and)\s*)[A-Z][A-Za-z.]+(?:(?:\s*(?:,|&|and)\s*)[A-Z][A-Za-z.]+)*\s+\d+m$/u, + '', + ) + // "Maxime Prades 2m" (exactly first + last before read-time) + .replace(/\s+[A-Z][a-z]+\s+[A-Z][a-z]+\s+\d+m$/u, ''); + if (next === text) break; + text = next; + } + return text; +} + +function cleanArticleTitle(title) { + let text = normalizeText(decodeEntities(title)); + text = text.replace( + /^(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},\s+\d{4}\s*[·•\-–|]?\s*(?:Research|Product|Company|company|product|Features|Announcements|News)?\s*/i, + '', + ); + text = text.replace( + /^(?:Featured|Announcements|Features|Product|News|Research|Company)\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},\s+\d{4}\s*/i, + '', + ); + text = text.replace( + /^(?:Featured|Announcements|Features|Product|News|Research|Company)\s+/i, + '', + ); + text = stripAuthorReadTimeCrumbs(text); + + // If a short headline is followed by a description sentence, keep the headline. + // Avoid bare "Cursor" here — titles like "… with Cursor for iOS" are valid. + const split = text.match( + /^(.{16,100}?)(?=\s+(?:We|The|How|This|A|An|Our|Built|I|Cursor is)\b)/, + ); + if (split?.[1] && !/[.!?]$/.test(split[1])) { + text = split[1]; + } else if (text.length > 110) { + const sentence = text.match(/^.{16,110}?(?:[.!?…]|$)/)?.[0]; + if (sentence) text = sentence; + } + return truncate(text, 120); +} + function tagValue(block, tag) { const cdata = block.match( new RegExp(`<${tag}[^>]*>\\s*\\s*`, 'i'), ); - if (cdata) return decodeEntities(cdata[1]); + if (cdata) return stripTags(cdata[1]); const plain = block.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)`, 'i')); return plain ? stripTags(plain[1]) : ''; } @@ -84,11 +231,11 @@ export function parseRssOrAtom(xml, { sourceName, baseUrl, limit = 6 } = {}) { for (const block of blocks) { const title = truncate(tagValue(block, 'title'), 180); const link = - safeHttpsUrl(tagValue(block, 'link'), baseUrl) || - safeHttpsUrl(tagAttr(block, 'link', 'href'), baseUrl) || - safeHttpsUrl(tagValue(block, 'guid'), baseUrl) || - safeHttpsUrl(tagValue(block, 'id'), baseUrl); - if (!title || !link) continue; + canonicalizeArticleUrl(tagValue(block, 'link'), baseUrl) || + canonicalizeArticleUrl(tagAttr(block, 'link', 'href'), baseUrl) || + canonicalizeArticleUrl(tagValue(block, 'guid'), baseUrl) || + canonicalizeArticleUrl(tagValue(block, 'id'), baseUrl); + if (!title || !link || isJunkTitle(title)) continue; const publishedAt = tagValue(block, 'pubDate') || @@ -96,12 +243,14 @@ export function parseRssOrAtom(xml, { sourceName, baseUrl, limit = 6 } = {}) { tagValue(block, 'updated') || tagValue(block, 'dc:date') || null; - const description = + const description = truncate( tagValue(block, 'description') || - tagValue(block, 'summary') || - tagValue(block, 'content:encoded') || - tagValue(block, 'content') || - ''; + tagValue(block, 'summary') || + tagValue(block, 'content:encoded') || + tagValue(block, 'content') || + '', + 500, + ); const guid = tagValue(block, 'guid') || tagValue(block, 'id') || link; items.push({ @@ -109,9 +258,9 @@ export function parseRssOrAtom(xml, { sourceName, baseUrl, limit = 6 } = {}) { title, url: link, guid, - description: truncate(description, 500), - content: truncate(description, 500), - transcript: truncate(description, 500), + description, + content: description, + transcript: description, publishedAt: publishedAt ? new Date(Date.parse(publishedAt) || Date.now()).toISOString() : undefined, }); if (items.length >= limit) break; @@ -121,28 +270,25 @@ export function parseRssOrAtom(xml, { sourceName, baseUrl, limit = 6 } = {}) { } export function parseBlogHtml(html, { sourceName, baseUrl, limit = 6 } = {}) { - const text = String(html || ''); + // Strip site chrome only. Do NOT strip
— many blog cards (e.g. Cursor) + // wrap the title/media inside a card-level
. + const text = String(html || '') + .replace(//gi, ' ') + .replace(//gi, ' '); const found = new Map(); - const patterns = [ - /]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, - ]; - - for (const pattern of patterns) { - for (const match of text.matchAll(pattern)) { - const href = match[1]; - const title = truncate(stripTags(match[2]), 180); - const url = safeHttpsUrl(href, baseUrl); - if (!url || !title || title.length < 12) continue; - if (!/\/(blog|news|engineering|posts|articles|research|changelog)\b/i.test(url) && - !/blog|news|engineering|post|article/i.test(href)) { - // Keep homepage-relative article-looking paths with dates or long slugs. - if (!/\/\d{4}\/\d{2}\//.test(url) && !/\/[a-z0-9-]{16,}\/?$/i.test(url)) continue; - } - if (/#(respond|comments)|\/tag\/|\/category\/|\/author\//i.test(url)) continue; - if (!found.has(url)) found.set(url, title); - if (found.size >= limit * 3) break; - } + // Document order; extractHeadingTitle already prefers h1–h4 / img[alt] inside the card. + const candidates = text.matchAll(/]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi); + + for (const match of candidates) { + const href = match[1]; + const inner = match[2] || ''; + const title = extractHeadingTitle(inner); + const url = canonicalizeArticleUrl(href, baseUrl); + if (!url || !title || isJunkTitle(title)) continue; + if (!looksLikeArticleUrl(url, baseUrl)) continue; + if (!found.has(url)) found.set(url, title); + if (found.size >= limit * 2) break; } return [...found.entries()].slice(0, limit).map(([url, title]) => ({ diff --git a/tests/feed-generator.test.mjs b/tests/feed-generator.test.mjs index c60ef9e..72983a5 100644 --- a/tests/feed-generator.test.mjs +++ b/tests/feed-generator.test.mjs @@ -23,6 +23,9 @@ describe('feed generator parsers', () => { assert.equal(items[0].title, 'Shipping smaller context windows'); assert.equal(items[0].url, 'https://example.com/posts/context-windows'); assert.match(items[0].description, /prompts/i); + assert.equal(items[0].description.includes('<'), false); + assert.equal(items[0].description.includes('<'), false); + assert.equal(items[0].description.includes(' { @@ -37,15 +40,25 @@ describe('feed generator parsers', () => { assert.equal(items[0].url, 'https://example.com/episodes/agents'); }); - it('parses blog HTML article links', () => { + it('parses blog HTML article links and ignores skip/nav junk', () => { const html = readFileSync(join(fixtures, 'blog.html'), 'utf8'); const items = parseBlogHtml(html, { sourceName: 'Cursor Blog', baseUrl: 'https://cursor.com/blog', }); - assert.ok(items.length >= 1); - assert.equal(items[0].url, 'https://cursor.com/blog/agent-harness'); - assert.match(items[0].title, /agent harness/i); + assert.equal(items.length, 3); + assert.deepEqual( + items.map((item) => [item.url, item.title]), + [ + ['https://cursor.com/blog/agent-harness', 'Building a reliable agent harness'], + ['https://cursor.com/blog/tab-workstation', 'Designing a personal tab workstation'], + ['https://cursor.com/blog/ios-mobile-app', 'Build from anywhere with Cursor for iOS'], + ], + ); + assert.equal( + items.every((item) => !/跳到|Skip to|如何获得支持/i.test(item.title)), + true, + ); }); it('parses X syndication markup into tweets', () => { diff --git a/tests/fixtures/blog.html b/tests/fixtures/blog.html index 3c55612..7f6038a 100644 --- a/tests/fixtures/blog.html +++ b/tests/fixtures/blog.html @@ -1,10 +1,35 @@ -
- Building a reliable agent harness - Designing a personal tab workstation + Skip to main content + 跳到页脚 + +
+ +
+ Building a reliable agent harness +

How we keep agents productive without expanding blast radius.

+ Cursor Team · 3m +
+ + +

Designing a personal tab workstation

+ Announcements +

Extra card body that should not become the title.

+ + +
+ Build from anywhere with Cursor for iOS +

Cursor is available as a native iOS app on your phone.

+ Chris, Rikki & Kevin · 7m +
+ + 跳到主要内容 + 如何获得支持
+
Footer
diff --git a/tests/fixtures/blog.rss.xml b/tests/fixtures/blog.rss.xml index 5c88789..c5ed018 100644 --- a/tests/fixtures/blog.rss.xml +++ b/tests/fixtures/blog.rss.xml @@ -7,7 +7,7 @@ https://example.com/posts/context-windows https://example.com/posts/context-windows Mon, 10 Aug 2026 09:00:00 GMT - Practical notes on trimming prompts and keeping the edit loop fast. + Local-first browser tools From 1f15899ef0201f2939d5c0ec4b7489f882498678 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 14 Aug 2026 13:04:57 +0800 Subject: [PATCH 3/6] Add conference video feeds for AI Engineer, Compile, and Config. Subscribe to the YouTube RSS sources in the AI Brief so recent talk uploads surface alongside blogs and podcasts. Co-authored-by: Cursor --- .github/workflows/generate-builder-feeds.yml | 3 +- AGENTS.md | 2 +- README.md | 2 +- builder-feeds/README.md | 2 +- builder-feeds/sources.json | 14 ++++ extension/app.js | 3 +- extension/builder-digest.js | 34 ++++++++- extension/style.css | 5 ++ scripts/generate-builder-feeds.mjs | 2 +- scripts/lib/feed-generator.mjs | 75 +++++++++++++++----- store/listing-en.md | 2 +- store/review-notes.md | 1 + tests/builder-digest.test.js | 66 ++++++++++++++--- tests/feed-generator.test.mjs | 16 +++++ tests/fixtures/sources.json | 6 ++ tests/fixtures/youtube.atom.xml | 16 +++++ 16 files changed, 213 insertions(+), 36 deletions(-) create mode 100644 tests/fixtures/youtube.atom.xml diff --git a/.github/workflows/generate-builder-feeds.yml b/.github/workflows/generate-builder-feeds.yml index 3a0f5b4..150c98b 100644 --- a/.github/workflows/generate-builder-feeds.yml +++ b/.github/workflows/generate-builder-feeds.yml @@ -58,11 +58,12 @@ jobs: cp builder-feeds/generated/feed-x.json "$PUBLISH_DIR/feed-x.json" cp builder-feeds/generated/feed-blogs.json "$PUBLISH_DIR/feed-blogs.json" cp builder-feeds/generated/feed-podcasts.json "$PUBLISH_DIR/feed-podcasts.json" + cp builder-feeds/generated/feed-videos.json "$PUBLISH_DIR/feed-videos.json" cp builder-feeds/generated/generation-report.json "$PUBLISH_DIR/generation-report.json" git -C "$PUBLISH_DIR" config user.name "github-actions[bot]" git -C "$PUBLISH_DIR" config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git -C "$PUBLISH_DIR" add feed-x.json feed-blogs.json feed-podcasts.json generation-report.json + git -C "$PUBLISH_DIR" add feed-x.json feed-blogs.json feed-podcasts.json feed-videos.json generation-report.json if git -C "$PUBLISH_DIR" diff --cached --quiet; then echo "No feed changes to publish." diff --git a/AGENTS.md b/AGENTS.md index ccbc478..2c18742 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ Before doing anything technical, tell the user what they're about to get: > - **Save for later** bookmark individual tabs to a checklist before closing them > - **Modular workspace cards** can be collapsed or hidden, with preferences stored locally > - **Daily horoscope** generates a private, deterministic reading from the local date and selected zodiac sign -> - **AI Builder daily brief** opens as an independent full-height drawer with public updates from builders, podcasts, and engineering blogs +> - **AI Builder daily brief** opens as an independent full-height drawer with public updates from builders, podcasts, engineering blogs, and conference videos > - **Local-first** tab and saved-item data never leaves the browser > > It's just a Chrome extension. Setup takes about 1 minute. diff --git a/README.md b/README.md index b8162d4..48e389b 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Zero Tab publishes AI Builder JSON feeds from this repository's `feeds` branch. npm run generate:builder-feeds ``` -Sources live in [`builder-feeds/sources.json`](builder-feeds/sources.json). GitHub Actions runs [`.github/workflows/generate-builder-feeds.yml`](.github/workflows/generate-builder-feeds.yml) daily and on `workflow_dispatch`, then commits `feed-x.json`, `feed-blogs.json`, and `feed-podcasts.json` to the `feeds` branch root. The extension reads those files from `raw.githubusercontent.com`. +Sources live in [`builder-feeds/sources.json`](builder-feeds/sources.json). GitHub Actions runs [`.github/workflows/generate-builder-feeds.yml`](.github/workflows/generate-builder-feeds.yml) daily and on `workflow_dispatch`, then commits `feed-x.json`, `feed-blogs.json`, `feed-podcasts.json`, and `feed-videos.json` to the `feeds` branch root. The extension reads those files from `raw.githubusercontent.com`. ## Attribution and license diff --git a/builder-feeds/README.md b/builder-feeds/README.md index d127c76..0a57fe1 100644 --- a/builder-feeds/README.md +++ b/builder-feeds/README.md @@ -2,7 +2,7 @@ Curated source list and local generation output for the AI Builder Daily Report. -- [`sources.json`](sources.json) — X handles, blogs (RSS and/or HTML), podcasts (RSS) +- [`sources.json`](sources.json) — X handles, blogs (RSS and/or HTML), podcasts (RSS), conference videos (YouTube RSS) - `generated/` — local output of `npm run generate:builder-feeds` (gitignored) - Published artifacts live on the repository `feeds` branch root diff --git a/builder-feeds/sources.json b/builder-feeds/sources.json index ef0118d..215cc3a 100644 --- a/builder-feeds/sources.json +++ b/builder-feeds/sources.json @@ -52,5 +52,19 @@ "name": "Practical AI", "rssUrl": "https://changelog.com/practicalai/feed" } + ], + "videos": [ + { + "name": "AI Engineer", + "rssUrl": "https://www.youtube.com/feeds/videos.xml?channel_id=UCLKPca3kwwd-B59HNr-_lvA" + }, + { + "name": "Cursor Compile", + "rssUrl": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLuI2ZfvGpzwDCn0njJpjZ3ZiEpCqhN7BK" + }, + { + "name": "Figma Config", + "rssUrl": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLXDU_eVOJTx6erPKfFHtCNbyCmcCn4zrp" + } ] } diff --git a/extension/app.js b/extension/app.js index dcb220c..0cfe70f 100644 --- a/extension/app.js +++ b/extension/app.js @@ -1313,6 +1313,7 @@ function formatDigestDate(dateStr) { function digestKindLabel(kind) { if (kind === 'blog') return 'Blog'; if (kind === 'podcast') return 'Podcast'; + if (kind === 'video') return 'Video'; return 'X'; } @@ -1372,7 +1373,7 @@ function renderDigestItemCard(item, state, stale, targetLanguage) { const safeExcerpt = escapeHtml(localized?.excerpt || item.excerpt); const safeSource = escapeHtml(item.source); const safeItemId = escapeHtml(item.id); - const kind = ['x', 'blog', 'podcast'].includes(item.kind) ? item.kind : 'x'; + const kind = ['x', 'blog', 'podcast', 'video'].includes(item.kind) ? item.kind : 'x'; const dateLabel = escapeHtml(formatDigestDate(item.publishedAt)); const isRead = state.readIds?.includes(item.id); diff --git a/extension/builder-digest.js b/extension/builder-digest.js index b43ad44..fd63de2 100644 --- a/extension/builder-digest.js +++ b/extension/builder-digest.js @@ -11,9 +11,11 @@ x: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-x.json', podcasts: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-podcasts.json', blogs: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-blogs.json', + videos: 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-videos.json', }; const MAX_RESPONSE_CHARS = 1_500_000; const CACHE_RETENTION_MS = 48 * 60 * 60 * 1000; + const VIDEO_RETENTION_MS = 90 * 24 * 60 * 60 * 1000; const RETRY_COOLDOWN_MS = 15 * 60 * 1000; const KEYWORDS = /\b(launch|launched|release|released|ship|shipped|announce|model|agent|coding|code|api|open[\s-]?source|research|benchmark|security|product|tool|framework|developer|build|robot|autonom)/i; @@ -79,6 +81,7 @@ const xFeed = feeds?.x; const podcastFeed = feeds?.podcasts; const blogFeed = feeds?.blogs; + const videoFeed = feeds?.videos; for (const builder of Array.isArray(xFeed?.x) ? xFeed.x : []) { for (const tweet of Array.isArray(builder?.tweets) ? builder.tweets : []) { @@ -144,16 +147,41 @@ }); } + for (const video of Array.isArray(videoFeed?.videos) ? videoFeed.videos : []) { + const url = safeHttpsUrl(video?.url); + const title = truncate(video?.title, 180); + if (!url || !title) continue; + + const fallbackTime = timestamp(videoFeed?.generatedAt, nowMs); + const publishedAt = new Date(timestamp(video?.publishedAt, fallbackTime)).toISOString(); + const excerpt = truncate(video?.transcript, 340); + const guid = normalizeText(video?.guid) || url; + items.push({ + id: `video:${guid}`, + kind: 'video', + source: truncate(video?.name || 'Conference talk', 80), + title, + excerpt, + url, + publishedAt, + score: 64 + keywordScore(`${title} ${excerpt}`) + recencyScore(publishedAt, nowMs), + }); + } + return items.sort((a, b) => b.score - a.score || b.publishedAt.localeCompare(a.publishedAt)); } + function retentionMsFor(item) { + return item?.kind === 'video' ? VIDEO_RETENTION_MS : CACHE_RETENTION_MS; + } + function mergeItems(previous, incoming, now = new Date(), limit = Infinity) { - const cutoff = now.getTime() - CACHE_RETENTION_MS; + const nowMs = now.getTime(); const merged = new Map(); for (const item of [...(Array.isArray(previous) ? previous : []), ...(Array.isArray(incoming) ? incoming : [])]) { if (!item?.id || !safeHttpsUrl(item.url)) continue; - if (timestamp(item.publishedAt, now.getTime()) < cutoff) continue; + if (timestamp(item.publishedAt, nowMs) < nowMs - retentionMsFor(item)) continue; merged.set(item.id, item); } @@ -168,7 +196,7 @@ const selected = []; const selectedIds = new Set(); - for (const kind of ['x', 'blog', 'podcast']) { + for (const kind of ['x', 'blog', 'podcast', 'video']) { const item = sorted.find(candidate => candidate.kind === kind); if (item && !selectedIds.has(item.id)) { selected.push(item); diff --git a/extension/style.css b/extension/style.css index 743f103..8d5319d 100644 --- a/extension/style.css +++ b/extension/style.css @@ -605,6 +605,11 @@ header { background: var(--orange-tint); } +.digest-kind[data-kind="video"] { + color: var(--green); + background: var(--green-tint); +} + .digest-freshness.is-stale { color: var(--orange); background: var(--orange-tint); diff --git a/scripts/generate-builder-feeds.mjs b/scripts/generate-builder-feeds.mjs index 4fb8175..e06469d 100644 --- a/scripts/generate-builder-feeds.mjs +++ b/scripts/generate-builder-feeds.mjs @@ -9,7 +9,7 @@ if (!result.hasData) { } console.log( - `Generated feeds: x=${result.feedX.x.length} blogs=${result.feedBlogs.blogs.length} podcasts=${result.feedPodcasts.podcasts.length}`, + `Generated feeds: x=${result.feedX.x.length} blogs=${result.feedBlogs.blogs.length} podcasts=${result.feedPodcasts.podcasts.length} videos=${result.feedVideos.videos.length}`, ); if (result.errors.length) { console.warn(`Completed with ${result.errors.length} source warning(s).`); diff --git a/scripts/lib/feed-generator.mjs b/scripts/lib/feed-generator.mjs index 5f9919e..940d91d 100644 --- a/scripts/lib/feed-generator.mjs +++ b/scripts/lib/feed-generator.mjs @@ -11,6 +11,7 @@ const FETCH_TIMEOUT_MS = 20_000; const MAX_TWEETS_PER_HANDLE = 8; const MAX_BLOG_ITEMS_PER_SOURCE = 6; const MAX_PODCAST_ITEMS_PER_SOURCE = 4; +const MAX_VIDEO_ITEMS_PER_SOURCE = 6; export function normalizeText(value) { return String(value || '') @@ -69,7 +70,7 @@ const NAV_TITLE = function isJunkTitle(title) { const text = normalizeText(title); - if (!text || text.length < 16 || text.length > 160) return true; + if (!text || text.length < 8 || text.length > 200) return true; if (SKIP_LINK_TITLE.test(text)) return true; if (NAV_TITLE.test(text)) return true; if (/^(skip to|jump to)\b/i.test(text)) return true; @@ -96,7 +97,16 @@ function canonicalizeArticleUrl(value, baseUrl) { } } url.hash = ''; - url.search = ''; + // Keep YouTube watch IDs; stripping all search params breaks video links. + if (/^(www\.)?youtube\.com$/i.test(url.hostname) && url.pathname === '/watch') { + const videoId = url.searchParams.get('v'); + url.search = ''; + if (videoId) url.searchParams.set('v', videoId); + } else if (/^(www\.)?youtu\.be$/i.test(url.hostname)) { + url.search = ''; + } else { + url.search = ''; + } return url.href.replace(/\/$/, ''); } catch { return ''; @@ -246,6 +256,7 @@ export function parseRssOrAtom(xml, { sourceName, baseUrl, limit = 6 } = {}) { const description = truncate( tagValue(block, 'description') || tagValue(block, 'summary') || + tagValue(block, 'media:description') || tagValue(block, 'content:encoded') || tagValue(block, 'content') || '', @@ -498,27 +509,27 @@ export async function collectBlogFeed(sources, options = {}) { return { blogs, errors }; } -export async function collectPodcastFeed(sources, options = {}) { - const podcasts = []; +async function collectRssListFeed(sources, { kind, limit, options = {} } = {}) { + const items = []; const errors = []; for (const source of sources || []) { try { if (!source.rssUrl) { - errors.push(`podcast:${source.name || 'unknown'}: missing rssUrl`); + errors.push(`${kind}:${source.name || 'unknown'}: missing rssUrl`); continue; } const xml = await fetchText(source.rssUrl, options); - const items = parseRssOrAtom(xml, { + const parsed = parseRssOrAtom(xml, { sourceName: source.name, baseUrl: source.rssUrl, - limit: MAX_PODCAST_ITEMS_PER_SOURCE, + limit, }); - if (!items.length) { - errors.push(`podcast:${source.name}: empty RSS`); + if (!parsed.length) { + errors.push(`${kind}:${source.name}: empty RSS`); continue; } - for (const item of items) { - podcasts.push({ + for (const item of parsed) { + items.push({ name: item.name, title: item.title, url: item.url, @@ -528,10 +539,28 @@ export async function collectPodcastFeed(sources, options = {}) { }); } } catch (error) { - errors.push(`podcast:${source.name || source.rssUrl}: ${error.message}`); + errors.push(`${kind}:${source.name || source.rssUrl}: ${error.message}`); } } - return { podcasts, errors }; + return { items, errors }; +} + +export async function collectPodcastFeed(sources, options = {}) { + const result = await collectRssListFeed(sources, { + kind: 'podcast', + limit: MAX_PODCAST_ITEMS_PER_SOURCE, + options, + }); + return { podcasts: result.items, errors: result.errors }; +} + +export async function collectVideoFeed(sources, options = {}) { + const result = await collectRssListFeed(sources, { + kind: 'video', + limit: MAX_VIDEO_ITEMS_PER_SOURCE, + options, + }); + return { videos: result.items, errors: result.errors }; } export function atomicWriteJson(filePath, value) { @@ -551,26 +580,35 @@ export async function generateBuilderFeeds({ const generatedAt = now.toISOString(); const options = { fetchImpl }; - const [xResult, blogResult, podcastResult] = await Promise.all([ + const [xResult, blogResult, podcastResult, videoResult] = await Promise.all([ collectXFeed(sources.x, options), collectBlogFeed(sources.blogs, options), collectPodcastFeed(sources.podcasts, options), + collectVideoFeed(sources.videos, options), ]); const feedX = { generatedAt, x: xResult.builders }; const feedBlogs = { generatedAt, blogs: blogResult.blogs }; const feedPodcasts = { generatedAt, podcasts: podcastResult.podcasts }; - const errors = [...xResult.errors, ...blogResult.errors, ...podcastResult.errors]; + const feedVideos = { generatedAt, videos: videoResult.videos }; + const errors = [ + ...xResult.errors, + ...blogResult.errors, + ...podcastResult.errors, + ...videoResult.errors, + ]; const hasData = feedX.x.some((builder) => builder.tweets?.length) || feedBlogs.blogs.length > 0 || - feedPodcasts.podcasts.length > 0; + feedPodcasts.podcasts.length > 0 || + feedVideos.videos.length > 0; mkdirSync(outDir, { recursive: true }); atomicWriteJson(join(outDir, 'feed-x.json'), feedX); atomicWriteJson(join(outDir, 'feed-blogs.json'), feedBlogs); atomicWriteJson(join(outDir, 'feed-podcasts.json'), feedPodcasts); + atomicWriteJson(join(outDir, 'feed-videos.json'), feedVideos); atomicWriteJson(join(outDir, 'generation-report.json'), { generatedAt, hasData, @@ -579,11 +617,12 @@ export async function generateBuilderFeeds({ xTweets: feedX.x.reduce((sum, builder) => sum + (builder.tweets?.length || 0), 0), blogs: feedBlogs.blogs.length, podcasts: feedPodcasts.podcasts.length, + videos: feedVideos.videos.length, }, errors, }); - return { hasData, errors, feedX, feedBlogs, feedPodcasts }; + return { hasData, errors, feedX, feedBlogs, feedPodcasts, feedVideos }; } const isMain = process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1]); @@ -596,7 +635,7 @@ if (isMain) { process.exit(1); } console.log( - `Generated feeds: x=${result.feedX.x.length} blogs=${result.feedBlogs.blogs.length} podcasts=${result.feedPodcasts.podcasts.length}`, + `Generated feeds: x=${result.feedX.x.length} blogs=${result.feedBlogs.blogs.length} podcasts=${result.feedPodcasts.podcasts.length} videos=${result.feedVideos.videos.length}`, ); if (result.errors.length) { console.warn(`Completed with ${result.errors.length} source warning(s).`); diff --git a/store/listing-en.md b/store/listing-en.md index d9a00db..4a5225c 100644 --- a/store/listing-en.md +++ b/store/listing-en.md @@ -32,7 +32,7 @@ Move a tab into a local Saved for later checklist before closing it. Completed i ### Optional AI Builder Daily Report -Enable a chronological feed of public updates from selected AI builders, engineering blogs, and podcasts. The report is disabled by default and requests access only to public JSON files on GitHub when enabled. +Enable a chronological feed of public updates from selected AI builders, engineering blogs, podcasts, and conference videos. The report is disabled by default and requests access only to public JSON files on GitHub when enabled. Cards can be marked as read, opened at their original source, and translated with Chrome's on-device Translator API when supported. diff --git a/store/review-notes.md b/store/review-notes.md index 04cdf37..2429a62 100644 --- a/store/review-notes.md +++ b/store/review-notes.md @@ -29,6 +29,7 @@ Requested only after the user clicks **Enable AI Builder Daily Report**. Used on - `beforeload/zero-tab/feeds/feed-x.json` - `beforeload/zero-tab/feeds/feed-podcasts.json` +- `beforeload/zero-tab/feeds/feed-videos.json` - `beforeload/zero-tab/feeds/feed-blogs.json` The files contain data, not executable logic. All parsing, ranking, rendering, and interaction logic is packaged in the extension. diff --git a/tests/builder-digest.test.js b/tests/builder-digest.test.js index d3a2f5d..c49f089 100644 --- a/tests/builder-digest.test.js +++ b/tests/builder-digest.test.js @@ -42,10 +42,21 @@ test('normalizes and ranks all supported feed types', () => { transcript: 'Today we discuss product iteration and shipping reliable AI tools.', }], }, + videos: { + generatedAt: NOW.toISOString(), + videos: [{ + name: 'Cursor Compile', + title: 'Opening Keynote, Michael Truell | Compile 26', + guid: 'yt:video:compile1', + url: 'https://www.youtube.com/watch?v=compile1', + transcript: 'Cursor Compile conference keynote about shipping agent coding tools.', + publishedAt: '2026-06-20T17:00:00.000Z', + }], + }, }, NOW); - assert.equal(items.length, 3); - assert.deepEqual(new Set(items.map(item => item.kind)), new Set(['x', 'blog', 'podcast'])); + assert.equal(items.length, 4); + assert.deepEqual(new Set(items.map(item => item.kind)), new Set(['x', 'blog', 'podcast', 'video'])); assert.ok(items.every(item => item.url.startsWith('https://'))); assert.ok(items.every(item => Number.isFinite(item.score))); }); @@ -63,10 +74,11 @@ test('keeps a balanced top selection when sources exist', () => { { id: 'x:2', kind: 'x', score: 90 }, { id: 'blog:1', kind: 'blog', score: 50 }, { id: 'podcast:1', kind: 'podcast', score: 40 }, + { id: 'video:1', kind: 'video', score: 35 }, ]; - const selected = digest.selectTopItems(items, 3); - assert.deepEqual(new Set(selected.map(item => item.kind)), new Set(['x', 'blog', 'podcast'])); + const selected = digest.selectTopItems(items, 4); + assert.deepEqual(new Set(selected.map(item => item.kind)), new Set(['x', 'blog', 'podcast', 'video'])); }); test('orders report items by published time descending', () => { @@ -104,6 +116,36 @@ test('merges duplicate items and drops entries older than 48 hours', () => { assert.equal(merged[0].score, 30); }); +test('keeps conference videos for 90 days while dropping older tweets', () => { + const recentTweet = { + id: 'x:recent', + kind: 'x', + url: 'https://x.com/example/status/recent', + publishedAt: '2026-07-25T07:00:00.000Z', + score: 10, + }; + const oldTweet = { + id: 'x:old', + kind: 'x', + url: 'https://x.com/example/status/old', + publishedAt: '2026-07-20T07:00:00.000Z', + score: 100, + }; + const conferenceVideo = { + id: 'video:compile', + kind: 'video', + url: 'https://www.youtube.com/watch?v=compile1', + publishedAt: '2026-06-20T17:00:00.000Z', + score: 64, + }; + + const merged = digest.mergeItems([recentTweet, oldTweet, conferenceVideo], [], NOW); + assert.deepEqual( + merged.map((item) => item.id).sort(), + ['video:compile', 'x:recent'], + ); +}); + test('detects local-day cache hits and stale feeds', () => { // Use explicit UTC instants so the local-day check is stable across CI timezones. assert.equal( @@ -137,7 +179,7 @@ test('returns partial feed results when one source fails', async () => { }; const result = await digest.fetchFeeds({ fetchImpl, timeoutMs: 100 }); - assert.deepEqual(Object.keys(result.feeds).sort(), ['blogs', 'x']); + assert.deepEqual(Object.keys(result.feeds).sort(), ['blogs', 'videos', 'x']); assert.equal(result.errors.length, 1); assert.match(result.errors[0], /podcasts/); }); @@ -156,6 +198,10 @@ test('points AI Builder feeds at this repository feeds branch', () => { digest.FEED_URLS.podcasts, 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-podcasts.json', ); + assert.equal( + digest.FEED_URLS.videos, + 'https://raw.githubusercontent.com/beforeload/zero-tab/feeds/feed-videos.json', + ); }); test('requests the declared raw GitHub origin without a wildcard path', async () => { @@ -221,6 +267,8 @@ test('enables optional permission, caches a refresh, and skips a second same-day } : url.includes('feed-podcasts') ? { generatedAt: NOW.toISOString(), podcasts: [] } + : url.includes('feed-videos') + ? { generatedAt: NOW.toISOString(), videos: [] } : { generatedAt: NOW.toISOString(), blogs: [] }; return { ok: true, text: async () => JSON.stringify(payload) }; }; @@ -231,7 +279,7 @@ test('enables optional permission, caches a refresh, and skips a second same-day const first = await digest.refresh({ force: true, now: NOW, fetchImpl }); assert.equal(first.items.length, 1); - assert.equal(fetchCount, 3); + assert.equal(fetchCount, 4); await digest.refresh({ now: new Date('2026-07-25T12:00:00.000Z'), @@ -239,7 +287,7 @@ test('enables optional permission, caches a refresh, and skips a second same-day throw new Error('same-day refresh should use cache'); }, }); - assert.equal(fetchCount, 3); + assert.equal(fetchCount, 4); } finally { delete global.chrome; } @@ -287,6 +335,8 @@ test('serializes concurrent refreshes and lets the second caller reuse the new c ? { generatedAt: NOW.toISOString(), x: [] } : url.includes('feed-podcasts') ? { generatedAt: NOW.toISOString(), podcasts: [] } + : url.includes('feed-videos') + ? { generatedAt: NOW.toISOString(), videos: [] } : { generatedAt: NOW.toISOString(), blogs: [] }; return { ok: true, text: async () => JSON.stringify(payload) }; }; @@ -296,7 +346,7 @@ test('serializes concurrent refreshes and lets the second caller reuse the new c digest.refresh({ force: true, now: NOW, fetchImpl }), digest.refresh({ now: NOW, fetchImpl }), ]); - assert.equal(fetchCount, 3); + assert.equal(fetchCount, 4); } finally { delete global.chrome; if (originalNavigator) { diff --git a/tests/feed-generator.test.mjs b/tests/feed-generator.test.mjs index 72983a5..0d587c0 100644 --- a/tests/feed-generator.test.mjs +++ b/tests/feed-generator.test.mjs @@ -61,6 +61,19 @@ describe('feed generator parsers', () => { ); }); + it('parses YouTube Atom video entries and keeps watch IDs', () => { + const xml = readFileSync(join(fixtures, 'youtube.atom.xml'), 'utf8'); + const items = parseRssOrAtom(xml, { + sourceName: 'Cursor Compile', + baseUrl: 'https://www.youtube.com/feeds/videos.xml?playlist_id=example', + limit: 1, + }); + assert.equal(items.length, 1); + assert.equal(items[0].title, 'Opening Keynote, Michael Truell | Compile 26'); + assert.equal(items[0].url, 'https://www.youtube.com/watch?v=abc123Compile'); + assert.match(items[0].description, /Compile keynote/i); + }); + it('parses X syndication markup into tweets', () => { const html = readFileSync(join(fixtures, 'x-syndication.html'), 'utf8'); const builder = parseXSyndicationHtml(html, { @@ -80,6 +93,7 @@ describe('feed generator parsers', () => { readFileSync(join(fixtures, 'x-syndication.html'), 'utf8'), 'https://example.com/feed.xml': readFileSync(join(fixtures, 'blog.rss.xml'), 'utf8'), 'https://example.com/podcast.xml': readFileSync(join(fixtures, 'podcast.atom.xml'), 'utf8'), + 'https://example.com/youtube.xml': readFileSync(join(fixtures, 'youtube.atom.xml'), 'utf8'), }; const fetchImpl = async (url) => { const body = fixturesByUrl[url]; @@ -104,6 +118,8 @@ describe('feed generator parsers', () => { assert.equal(result.feedX.x[0].tweets.length, 1); assert.ok(result.feedBlogs.blogs.length >= 1); assert.ok(result.feedPodcasts.podcasts.length >= 1); + assert.ok(result.feedVideos.videos.length >= 1); + assert.equal(result.feedVideos.videos[0].url, 'https://www.youtube.com/watch?v=abc123Compile'); const writtenX = JSON.parse(readFileSync(join(outDir, 'feed-x.json'), 'utf8')); assert.equal(writtenX.generatedAt, '2026-08-11T12:00:00.000Z'); diff --git a/tests/fixtures/sources.json b/tests/fixtures/sources.json index 0f6da74..ca491ac 100644 --- a/tests/fixtures/sources.json +++ b/tests/fixtures/sources.json @@ -13,5 +13,11 @@ "name": "Builders Podcast", "rssUrl": "https://example.com/podcast.xml" } + ], + "videos": [ + { + "name": "Cursor Compile", + "rssUrl": "https://example.com/youtube.xml" + } ] } diff --git a/tests/fixtures/youtube.atom.xml b/tests/fixtures/youtube.atom.xml new file mode 100644 index 0000000..f9ee6d2 --- /dev/null +++ b/tests/fixtures/youtube.atom.xml @@ -0,0 +1,16 @@ + + + Cursor Compile + + yt:video:abc123Compile + abc123Compile + Opening Keynote, Michael Truell | Compile 26 + + 2026-06-20T17:00:00+00:00 + + Cursor's first Compile keynote. + + + From eeb90bc4268ce790e5114b7a03804ce426fe04b5 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 14 Aug 2026 13:13:37 +0800 Subject: [PATCH 4/6] Allow feed generation workflow to run from the triggering branch. workflow_dispatch from a feature branch can publish with that branch's sources and generator. Co-authored-by: Cursor --- .github/workflows/generate-builder-feeds.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-builder-feeds.yml b/.github/workflows/generate-builder-feeds.yml index 150c98b..ff22048 100644 --- a/.github/workflows/generate-builder-feeds.yml +++ b/.github/workflows/generate-builder-feeds.yml @@ -20,10 +20,12 @@ jobs: timeout-minutes: 20 steps: - - name: Checkout main + - name: Checkout uses: actions/checkout@v4 with: - ref: main + # Use the triggering ref so workflow_dispatch from a feature branch + # can publish with that branch's generator/sources. + ref: ${{ github.sha }} fetch-depth: 0 - name: Setup Node.js From 081b046333dfa5e466461cb21f89b0bb75cc83e6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 15 Aug 2026 16:45:06 +0800 Subject: [PATCH 5/6] Fix X feed scraping so Brief shows real tweets, not profile chrome. Filter jina markdown junk, clean link/image syntax, and use tweet text as the card title. Co-authored-by: Cursor --- extension/builder-digest.js | 41 ++++++++++-- scripts/lib/feed-generator.mjs | 111 ++++++++++++++++++++++++--------- tests/builder-digest.test.js | 40 ++++++++++++ tests/feed-generator.test.mjs | 12 ++++ tests/fixtures/x-jina.md | 15 +++++ 5 files changed, 183 insertions(+), 36 deletions(-) create mode 100644 tests/fixtures/x-jina.md diff --git a/extension/builder-digest.js b/extension/builder-digest.js index fd63de2..548611d 100644 --- a/extension/builder-digest.js +++ b/extension/builder-digest.js @@ -45,6 +45,35 @@ return `${text.slice(0, Math.max(0, maxLength - 1)).trimEnd()}…`; } + function cleanTweetText(value) { + let text = String(value || ''); + text = text.replace(/!\[[^\]]*\]\([^)]*\)/g, ' '); + text = text.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1'); + text = text.replace(/^#{1,6}\s+/gm, ''); + text = text.replace(/^\*\s+/gm, ''); + text = text.replace(/\*\*|__/g, ''); + text = text.replace(/<[^>]+>/g, ' '); + return normalizeText(text); + } + + function isJunkTweetText(value) { + const text = cleanTweetText(value); + if (!text || text.length < 16 || text.length > 500) return true; + if (/^(log in or sign up|sign up for x|create an account)\b/i.test(text)) return true; + if (/^joined (jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i.test(text)) return true; + if (/\bfollowing\b/i.test(text) && /\bfollowers?\b/i.test(text)) return true; + if (/pbs\.twimg\.com\/profile_images/i.test(text)) return true; + if (/\buser avatar\b/i.test(text)) return true; + if (/^image\s+\d+\b/i.test(text)) return true; + if (/^(posts?|replies|highlights|media|likes|articles|subscriptions)\b/i.test(text)) return true; + if (/^(san francisco|singapore|new york|london|seattle|remote)\b/i.test(text) && text.length < 48) { + return true; + } + if (/^(?:[a-z0-9-]+\.)+[a-z]{2,}(?:\/\S*)?$/i.test(text)) return true; + if (/^@?[A-Za-z0-9_]{2,40}$/.test(text)) return true; + return false; + } + function safeHttpsUrl(value) { try { const url = new URL(value); @@ -87,21 +116,23 @@ for (const tweet of Array.isArray(builder?.tweets) ? builder.tweets : []) { const url = safeHttpsUrl(tweet?.url); const id = normalizeText(tweet?.id); - const excerpt = truncate(tweet?.text, 300); - if (!id || !url || !excerpt) continue; + const text = cleanTweetText(tweet?.text); + if (!id || !url || !text || isJunkTweetText(text)) continue; const name = truncate(builder?.name || builder?.handle || 'AI Builder', 80); const handle = truncate(builder?.handle, 40); const publishedAt = new Date(timestamp(tweet?.createdAt, nowMs)).toISOString(); + const title = truncate(text, 180); + const excerpt = text.length > 180 ? truncate(text, 320) : ''; items.push({ id: `x:${id}`, kind: 'x', - source: name, - title: handle ? `${name} (@${handle})` : name, + source: handle ? `${name} (@${handle})` : name, + title, excerpt, url, publishedAt, - score: 40 + engagementScore(tweet) + keywordScore(excerpt) + recencyScore(publishedAt, nowMs), + score: 40 + engagementScore(tweet) + keywordScore(text) + recencyScore(publishedAt, nowMs), }); } } diff --git a/scripts/lib/feed-generator.mjs b/scripts/lib/feed-generator.mjs index 940d91d..5fb93dc 100644 --- a/scripts/lib/feed-generator.mjs +++ b/scripts/lib/feed-generator.mjs @@ -311,6 +311,71 @@ export function parseBlogHtml(html, { sourceName, baseUrl, limit = 6 } = {}) { })); } +export function cleanTweetText(value) { + let text = String(value || ''); + text = text.replace(/!\[[^\]]*\]\([^)]*\)/g, ' '); + text = text.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1'); + text = text.replace(/^#{1,6}\s+/gm, ''); + text = text.replace(/^\*\s+/gm, ''); + text = text.replace(/\*\*|__/g, ''); + text = stripTags(text); + return normalizeText(text); +} + +export function isJunkTweetText(value) { + const text = cleanTweetText(value); + if (!text || text.length < 16 || text.length > 500) return true; + if (/^(log in or sign up|sign up for x|create an account)\b/i.test(text)) return true; + if (/^joined (jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i.test(text)) return true; + if (/\bfollowing\b/i.test(text) && /\bfollowers?\b/i.test(text)) return true; + if (/pbs\.twimg\.com\/profile_images/i.test(text)) return true; + if (/\buser avatar\b/i.test(text)) return true; + if (/^image\s+\d+\b/i.test(text)) return true; + if (/^(posts?|replies|highlights|media|likes|articles|subscriptions)\b/i.test(text)) return true; + if (/^(san francisco|singapore|new york|london|seattle|remote)\b/i.test(text) && text.length < 48) { + return true; + } + // Bare domain / vanity URL profile fields. + if (/^(?:[a-z0-9-]+\.)+[a-z]{2,}(?:\/\S*)?$/i.test(text)) return true; + // Profile chrome leftovers that still contain the handle after cleaning. + if (/^@?[A-Za-z0-9_]{2,40}$/.test(text)) return true; + return false; +} + +function extractTweetBodyNearMatch(text, matchIndex, permalink, previousIndex = 0) { + const around = text.slice(Math.max(0, matchIndex - 900), matchIndex + 1400); + const textMatch = + around.match(/data-tweet-text=["']([^"']+)["']/i) || + around.match(/]*class=["'][^"']*tweet-text[^"']*["'][^>]*>([\s\S]*?)<\/p>/i) || + around.match(/"full_text"\s*:\s*"((?:\\.|[^"\\])*)"/) || + around.match(/"text"\s*:\s*"((?:\\.|[^"\\])*)"/); + + if (textMatch) { + const body = cleanTweetText( + textMatch[1] + .replace(/\\n/g, ' ') + .replace(/\\"/g, '"') + .replace(/\\u([0-9a-f]{4})/gi, (_, hex) => + String.fromCharCode(Number.parseInt(hex, 16)), + ), + ); + if (body && !isJunkTweetText(body)) return body; + } + + // jina.ai markdown: only inspect text between the previous permalink and this one. + const before = text.slice(Math.max(previousIndex, matchIndex - 700), matchIndex); + const candidates = before + .split('\n') + .map((part) => cleanTweetText(part)) + .filter((part) => part.length >= 16) + .filter((part) => !/^https?:\/\//i.test(part)) + .filter((part) => !isJunkTweetText(part)) + .filter((part) => !part.includes(permalink)); + + if (!candidates.length) return ''; + return candidates.sort((a, b) => b.length - a.length)[0]; +} + export function parseXSyndicationHtml(html, { name, handle } = {}) { const text = String(html || ''); const tweets = []; @@ -323,41 +388,25 @@ export function parseXSyndicationHtml(html, { name, handle } = {}) { ), ]; + let previousIndex = 0; for (const match of permalinks) { const tweetHandle = match[1]; const id = match[2]; - if (expectedHandle && tweetHandle.toLowerCase() !== expectedHandle) continue; - if (seen.has(id)) continue; + const nextIndex = match.index + match[0].length; + if (expectedHandle && tweetHandle.toLowerCase() !== expectedHandle) { + previousIndex = nextIndex; + continue; + } + if (seen.has(id)) { + previousIndex = nextIndex; + continue; + } seen.add(id); const around = text.slice(Math.max(0, match.index - 800), match.index + 1200); - const textMatch = - around.match(/data-tweet-text=["']([^"']+)["']/i) || - around.match(/]*class=["'][^"']*tweet-text[^"']*["'][^>]*>([\s\S]*?)<\/p>/i) || - around.match(/"full_text"\s*:\s*"((?:\\.|[^"\\])*)"/) || - around.match(/"text"\s*:\s*"((?:\\.|[^"\\])*)"/) || - around.match(/\n([^\n]{20,280})\n/); - let body = ''; - if (textMatch) { - body = textMatch[1] - .replace(/\\n/g, ' ') - .replace(/\\"/g, '"') - .replace(/\\u([0-9a-f]{4})/gi, (_, hex) => - String.fromCharCode(Number.parseInt(hex, 16)), - ); - body = stripTags(body); - } - if (!body) { - // jina.ai markdown often has the tweet body on the previous lines. - const before = text.slice(Math.max(0, match.index - 400), match.index); - const line = before - .split('\n') - .map((part) => normalizeText(part)) - .filter((part) => part.length >= 24 && !/^https?:\/\//i.test(part) && !/^@/.test(part)) - .at(-1); - body = line || ''; - } - if (!body) continue; + const body = extractTweetBodyNearMatch(text, match.index, match[0], previousIndex); + previousIndex = nextIndex; + if (!body || isJunkTweetText(body)) continue; const created = around.match(/datetime=["']([^"']+)["']/i)?.[1] || @@ -385,9 +434,9 @@ export function parseXSyndicationHtml(html, { name, handle } = {}) { const payload = JSON.parse(jsonMatch[0]); for (const tweet of Array.isArray(payload.tweets) ? payload.tweets : []) { const id = String(tweet.id_str || tweet.id || ''); - const body = normalizeText(tweet.full_text || tweet.text || ''); + const body = cleanTweetText(tweet.full_text || tweet.text || ''); const tweetHandle = tweet.user?.screen_name || handle; - if (!id || !body || !tweetHandle) continue; + if (!id || !body || !tweetHandle || isJunkTweetText(body)) continue; tweets.push({ id, text: truncate(body, 400), diff --git a/tests/builder-digest.test.js b/tests/builder-digest.test.js index c49f089..8bf76a1 100644 --- a/tests/builder-digest.test.js +++ b/tests/builder-digest.test.js @@ -57,10 +57,50 @@ test('normalizes and ranks all supported feed types', () => { assert.equal(items.length, 4); assert.deepEqual(new Set(items.map(item => item.kind)), new Set(['x', 'blog', 'podcast', 'video'])); + const xItem = items.find(item => item.kind === 'x'); + assert.equal(xItem.title, 'We launched a new open-source agent framework today.'); + assert.equal(xItem.source, 'Builder (@builder)'); + assert.equal(xItem.excerpt, ''); assert.ok(items.every(item => item.url.startsWith('https://'))); assert.ok(items.every(item => Number.isFinite(item.score))); }); +test('drops junk X markdown profile chrome from feed normalization', () => { + const items = digest.normalizeFeeds({ + x: { + generatedAt: NOW.toISOString(), + x: [{ + name: 'Swyx', + handle: 'swyx', + tweets: [ + { + id: 'good', + text: 'Shipping agents that actually stay useful for a week', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/swyx/status/good', + }, + { + id: 'bad-avatar', + text: '* [![Image 8: user avatar](https://pbs.twimg.com/profile_images/x.jpg)](https://x.com/swyx)', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/swyx/status/bad-avatar', + }, + { + id: 'bad-login', + text: '## Log in or sign up for X', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/swyx/status/bad-login', + }, + ], + }], + }, + }, NOW); + + assert.equal(items.length, 1); + assert.equal(items[0].id, 'x:good'); + assert.equal(items[0].title, 'Shipping agents that actually stay useful for a week'); +}); + test('rejects unsafe links and control characters', () => { assert.equal(digest.safeHttpsUrl('javascript:alert(1)'), ''); assert.equal(digest.safeHttpsUrl('http://example.com'), ''); diff --git a/tests/feed-generator.test.mjs b/tests/feed-generator.test.mjs index 0d587c0..c3c9535 100644 --- a/tests/feed-generator.test.mjs +++ b/tests/feed-generator.test.mjs @@ -87,6 +87,18 @@ describe('feed generator parsers', () => { assert.equal(builder.tweets[0].url, 'https://x.com/simonw/status/1234567890'); }); + it('parses jina markdown timelines and ignores profile chrome', () => { + const markdown = readFileSync(join(fixtures, 'x-jina.md'), 'utf8'); + const builder = parseXSyndicationHtml(markdown, { + name: 'Swyx', + handle: 'swyx', + }); + assert.equal(builder.tweets.length, 1); + assert.equal(builder.tweets[0].id, '2088381680478540096'); + assert.equal(builder.tweets[0].text, 'Shipping agents that actually stay useful for a week'); + assert.equal(builder.tweets.every((tweet) => !/avatar|Log in|Joined|Following/i.test(tweet.text)), true); + }); + it('writes feed JSON through a mocked fetch layer', async () => { const fixturesByUrl = { 'https://cdn.syndication.twimg.com/timeline/profile?screen_name=simonw': diff --git a/tests/fixtures/x-jina.md b/tests/fixtures/x-jina.md new file mode 100644 index 0000000..dfad676 --- /dev/null +++ b/tests/fixtures/x-jina.md @@ -0,0 +1,15 @@ +Title: swyx (swyx) + +Joined November 2007 +san francisco / singapore + +[664 Following](https://x.com/swyx/following)[100K Followers](https://x.com/swyx/verified_followers) + +## Log in or sign up for X + +* [![Image 8: user avatar](https://pbs.twimg.com/profile_images/2073162797354217472/hNny55eF_normal.jpg)](https://x.com/swyx) [swyx](https://x.com/swyx) +Shipping agents that actually stay useful for a week +https://x.com/swyx/status/2088381680478540096 + +* [![Image 13: user avatar](https://pbs.twimg.com/profile_images/2073162797354217472/hNny55eF_normal.jpg)](https://x.com/swyx) [swyx](https://x.com/swyx) +https://x.com/swyx/status/2088358628000768263 From 38c0900c0f0e0b340925f323aea57b7ed3a13b07 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 20 Aug 2026 22:47:49 +0800 Subject: [PATCH 6/6] Tighten X tweet junk filters for avatar/date chrome and JSON debris. Reject cleaned leftovers like "Name @handle [Aug 15](" and prefer structured full_text extraction over scraped profile markdown. Co-authored-by: Cursor --- extension/builder-digest.js | 26 ++++++-- scripts/lib/feed-generator.mjs | 115 ++++++++++++++++++++++++++++----- tests/builder-digest.test.js | 14 +++- 3 files changed, 133 insertions(+), 22 deletions(-) diff --git a/extension/builder-digest.js b/extension/builder-digest.js index 548611d..c773c57 100644 --- a/extension/builder-digest.js +++ b/extension/builder-digest.js @@ -49,6 +49,8 @@ let text = String(value || ''); text = text.replace(/!\[[^\]]*\]\([^)]*\)/g, ' '); text = text.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1'); + text = text.replace(/\[[^\]]*\]\([^)]*$/g, ' '); + text = text.replace(/\[[^\]]*\]\(/g, ' '); text = text.replace(/^#{1,6}\s+/gm, ''); text = text.replace(/^\*\s+/gm, ''); text = text.replace(/\*\*|__/g, ''); @@ -57,13 +59,19 @@ } function isJunkTweetText(value) { + const raw = String(value || ''); + if (/pbs\.twimg\.com\/profile_images/i.test(raw)) return true; + if (/\buser avatar\b/i.test(raw)) return true; + if (/!\[[^\]]*\]\([^)]*profile_images/i.test(raw)) return true; + if (/is_blue_verified|entry_id|conversation_id_str|withheld_in_countries|"sort_index"/i.test(raw)) { + return true; + } + const text = cleanTweetText(value); - if (!text || text.length < 16 || text.length > 500) return true; + if (!text || text.length < 20 || text.length > 400) return true; if (/^(log in or sign up|sign up for x|create an account)\b/i.test(text)) return true; if (/^joined (jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i.test(text)) return true; if (/\bfollowing\b/i.test(text) && /\bfollowers?\b/i.test(text)) return true; - if (/pbs\.twimg\.com\/profile_images/i.test(text)) return true; - if (/\buser avatar\b/i.test(text)) return true; if (/^image\s+\d+\b/i.test(text)) return true; if (/^(posts?|replies|highlights|media|likes|articles|subscriptions)\b/i.test(text)) return true; if (/^(san francisco|singapore|new york|london|seattle|remote)\b/i.test(text) && text.length < 48) { @@ -71,6 +79,16 @@ } if (/^(?:[a-z0-9-]+\.)+[a-z]{2,}(?:\/\S*)?$/i.test(text)) return true; if (/^@?[A-Za-z0-9_]{2,40}$/.test(text)) return true; + if ( + /^.{2,60}\s@\w{1,40}\s+(?:\[)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i.test( + text, + ) + ) { + return true; + } + if (/^[\p{L}\p{N}.''\-\s]{2,60}\s@\w{1,40}$/u.test(text)) return true; + if (/\[[^\]]*$/.test(text) || /\]\($/.test(text) || /\[[^\]]*\]\($/.test(text)) return true; + if (/[{}=]|\\u00|"type":/.test(text)) return true; return false; } @@ -117,7 +135,7 @@ const url = safeHttpsUrl(tweet?.url); const id = normalizeText(tweet?.id); const text = cleanTweetText(tweet?.text); - if (!id || !url || !text || isJunkTweetText(text)) continue; + if (!id || !url || !text || isJunkTweetText(tweet?.text) || isJunkTweetText(text)) continue; const name = truncate(builder?.name || builder?.handle || 'AI Builder', 80); const handle = truncate(builder?.handle, 40); diff --git a/scripts/lib/feed-generator.mjs b/scripts/lib/feed-generator.mjs index 5fb93dc..bf7063c 100644 --- a/scripts/lib/feed-generator.mjs +++ b/scripts/lib/feed-generator.mjs @@ -315,6 +315,9 @@ export function cleanTweetText(value) { let text = String(value || ''); text = text.replace(/!\[[^\]]*\]\([^)]*\)/g, ' '); text = text.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1'); + // Drop broken/incomplete markdown links left by truncated scrapes: [Aug 15]( + text = text.replace(/\[[^\]]*\]\([^)]*$/g, ' '); + text = text.replace(/\[[^\]]*\]\(/g, ' '); text = text.replace(/^#{1,6}\s+/gm, ''); text = text.replace(/^\*\s+/gm, ''); text = text.replace(/\*\*|__/g, ''); @@ -323,13 +326,20 @@ export function cleanTweetText(value) { } export function isJunkTweetText(value) { + const raw = String(value || ''); + // Check raw first — cleaning strips the evidence from avatar/profile chrome. + if (/pbs\.twimg\.com\/profile_images/i.test(raw)) return true; + if (/\buser avatar\b/i.test(raw)) return true; + if (/!\[[^\]]*\]\([^)]*profile_images/i.test(raw)) return true; + if (/is_blue_verified|entry_id|conversation_id_str|withheld_in_countries|"sort_index"/i.test(raw)) { + return true; + } + const text = cleanTweetText(value); - if (!text || text.length < 16 || text.length > 500) return true; + if (!text || text.length < 20 || text.length > 400) return true; if (/^(log in or sign up|sign up for x|create an account)\b/i.test(text)) return true; if (/^joined (jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i.test(text)) return true; if (/\bfollowing\b/i.test(text) && /\bfollowers?\b/i.test(text)) return true; - if (/pbs\.twimg\.com\/profile_images/i.test(text)) return true; - if (/\buser avatar\b/i.test(text)) return true; if (/^image\s+\d+\b/i.test(text)) return true; if (/^(posts?|replies|highlights|media|likes|articles|subscriptions)\b/i.test(text)) return true; if (/^(san francisco|singapore|new york|london|seattle|remote)\b/i.test(text) && text.length < 48) { @@ -337,29 +347,89 @@ export function isJunkTweetText(value) { } // Bare domain / vanity URL profile fields. if (/^(?:[a-z0-9-]+\.)+[a-z]{2,}(?:\/\S*)?$/i.test(text)) return true; - // Profile chrome leftovers that still contain the handle after cleaning. if (/^@?[A-Za-z0-9_]{2,40}$/.test(text)) return true; + // "Simon Willison @simonw Aug 15" / "Name @handle [Aug 15](" profile chrome. + if ( + /^.{2,60}\s@\w{1,40}\s+(?:\[)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i.test( + text, + ) + ) { + return true; + } + // Author line with no tweet body left after cleaning: "Simon Willison @simonw" + if (/^[\p{L}\p{N}.''\-\s]{2,60}\s@\w{1,40}$/u.test(text)) return true; + // Leftover truncated markdown crumbs. + if (/\[[^\]]*$/.test(text) || /\]\($/.test(text) || /\[[^\]]*\]\($/.test(text)) return true; + // JSON/API debris that slipped through. + if (/[{}=]|\\u00|"type":/.test(text)) return true; return false; } +function decodeJsonString(value) { + return String(value || '') + .replace(/\\n/g, ' ') + .replace(/\\"/g, '"') + .replace(/\\\//g, '/') + .replace(/\\u([0-9a-f]{4})/gi, (_, hex) => + String.fromCharCode(Number.parseInt(hex, 16)), + ); +} + +function extractTweetsFromStructuredJson(text, { handle, limit = MAX_TWEETS_PER_HANDLE } = {}) { + const tweets = []; + const seen = new Set(); + const expectedHandle = String(handle || '').replace(/^@/, '').toLowerCase(); + + const candidates = [ + ...text.matchAll(/"id_str"\s*:\s*"(\d+)"[\s\S]{0,1200}?"full_text"\s*:\s*"((?:\\.|[^"\\])*)"/g), + ...text.matchAll(/"full_text"\s*:\s*"((?:\\.|[^"\\])*)"[\s\S]{0,800}?"id_str"\s*:\s*"(\d+)"/g), + ...text.matchAll(/"id_str"\s*:\s*"(\d+)"[\s\S]{0,1200}?"text"\s*:\s*"((?:\\.|[^"\\])*)"/g), + ]; + + for (const match of candidates) { + const looksLikeFullTextFirst = match[0].trimStart().startsWith('"full_text"'); + const id = looksLikeFullTextFirst ? match[2] : match[1]; + const rawBody = looksLikeFullTextFirst ? match[1] : match[2]; + if (!id || seen.has(id)) continue; + const body = cleanTweetText(decodeJsonString(rawBody)); + if (!body || isJunkTweetText(rawBody) || isJunkTweetText(body)) continue; + + const around = text.slice(Math.max(0, match.index - 400), match.index + 1600); + const screen = + around.match(/"screen_name"\s*:\s*"([A-Za-z0-9_]+)"/)?.[1] || handle || ''; + if (expectedHandle && screen && screen.toLowerCase() !== expectedHandle) continue; + + seen.add(id); + const created = around.match(/"created_at"\s*:\s*"([^"]+)"/)?.[1] || null; + tweets.push({ + id, + text: truncate(body, 400), + createdAt: created + ? new Date(Date.parse(created) || Date.now()).toISOString() + : new Date().toISOString(), + url: `https://x.com/${screen || handle}/status/${id}`, + likes: Number(around.match(/"favorite_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + retweets: Number(around.match(/"retweet_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + replies: Number(around.match(/"reply_count"\s*:\s*(\d+)/)?.[1] || 0) || undefined, + }); + if (tweets.length >= limit) break; + } + return tweets; +} + function extractTweetBodyNearMatch(text, matchIndex, permalink, previousIndex = 0) { + // JSON timelines should be handled by extractTweetsFromStructuredJson. + if (/"is_blue_verified"|"entry_id"|"conversation_id_str"/.test(text)) return ''; + const around = text.slice(Math.max(0, matchIndex - 900), matchIndex + 1400); const textMatch = around.match(/data-tweet-text=["']([^"']+)["']/i) || around.match(/]*class=["'][^"']*tweet-text[^"']*["'][^>]*>([\s\S]*?)<\/p>/i) || - around.match(/"full_text"\s*:\s*"((?:\\.|[^"\\])*)"/) || - around.match(/"text"\s*:\s*"((?:\\.|[^"\\])*)"/); + around.match(/"full_text"\s*:\s*"((?:\\.|[^"\\])*)"/); if (textMatch) { - const body = cleanTweetText( - textMatch[1] - .replace(/\\n/g, ' ') - .replace(/\\"/g, '"') - .replace(/\\u([0-9a-f]{4})/gi, (_, hex) => - String.fromCharCode(Number.parseInt(hex, 16)), - ), - ); - if (body && !isJunkTweetText(body)) return body; + const body = cleanTweetText(decodeJsonString(textMatch[1])); + if (body && !isJunkTweetText(body) && !isJunkTweetText(textMatch[1])) return body; } // jina.ai markdown: only inspect text between the previous permalink and this one. @@ -367,7 +437,7 @@ function extractTweetBodyNearMatch(text, matchIndex, permalink, previousIndex = const candidates = before .split('\n') .map((part) => cleanTweetText(part)) - .filter((part) => part.length >= 16) + .filter((part) => part.length >= 20) .filter((part) => !/^https?:\/\//i.test(part)) .filter((part) => !isJunkTweetText(part)) .filter((part) => !part.includes(permalink)); @@ -378,9 +448,20 @@ function extractTweetBodyNearMatch(text, matchIndex, permalink, previousIndex = export function parseXSyndicationHtml(html, { name, handle } = {}) { const text = String(html || ''); + const expectedHandle = String(handle || '').replace(/^@/, '').toLowerCase(); + + // Prefer structured JSON tweet objects when present (syndication/timeline payloads). + const fromJson = extractTweetsFromStructuredJson(text, { handle: expectedHandle }); + if (fromJson.length) { + return { + name: truncate(name || handle || 'AI Builder', 80), + handle: truncate(expectedHandle, 40), + tweets: fromJson, + }; + } + const tweets = []; const seen = new Set(); - const expectedHandle = String(handle || '').replace(/^@/, '').toLowerCase(); const permalinks = [ ...text.matchAll( diff --git a/tests/builder-digest.test.js b/tests/builder-digest.test.js index 8bf76a1..bbbd347 100644 --- a/tests/builder-digest.test.js +++ b/tests/builder-digest.test.js @@ -81,10 +81,22 @@ test('drops junk X markdown profile chrome from feed normalization', () => { }, { id: 'bad-avatar', - text: '* [![Image 8: user avatar](https://pbs.twimg.com/profile_images/x.jpg)](https://x.com/swyx)', + text: '* [![Image 8: user avatar](https://pbs.twimg.com/profile_images/x.jpg)](https://x.com/swyx) [swyx](https://x.com/swyx) [Aug 15](', createdAt: '2026-07-25T07:00:00.000Z', url: 'https://x.com/swyx/status/bad-avatar', }, + { + id: 'bad-date-chrome', + text: 'Simon Willison @simonw [Aug 15](', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/simonw/status/bad-date-chrome', + }, + { + id: 'bad-json', + text: ',"is_blue_verified":true}}}},{"type":"tweet","entry_id":"tweet-1"', + createdAt: '2026-07-25T07:00:00.000Z', + url: 'https://x.com/swyx/status/bad-json', + }, { id: 'bad-login', text: '## Log in or sign up for X',