From dce23536c2056ffdea967c8087c92ad585397ad8 Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Tue, 29 Sep 2026 07:09:06 +0900 Subject: [PATCH] Read a reddit short or share link as the feed of the post it opens fetchPage swaps a reddit.com page URL for its Atom feed, since the HTML page is a login wall for a reader with no cookies. It only looked at the URL as typed. A redd.it short link and the app's /r//s/ share link are both a 301 to the post page, so the redirect landed on the wall and oc reported no readable content, while the post URL they point at reads fine. The swap now runs on every redirect hop too, through followRedirects, with the same rule: a logged-in session still gets the page. redd.it also joins the firefox-first hosts: the identity is picked once for the whole chain, and the reddit.com hop answers chrome with a 403, so starting with chrome spent two extra requests against reddit's rate limit on every short link. Co-Authored-By: Claude Opus 5.5 --- src/fetch.js | 35 +++++++++++++++++++++++++---------- tests/fetch.test.js | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/src/fetch.js b/src/fetch.js index afbf5c4..cfaa88e 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -623,15 +623,24 @@ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) { * Content-Type the server declared */ export async function fetchPage(url, { jar } = {}) { - let target = /^https?:\/\//i.test(url) ? url : `https://${url}`; - // A reader with no cookies for reddit.com gets the feed where the page - // would be a login wall; one who logged in gets the page it asked for. - if (!jar?.cookieHeaderFor(target)) target = redditFeedURL(target) ?? target; + const target = readableURL(/^https?:\/\//i.test(url) ? url : `https://${url}`, jar); await assertSafeTarget(target); const impers = await loadImpers(); return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar); } +/** + * The URL to ask for in place of this one. A reader with no cookies for + * reddit.com gets the feed where the page would be a login wall; one who + * logged in gets the page it asked for. Every redirect hop goes through this + * too: redd.it short links and the app's /r//s/ share links answer + * with a 301 to the post page, the very wall the feed steps around. + * @param {string} url + * @param {{ cookieHeaderFor(url: string): string|undefined }} [jar] + * @returns {string} + */ +const readableURL = (url, jar) => (jar?.cookieHeaderFor(url) ? url : redditFeedURL(url) ?? url); + /** * Follow redirects one hop at a time, validating each destination before the * next request goes out. @@ -644,9 +653,11 @@ export async function fetchPage(url, { jar } = {}) { * process. * @param {(url: string) => Promise} get - one request, redirects not followed * @param {string} start + * @param {{onResponse?: (url: string, res: any) => void, rewrite?: (url: string) => string}} [opts] - + * rewrite names the URL to ask for in place of a hop's Location * @returns {Promise<{res: any, url: string}>} the first non-redirect response */ -export async function followRedirects(get, start, { onResponse } = {}) { +export async function followRedirects(get, start, { onResponse, rewrite } = {}) { let current = start; for (let i = 0; ; i++) { if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`); @@ -655,7 +666,8 @@ export async function followRedirects(get, start, { onResponse } = {}) { const status = res.status ?? res.statusCode ?? 0; const location = res.headers.get('location'); if (status >= 300 && status < 400 && location) { - current = new URL(location, current).toString(); + const next = new URL(location, current).toString(); + current = rewrite ? rewrite(next) : next; await assertSafeTarget(current); continue; } @@ -688,8 +700,9 @@ function captureSetCookie(jar, url, res) { // letting firefox through. reddit.com started doing this in 2026 (#52), so // starting with chrome there would turn every read into two requests against // a per-address rate limit of about ten a minute. Subdomains inherit the -// entry. -const FIREFOX_FIRST_HOSTS = ['reddit.com']; +// entry. A redd.it short link is a 301 to a reddit.com post, and the identity +// is picked once for the whole chain, so it starts the same way. +const FIREFOX_FIRST_HOSTS = ['reddit.com', 'redd.it']; // Reddit has sent logged-out readers of its HTML pages to a login page since // June 2026 (#52), while the Atom feed beside each of those pages still @@ -778,9 +791,10 @@ export async function viaImpers(impers, target, jar) { headers: jarHeaders(jar, url, {}), }); const onResponse = (url, res) => captureSetCookie(jar, url, res); + const rewrite = (url) => readableURL(url, jar); const attempt = async (impersonate) => { try { - const { res } = await followRedirects(asking(impersonate), target, { onResponse }); + const { res } = await followRedirects(asking(impersonate), target, { onResponse, rewrite }); return { res, status: res.status ?? res.statusCode ?? 0 }; } catch (err) { if (err?.name !== 'ImpersonateError') throw err; @@ -827,7 +841,8 @@ async function viaFetch(target, jar) { : fetch(url, { redirect: 'manual', headers }); }; const onResponse = (url, res) => captureSetCookie(jar, url, res); - const { res, url: current } = await followRedirects(get, target, { onResponse }); + const rewrite = (url) => readableURL(url, jar); + const { res, url: current } = await followRedirects(get, target, { onResponse, rewrite }); if (!res.ok) { throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`); } diff --git a/tests/fetch.test.js b/tests/fetch.test.js index 45ee84d..bea2df7 100644 --- a/tests/fetch.test.js +++ b/tests/fetch.test.js @@ -1036,6 +1036,7 @@ test('reddit.com is asked with the firefox fingerprint first', () => withoutProx assert.deepEqual(identityOrder('https://www.reddit.com/r/ClaudeAI/.rss'), ['firefox', 'chrome']); assert.deepEqual(identityOrder('https://old.reddit.com/r/ClaudeAI/'), ['firefox', 'chrome']); assert.deepEqual(identityOrder('https://reddit.com/'), ['firefox', 'chrome']); + assert.deepEqual(identityOrder('https://redd.it/1w48zcr'), ['firefox', 'chrome']); assert.deepEqual(identityOrder('https://notreddit.com/'), ['chrome', 'firefox']); assert.deepEqual(identityOrder('https://reddit.com.example/'), ['chrome', 'firefox']); assert.deepEqual(identityOrder('https://news.ycombinator.com/'), ['chrome', 'firefox']); @@ -1074,6 +1075,39 @@ test('reddit.com page URLs are read as their atom feeds, feeds and everything el assert.equal(redditFeedURL('not a url'), null); }); +test('a reddit short or share link is read as the feed of the post it redirects to', () => withoutProxyEnv(async () => { + // redd.it/ and the app's /r//s/ share links answer with a + // 301 to the post page, which is the login wall the feed mapping exists to + // step around. Mapping only the URL as typed left both at that wall. + const redirecting = () => { + const asked = []; + const get = (url) => { + asked.push(url); + if (url === 'https://redd.it/abc123') { + return Promise.resolve({ status: 301, headers: new Map([['location', 'https://www.reddit.com/comments/abc123']]), url }); + } + return Promise.resolve({ + status: 200, + headers: new Map([['content-type', 'application/atom+xml']]), + text: () => Promise.resolve('post'), + url, + }); + }; + return { get, asked }; + }; + const impers = redirecting(); + const page = await viaImpers(impers, 'https://redd.it/abc123'); + assert.deepEqual(impers.asked, ['https://redd.it/abc123', 'https://www.reddit.com/comments/abc123/.rss']); + assert.equal(page.url, 'https://www.reddit.com/comments/abc123/.rss'); + + // A reader logged in to reddit.com asked for the page, and gets it. + const { jarFromCookieHeader, createJarHandle } = await import('../src/cookies.js'); + const jar = createJarHandle('test', jarFromCookieHeader('reddit_session=x', 'reddit.com')); + const logged = redirecting(); + await viaImpers(logged, 'https://redd.it/abc123', jar); + assert.deepEqual(logged.asked, ['https://redd.it/abc123', 'https://www.reddit.com/comments/abc123']); +})); + test('a refused firefox fingerprint on reddit.com falls back to chrome', () => withoutProxyEnv(async () => { const impers = fakeImpers(['firefox']); const page = await viaImpers(impers, 'https://www.reddit.com/r/ClaudeAI/.rss');