Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 25 additions & 10 deletions src/fetch.js
Original file line number Diff line number Diff line change
Expand Up @@ -623,15 +623,24 @@ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
* Content-Type the server declared
*/
export async function fetchPage(url, { jar } = {}) {
let target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
// A reader with no cookies for reddit.com gets the feed where the page
// would be a login wall; one who logged in gets the page it asked for.
if (!jar?.cookieHeaderFor(target)) target = redditFeedURL(target) ?? target;
const target = readableURL(/^https?:\/\//i.test(url) ? url : `https://${url}`, jar);
await assertSafeTarget(target);
const impers = await loadImpers();
return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
}

/**
* The URL to ask for in place of this one. A reader with no cookies for
* reddit.com gets the feed where the page would be a login wall; one who
* logged in gets the page it asked for. Every redirect hop goes through this
* too: redd.it short links and the app's /r/<sub>/s/<code> share links answer
* with a 301 to the post page, the very wall the feed steps around.
* @param {string} url
* @param {{ cookieHeaderFor(url: string): string|undefined }} [jar]
* @returns {string}
*/
const readableURL = (url, jar) => (jar?.cookieHeaderFor(url) ? url : redditFeedURL(url) ?? url);

/**
* Follow redirects one hop at a time, validating each destination before the
* next request goes out.
Expand All @@ -644,9 +653,11 @@ export async function fetchPage(url, { jar } = {}) {
* process.
* @param {(url: string) => Promise<any>} get - one request, redirects not followed
* @param {string} start
* @param {{onResponse?: (url: string, res: any) => void, rewrite?: (url: string) => string}} [opts] -
* rewrite names the URL to ask for in place of a hop's Location
* @returns {Promise<{res: any, url: string}>} the first non-redirect response
*/
export async function followRedirects(get, start, { onResponse } = {}) {
export async function followRedirects(get, start, { onResponse, rewrite } = {}) {
let current = start;
for (let i = 0; ; i++) {
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
Expand All @@ -655,7 +666,8 @@ export async function followRedirects(get, start, { onResponse } = {}) {
const status = res.status ?? res.statusCode ?? 0;
const location = res.headers.get('location');
if (status >= 300 && status < 400 && location) {
current = new URL(location, current).toString();
const next = new URL(location, current).toString();
current = rewrite ? rewrite(next) : next;
await assertSafeTarget(current);
continue;
}
Expand Down Expand Up @@ -688,8 +700,9 @@ function captureSetCookie(jar, url, res) {
// letting firefox through. reddit.com started doing this in 2026 (#52), so
// starting with chrome there would turn every read into two requests against
// a per-address rate limit of about ten a minute. Subdomains inherit the
// entry.
const FIREFOX_FIRST_HOSTS = ['reddit.com'];
// entry. A redd.it short link is a 301 to a reddit.com post, and the identity
// is picked once for the whole chain, so it starts the same way.
const FIREFOX_FIRST_HOSTS = ['reddit.com', 'redd.it'];

// Reddit has sent logged-out readers of its HTML pages to a login page since
// June 2026 (#52), while the Atom feed beside each of those pages still
Expand Down Expand Up @@ -778,9 +791,10 @@ export async function viaImpers(impers, target, jar) {
headers: jarHeaders(jar, url, {}),
});
const onResponse = (url, res) => captureSetCookie(jar, url, res);
const rewrite = (url) => readableURL(url, jar);
const attempt = async (impersonate) => {
try {
const { res } = await followRedirects(asking(impersonate), target, { onResponse });
const { res } = await followRedirects(asking(impersonate), target, { onResponse, rewrite });
return { res, status: res.status ?? res.statusCode ?? 0 };
} catch (err) {
if (err?.name !== 'ImpersonateError') throw err;
Expand Down Expand Up @@ -827,7 +841,8 @@ async function viaFetch(target, jar) {
: fetch(url, { redirect: 'manual', headers });
};
const onResponse = (url, res) => captureSetCookie(jar, url, res);
const { res, url: current } = await followRedirects(get, target, { onResponse });
const rewrite = (url) => readableURL(url, jar);
const { res, url: current } = await followRedirects(get, target, { onResponse, rewrite });
if (!res.ok) {
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
}
Expand Down
34 changes: 34 additions & 0 deletions tests/fetch.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -1036,6 +1036,7 @@ test('reddit.com is asked with the firefox fingerprint first', () => withoutProx
assert.deepEqual(identityOrder('https://www.reddit.com/r/ClaudeAI/.rss'), ['firefox', 'chrome']);
assert.deepEqual(identityOrder('https://old.reddit.com/r/ClaudeAI/'), ['firefox', 'chrome']);
assert.deepEqual(identityOrder('https://reddit.com/'), ['firefox', 'chrome']);
assert.deepEqual(identityOrder('https://redd.it/1w48zcr'), ['firefox', 'chrome']);
assert.deepEqual(identityOrder('https://notreddit.com/'), ['chrome', 'firefox']);
assert.deepEqual(identityOrder('https://reddit.com.example/'), ['chrome', 'firefox']);
assert.deepEqual(identityOrder('https://news.ycombinator.com/'), ['chrome', 'firefox']);
Expand Down Expand Up @@ -1074,6 +1075,39 @@ test('reddit.com page URLs are read as their atom feeds, feeds and everything el
assert.equal(redditFeedURL('not a url'), null);
});

test('a reddit short or share link is read as the feed of the post it redirects to', () => withoutProxyEnv(async () => {
// redd.it/<id> and the app's /r/<sub>/s/<code> share links answer with a
// 301 to the post page, which is the login wall the feed mapping exists to
// step around. Mapping only the URL as typed left both at that wall.
const redirecting = () => {
const asked = [];
const get = (url) => {
asked.push(url);
if (url === 'https://redd.it/abc123') {
return Promise.resolve({ status: 301, headers: new Map([['location', 'https://www.reddit.com/comments/abc123']]), url });
}
return Promise.resolve({
status: 200,
headers: new Map([['content-type', 'application/atom+xml']]),
text: () => Promise.resolve('<feed><title>post</title></feed>'),
url,
});
};
return { get, asked };
};
const impers = redirecting();
const page = await viaImpers(impers, 'https://redd.it/abc123');
assert.deepEqual(impers.asked, ['https://redd.it/abc123', 'https://www.reddit.com/comments/abc123/.rss']);
assert.equal(page.url, 'https://www.reddit.com/comments/abc123/.rss');

// A reader logged in to reddit.com asked for the page, and gets it.
const { jarFromCookieHeader, createJarHandle } = await import('../src/cookies.js');
const jar = createJarHandle('test', jarFromCookieHeader('reddit_session=x', 'reddit.com'));
const logged = redirecting();
await viaImpers(logged, 'https://redd.it/abc123', jar);
assert.deepEqual(logged.asked, ['https://redd.it/abc123', 'https://www.reddit.com/comments/abc123']);
}));

test('a refused firefox fingerprint on reddit.com falls back to chrome', () => withoutProxyEnv(async () => {
const impers = fakeImpers(['firefox']);
const page = await viaImpers(impers, 'https://www.reddit.com/r/ClaudeAI/.rss');
Expand Down
Loading