Replaced Snoowrap with public API calls in response to Reddit revoking API keys. This was largely done using Claude Code, extensive manual review and major version bump needed if merged into main.
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
'use strict';
|
||||
|
||||
const fetch = require('node-fetch');
|
||||
const Bottleneck = require('bottleneck');
|
||||
const config = require('config');
|
||||
|
||||
const { version } = require('../../package.json');
|
||||
const logger = require('../logger')(__filename);
|
||||
|
||||
const BASE_URL = 'https://www.reddit.com';
|
||||
const MAX_RETRIES = 5;
|
||||
|
||||
// deliberately much heavier throttling than the old OAuth limiter (src/limiter.js reddit: 30/min) -
|
||||
// this is unauthenticated/public access and getting flagged as a bot defeats the point
|
||||
const limiter = new Bottleneck({
|
||||
reservoir: config.reddit.throttle?.reservoir ?? 6,
|
||||
reservoirRefreshAmount: config.reddit.throttle?.reservoir ?? 6,
|
||||
reservoirRefreshInterval: 60000,
|
||||
maxConcurrent: 1,
|
||||
minTime: config.reddit.throttle?.minTime ?? 1500,
|
||||
});
|
||||
|
||||
// parses a Netscape/Mozilla-format cookie jar dump (domain, includeSubdomains, path,
|
||||
// secure, expiry, name, value - tab separated) into a `name=value; name2=value2` header
|
||||
function parseCookieJar(raw) {
|
||||
const now = Date.now() / 1000;
|
||||
|
||||
return raw
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line && !line.startsWith('#'))
|
||||
.map((line) => line.split('\t'))
|
||||
.filter((fields) => fields.length === 7)
|
||||
.filter(([, , , , expiry]) => expiry === '0' || Number(expiry) > now)
|
||||
.filter(([domain]) => domain.replace(/^\./, '').endsWith('reddit.com'))
|
||||
.map(([, , , , , name, value]) => `${name}=${value}`)
|
||||
.join('; ');
|
||||
}
|
||||
|
||||
function soonestExpiry(raw) {
|
||||
const expiries = raw
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line && !line.startsWith('#'))
|
||||
.map((line) => line.split('\t'))
|
||||
.filter((fields) => fields.length === 7)
|
||||
.map(([, , , , expiry]) => Number(expiry))
|
||||
.filter((expiry) => expiry > 0);
|
||||
|
||||
return expiries.length ? Math.min(...expiries) : null;
|
||||
}
|
||||
|
||||
let cachedCookieHeader;
|
||||
|
||||
function cookieHeader() {
|
||||
if (cachedCookieHeader === undefined) {
|
||||
cachedCookieHeader = config.reddit.cookies
|
||||
? parseCookieJar(config.reddit.cookies)
|
||||
: (config.reddit.cookie || null);
|
||||
|
||||
logger.info(cachedCookieHeader
|
||||
? `Using ${cachedCookieHeader.split(';').length} reddit cookie(s) for authenticated requests`
|
||||
: 'No reddit cookies configured, requesting unauthenticated');
|
||||
|
||||
if (config.reddit.cookies) {
|
||||
const expiry = soonestExpiry(config.reddit.cookies);
|
||||
const hoursLeft = expiry ? (expiry - Date.now() / 1000) / 3600 : null;
|
||||
|
||||
if (hoursLeft !== null && hoursLeft < 48) {
|
||||
logger.warn(`Soonest-expiring reddit cookie expires in ${hoursLeft.toFixed(1)}h (${new Date(expiry * 1000).toISOString()}) - refresh config/cookies.js before then`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cachedCookieHeader;
|
||||
}
|
||||
|
||||
function headers() {
|
||||
const requestHeaders = {
|
||||
'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
|
||||
accept: 'application/json',
|
||||
};
|
||||
|
||||
const cookie = cookieHeader();
|
||||
|
||||
if (cookie) {
|
||||
requestHeaders.cookie = cookie;
|
||||
}
|
||||
|
||||
return requestHeaders;
|
||||
}
|
||||
|
||||
function buildUrl(path, params) {
|
||||
const url = new URL(`${BASE_URL}${path}`);
|
||||
|
||||
Object.entries({ raw_json: 1, ...params }).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, value);
|
||||
}
|
||||
});
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
async function wait(seconds) {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, seconds * 1000);
|
||||
});
|
||||
}
|
||||
|
||||
async function requestJson(path, params = {}, attempt = 0) {
|
||||
const url = buildUrl(path, params);
|
||||
|
||||
return limiter.schedule(async () => {
|
||||
const res = await fetch(url.toString(), { headers: headers() });
|
||||
const contentType = res.headers.get('content-type') || '';
|
||||
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
if (attempt >= MAX_RETRIES) {
|
||||
throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
|
||||
}
|
||||
|
||||
const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;
|
||||
|
||||
logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);
|
||||
|
||||
await wait(retryAfter);
|
||||
|
||||
return requestJson(path, params, attempt + 1);
|
||||
}
|
||||
|
||||
if (!contentType.includes('json')) {
|
||||
// reddit serves an HTML interstitial/block page instead of JSON when it decides
|
||||
// the request looks like a bot - this is a distinct failure mode from a genuine
|
||||
// 4xx and should be logged/handled separately rather than treated as "bad request"
|
||||
const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
|
||||
|
||||
throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
if (!res.ok) {
|
||||
const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);
|
||||
|
||||
throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
|
||||
}
|
||||
|
||||
return data;
|
||||
});
|
||||
}
|
||||
|
||||
async function getUser(username) {
|
||||
const data = await requestJson(`/user/${username}/about.json`);
|
||||
|
||||
return data.data;
|
||||
}
|
||||
|
||||
async function getSubmissions(username, { sort = 'new', limit = Infinity } = {}) {
|
||||
const submissions = [];
|
||||
let after = null;
|
||||
|
||||
do {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const data = await requestJson(`/user/${username}/submitted.json`, {
|
||||
sort,
|
||||
limit: Math.min(100, Number.isFinite(limit) ? limit - submissions.length : 100),
|
||||
after,
|
||||
});
|
||||
|
||||
const page = data.data.children.map((child) => child.data);
|
||||
|
||||
submissions.push(...page);
|
||||
after = data.data.after;
|
||||
} while (after && submissions.length < limit);
|
||||
|
||||
return submissions;
|
||||
}
|
||||
|
||||
// reddit's raw comment listing is a tree of {kind, data} "things": kind 't1' is a comment
|
||||
// (whose own `replies` field is either '' or a nested Listing of more things), kind 'more'
|
||||
// is a stub meaning "N additional comment IDs exist here, fetch them separately".
|
||||
function flattenThings(things) {
|
||||
const comments = [];
|
||||
const moreIds = [];
|
||||
|
||||
things.forEach((thing) => {
|
||||
if (thing.kind === 't1') {
|
||||
comments.push(thing.data);
|
||||
|
||||
if (thing.data.replies?.data?.children) {
|
||||
const nested = flattenThings(thing.data.replies.data.children);
|
||||
|
||||
comments.push(...nested.comments);
|
||||
moreIds.push(...nested.moreIds);
|
||||
}
|
||||
} else if (thing.kind === 'more' && thing.data.id !== '_') {
|
||||
moreIds.push(...thing.data.children);
|
||||
}
|
||||
});
|
||||
|
||||
return { comments, moreIds };
|
||||
}
|
||||
|
||||
async function moreChildren(linkId, ids) {
|
||||
const unique = [...new Set(ids)];
|
||||
const results = [];
|
||||
|
||||
for (let offset = 0; offset < unique.length; offset += 100) {
|
||||
const chunk = unique.slice(offset, offset + 100);
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const data = await requestJson('/api/morechildren.json', {
|
||||
api_type: 'json',
|
||||
link_id: linkId,
|
||||
children: chunk.join(','),
|
||||
limit_children: false,
|
||||
});
|
||||
|
||||
results.push(...(data.json?.data?.things || []));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// re-fetches 'more' stubs and re-attaches them by parent_id; morechildren can itself return
|
||||
// further 'more' stubs on very deep threads, so this recurses with a depth cap rather than
|
||||
// looping forever
|
||||
async function resolveMore(moreIds, linkId, depth = 0) {
|
||||
if (!moreIds.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (depth >= 4) {
|
||||
logger.warn(`Gave up resolving ${moreIds.length} deeply-nested 'more comments' stubs for '${linkId}' after ${depth} rounds`);
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
const more = await moreChildren(linkId, moreIds);
|
||||
const resolvedComments = more.filter((thing) => thing.kind === 't1').map((thing) => thing.data);
|
||||
const nestedMoreIds = more.filter((thing) => thing.kind === 'more').flatMap((thing) => thing.data.children);
|
||||
|
||||
return [...resolvedComments, ...await resolveMore(nestedMoreIds, linkId, depth + 1)];
|
||||
}
|
||||
|
||||
function buildCommentTree(comments, parentId) {
|
||||
return comments
|
||||
.filter((comment) => comment.parent_id === parentId)
|
||||
.map((comment) => ({
|
||||
...comment,
|
||||
replies: buildCommentTree(comments, `t1_${comment.id}`),
|
||||
}));
|
||||
}
|
||||
|
||||
// post metadata only - cheap, no comment-tree resolution (limit: 0 asks reddit to skip
|
||||
// returning comments too, but we ignore the second listing element regardless)
|
||||
async function getSubmission(id) {
|
||||
const [postListing] = await requestJson(`/comments/${id}.json`, { limit: 0 });
|
||||
|
||||
return postListing.data.children[0].data;
|
||||
}
|
||||
|
||||
// post + fully-resolved comment tree, incl. 'more comments' stubs - expensive on busy
|
||||
// threads (each round of stubs is its own throttled request), only use where the full
|
||||
// thread is actually needed
|
||||
async function getSubmissionWithComments(id) {
|
||||
const linkId = `t3_${id}`;
|
||||
const [postListing, commentsListing] = await requestJson(`/comments/${id}.json`);
|
||||
const post = postListing.data.children[0].data;
|
||||
|
||||
const { comments, moreIds } = flattenThings(commentsListing.data.children);
|
||||
const resolved = [...comments, ...await resolveMore(moreIds, linkId)];
|
||||
|
||||
return { post, comments: buildCommentTree(resolved, linkId) };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
requestJson,
|
||||
getUser,
|
||||
getSubmissions,
|
||||
getSubmission,
|
||||
getSubmissionWithComments,
|
||||
};
|
||||
Reference in New Issue
Block a user