From 2e4ba27d98be470b3b4139f30d910c132c24b290 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Fri, 7 Aug 2026 23:37:26 +0200 Subject: [PATCH] Replaced Snoowrap with public API calls in response to Reddit revoking API keys. This was largely done using Claude Code, extensive manual review and major version bump needed if merged into main. --- config/default.js | 22 ++- package.json | 1 - src/app.js | 3 +- src/curate/posts.js | 5 +- src/curate/user.js | 20 +-- src/limiter.js | 8 +- src/methods/self.js | 22 +-- src/reddit/client.js | 284 ++++++++++++++++++++++++++++++++++++ src/sources/getPosts.js | 19 ++- src/sources/getUserPosts.js | 15 +- 10 files changed, 336 insertions(+), 63 deletions(-) create mode 100644 src/reddit/client.js diff --git a/config/default.js b/config/default.js index 962ab49..4f513dc 100644 --- a/config/default.js +++ b/config/default.js @@ -77,14 +77,20 @@ module.exports = { interval: 100, }, reddit: { - api: { - userAgent: 'ripunzel', - clientId: '1234567abcdefg', - access_token: 'abcD123eFg45Hi6J7klmnop8qr9', - token_type: 'bearer', - expires_in: 3600, - refresh_token: '1234567-A-Bc-defg8912hij-klm345opqr', - scope: 'history identity mysubreddits read subscribe', + // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to + // get past reddit's anonymous-traffic bot challenge on www.reddit.com. Needs periodic + // refreshing - src/reddit/client.js logs a warning once the soonest-expiring cookie + // in here is within 48h of expiring. + cookies: null, + // fallback/simpler alternative to `cookies` above: a single pre-built `name=value; ...` + // Cookie header string. Only used if `cookies` is not set. + cookie: null, + userAgent: 'ripunzel', + throttle: { + // deliberately conservative - this is unauthenticated-looking public access, + // not an approved API key, getting flagged as a bot defeats the point + reservoir: 6, + minTime: 1500, }, }, methods: { diff --git a/package.json b/package.json index 784da5b..2fa7918 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,6 @@ "object.omit": "^3.0.0", "object.pick": "^1.3.0", "sharp-phash": "^2.1.0", - "snoowrap": "^1.23.0", "template-format": "^1.2.4", "unprint": "^0.8.2", "url-pattern": "^1.0.3", diff --git a/src/app.js b/src/app.js index aebcad4..4cc361e 100644 --- a/src/app.js +++ b/src/app.js @@ -1,7 +1,6 @@ 'use strict'; const config = require('config'); -const Snoowrap = require('snoowrap'); const fs = require('fs-extra'); const Promise = require('bluebird'); const exiftool = require('node-exiftool'); @@ -10,7 +9,7 @@ const cron = require('node-cron'); require('array.prototype.flatten').shim(); -const reddit = new Snoowrap(config.reddit.api); +const reddit = require('./reddit/client'); const args = require('./cli')(); const logger = require('./logger')(__filename); diff --git a/src/curate/posts.js b/src/curate/posts.js index 07ff6c8..02ddc96 100644 --- a/src/curate/posts.js +++ b/src/curate/posts.js @@ -57,12 +57,13 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) permalink, url: post.url, datetime: new Date(post.created_utc * 1000), - subreddit: post.subreddit.display_name, + subreddit: post.subreddit, score: post.score, preview: post.preview ? post.preview.images.map((image) => image.source) : null, host, direct: post.direct, - comments: post.comments, + // count only - the full thread (when needed) is fetched separately via methods/self.js + comments: post.num_comments, hash: hashPost(post), }; diff --git a/src/curate/user.js b/src/curate/user.js index b51b512..d0cb835 100644 --- a/src/curate/user.js +++ b/src/curate/user.js @@ -9,23 +9,23 @@ function curateUser(user) { gold: user.is_gold, verified: user.verified, verifiedEmail: user.has_verified_email, - fallback: false + fallback: false, }; - if(user.subreddit) { + if (user.subreddit) { Object.assign(curatedUser, { profile: { - id: user.subreddit.display_name.name, - title: user.subreddit.display_name.title, - image: user.subreddit.display_name.icon_img, - banner: user.subreddit.display_name.banner_img, - description: user.subreddit.display_name.public_description, - over18: user.subreddit.display_name.over_18 - } + id: user.subreddit.display_name, + title: user.subreddit.title, + image: user.subreddit.icon_img, + banner: user.subreddit.banner_img, + description: user.subreddit.public_description, + over18: user.subreddit.over_18, + }, }); } return curatedUser; -}; +} module.exports = curateUser; diff --git a/src/limiter.js b/src/limiter.js index 1c4a22e..ba21479 100644 --- a/src/limiter.js +++ b/src/limiter.js @@ -5,13 +5,7 @@ const Bottleneck = require('bottleneck'); const args = require('./cli')(); module.exports = { - reddit: new Bottleneck({ - reservoir: 30, - reservoirRefreshAmount: 30, - reservoirRefreshInterval: 60000, - maxConcurrent: 1, - minTime: 100, - }), + // reddit requests are throttled inside src/reddit/client.js itself (config.reddit.throttle) items: new Bottleneck({ maxConcurrent: args.concurrency, minTime: args.interval, diff --git a/src/methods/self.js b/src/methods/self.js index 31895dd..4e45934 100644 --- a/src/methods/self.js +++ b/src/methods/self.js @@ -4,11 +4,11 @@ function curateComments(comments) { return comments.map((comment) => ({ id: comment.id, url: `https://reddit.com${comment.permalink}`, - author: comment.author.name, + author: comment.author, body: comment.body, html: comment.body_html, score: comment.score, - datetime: new Date(comment.created * 1000), + datetime: new Date(comment.created_utc * 1000), edited: comment.edited, controversiality: comment.controversiality, gilded: comment.gilded, @@ -22,18 +22,12 @@ function curateComments(comments) { } async function getFullPost(postId, reddit) { - return reddit - .getSubmission(postId) - .expandReplies({ - limit: Infinity, - depth: Infinity, - }) - .fetch(); + return reddit.getSubmissionWithComments(postId); } async function self(host, originalPost, { reddit }) { - const post = await getFullPost(originalPost.id, reddit) || originalPost; - const curatedComments = curateComments(post.comments); + const { post, comments } = await getFullPost(originalPost.id, reddit); + const curatedComments = curateComments(comments); return { album: null, @@ -41,9 +35,9 @@ async function self(host, originalPost, { reddit }) { id: post.id, url: post.url, title: post.title, - text: post.text, - author: post.author.name, - datetime: post.datetime, + text: post.selftext, + author: post.author, + datetime: new Date(post.created_utc * 1000), comments: curatedComments, type: 'text/plain', self: true, diff --git a/src/reddit/client.js b/src/reddit/client.js new file mode 100644 index 0000000..b326dc4 --- /dev/null +++ b/src/reddit/client.js @@ -0,0 +1,284 @@ +'use strict'; + +const fetch = require('node-fetch'); +const Bottleneck = require('bottleneck'); +const config = require('config'); + +const { version } = require('../../package.json'); +const logger = require('../logger')(__filename); + +const BASE_URL = 'https://www.reddit.com'; +const MAX_RETRIES = 5; + +// deliberately much heavier throttling than the old OAuth limiter (src/limiter.js reddit: 30/min) - +// this is unauthenticated/public access and getting flagged as a bot defeats the point +const limiter = new Bottleneck({ + reservoir: config.reddit.throttle?.reservoir ?? 6, + reservoirRefreshAmount: config.reddit.throttle?.reservoir ?? 6, + reservoirRefreshInterval: 60000, + maxConcurrent: 1, + minTime: config.reddit.throttle?.minTime ?? 1500, +}); + +// parses a Netscape/Mozilla-format cookie jar dump (domain, includeSubdomains, path, +// secure, expiry, name, value - tab separated) into a `name=value; name2=value2` header +function parseCookieJar(raw) { + const now = Date.now() / 1000; + + return raw + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .map((line) => line.split('\t')) + .filter((fields) => fields.length === 7) + .filter(([, , , , expiry]) => expiry === '0' || Number(expiry) > now) + .filter(([domain]) => domain.replace(/^\./, '').endsWith('reddit.com')) + .map(([, , , , , name, value]) => `${name}=${value}`) + .join('; '); +} + +function soonestExpiry(raw) { + const expiries = raw + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .map((line) => line.split('\t')) + .filter((fields) => fields.length === 7) + .map(([, , , , expiry]) => Number(expiry)) + .filter((expiry) => expiry > 0); + + return expiries.length ? Math.min(...expiries) : null; +} + +let cachedCookieHeader; + +function cookieHeader() { + if (cachedCookieHeader === undefined) { + cachedCookieHeader = config.reddit.cookies + ? parseCookieJar(config.reddit.cookies) + : (config.reddit.cookie || null); + + logger.info(cachedCookieHeader + ? `Using ${cachedCookieHeader.split(';').length} reddit cookie(s) for authenticated requests` + : 'No reddit cookies configured, requesting unauthenticated'); + + if (config.reddit.cookies) { + const expiry = soonestExpiry(config.reddit.cookies); + const hoursLeft = expiry ? (expiry - Date.now() / 1000) / 3600 : null; + + if (hoursLeft !== null && hoursLeft < 48) { + logger.warn(`Soonest-expiring reddit cookie expires in ${hoursLeft.toFixed(1)}h (${new Date(expiry * 1000).toISOString()}) - refresh config/cookies.js before then`); + } + } + } + + return cachedCookieHeader; +} + +function headers() { + const requestHeaders = { + 'user-agent': config.reddit.userAgent || `ripunzel/${version}`, + accept: 'application/json', + }; + + const cookie = cookieHeader(); + + if (cookie) { + requestHeaders.cookie = cookie; + } + + return requestHeaders; +} + +function buildUrl(path, params) { + const url = new URL(`${BASE_URL}${path}`); + + Object.entries({ raw_json: 1, ...params }).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, value); + } + }); + + return url; +} + +async function wait(seconds) { + return new Promise((resolve) => { + setTimeout(resolve, seconds * 1000); + }); +} + +async function requestJson(path, params = {}, attempt = 0) { + const url = buildUrl(path, params); + + return limiter.schedule(async () => { + const res = await fetch(url.toString(), { headers: headers() }); + const contentType = res.headers.get('content-type') || ''; + + if (res.status === 429 || res.status === 503) { + if (attempt >= MAX_RETRIES) { + throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`); + } + + const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt; + + logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`); + + await wait(retryAfter); + + return requestJson(path, params, attempt + 1); + } + + if (!contentType.includes('json')) { + // reddit serves an HTML interstitial/block page instead of JSON when it decides + // the request looks like a bot - this is a distinct failure mode from a genuine + // 4xx and should be logged/handled separately rather than treated as "bad request" + const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)'; + + throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`); + } + + const data = await res.json(); + + if (!res.ok) { + const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200); + + throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`); + } + + return data; + }); +} + +async function getUser(username) { + const data = await requestJson(`/user/${username}/about.json`); + + return data.data; +} + +async function getSubmissions(username, { sort = 'new', limit = Infinity } = {}) { + const submissions = []; + let after = null; + + do { + // eslint-disable-next-line no-await-in-loop + const data = await requestJson(`/user/${username}/submitted.json`, { + sort, + limit: Math.min(100, Number.isFinite(limit) ? limit - submissions.length : 100), + after, + }); + + const page = data.data.children.map((child) => child.data); + + submissions.push(...page); + after = data.data.after; + } while (after && submissions.length < limit); + + return submissions; +} + +// reddit's raw comment listing is a tree of {kind, data} "things": kind 't1' is a comment +// (whose own `replies` field is either '' or a nested Listing of more things), kind 'more' +// is a stub meaning "N additional comment IDs exist here, fetch them separately". +function flattenThings(things) { + const comments = []; + const moreIds = []; + + things.forEach((thing) => { + if (thing.kind === 't1') { + comments.push(thing.data); + + if (thing.data.replies?.data?.children) { + const nested = flattenThings(thing.data.replies.data.children); + + comments.push(...nested.comments); + moreIds.push(...nested.moreIds); + } + } else if (thing.kind === 'more' && thing.data.id !== '_') { + moreIds.push(...thing.data.children); + } + }); + + return { comments, moreIds }; +} + +async function moreChildren(linkId, ids) { + const unique = [...new Set(ids)]; + const results = []; + + for (let offset = 0; offset < unique.length; offset += 100) { + const chunk = unique.slice(offset, offset + 100); + + // eslint-disable-next-line no-await-in-loop + const data = await requestJson('/api/morechildren.json', { + api_type: 'json', + link_id: linkId, + children: chunk.join(','), + limit_children: false, + }); + + results.push(...(data.json?.data?.things || [])); + } + + return results; +} + +// re-fetches 'more' stubs and re-attaches them by parent_id; morechildren can itself return +// further 'more' stubs on very deep threads, so this recurses with a depth cap rather than +// looping forever +async function resolveMore(moreIds, linkId, depth = 0) { + if (!moreIds.length) { + return []; + } + + if (depth >= 4) { + logger.warn(`Gave up resolving ${moreIds.length} deeply-nested 'more comments' stubs for '${linkId}' after ${depth} rounds`); + + return []; + } + + const more = await moreChildren(linkId, moreIds); + const resolvedComments = more.filter((thing) => thing.kind === 't1').map((thing) => thing.data); + const nestedMoreIds = more.filter((thing) => thing.kind === 'more').flatMap((thing) => thing.data.children); + + return [...resolvedComments, ...await resolveMore(nestedMoreIds, linkId, depth + 1)]; +} + +function buildCommentTree(comments, parentId) { + return comments + .filter((comment) => comment.parent_id === parentId) + .map((comment) => ({ + ...comment, + replies: buildCommentTree(comments, `t1_${comment.id}`), + })); +} + +// post metadata only - cheap, no comment-tree resolution (limit: 0 asks reddit to skip +// returning comments too, but we ignore the second listing element regardless) +async function getSubmission(id) { + const [postListing] = await requestJson(`/comments/${id}.json`, { limit: 0 }); + + return postListing.data.children[0].data; +} + +// post + fully-resolved comment tree, incl. 'more comments' stubs - expensive on busy +// threads (each round of stubs is its own throttled request), only use where the full +// thread is actually needed +async function getSubmissionWithComments(id) { + const linkId = `t3_${id}`; + const [postListing, commentsListing] = await requestJson(`/comments/${id}.json`); + const post = postListing.data.children[0].data; + + const { comments, moreIds } = flattenThings(commentsListing.data.children); + const resolved = [...comments, ...await resolveMore(moreIds, linkId)]; + + return { post, comments: buildCommentTree(resolved, linkId) }; +} + +module.exports = { + requestJson, + getUser, + getSubmissions, + getSubmission, + getSubmissionWithComments, +}; diff --git a/src/sources/getPosts.js b/src/sources/getPosts.js index e5dd7f4..5403fb4 100644 --- a/src/sources/getPosts.js +++ b/src/sources/getPosts.js @@ -6,11 +6,10 @@ const getIndex = require('./getIndex'); const curateUser = require('../curate/user'); const logger = require('../logger')(__filename); -const limiter = require('../limiter').reddit; async function getUser(username, reddit) { try { - const user = await limiter.schedule(async () => reddit.getUser(username).fetch()); + const user = await reddit.getUser(username); return curateUser(user); } catch (error) { @@ -25,22 +24,22 @@ async function getUser(username, reddit) { const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) { return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => { - const post = await limiter.schedule(async () => reddit.getSubmission(postId).fetch()); + const post = await reddit.getSubmission(postId); post.direct = true; - if (accUserPosts[post.author.name]) { + if (accUserPosts[post.author]) { return { ...accUserPosts, - [post.author.name]: { - ...accUserPosts[post.author.name], - posts: [...accUserPosts[post.author.name].posts, post], + [post.author]: { + ...accUserPosts[post.author], + posts: [...accUserPosts[post.author].posts, post], }, }; } // don't attempt to fetch deleted user - if (post.author.name === '[deleted]') { + if (post.author === '[deleted]') { return { ...accUserPosts, '[deleted]': { @@ -56,12 +55,12 @@ const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) { }; } - const user = await getUser(post.author.name, reddit); + const user = await getUser(post.author, reddit); const { profile, posts: indexed } = await getIndex(user); return { ...accUserPosts, - [post.author.name]: { + [post.author]: { ...user, posts: [post], indexed: { diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index 0d27200..4451e05 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -7,11 +7,10 @@ const getArchivePostIds = require('../archives/getArchivePostIds'); const curateUser = require('../curate/user'); const logger = require('../logger')(__filename); -const limiter = require('../limiter').reddit; async function getUser(username, reddit) { try { - const user = await limiter.schedule(async () => reddit.getUser(username).fetch()); + const user = await reddit.getUser(username); logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`); @@ -28,12 +27,10 @@ async function getUser(username, reddit) { async function getPosts(username, reddit, args) { try { - const submissions = await limiter.schedule(async () => reddit - .getUser(username) - .getSubmissions({ - sort: args.sort, - limit: Infinity, - })); + const submissions = await reddit.getSubmissions(username, { + sort: args.sort, + limit: Infinity, + }); logger.info(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`); @@ -48,7 +45,7 @@ async function getPosts(username, reddit, args) { async function getArchivedPosts(username, posts, reddit) { const postIds = await getArchivePostIds(username, posts.map((post) => post.id)); - return Promise.all(postIds.map((postId) => limiter.schedule(async () => reddit.getSubmission(postId).fetch()))); + return Promise.all(postIds.map((postId) => reddit.getSubmission(postId))); } function getUserPostsWrap(reddit, args) {