Replaced Snoowrap with public API calls in response to Reddit revoking API keys. This was largely done using Claude Code, extensive manual review and major version bump needed if merged into main.

config/default.js

@@ -77,14 +77,20 @@ module.exports = {
         interval: 100,
     },
     reddit: {
-        api: {
+        // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to
+        // get past reddit's anonymous-traffic bot challenge on www.reddit.com. Needs periodic
+        // refreshing - src/reddit/client.js logs a warning once the soonest-expiring cookie
+        // in here is within 48h of expiring.
+        cookies: null,
+        // fallback/simpler alternative to `cookies` above: a single pre-built `name=value; ...`
+        // Cookie header string. Only used if `cookies` is not set.
+        cookie: null,
         userAgent: 'ripunzel',
-            clientId: '1234567abcdefg',
+        throttle: {
-            access_token: 'abcD123eFg45Hi6J7klmnop8qr9',
+            // deliberately conservative - this is unauthenticated-looking public access,
-            token_type: 'bearer',
+            // not an approved API key, getting flagged as a bot defeats the point
-            expires_in: 3600,
+            reservoir: 6,
-            refresh_token: '1234567-A-Bc-defg8912hij-klm345opqr',
+            minTime: 1500,
-            scope: 'history identity mysubreddits read subscribe',
         },
     },
     methods: {

package.json

@@ -49,7 +49,6 @@
     "object.omit": "^3.0.0",
     "object.pick": "^1.3.0",
     "sharp-phash": "^2.1.0",
-    "snoowrap": "^1.23.0",
     "template-format": "^1.2.4",
     "unprint": "^0.8.2",
     "url-pattern": "^1.0.3",

src/app.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const Snoowrap = require('snoowrap');
 const fs = require('fs-extra');
 const Promise = require('bluebird');
 const exiftool = require('node-exiftool');
@@ -10,7 +9,7 @@ const cron = require('node-cron');
 
 require('array.prototype.flatten').shim();
 
-const reddit = new Snoowrap(config.reddit.api);
+const reddit = require('./reddit/client');
 const args = require('./cli')();
 const logger = require('./logger')(__filename);
 

src/curate/posts.js

@@ -57,12 +57,13 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
         permalink,
         url: post.url,
         datetime: new Date(post.created_utc * 1000),
-        subreddit: post.subreddit.display_name,
+        subreddit: post.subreddit,
         score: post.score,
         preview: post.preview ? post.preview.images.map((image) => image.source) : null,
         host,
         direct: post.direct,
-        comments: post.comments,
+        // count only - the full thread (when needed) is fetched separately via methods/self.js
+        comments: post.num_comments,
         hash: hashPost(post),
     };
 

src/curate/user.js

@@ -9,23 +9,23 @@ function curateUser(user) {
         gold: user.is_gold,
         verified: user.verified,
         verifiedEmail: user.has_verified_email,
-        fallback: false
+        fallback: false,
     };
 
     if (user.subreddit) {
         Object.assign(curatedUser, {
             profile: {
-                id: user.subreddit.display_name.name,
+                id: user.subreddit.display_name,
-                title: user.subreddit.display_name.title,
+                title: user.subreddit.title,
-                image: user.subreddit.display_name.icon_img,
+                image: user.subreddit.icon_img,
-                banner: user.subreddit.display_name.banner_img,
+                banner: user.subreddit.banner_img,
-                description: user.subreddit.display_name.public_description,
+                description: user.subreddit.public_description,
-                over18: user.subreddit.display_name.over_18
+                over18: user.subreddit.over_18,
-            }
+            },
         });
     }
 
     return curatedUser;
-};
+}
 
 module.exports = curateUser;

src/limiter.js

@@ -5,13 +5,7 @@ const Bottleneck = require('bottleneck');
 const args = require('./cli')();
 
 module.exports = {
-    reddit: new Bottleneck({
+    // reddit requests are throttled inside src/reddit/client.js itself (config.reddit.throttle)
-        reservoir: 30,
-        reservoirRefreshAmount: 30,
-        reservoirRefreshInterval: 60000,
-        maxConcurrent: 1,
-        minTime: 100,
-    }),
     items: new Bottleneck({
         maxConcurrent: args.concurrency,
         minTime: args.interval,

src/methods/self.js

@@ -4,11 +4,11 @@ function curateComments(comments) {
     return comments.map((comment) => ({
         id: comment.id,
         url: `https://reddit.com${comment.permalink}`,
-        author: comment.author.name,
+        author: comment.author,
         body: comment.body,
         html: comment.body_html,
         score: comment.score,
-        datetime: new Date(comment.created * 1000),
+        datetime: new Date(comment.created_utc * 1000),
         edited: comment.edited,
         controversiality: comment.controversiality,
         gilded: comment.gilded,
@@ -22,18 +22,12 @@ function curateComments(comments) {
 }
 
 async function getFullPost(postId, reddit) {
-    return reddit
+    return reddit.getSubmissionWithComments(postId);
-        .getSubmission(postId)
-        .expandReplies({
-            limit: Infinity,
-            depth: Infinity,
-        })
-        .fetch();
 }
 
 async function self(host, originalPost, { reddit }) {
-    const post = await getFullPost(originalPost.id, reddit) || originalPost;
+    const { post, comments } = await getFullPost(originalPost.id, reddit);
-    const curatedComments = curateComments(post.comments);
+    const curatedComments = curateComments(comments);
 
     return {
         album: null,
@@ -41,9 +35,9 @@ async function self(host, originalPost, { reddit }) {
             id: post.id,
             url: post.url,
             title: post.title,
-            text: post.text,
+            text: post.selftext,
-            author: post.author.name,
+            author: post.author,
-            datetime: post.datetime,
+            datetime: new Date(post.created_utc * 1000),
             comments: curatedComments,
             type: 'text/plain',
             self: true,

src/reddit/client.js

@@ -0,0 +1,284 @@
+'use strict';
+
+const fetch = require('node-fetch');
+const Bottleneck = require('bottleneck');
+const config = require('config');
+
+const { version } = require('../../package.json');
+const logger = require('../logger')(__filename);
+
+const BASE_URL = 'https://www.reddit.com';
+const MAX_RETRIES = 5;
+
+// deliberately much heavier throttling than the old OAuth limiter (src/limiter.js reddit: 30/min) -
+// this is unauthenticated/public access and getting flagged as a bot defeats the point
+const limiter = new Bottleneck({
+    reservoir: config.reddit.throttle?.reservoir ?? 6,
+    reservoirRefreshAmount: config.reddit.throttle?.reservoir ?? 6,
+    reservoirRefreshInterval: 60000,
+    maxConcurrent: 1,
+    minTime: config.reddit.throttle?.minTime ?? 1500,
+});
+
+// parses a Netscape/Mozilla-format cookie jar dump (domain, includeSubdomains, path,
+// secure, expiry, name, value - tab separated) into a `name=value; name2=value2` header
+function parseCookieJar(raw) {
+    const now = Date.now() / 1000;
+
+    return raw
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith('#'))
+        .map((line) => line.split('\t'))
+        .filter((fields) => fields.length === 7)
+        .filter(([, , , , expiry]) => expiry === '0' || Number(expiry) > now)
+        .filter(([domain]) => domain.replace(/^\./, '').endsWith('reddit.com'))
+        .map(([, , , , , name, value]) => `${name}=${value}`)
+        .join('; ');
+}
+
+function soonestExpiry(raw) {
+    const expiries = raw
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith('#'))
+        .map((line) => line.split('\t'))
+        .filter((fields) => fields.length === 7)
+        .map(([, , , , expiry]) => Number(expiry))
+        .filter((expiry) => expiry > 0);
+
+    return expiries.length ? Math.min(...expiries) : null;
+}
+
+let cachedCookieHeader;
+
+function cookieHeader() {
+    if (cachedCookieHeader === undefined) {
+        cachedCookieHeader = config.reddit.cookies
+            ? parseCookieJar(config.reddit.cookies)
+            : (config.reddit.cookie || null);
+
+        logger.info(cachedCookieHeader
+            ? `Using ${cachedCookieHeader.split(';').length} reddit cookie(s) for authenticated requests`
+            : 'No reddit cookies configured, requesting unauthenticated');
+
+        if (config.reddit.cookies) {
+            const expiry = soonestExpiry(config.reddit.cookies);
+            const hoursLeft = expiry ? (expiry - Date.now() / 1000) / 3600 : null;
+
+            if (hoursLeft !== null && hoursLeft < 48) {
+                logger.warn(`Soonest-expiring reddit cookie expires in ${hoursLeft.toFixed(1)}h (${new Date(expiry * 1000).toISOString()}) - refresh config/cookies.js before then`);
+            }
+        }
+    }
+
+    return cachedCookieHeader;
+}
+
+function headers() {
+    const requestHeaders = {
+        'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
+        accept: 'application/json',
+    };
+
+    const cookie = cookieHeader();
+
+    if (cookie) {
+        requestHeaders.cookie = cookie;
+    }
+
+    return requestHeaders;
+}
+
+function buildUrl(path, params) {
+    const url = new URL(`${BASE_URL}${path}`);
+
+    Object.entries({ raw_json: 1, ...params }).forEach(([key, value]) => {
+        if (value !== undefined && value !== null) {
+            url.searchParams.set(key, value);
+        }
+    });
+
+    return url;
+}
+
+async function wait(seconds) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, seconds * 1000);
+    });
+}
+
+async function requestJson(path, params = {}, attempt = 0) {
+    const url = buildUrl(path, params);
+
+    return limiter.schedule(async () => {
+        const res = await fetch(url.toString(), { headers: headers() });
+        const contentType = res.headers.get('content-type') || '';
+
+        if (res.status === 429 || res.status === 503) {
+            if (attempt >= MAX_RETRIES) {
+                throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
+            }
+
+            const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;
+
+            logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);
+
+            await wait(retryAfter);
+
+            return requestJson(path, params, attempt + 1);
+        }
+
+        if (!contentType.includes('json')) {
+            // reddit serves an HTML interstitial/block page instead of JSON when it decides
+            // the request looks like a bot - this is a distinct failure mode from a genuine
+            // 4xx and should be logged/handled separately rather than treated as "bad request"
+            const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
+
+            throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
+        }
+
+        const data = await res.json();
+
+        if (!res.ok) {
+            const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);
+
+            throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
+        }
+
+        return data;
+    });
+}
+
+async function getUser(username) {
+    const data = await requestJson(`/user/${username}/about.json`);
+
+    return data.data;
+}
+
+async function getSubmissions(username, { sort = 'new', limit = Infinity } = {}) {
+    const submissions = [];
+    let after = null;
+
+    do {
+        // eslint-disable-next-line no-await-in-loop
+        const data = await requestJson(`/user/${username}/submitted.json`, {
+            sort,
+            limit: Math.min(100, Number.isFinite(limit) ? limit - submissions.length : 100),
+            after,
+        });
+
+        const page = data.data.children.map((child) => child.data);
+
+        submissions.push(...page);
+        after = data.data.after;
+    } while (after && submissions.length < limit);
+
+    return submissions;
+}
+
+// reddit's raw comment listing is a tree of {kind, data} "things": kind 't1' is a comment
+// (whose own `replies` field is either '' or a nested Listing of more things), kind 'more'
+// is a stub meaning "N additional comment IDs exist here, fetch them separately".
+function flattenThings(things) {
+    const comments = [];
+    const moreIds = [];
+
+    things.forEach((thing) => {
+        if (thing.kind === 't1') {
+            comments.push(thing.data);
+
+            if (thing.data.replies?.data?.children) {
+                const nested = flattenThings(thing.data.replies.data.children);
+
+                comments.push(...nested.comments);
+                moreIds.push(...nested.moreIds);
+            }
+        } else if (thing.kind === 'more' && thing.data.id !== '_') {
+            moreIds.push(...thing.data.children);
+        }
+    });
+
+    return { comments, moreIds };
+}
+
+async function moreChildren(linkId, ids) {
+    const unique = [...new Set(ids)];
+    const results = [];
+
+    for (let offset = 0; offset < unique.length; offset += 100) {
+        const chunk = unique.slice(offset, offset + 100);
+
+        // eslint-disable-next-line no-await-in-loop
+        const data = await requestJson('/api/morechildren.json', {
+            api_type: 'json',
+            link_id: linkId,
+            children: chunk.join(','),
+            limit_children: false,
+        });
+
+        results.push(...(data.json?.data?.things || []));
+    }
+
+    return results;
+}
+
+// re-fetches 'more' stubs and re-attaches them by parent_id; morechildren can itself return
+// further 'more' stubs on very deep threads, so this recurses with a depth cap rather than
+// looping forever
+async function resolveMore(moreIds, linkId, depth = 0) {
+    if (!moreIds.length) {
+        return [];
+    }
+
+    if (depth >= 4) {
+        logger.warn(`Gave up resolving ${moreIds.length} deeply-nested 'more comments' stubs for '${linkId}' after ${depth} rounds`);
+
+        return [];
+    }
+
+    const more = await moreChildren(linkId, moreIds);
+    const resolvedComments = more.filter((thing) => thing.kind === 't1').map((thing) => thing.data);
+    const nestedMoreIds = more.filter((thing) => thing.kind === 'more').flatMap((thing) => thing.data.children);
+
+    return [...resolvedComments, ...await resolveMore(nestedMoreIds, linkId, depth + 1)];
+}
+
+function buildCommentTree(comments, parentId) {
+    return comments
+        .filter((comment) => comment.parent_id === parentId)
+        .map((comment) => ({
+            ...comment,
+            replies: buildCommentTree(comments, `t1_${comment.id}`),
+        }));
+}
+
+// post metadata only - cheap, no comment-tree resolution (limit: 0 asks reddit to skip
+// returning comments too, but we ignore the second listing element regardless)
+async function getSubmission(id) {
+    const [postListing] = await requestJson(`/comments/${id}.json`, { limit: 0 });
+
+    return postListing.data.children[0].data;
+}
+
+// post + fully-resolved comment tree, incl. 'more comments' stubs - expensive on busy
+// threads (each round of stubs is its own throttled request), only use where the full
+// thread is actually needed
+async function getSubmissionWithComments(id) {
+    const linkId = `t3_${id}`;
+    const [postListing, commentsListing] = await requestJson(`/comments/${id}.json`);
+    const post = postListing.data.children[0].data;
+
+    const { comments, moreIds } = flattenThings(commentsListing.data.children);
+    const resolved = [...comments, ...await resolveMore(moreIds, linkId)];
+
+    return { post, comments: buildCommentTree(resolved, linkId) };
+}
+
+module.exports = {
+    requestJson,
+    getUser,
+    getSubmissions,
+    getSubmission,
+    getSubmissionWithComments,
+};

src/sources/getPosts.js

@@ -6,11 +6,10 @@ const getIndex = require('./getIndex');
 const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').reddit;
 
 async function getUser(username, reddit) {
     try {
-        const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
+        const user = await reddit.getUser(username);
 
         return curateUser(user);
     } catch (error) {
@@ -25,22 +24,22 @@ async function getUser(username, reddit) {
 
 const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) {
     return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => {
-        const post = await limiter.schedule(async () => reddit.getSubmission(postId).fetch());
+        const post = await reddit.getSubmission(postId);
 
         post.direct = true;
 
-        if (accUserPosts[post.author.name]) {
+        if (accUserPosts[post.author]) {
             return {
                 ...accUserPosts,
-                [post.author.name]: {
+                [post.author]: {
-                    ...accUserPosts[post.author.name],
+                    ...accUserPosts[post.author],
-                    posts: [...accUserPosts[post.author.name].posts, post],
+                    posts: [...accUserPosts[post.author].posts, post],
                 },
             };
         }
 
         // don't attempt to fetch deleted user
-        if (post.author.name === '[deleted]') {
+        if (post.author === '[deleted]') {
             return {
                 ...accUserPosts,
                 '[deleted]': {
@@ -56,12 +55,12 @@ const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) {
             };
         }
 
-        const user = await getUser(post.author.name, reddit);
+        const user = await getUser(post.author, reddit);
         const { profile, posts: indexed } = await getIndex(user);
 
         return {
             ...accUserPosts,
-            [post.author.name]: {
+            [post.author]: {
                 ...user,
                 posts: [post],
                 indexed: {

src/sources/getUserPosts.js

@@ -7,11 +7,10 @@ const getArchivePostIds = require('../archives/getArchivePostIds');
 const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').reddit;
 
 async function getUser(username, reddit) {
     try {
-        const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
+        const user = await reddit.getUser(username);
 
         logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
 
@@ -28,12 +27,10 @@ async function getUser(username, reddit) {
 
 async function getPosts(username, reddit, args) {
     try {
-        const submissions = await limiter.schedule(async () => reddit
+        const submissions = await reddit.getSubmissions(username, {
-            .getUser(username)
-            .getSubmissions({
             sort: args.sort,
             limit: Infinity,
-            }));
+        });
 
         logger.info(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);
 
@@ -48,7 +45,7 @@ async function getPosts(username, reddit, args) {
 async function getArchivedPosts(username, posts, reddit) {
     const postIds = await getArchivePostIds(username, posts.map((post) => post.id));
 
-    return Promise.all(postIds.map((postId) => limiter.schedule(async () => reddit.getSubmission(postId).fetch())));
+    return Promise.all(postIds.map((postId) => reddit.getSubmission(postId)));
 }
 
 function getUserPostsWrap(reddit, args) {