Bumped to Node 24 and upgraded dependencies.

.nvmrc

@@ -1 +1 @@
-20.19.6
+24.19.0

config/default.js

@@ -44,7 +44,8 @@ module.exports = {
             gold: '★',
             over18: '♥',
         },
-        dateFormat: 'YYYYMMDD',
+        // date-fns v2+ token syntax (was 'YYYYMMDD' under date-fns v1)
+        dateFormat: 'yyyyMMdd',
         truncate: {
             limit: 250,
             truncator: '...',
@@ -77,14 +78,20 @@ module.exports = {
         interval: 100,
     },
     reddit: {
-        api: {
+        // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to
+        // get past reddit's anonymous-traffic bot challenge on www.reddit.com. Needs periodic
+        // refreshing - src/reddit/client.js logs a warning once the soonest-expiring cookie
+        // in here is within 48h of expiring.
+        cookies: null,
+        // fallback/simpler alternative to `cookies` above: a single pre-built `name=value; ...`
+        // Cookie header string. Only used if `cookies` is not set.
+        cookie: null,
         userAgent: 'ripunzel',
-            clientId: '1234567abcdefg',
+        throttle: {
-            access_token: 'abcD123eFg45Hi6J7klmnop8qr9',
+            // deliberately conservative - this is unauthenticated-looking public access,
-            token_type: 'bearer',
+            // not an approved API key, getting flagged as a bot defeats the point
-            expires_in: 3600,
+            reservoir: 6,
-            refresh_token: '1234567-A-Bc-defg8912hij-klm345opqr',
+            minTime: 1500,
-            scope: 'history identity mysubreddits read subscribe',
         },
     },
     methods: {

package.json

@@ -27,41 +27,41 @@
   ],
   "author": "Niels Simenon",
   "license": "ISC",
+  "engines": {
+    "node": ">=24.19.0"
+  },
   "dependencies": {
     "array.prototype.flatten": "^1.2.1",
-    "bhttp": "^1.2.4",
+    "bhttp": "^1.2.8",
-    "blake2": "^4.1.1",
+    "blake2": "^5.0.1",
-    "bluebird": "^3.5.1",
+    "bluebird": "^3.7.2",
     "bottleneck": "^2.19.5",
-    "cheerio": "^1.0.0-rc.2",
+    "cheerio": "^1.2.0",
-    "config": "^1.30.0",
+    "config": "^4.4.2",
-    "date-fns": "^1.29.0",
+    "date-fns": "^4.4.0",
     "dist-exiftool": "^10.53.0",
-    "fluent-ffmpeg": "^2.1.2",
+    "fluent-ffmpeg": "^2.1.3",
-    "fs-extra": "^5.0.0",
+    "fs-extra": "^11.4.0",
-    "js-yaml": "^3.12.0",
+    "js-yaml": "^5.2.3",
-    "jsdom": "^15.2.0",
     "mime": "^3.0.0",
-    "mime-types": "^2.1.18",
+    "mime-types": "^3.0.2",
-    "node-cron": "^1.2.1",
+    "node-cron": "^4.6.0",
     "node-exiftool": "^2.3.0",
-    "node-fetch": "^2.1.2",
     "object.omit": "^3.0.0",
     "object.pick": "^1.3.0",
-    "sharp-phash": "^2.1.0",
+    "sharp-phash": "^2.2.0",
-    "snoowrap": "^1.23.0",
+    "template-format": "^1.2.5",
-    "template-format": "^1.2.4",
+    "unprint": "^0.19.20",
-    "unprint": "^0.8.2",
     "url-pattern": "^1.0.3",
-    "winston": "^3.3.3",
+    "winston": "^3.19.0",
-    "winston-daily-rotate-file": "^4.7.1",
+    "winston-daily-rotate-file": "^5.0.0",
-    "yargs": "^11.0.0",
+    "yargs": "^17.7.2",
-    "youtube-dl-exec": "^2.2.3"
+    "youtube-dl-exec": "^3.1.12"
   },
   "devDependencies": {
-    "@babel/eslint-parser": "^7.19.1",
+    "@babel/eslint-parser": "^7.29.7",
-    "eslint": "^8.34.0",
+    "eslint": "^8.57.1",
     "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-plugin-import": "^2.27.5"
+    "eslint-plugin-import": "^2.32.0"
   }
 }

src/app.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const Snoowrap = require('snoowrap');
 const fs = require('fs-extra');
 const Promise = require('bluebird');
 const exiftool = require('node-exiftool');
@@ -10,7 +9,7 @@ const cron = require('node-cron');
 
 require('array.prototype.flatten').shim();
 
-const reddit = new Snoowrap(config.reddit.api);
+const reddit = require('./reddit/client');
 const args = require('./cli')();
 const logger = require('./logger')(__filename);
 
@@ -141,8 +140,10 @@ function fetchSavePosts(userPosts, ep) {
 }
 
 async function initApp() {
+    let ep;
+
     try {
-        const ep = new exiftool.ExiftoolProcess(exiftoolBin);
+        ep = new exiftool.ExiftoolProcess(exiftoolBin);
         await ep.open();
 
         if (args.fetch || args.fileDirect) {
@@ -155,8 +156,6 @@ async function initApp() {
             await fetchSavePosts(userPosts, ep);
         }
 
-        await ep.close();
-
         if (args.watch) {
             logger.info(`Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`);
         }
@@ -166,6 +165,13 @@ async function initApp() {
         } else {
             logger.error(error.message);
         }
+    } finally {
+        // the exiftool child process keeps the event loop alive - if it's never closed
+        // (e.g. because an error above was thrown before we got to it), the process hangs
+        // forever after logging the error instead of actually exiting
+        if (ep) {
+            await ep.close().catch((error) => logger.error(`Could not close exiftool process: ${error.message}`));
+        }
     }
 }
 

src/archives/ip.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const $ = require('cheerio');
 
 function findOnIp(username, page = 1, acc = []) {

src/cli.js

@@ -124,6 +124,11 @@ function getArgs() {
             type: 'boolean',
             default: config.fetch.archives.search,
         })
+        .option('debug', {
+            describe: 'Log full stack traces instead of just error messages',
+            type: 'boolean',
+            default: false,
+        })
         .argv;
 
     return {

src/curate/posts.js

@@ -57,12 +57,13 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
         permalink,
         url: post.url,
         datetime: new Date(post.created_utc * 1000),
-        subreddit: post.subreddit.display_name,
+        subreddit: post.subreddit,
         score: post.score,
         preview: post.preview ? post.preview.images.map((image) => image.source) : null,
         host,
         direct: post.direct,
-        comments: post.comments,
+        // count only - the full thread (when needed) is fetched separately via methods/self.js
+        comments: post.num_comments,
         hash: hashPost(post),
     };
 

src/curate/user.js

@@ -9,23 +9,23 @@ function curateUser(user) {
         gold: user.is_gold,
         verified: user.verified,
         verifiedEmail: user.has_verified_email,
-        fallback: false
+        fallback: false,
     };
 
-    if(user.subreddit) {
+    if (user.subreddit) {
         Object.assign(curatedUser, {
             profile: {
-                id: user.subreddit.display_name.name,
+                id: user.subreddit.display_name,
-                title: user.subreddit.display_name.title,
+                title: user.subreddit.title,
-                image: user.subreddit.display_name.icon_img,
+                image: user.subreddit.icon_img,
-                banner: user.subreddit.display_name.banner_img,
+                banner: user.subreddit.banner_img,
-                description: user.subreddit.display_name.public_description,
+                description: user.subreddit.public_description,
-                over18: user.subreddit.display_name.over_18
+                over18: user.subreddit.over_18,
-            }
+            },
         });
     }
 
     return curatedUser;
-};
+}
 
 module.exports = curateUser;

src/fetch/content.js

@@ -46,7 +46,7 @@ function selfPostToText(item, post) {
         comments: curateComments(item.comments),
     };
 
-    return yaml.safeDump(curatedPost);
+    return yaml.dump(curatedPost);
 }
 
 async function getBuffers(item, context) {
@@ -117,6 +117,8 @@ async function fetchSaveUserContent(user, ep, args) {
 
         const items = await Promise.map(post.content.items, async (originalItem, index) => {
             const item = { ...originalItem, index };
+
+            try {
                 const buffers = await getBuffers(item, { post, host: post.host, headers: post.content.headers || item.headers });
 
                 // no buffers, ignore item
@@ -149,6 +151,17 @@ async function fetchSaveUserContent(user, ep, args) {
                 await addMeta(filepath, item, post, user, ep);
 
                 return item;
+            } catch (error) {
+                // don't let one bad item (e.g. a post with a missing/invalid date) abort the
+                // entire run - log it with enough context to find it again and move on
+                logger.error(`Failed to fetch/save item ${index} for post '${post.id}' (${post.permalink}): ${error.message}`);
+
+                if (args.debug) {
+                    logger.error(error.stack);
+                }
+
+                return null;
+            }
         });
 
         return {

src/interpolate.js

@@ -9,6 +9,24 @@ const format = require('template-format');
 
 const args = require('./cli')();
 
+// date-fns throws a bare, contextless "Invalid time value" RangeError for any invalid date
+// passed to format() - wrap it so a bad timestamp somewhere upstream (a post, item, user or
+// album with a malformed created/createDate field) points back at its source instead.
+// No date at all (e.g. profileDetails.js's item-less-of-a-datetime avatar interpolation) is a
+// legitimate case, not a bug - that just interpolates to an empty string, same as any other
+// unset field like title/description elsewhere in this function.
+function safeFormat(date, dateFormat, label) {
+    if (date === null || date === undefined) {
+        return null;
+    }
+
+    if (!(date instanceof Date) || Number.isNaN(date.getTime())) {
+        throw new Error(`Cannot format ${label} date for filename - invalid datetime (${date})`);
+    }
+
+    return dateFns.format(date, dateFormat);
+}
+
 function interpolate(pattern, item = null, content = null, host = null, post = null, user = null, strip = true, dateFormat = config.library.dateFormat) {
     const data = {
         tags: {},
@@ -20,7 +38,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: item.id,
                 title: item.title && item.title.slice(0, config.library.titleLength),
                 description: item.description,
-                date: dateFns.format(item.datetime, dateFormat),
+                date: safeFormat(item.datetime, dateFormat, 'item'),
                 index: item.index + config.library.indexOffset,
             },
             ext: item.type
@@ -49,7 +67,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: content.album.id,
                 title: content.album.title && content.album.title.slice(0, config.library.titleLength),
                 description: content.album.description,
-                date: dateFns.format(content.album.datetime, dateFormat),
+                date: safeFormat(content.album.datetime, dateFormat, 'album'),
             },
         });
     }
@@ -60,7 +78,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: post.id,
                 title: post.title && post.title.slice(0, config.library.titleLength),
                 url: post.url,
-                date: dateFns.format(post.datetime, dateFormat),
+                date: safeFormat(post.datetime, dateFormat, 'post'),
                 index: post.index + config.library.indexOffset,
                 hash: post.hash,
                 score: post.score,
@@ -75,7 +93,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 name: user.name,
                 username: user.name,
                 id: user.id,
-                created: dateFns.format(user.created, dateFormat),
+                created: safeFormat(user.created, dateFormat, 'user'),
             },
             tags: {
                 ...data.tags,

src/limiter.js

@@ -5,13 +5,7 @@ const Bottleneck = require('bottleneck');
 const args = require('./cli')();
 
 module.exports = {
-    reddit: new Bottleneck({
+    // reddit requests are throttled inside src/reddit/client.js itself (config.reddit.throttle)
-        reservoir: 30,
-        reservoirRefreshAmount: 30,
-        reservoirRefreshInterval: 60000,
-        maxConcurrent: 1,
-        minTime: 100,
-    }),
     items: new Bottleneck({
         maxConcurrent: args.concurrency,
         minTime: args.interval,

src/methods/erome.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
 const logger = require('../logger')(__filename);

src/methods/eroshareAlbum.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 
 async function eroshareAlbum(host) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);

src/methods/eroshareItem.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
 async function eroshareItem(host, post) {

src/methods/imgurAlbum.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 
 const logger = require('../logger')(__filename);
 const { fetchPredata } = require('./imgurImage');

src/methods/imgurImage.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 
 async function fetchPredata() {
     const data = {

src/methods/redditAlbum.js

@@ -1,18 +1,32 @@
 'use strict';
 
-const mime = require('mime');
+// reddit gallery images: `s.u` (still) or `s.gif`/`s.mp4` (animated) - URLs come back
-const bhttp = require('bhttp');
+// HTML-entity-encoded (e.g. '&' instead of '&')
-const { JSDOM } = require('jsdom');
+function resolveMediaUrl(media) {
+    const url = media?.s?.u || media?.s?.gif || media?.s?.mp4;
 
-async function redditAlbum(host, post) {
+    return url ? url.replace(/&/g, '&') : null;
-    const res = await bhttp.get(host.url);
+}
 
-    if (res.statusCode !== 200) {
+async function redditAlbum(host, post, { reddit }) {
-        throw new Error(res.body.toString());
+    // fetched through the authenticated/throttled client, not a raw fetch()/bhttp request -
-    }
+    // reddit's anonymous-traffic bot challenge blocks plain unauthenticated .json requests
+    const data = await reddit.getSubmission(post.id);
 
-    const { document } = new JSDOM(res.body.toString(), { runScripts: 'dangerously' }).window;
+    const items = (data.gallery_data?.items || [])
-    const items = Array.from(document.querySelectorAll('li a'), el => el.href);
+        .filter((item) => !item.is_deleted)
+        .map((item) => {
+            const media = data.media_metadata?.[item.media_id];
+            const url = resolveMediaUrl(media);
+
+            return url && {
+                id: item.media_id,
+                url,
+                datetime: post.datetime,
+                type: media.m || 'image/jpeg',
+            };
+        })
+        .filter(Boolean);
 
     return {
         album: {
@@ -20,12 +34,7 @@ async function redditAlbum(host, post) {
             url: host.url,
             title: post.title,
         },
-        items: items.map((url) => ({
+        items,
-            id: new URL(url).pathname.match(/\/(.*).\w+$/)?.[1],
-            url,
-            datetime: post.datetime,
-            type: mime.getType(url) || 'image/jpeg',
-        })),
     };
 }
 

src/methods/redditVideo.js

@@ -1,23 +1,48 @@
 'use strict';
 
-const fetch = require('node-fetch');
+// the DASH manifest lists the actual audio track filename - reddit has changed this naming
+// scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the
+// manifest rather than guessing a hardcoded suffix off the video URL again
+function findAudioUrl(manifest, videoId) {
+    const audioSection = manifest.match(/<AdaptationSet[^>]*contentType="audio"[^>]*>([\s\S]*?)<\/AdaptationSet>/);
 
-async function redditVideo(host, post) {
+    if (!audioSection) {
-    const res = await fetch(`${post.permalink}.json`);
+        return null;
-    const [{ data }] = await res.json();
+    }
 
-    const videoUrl = data.children[0].data.media.reddit_video.fallback_url;
+    const baseUrls = [...audioSection[1].matchAll(/<BaseURL>([^<]+)<\/BaseURL>/g)].map((match) => match[1]);
-    const audioUrl = `${videoUrl.split('/').slice(0, -1).join('/')}/audio`;
+    // representations are listed lowest to highest bandwidth - take the best one
+    const audioFile = baseUrls[baseUrls.length - 1];
 
-    const audioRes = await fetch(audioUrl, {
+    return audioFile ? `https://v.redd.it/${videoId}/${audioFile}` : null;
-        method: 'HEAD',
+}
-    });
+
+async function findAudio(video, videoId) {
+    if (!video.has_audio || !video.dash_url) {
+        return null;
+    }
+
+    const res = await fetch(video.dash_url);
+
+    if (!res.ok) {
+        return null;
+    }
+
+    return findAudioUrl(await res.text(), videoId);
+}
+
+async function redditVideo(host, post, { reddit }) {
+    // fetched through the authenticated/throttled client, not a raw fetch() - reddit's
+    // anonymous-traffic bot challenge blocks plain unauthenticated .json requests
+    const data = await reddit.getSubmission(post.id);
+    const video = data.media.reddit_video;
+    const videoId = post.host.id || post.id;
 
     const item = {
         album: null,
         items: [{
-            id: post.host.id || post.id,
+            id: videoId,
-            url: videoUrl,
+            url: video.fallback_url,
             title: post.title,
             datetime: post.datetime,
             type: 'video/mp4',
@@ -25,7 +50,9 @@ async function redditVideo(host, post) {
         }],
     };
 
-    if (audioRes.status === 200) {
+    const audioUrl = await findAudio(video, videoId);
+
+    if (audioUrl) {
         item.items[0].mux = [audioUrl];
     }
 

src/methods/redgifs.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const mime = require('mime');
 // const unprint = require('unprint');
 
@@ -28,6 +27,11 @@ async function fetchPredata() {
 }
 
 function scrapeGallery(data, { predata }) {
+    if (!data.gifs.length) {
+        // Math.min() of an empty array is Infinity, which becomes an invalid date below
+        return null;
+    }
+
     const oldestDate = Math.min(...data.gifs.map((gif) => gif.createDate));
 
     const curated = {

src/methods/self.js

@@ -4,11 +4,11 @@ function curateComments(comments) {
     return comments.map((comment) => ({
         id: comment.id,
         url: `https://reddit.com${comment.permalink}`,
-        author: comment.author.name,
+        author: comment.author,
         body: comment.body,
         html: comment.body_html,
         score: comment.score,
-        datetime: new Date(comment.created * 1000),
+        datetime: new Date(comment.created_utc * 1000),
         edited: comment.edited,
         controversiality: comment.controversiality,
         gilded: comment.gilded,
@@ -22,18 +22,12 @@ function curateComments(comments) {
 }
 
 async function getFullPost(postId, reddit) {
-    return reddit
+    return reddit.getSubmissionWithComments(postId);
-        .getSubmission(postId)
-        .expandReplies({
-            limit: Infinity,
-            depth: Infinity,
-        })
-        .fetch();
 }
 
 async function self(host, originalPost, { reddit }) {
-    const post = await getFullPost(originalPost.id, reddit) || originalPost;
+    const { post, comments } = await getFullPost(originalPost.id, reddit);
-    const curatedComments = curateComments(post.comments);
+    const curatedComments = curateComments(comments);
 
     return {
         album: null,
@@ -41,9 +35,9 @@ async function self(host, originalPost, { reddit }) {
             id: post.id,
             url: post.url,
             title: post.title,
-            text: post.text,
+            text: post.selftext,
-            author: post.author.name,
+            author: post.author,
-            datetime: post.datetime,
+            datetime: new Date(post.created_utc * 1000),
             comments: curatedComments,
             type: 'text/plain',
             self: true,

src/methods/tube.js

@@ -25,7 +25,7 @@ async function tube(host, post) {
                     title: data.fulltitle || data.title,
                     description: data.description,
                     type: `video/${data.ext}`,
-                    datetime: dateFns.parse(data.upload_date, 'YYYYMMDD'),
+                    datetime: dateFns.parse(data.upload_date, 'yyyyMMdd', new Date()),
                     original: data,
                 },
             ],

src/methods/vidbleAlbum.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 const UrlPattern = require('url-pattern');
 const cheerio = require('cheerio');
 const mime = require('mime-types');

src/methods/vidbleImage.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const $ = require('cheerio');
 const mime = require('mime-types');
 

src/methods/vidbleVideo.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
 async function vidbleVideo(host, post) {

src/reddit/client.js

@@ -0,0 +1,283 @@
+'use strict';
+
+const Bottleneck = require('bottleneck');
+const config = require('config');
+
+const { version } = require('../../package.json');
+const logger = require('../logger')(__filename);
+
+const BASE_URL = 'https://www.reddit.com';
+const MAX_RETRIES = 5;
+
+// deliberately much heavier throttling than the old OAuth limiter (src/limiter.js reddit: 30/min) -
+// this is unauthenticated/public access and getting flagged as a bot defeats the point
+const limiter = new Bottleneck({
+    reservoir: config.reddit.throttle?.reservoir ?? 6,
+    reservoirRefreshAmount: config.reddit.throttle?.reservoir ?? 6,
+    reservoirRefreshInterval: 60000,
+    maxConcurrent: 1,
+    minTime: config.reddit.throttle?.minTime ?? 1500,
+});
+
+// parses a Netscape/Mozilla-format cookie jar dump (domain, includeSubdomains, path,
+// secure, expiry, name, value - tab separated) into a `name=value; name2=value2` header
+function parseCookieJar(raw) {
+    const now = Date.now() / 1000;
+
+    return raw
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith('#'))
+        .map((line) => line.split('\t'))
+        .filter((fields) => fields.length === 7)
+        .filter(([, , , , expiry]) => expiry === '0' || Number(expiry) > now)
+        .filter(([domain]) => domain.replace(/^\./, '').endsWith('reddit.com'))
+        .map(([, , , , , name, value]) => `${name}=${value}`)
+        .join('; ');
+}
+
+function soonestExpiry(raw) {
+    const expiries = raw
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith('#'))
+        .map((line) => line.split('\t'))
+        .filter((fields) => fields.length === 7)
+        .map(([, , , , expiry]) => Number(expiry))
+        .filter((expiry) => expiry > 0);
+
+    return expiries.length ? Math.min(...expiries) : null;
+}
+
+let cachedCookieHeader;
+
+function cookieHeader() {
+    if (cachedCookieHeader === undefined) {
+        cachedCookieHeader = config.reddit.cookies
+            ? parseCookieJar(config.reddit.cookies)
+            : (config.reddit.cookie || null);
+
+        logger.info(cachedCookieHeader
+            ? `Using ${cachedCookieHeader.split(';').length} reddit cookie(s) for authenticated requests`
+            : 'No reddit cookies configured, requesting unauthenticated');
+
+        if (config.reddit.cookies) {
+            const expiry = soonestExpiry(config.reddit.cookies);
+            const hoursLeft = expiry ? (expiry - Date.now() / 1000) / 3600 : null;
+
+            if (hoursLeft !== null && hoursLeft < 48) {
+                logger.warn(`Soonest-expiring reddit cookie expires in ${hoursLeft.toFixed(1)}h (${new Date(expiry * 1000).toISOString()}) - refresh config/cookies.js before then`);
+            }
+        }
+    }
+
+    return cachedCookieHeader;
+}
+
+function headers() {
+    const requestHeaders = {
+        'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
+        accept: 'application/json',
+    };
+
+    const cookie = cookieHeader();
+
+    if (cookie) {
+        requestHeaders.cookie = cookie;
+    }
+
+    return requestHeaders;
+}
+
+function buildUrl(path, params) {
+    const url = new URL(`${BASE_URL}${path}`);
+
+    Object.entries({ raw_json: 1, ...params }).forEach(([key, value]) => {
+        if (value !== undefined && value !== null) {
+            url.searchParams.set(key, value);
+        }
+    });
+
+    return url;
+}
+
+async function wait(seconds) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, seconds * 1000);
+    });
+}
+
+async function requestJson(path, params = {}, attempt = 0) {
+    const url = buildUrl(path, params);
+
+    return limiter.schedule(async () => {
+        const res = await fetch(url.toString(), { headers: headers() });
+        const contentType = res.headers.get('content-type') || '';
+
+        if (res.status === 429 || res.status === 503) {
+            if (attempt >= MAX_RETRIES) {
+                throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
+            }
+
+            const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;
+
+            logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);
+
+            await wait(retryAfter);
+
+            return requestJson(path, params, attempt + 1);
+        }
+
+        if (!contentType.includes('json')) {
+            // reddit serves an HTML interstitial/block page instead of JSON when it decides
+            // the request looks like a bot - this is a distinct failure mode from a genuine
+            // 4xx and should be logged/handled separately rather than treated as "bad request"
+            const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
+
+            throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
+        }
+
+        const data = await res.json();
+
+        if (!res.ok) {
+            const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);
+
+            throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
+        }
+
+        return data;
+    });
+}
+
+async function getUser(username) {
+    const data = await requestJson(`/user/${username}/about.json`);
+
+    return data.data;
+}
+
+async function getSubmissions(username, { sort = 'new', limit = Infinity } = {}) {
+    const submissions = [];
+    let after = null;
+
+    do {
+        // eslint-disable-next-line no-await-in-loop
+        const data = await requestJson(`/user/${username}/submitted.json`, {
+            sort,
+            limit: Math.min(100, Number.isFinite(limit) ? limit - submissions.length : 100),
+            after,
+        });
+
+        const page = data.data.children.map((child) => child.data);
+
+        submissions.push(...page);
+        after = data.data.after;
+    } while (after && submissions.length < limit);
+
+    return submissions;
+}
+
+// reddit's raw comment listing is a tree of {kind, data} "things": kind 't1' is a comment
+// (whose own `replies` field is either '' or a nested Listing of more things), kind 'more'
+// is a stub meaning "N additional comment IDs exist here, fetch them separately".
+function flattenThings(things) {
+    const comments = [];
+    const moreIds = [];
+
+    things.forEach((thing) => {
+        if (thing.kind === 't1') {
+            comments.push(thing.data);
+
+            if (thing.data.replies?.data?.children) {
+                const nested = flattenThings(thing.data.replies.data.children);
+
+                comments.push(...nested.comments);
+                moreIds.push(...nested.moreIds);
+            }
+        } else if (thing.kind === 'more' && thing.data.id !== '_') {
+            moreIds.push(...thing.data.children);
+        }
+    });
+
+    return { comments, moreIds };
+}
+
+async function moreChildren(linkId, ids) {
+    const unique = [...new Set(ids)];
+    const results = [];
+
+    for (let offset = 0; offset < unique.length; offset += 100) {
+        const chunk = unique.slice(offset, offset + 100);
+
+        // eslint-disable-next-line no-await-in-loop
+        const data = await requestJson('/api/morechildren.json', {
+            api_type: 'json',
+            link_id: linkId,
+            children: chunk.join(','),
+            limit_children: false,
+        });
+
+        results.push(...(data.json?.data?.things || []));
+    }
+
+    return results;
+}
+
+// re-fetches 'more' stubs and re-attaches them by parent_id; morechildren can itself return
+// further 'more' stubs on very deep threads, so this recurses with a depth cap rather than
+// looping forever
+async function resolveMore(moreIds, linkId, depth = 0) {
+    if (!moreIds.length) {
+        return [];
+    }
+
+    if (depth >= 4) {
+        logger.warn(`Gave up resolving ${moreIds.length} deeply-nested 'more comments' stubs for '${linkId}' after ${depth} rounds`);
+
+        return [];
+    }
+
+    const more = await moreChildren(linkId, moreIds);
+    const resolvedComments = more.filter((thing) => thing.kind === 't1').map((thing) => thing.data);
+    const nestedMoreIds = more.filter((thing) => thing.kind === 'more').flatMap((thing) => thing.data.children);
+
+    return [...resolvedComments, ...await resolveMore(nestedMoreIds, linkId, depth + 1)];
+}
+
+function buildCommentTree(comments, parentId) {
+    return comments
+        .filter((comment) => comment.parent_id === parentId)
+        .map((comment) => ({
+            ...comment,
+            replies: buildCommentTree(comments, `t1_${comment.id}`),
+        }));
+}
+
+// post metadata only - cheap, no comment-tree resolution (limit: 0 asks reddit to skip
+// returning comments too, but we ignore the second listing element regardless)
+async function getSubmission(id) {
+    const [postListing] = await requestJson(`/comments/${id}.json`, { limit: 0 });
+
+    return postListing.data.children[0].data;
+}
+
+// post + fully-resolved comment tree, incl. 'more comments' stubs - expensive on busy
+// threads (each round of stubs is its own throttled request), only use where the full
+// thread is actually needed
+async function getSubmissionWithComments(id) {
+    const linkId = `t3_${id}`;
+    const [postListing, commentsListing] = await requestJson(`/comments/${id}.json`);
+    const post = postListing.data.children[0].data;
+
+    const { comments, moreIds } = flattenThings(commentsListing.data.children);
+    const resolved = [...comments, ...await resolveMore(moreIds, linkId)];
+
+    return { post, comments: buildCommentTree(resolved, linkId) };
+}
+
+module.exports = {
+    requestJson,
+    getUser,
+    getSubmissions,
+    getSubmission,
+    getSubmissionWithComments,
+};

src/save/mux.js

@@ -18,7 +18,7 @@ async function mux(target, sources) {
 
                 resolve(stdout);
             })
-            .on('error', () => reject)
+            .on('error', (error) => reject(error))
             .save(target);
     }).then(() => Promise.all(sources.map((source) => fs.remove(source))).then(() => {
         logger.verbose(`Cleaned up temporary files for '${target}'`);

src/save/writeToIndex.js

@@ -61,7 +61,7 @@ async function writeToIndex(posts, profilePaths, user, args) {
     }
 
     try {
-        const yamlIndex = yaml.safeDump(data, { skipInvalid: true });
+        const yamlIndex = yaml.dump(data, { skipInvalid: true });
         const saved = await save(filepath, Buffer.from(yamlIndex, 'utf8'));
 
         logger.info(`Saved index with ${posts.length} new posts for ${user.name}`);

src/sources/getIndex.js

@@ -13,7 +13,7 @@ async function getIndex(user) {
     try {
         const indexFile = await fs.readFile(indexFilePath, 'utf8');
 
-        return yaml.safeLoad(indexFile, { json: true }); // allow duplicate keys
+        return yaml.load(indexFile, { json: true }); // allow duplicate keys
     } catch (error) {
         logger.info(`No index file found for '${user.name}' at '${indexFilePath}': ${error.message}`);
 

src/sources/getPosts.js

@@ -6,11 +6,10 @@ const getIndex = require('./getIndex');
 const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').reddit;
 
 async function getUser(username, reddit) {
     try {
-        const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
+        const user = await reddit.getUser(username);
 
         return curateUser(user);
     } catch (error) {
@@ -25,22 +24,22 @@ async function getUser(username, reddit) {
 
 const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) {
     return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => {
-        const post = await limiter.schedule(async () => reddit.getSubmission(postId).fetch());
+        const post = await reddit.getSubmission(postId);
 
         post.direct = true;
 
-        if (accUserPosts[post.author.name]) {
+        if (accUserPosts[post.author]) {
             return {
                 ...accUserPosts,
-                [post.author.name]: {
+                [post.author]: {
-                    ...accUserPosts[post.author.name],
+                    ...accUserPosts[post.author],
-                    posts: [...accUserPosts[post.author.name].posts, post],
+                    posts: [...accUserPosts[post.author].posts, post],
                 },
             };
         }
 
         // don't attempt to fetch deleted user
-        if (post.author.name === '[deleted]') {
+        if (post.author === '[deleted]') {
             return {
                 ...accUserPosts,
                 '[deleted]': {
@@ -56,12 +55,12 @@ const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) {
             };
         }
 
-        const user = await getUser(post.author.name, reddit);
+        const user = await getUser(post.author, reddit);
         const { profile, posts: indexed } = await getIndex(user);
 
         return {
             ...accUserPosts,
-            [post.author.name]: {
+            [post.author]: {
                 ...user,
                 posts: [post],
                 indexed: {

src/sources/getUserPosts.js

@@ -7,11 +7,10 @@ const getArchivePostIds = require('../archives/getArchivePostIds');
 const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').reddit;
 
 async function getUser(username, reddit) {
     try {
-        const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
+        const user = await reddit.getUser(username);
 
         logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
 
@@ -28,12 +27,10 @@ async function getUser(username, reddit) {
 
 async function getPosts(username, reddit, args) {
     try {
-        const submissions = await limiter.schedule(async () => reddit
+        const submissions = await reddit.getSubmissions(username, {
-            .getUser(username)
-            .getSubmissions({
             sort: args.sort,
             limit: Infinity,
-            }));
+        });
 
         logger.info(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);
 
@@ -48,7 +45,7 @@ async function getPosts(username, reddit, args) {
 async function getArchivedPosts(username, posts, reddit) {
     const postIds = await getArchivePostIds(username, posts.map((post) => post.id));
 
-    return Promise.all(postIds.map((postId) => limiter.schedule(async () => reddit.getSubmission(postId).fetch())));
+    return Promise.all(postIds.map((postId) => reddit.getSubmission(postId)));
 }
 
 function getUserPostsWrap(reddit, args) {