Storing social links in profile file, changed format to YAML.

.nvmrc

@@ -1 +1 @@
-20.19.6
+24.19.0

config/default.js

@@ -44,7 +44,8 @@ module.exports = {
             gold: '★',
             over18: '♥',
         },
-        dateFormat: 'YYYYMMDD',
+        // date-fns v2+ token syntax (was 'YYYYMMDD' under date-fns v1)
+        dateFormat: 'yyyyMMdd',
         truncate: {
             limit: 250,
             truncator: '...',
@@ -59,6 +60,13 @@ module.exports = {
         avoidDuplicates: true,
         retries: 3,
         concurrency: 10,
+        // wall-clock ceiling (ms) for a single item-download attempt (src/fetch/item.js).
+        // bhttp has no timeout of its own - a request that connects but then just stops
+        // responding (a stalled/misbehaving proxy tunnel is a common cause) would otherwise
+        // hang forever, and since the limiter's concurrency is capped, that one stuck request
+        // blocks every other download queued behind it indefinitely. This bounds each attempt
+        // so a stall fails (and retries, per `retries` above) instead of hanging forever.
+        timeout: 60000,
         watch: {
             schedule: '*/30 * * * *',
         },
@@ -73,18 +81,35 @@ module.exports = {
         level: 'info',
     },
     limiter: {
+        // default concurrency/interval for item/media downloads (src/fetch/item.js) - applies
+        // to any host without its own entry in `hosts` below
         concurrency: 10,
         interval: 100,
+        hosts: {
+            // per-host overrides of `concurrency`/`interval` above, for hosts that can safely
+            // be hit harder (most media CDNs) or need to be treated more cautiously than the
+            // default. Hostnames are matched by suffix, same as proxy.hosts above - a bare
+            // 'redgifs.com' also covers 'api.redgifs.com' and 'media.redgifs.com'. Unlike a
+            // single shared limiter, each entry here runs independently: a host that's
+            // currently stalling or being throttled doesn't block any other host's downloads.
+            // 'redgifs.com': { concurrency: 20, interval: 50 },
+        },
     },
     reddit: {
-        api: {
+        // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to
+        // get past reddit's anonymous-traffic bot challenge on www.reddit.com. Needs periodic
+        // refreshing - src/reddit/client.js logs a warning once the soonest-expiring cookie
+        // in here is within 48h of expiring.
+        cookies: null,
+        // fallback/simpler alternative to `cookies` above: a single pre-built `name=value; ...`
+        // Cookie header string. Only used if `cookies` is not set.
+        cookie: null,
         userAgent: 'ripunzel',
-            clientId: '1234567abcdefg',
-            access_token: 'abcD123eFg45Hi6J7klmnop8qr9',
-            token_type: 'bearer',
-            expires_in: 3600,
-            refresh_token: '1234567-A-Bc-defg8912hij-klm345opqr',
-            scope: 'history identity mysubreddits read subscribe',
+        throttle: {
+            // deliberately conservative - this is unauthenticated-looking public access,
+            // not an approved API key, getting flagged as a bot defeats the point
+            reservoir: 6,
+            minTime: 1500,
         },
     },
     methods: {
@@ -92,4 +117,26 @@ module.exports = {
             clientId: null,
         },
     },
+    proxy: {
+        use: true,
+        default: null,
+        // if a request fails (including timing out - see fetch.timeout above) and is about to
+        // be retried, retry it using whichever of proxied/direct is NOT what would normally
+        // apply for that host - e.g. the proxy itself is down or being blocked by the target
+        // site, or (the reverse) a host reachable directly is refusing unproxied traffic.
+        // Off by default: for a privacy-sensitive proxy (e.g. Tor) silently falling back to a
+        // direct connection on failure is very much not what you want.
+        retryFlipped: false,
+        hosts: {
+            // hostnames are matched by suffix, so e.g. 'reddit.com' also covers
+            // 'www.reddit.com', but not 'v.redd.it' or 'redd.it' - those need their own
+            // entry, or fall back to `default`. A host entry can be:
+            // - a proxy URL, to override `default` for that host
+            // - `true`, to explicitly use `default` for that host
+            // - `null`/`false`, to explicitly exclude that host from `default`
+            // 'reddit.com': true,
+            // 'v.redd.it': null,
+            // 'imgur.com': 'socks5://127.0.0.1:9050',
+        },
+    },
 };

package.json

@@ -27,41 +27,50 @@
   ],
   "author": "Niels Simenon",
   "license": "ISC",
+  "engines": {
+    "node": ">=24.19.0"
+  },
   "dependencies": {
     "array.prototype.flatten": "^1.2.1",
-    "bhttp": "^1.2.4",
-    "blake2": "^4.1.1",
-    "bluebird": "^3.5.1",
+    "bhttp": "^1.2.8",
+    "blake2": "^5.0.1",
+    "bluebird": "^3.7.2",
     "bottleneck": "^2.19.5",
-    "cheerio": "^1.0.0-rc.2",
-    "config": "^1.30.0",
-    "date-fns": "^1.29.0",
+    "cheerio": "^1.2.0",
+    "config": "^4.4.2",
+    "date-fns": "^4.4.0",
     "dist-exiftool": "^10.53.0",
-    "fluent-ffmpeg": "^2.1.2",
-    "fs-extra": "^5.0.0",
-    "js-yaml": "^3.12.0",
-    "jsdom": "^15.2.0",
+    "fluent-ffmpeg": "^2.1.3",
+    "fs-extra": "^11.4.0",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.6",
+    "js-yaml": "^5.2.3",
     "mime": "^3.0.0",
-    "mime-types": "^2.1.18",
-    "node-cron": "^1.2.1",
+    "mime-types": "^3.0.2",
+    "node-cron": "^4.6.0",
     "node-exiftool": "^2.3.0",
-    "node-fetch": "^2.1.2",
     "object.omit": "^3.0.0",
     "object.pick": "^1.3.0",
-    "sharp-phash": "^2.1.0",
-    "snoowrap": "^1.23.0",
-    "template-format": "^1.2.4",
-    "unprint": "^0.8.2",
+    "sharp-phash": "^2.2.0",
+    "socks-proxy-agent": "^8.0.5",
+    "template-format": "^1.2.5",
+    "undici": "^7.29.0",
+    "unprint": "^0.19.20",
     "url-pattern": "^1.0.3",
-    "winston": "^3.3.3",
-    "winston-daily-rotate-file": "^4.7.1",
-    "yargs": "^11.0.0",
-    "youtube-dl-exec": "^2.2.3"
+    "winston": "^3.19.0",
+    "winston-daily-rotate-file": "^5.0.0",
+    "yargs": "^17.7.2",
+    "youtube-dl-exec": "^3.1.12"
   },
   "devDependencies": {
-    "@babel/eslint-parser": "^7.19.1",
-    "eslint": "^8.34.0",
+    "@babel/eslint-parser": "^7.29.7",
+    "eslint": "^8.57.1",
     "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-plugin-import": "^2.27.5"
+    "eslint-plugin-import": "^2.32.0"
+  },
+  "allowScripts": {
+    "blake2@5.0.1": true,
+    "dist-exiftool@10.53.0": true,
+    "youtube-dl-exec@3.1.12": true
   }
 }

src/app.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const Snoowrap = require('snoowrap');
 const fs = require('fs-extra');
 const Promise = require('bluebird');
 const exiftool = require('node-exiftool');
@@ -10,7 +9,7 @@ const cron = require('node-cron');
 
 require('array.prototype.flatten').shim();
 
-const reddit = new Snoowrap(config.reddit.api);
+const reddit = require('./reddit/client');
 const args = require('./cli')();
 const logger = require('./logger')(__filename);
 
@@ -141,8 +140,10 @@ function fetchSavePosts(userPosts, ep) {
 }
 
 async function initApp() {
+    let ep;
+
     try {
-        const ep = new exiftool.ExiftoolProcess(exiftoolBin);
+        ep = new exiftool.ExiftoolProcess(exiftoolBin);
         await ep.open();
 
         if (args.fetch || args.fileDirect) {
@@ -155,8 +156,6 @@ async function initApp() {
             await fetchSavePosts(userPosts, ep);
         }
 
-        await ep.close();
-
         if (args.watch) {
             logger.info(`Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`);
         }
@@ -166,6 +165,13 @@ async function initApp() {
         } else {
             logger.error(error.message);
         }
+    } finally {
+        // the exiftool child process keeps the event loop alive - if it's never closed
+        // (e.g. because an error above was thrown before we got to it), the process hangs
+        // forever after logging the error instead of actually exiting
+        if (ep) {
+            await ep.close().catch((error) => logger.error(`Could not close exiftool process: ${error.message}`));
+        }
     }
 }
 

src/archives/ip.js

@@ -1,8 +1,9 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const $ = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 function findOnIp(username, page = 1, acc = []) {
     return Promise.resolve().then(() => {
         return fetch(`https://www.imageporn.net/user/${username}/all/${page}`);

src/cli.js

@@ -124,6 +124,11 @@ function getArgs() {
             type: 'boolean',
             default: config.fetch.archives.search,
         })
+        .option('debug', {
+            describe: 'Log full stack traces instead of just error messages',
+            type: 'boolean',
+            default: false,
+        })
         .argv;
 
     return {

src/curate/posts.js

@@ -57,12 +57,19 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
         permalink,
         url: post.url,
         datetime: new Date(post.created_utc * 1000),
-        subreddit: post.subreddit.display_name,
+        subreddit: post.subreddit,
         score: post.score,
         preview: post.preview ? post.preview.images.map((image) => image.source) : null,
         host,
         direct: post.direct,
-        comments: post.comments,
+        // reddit's submission listing (src/reddit/client.js's getSubmissions, fetched once up
+        // front) already embeds these for gallery posts - carrying them through lets
+        // methods/redditAlbum.js use them directly instead of re-fetching the same post
+        // individually through reddit's own separately-throttled client (see there)
+        mediaMetadata: post.media_metadata || null,
+        galleryData: post.gallery_data || null,
+        // count only - the full thread (when needed) is fetched separately via methods/self.js
+        comments: post.num_comments,
         hash: hashPost(post),
     };
 

src/curate/user.js

@@ -1,6 +1,6 @@
 'use strict';
 
-function curateUser(user) {
+function curateUser(user, socialLinks = []) {
     const curatedUser = {
         id: user.id,
         name: user.name,
@@ -9,23 +9,26 @@ function curateUser(user) {
         gold: user.is_gold,
         verified: user.verified,
         verifiedEmail: user.has_verified_email,
-        fallback: false
+        fallback: false,
     };
 
-    if(user.subreddit) {
+    if (user.subreddit) {
         Object.assign(curatedUser, {
             profile: {
-                id: user.subreddit.display_name.name,
-                title: user.subreddit.display_name.title,
-                image: user.subreddit.display_name.icon_img,
-                banner: user.subreddit.display_name.banner_img,
-                description: user.subreddit.display_name.public_description,
-                over18: user.subreddit.display_name.over_18
-            }
+                id: user.subreddit.display_name,
+                title: user.subreddit.title,
+                image: user.subreddit.icon_img,
+                banner: user.subreddit.banner_img,
+                description: user.subreddit.public_description,
+                over18: user.subreddit.over_18,
+                // the "Social Links" widget on their profile (Twitter/Instagram/OnlyFans/etc.) -
+                // see reddit/client.js's getSocialLinks()
+                links: socialLinks,
+            },
         });
     }
 
     return curatedUser;
-};
+}
 
 module.exports = curateUser;

src/fetch/content.js

@@ -46,7 +46,7 @@ function selfPostToText(item, post) {
         comments: curateComments(item.comments),
     };
 
-    return yaml.safeDump(curatedPost);
+    return yaml.dump(curatedPost);
 }
 
 async function getBuffers(item, context) {
@@ -117,6 +117,8 @@ async function fetchSaveUserContent(user, ep, args) {
 
         const items = await Promise.map(post.content.items, async (originalItem, index) => {
             const item = { ...originalItem, index };
+
+            try {
                 const buffers = await getBuffers(item, { post, host: post.host, headers: post.content.headers || item.headers });
 
                 // no buffers, ignore item
@@ -149,6 +151,17 @@ async function fetchSaveUserContent(user, ep, args) {
                 await addMeta(filepath, item, post, user, ep);
 
                 return item;
+            } catch (error) {
+                // don't let one bad item (e.g. a post with a missing/invalid date) abort the
+                // entire run - log it with enough context to find it again and move on
+                logger.error(`Failed to fetch/save item ${index} for post '${post.id}' (${post.permalink}): ${error.message}`);
+
+                if (args.debug) {
+                    logger.error(error.stack);
+                }
+
+                return null;
+            }
         });
 
         return {

src/fetch/item.js

@@ -6,8 +6,9 @@ const blake2 = require('blake2');
 const phash = require('sharp-phash');
 // const phashDistance = require('sharp-phash/distance');
 
+const { agentFor } = require('../proxy');
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').items;
+const limiterFor = require('../limiter');
 
 async function fetchItem(url, attempt, context) {
     async function retry(error) {
@@ -22,7 +23,25 @@ async function fetchItem(url, attempt, context) {
 
     try {
         // throw new Error('Failed it!');
-        const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers }));
+        // config.proxy.retryFlipped: once this url has already failed once, retry it via
+        // whichever of proxied/direct is NOT what it'd normally get - see resolveProxyUrl()
+        const flip = config.proxy.retryFlipped && attempt > 0;
+
+        const res = await limiterFor(url).schedule(() => bhttp.get(url, {
+            headers: context.headers,
+            agent: agentFor(url, { flip }),
+            // bhttp has no timeout at all by default - a server (or, more likely, a proxy
+            // tunnel) that connects fine but then just stops responding would otherwise hang
+            // forever. This isn't just a wall-clock ceiling on the *promise* (an external
+            // Promise.race achieves that, but leaves the real connection dangling) - bhttp
+            // calls req.abort() itself when this fires, actually releasing the socket. Without
+            // that, every stalled attempt leaks one connection onto the shared agent for that
+            // host; enough of those piling up (e.g. after several retries against a host that's
+            // currently black-holing everything, as seen with i.redd.it) exhausts it, and every
+            // *future* request to that host silently queues forever waiting for a socket that's
+            // never coming back - a freeze with no further log lines, not just a slow retry.
+            responseTimeout: config.fetch.timeout,
+        }));
 
         if (res.statusCode !== 200) {
             throw new Error(`Response not OK for ${url} (${res.statusCode})`);

src/interpolate.js

@@ -9,6 +9,35 @@ const format = require('template-format');
 
 const args = require('./cli')();
 
+// date-fns throws a bare, contextless "Invalid time value" RangeError for any invalid date
+// passed to format() - wrap it so a bad timestamp somewhere upstream (a post, item, user or
+// album with a malformed created/createDate field) points back at its source instead.
+// No date at all (e.g. profileDetails.js's item-less-of-a-datetime avatar interpolation) is a
+// legitimate case, not a bug - that just interpolates to an empty string, same as any other
+// unset field like title/description elsewhere in this function.
+function safeFormat(date, dateFormat, label) {
+    if (date === null || date === undefined) {
+        return null;
+    }
+
+    if (!(date instanceof Date) || Number.isNaN(date.getTime())) {
+        throw new Error(`Cannot format ${label} date for filename - invalid datetime (${date})`);
+    }
+
+    return dateFns.format(date, dateFormat);
+}
+
+// mime.extension() returns the boolean `false` (not undefined/null) for a mime type it
+// doesn't recognize - reddit's gallery API is a known offender, sending the non-standard
+// 'image/jpg' instead of 'image/jpeg' - which without this check ends up interpolated
+// verbatim as the literal filename extension '.false'. Fall back to whatever extension the
+// URL itself has instead, same as when there's no type at all.
+function resolveExt(item) {
+    const extFromType = item.type && mime.extension(item.type);
+
+    return extFromType ? `.${extFromType}` : path.extname(url.parse(item.url).pathname);
+}
+
 function interpolate(pattern, item = null, content = null, host = null, post = null, user = null, strip = true, dateFormat = config.library.dateFormat) {
     const data = {
         tags: {},
@@ -20,12 +49,10 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: item.id,
                 title: item.title && item.title.slice(0, config.library.titleLength),
                 description: item.description,
-                date: dateFns.format(item.datetime, dateFormat),
+                date: safeFormat(item.datetime, dateFormat, 'item'),
                 index: item.index + config.library.indexOffset,
             },
-            ext: item.type
-                ? `.${mime.extension(item.type)}`
-                : path.extname(url.parse(item.url).pathname),
+            ext: resolveExt(item),
             tags: {
                 ...data.tags,
                 extracted: item.extracted && config.library.tags.extracted,
@@ -49,7 +76,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: content.album.id,
                 title: content.album.title && content.album.title.slice(0, config.library.titleLength),
                 description: content.album.description,
-                date: dateFns.format(content.album.datetime, dateFormat),
+                date: safeFormat(content.album.datetime, dateFormat, 'album'),
             },
         });
     }
@@ -60,7 +87,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: post.id,
                 title: post.title && post.title.slice(0, config.library.titleLength),
                 url: post.url,
-                date: dateFns.format(post.datetime, dateFormat),
+                date: safeFormat(post.datetime, dateFormat, 'post'),
                 index: post.index + config.library.indexOffset,
                 hash: post.hash,
                 score: post.score,
@@ -75,7 +102,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 name: user.name,
                 username: user.name,
                 id: user.id,
-                created: dateFns.format(user.created, dateFormat),
+                created: safeFormat(user.created, dateFormat, 'user'),
             },
             tags: {
                 ...data.tags,

src/limiter.js

@@ -1,19 +1,50 @@
 'use strict';
 
 const Bottleneck = require('bottleneck');
+const config = require('config');
 
 const args = require('./cli')();
 
-module.exports = {
-    reddit: new Bottleneck({
-        reservoir: 30,
-        reservoirRefreshAmount: 30,
-        reservoirRefreshInterval: 60000,
-        maxConcurrent: 1,
-        minTime: 100,
-    }),
-    items: new Bottleneck({
+// same suffix matching as config.proxy.hosts (src/proxy.js): a bare hostname also matches its
+// subdomains, so 'redgifs.com' catches 'media.redgifs.com' but not the reverse
+function hostMatches(hostname, pattern) {
+    const normalized = pattern.replace(/^\./, '').toLowerCase();
+
+    return hostname === normalized || hostname.endsWith(`.${normalized}`);
+}
+
+// reddit's own API requests are throttled separately, inside src/reddit/client.js
+// (config.reddit.throttle) - this only governs item/media downloads (src/fetch/item.js)
+const defaultLimiter = new Bottleneck({
     maxConcurrent: args.concurrency,
     minTime: args.interval,
-    }),
-};
+});
+
+const hostLimiters = new Map();
+
+// config.limiter.hosts: most media CDNs (redgifs, imgur, i.redd.it, ...) don't need to be
+// treated as cautiously as reddit's own API, but a single shared limiter forces every host
+// down to whatever pace the most conservative one requires - worse, it means one host that's
+// currently stalling/throttling (see src/fetch/item.js's timeout handling) occupies the one
+// shared slot and blocks every *other* host's downloads behind it too, even though they have
+// nothing to do with each other. Giving a host its own limiter here lets it run at its own
+// concurrency/interval, independently of every other host's.
+function limiterFor(url) {
+    const { hostname } = new URL(url);
+    const hosts = config.limiter?.hosts || {};
+    const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
+
+    if (!matchedPattern) {
+        return defaultLimiter;
+    }
+
+    if (!hostLimiters.has(matchedPattern)) {
+        const { concurrency = args.concurrency, interval = args.interval } = hosts[matchedPattern];
+
+        hostLimiters.set(matchedPattern, new Bottleneck({ maxConcurrent: concurrency, minTime: interval }));
+    }
+
+    return hostLimiters.get(matchedPattern);
+}
+
+module.exports = limiterFor;

src/methods/erome.js

@@ -1,9 +1,9 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 
 const base = 'https://www.erome.com/';

src/methods/eroshareAlbum.js

@@ -1,7 +1,8 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
+
+const { fetch } = require('../proxy');
 
 async function eroshareAlbum(host) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);

src/methods/eroshareItem.js

@@ -1,8 +1,9 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 async function eroshareItem(host, post) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${host.id}`);
     if (!res.ok) {

src/methods/gfycat.js

@@ -1,15 +1,19 @@
 'use strict';
 
 const bhttp = require('bhttp');
+
+const { agentFor } = require('../proxy');
 const redgifs = require('./redgifs');
 
 async function gfycat(host) {
-    const res = await bhttp.get(`https://api.gfycat.com/v1/gfycats/${host.id}`);
+    const infoUrl = `https://api.gfycat.com/v1/gfycats/${host.id}`;
+    const res = await bhttp.get(infoUrl, { agent: agentFor(infoUrl) });
     const data = await res.body;
 
     if (data.errorMessage) {
         const redirectRes = await bhttp.head(host.url, {
             followRedirects: false,
+            agent: agentFor(host.url),
         });
 
         if (redirectRes.statusCode === 301) {

src/methods/imgurAlbum.js

@@ -1,8 +1,8 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 const { fetchPredata } = require('./imgurImage');
 

src/methods/imgurImage.js

@@ -1,7 +1,8 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
+
+const { fetch } = require('../proxy');
 
 async function fetchPredata() {
     const data = {

src/methods/redditAlbum.js

@@ -1,18 +1,57 @@
 'use strict';
 
-const mime = require('mime');
-const bhttp = require('bhttp');
-const { JSDOM } = require('jsdom');
+// reddit gallery images: `s.u` (still) or `s.gif`/`s.mp4` (animated) - URLs come back
+// HTML-entity-encoded (e.g. '&' instead of '&')
+function resolveMediaUrl(media) {
+    const url = media?.s?.u || media?.s?.gif || media?.s?.mp4;
 
-async function redditAlbum(host, post) {
-    const res = await bhttp.get(host.url);
+    return url ? url.replace(/&/g, '&') : null;
+}
 
-    if (res.statusCode !== 200) {
-        throw new Error(res.body.toString());
-    }
+// reddit sends the non-standard 'image/jpg' instead of the real IANA 'image/jpeg' for
+// gallery items - left as-is, this silently breaks anything matching against real mime
+// types downstream (interpolate.js's extension lookup ends up with a '.false' extension,
+// fetch/content.js's addMeta() skips embedding metadata since it only matches 'image/jpeg')
+function normalizeType(type) {
+    return type === 'image/jpg' ? 'image/jpeg' : type;
+}
 
-    const { document } = new JSDOM(res.body.toString(), { runScripts: 'dangerously' }).window;
-    const items = Array.from(document.querySelectorAll('li a'), el => el.href);
+async function redditAlbum(host, post, { reddit }) {
+    // reddit's submission listing already embeds media_metadata/gallery_data for gallery posts,
+    // and curate/posts.js carries them through as post.mediaMetadata/post.galleryData - reuse
+    // those instead of re-fetching the same post individually through reddit's own client
+    // (reservoir: 6/min, maxConcurrent: 1 - see src/reddit/client.js), which would otherwise cost
+    // one full throttled round trip per gallery post. Only fall back to a fresh fetch if the
+    // listing didn't have it for some reason (e.g. gallery_data null on the listing too).
+    const data = post.mediaMetadata
+        ? { media_metadata: post.mediaMetadata, gallery_data: post.galleryData }
+        : await reddit.getSubmission(post.id);
+    const mediaMetadata = data.media_metadata || {};
+
+    // gallery_data (the ordered {media_id, is_deleted, ...} list for this gallery) has been
+    // seen coming back null on a freshly-posted gallery even though media_metadata itself
+    // was already fully populated - fall back to media_metadata's own keys, in the order the
+    // JSON provided them (reddit sends them in gallery order, and object key order is
+    // preserved through JSON.parse), skipping anything not (yet, or any longer) valid
+    const galleryItems = data.gallery_data?.items
+        || Object.entries(mediaMetadata)
+            .filter(([, media]) => media.status === 'valid')
+            .map(([mediaId]) => ({ media_id: mediaId }));
+
+    const items = galleryItems
+        .filter((item) => !item.is_deleted)
+        .map((item) => {
+            const media = mediaMetadata[item.media_id];
+            const url = resolveMediaUrl(media);
+
+            return url && {
+                id: item.media_id,
+                url,
+                datetime: post.datetime,
+                type: normalizeType(media.m) || 'image/jpeg',
+            };
+        })
+        .filter(Boolean);
 
     return {
         album: {
@@ -20,12 +59,7 @@ async function redditAlbum(host, post) {
             url: host.url,
             title: post.title,
         },
-        items: items.map((url) => ({
-            id: new URL(url).pathname.match(/\/(.*).\w+$/)?.[1],
-            url,
-            datetime: post.datetime,
-            type: mime.getType(url) || 'image/jpeg',
-        })),
+        items,
     };
 }
 

src/methods/redditVideo.js

@@ -1,23 +1,50 @@
 'use strict';
 
-const fetch = require('node-fetch');
+const { fetch } = require('../proxy');
 
-async function redditVideo(host, post) {
-    const res = await fetch(`${post.permalink}.json`);
-    const [{ data }] = await res.json();
+// the DASH manifest lists the actual audio track filename - reddit has changed this naming
+// scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the
+// manifest rather than guessing a hardcoded suffix off the video URL again
+function findAudioUrl(manifest, videoId) {
+    const audioSection = manifest.match(/<AdaptationSet[^>]*contentType="audio"[^>]*>([\s\S]*?)<\/AdaptationSet>/);
 
-    const videoUrl = data.children[0].data.media.reddit_video.fallback_url;
-    const audioUrl = `${videoUrl.split('/').slice(0, -1).join('/')}/audio`;
+    if (!audioSection) {
+        return null;
+    }
 
-    const audioRes = await fetch(audioUrl, {
-        method: 'HEAD',
-    });
+    const baseUrls = [...audioSection[1].matchAll(/<BaseURL>([^<]+)<\/BaseURL>/g)].map((match) => match[1]);
+    // representations are listed lowest to highest bandwidth - take the best one
+    const audioFile = baseUrls[baseUrls.length - 1];
+
+    return audioFile ? `https://v.redd.it/${videoId}/${audioFile}` : null;
+}
+
+async function findAudio(video, videoId) {
+    if (!video.has_audio || !video.dash_url) {
+        return null;
+    }
+
+    const res = await fetch(video.dash_url);
+
+    if (!res.ok) {
+        return null;
+    }
+
+    return findAudioUrl(await res.text(), videoId);
+}
+
+async function redditVideo(host, post, { reddit }) {
+    // fetched through the authenticated/throttled client, not a raw fetch() - reddit's
+    // anonymous-traffic bot challenge blocks plain unauthenticated .json requests
+    const data = await reddit.getSubmission(post.id);
+    const video = data.media.reddit_video;
+    const videoId = post.host.id || post.id;
 
     const item = {
         album: null,
         items: [{
-            id: post.host.id || post.id,
-            url: videoUrl,
+            id: videoId,
+            url: video.fallback_url,
             title: post.title,
             datetime: post.datetime,
             type: 'video/mp4',
@@ -25,7 +52,9 @@ async function redditVideo(host, post) {
         }],
     };
 
-    if (audioRes.status === 200) {
+    const audioUrl = await findAudio(video, videoId);
+
+    if (audioUrl) {
         item.items[0].mux = [audioUrl];
     }
 

src/methods/redgifs.js

@@ -1,9 +1,9 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const mime = require('mime');
 // const unprint = require('unprint');
 
+const { fetch } = require('../proxy');
 const { version } = require('../../package.json');
 
 async function fetchPredata() {
@@ -28,6 +28,11 @@ async function fetchPredata() {
 }
 
 function scrapeGallery(data, { predata }) {
+    if (!data.gifs.length) {
+        // Math.min() of an empty array is Infinity, which becomes an invalid date below
+        return null;
+    }
+
     const oldestDate = Math.min(...data.gifs.map((gif) => gif.createDate));
 
     const curated = {

src/methods/self.js

@@ -4,11 +4,11 @@ function curateComments(comments) {
     return comments.map((comment) => ({
         id: comment.id,
         url: `https://reddit.com${comment.permalink}`,
-        author: comment.author.name,
+        author: comment.author,
         body: comment.body,
         html: comment.body_html,
         score: comment.score,
-        datetime: new Date(comment.created * 1000),
+        datetime: new Date(comment.created_utc * 1000),
         edited: comment.edited,
         controversiality: comment.controversiality,
         gilded: comment.gilded,
@@ -22,18 +22,12 @@ function curateComments(comments) {
 }
 
 async function getFullPost(postId, reddit) {
-    return reddit
-        .getSubmission(postId)
-        .expandReplies({
-            limit: Infinity,
-            depth: Infinity,
-        })
-        .fetch();
+    return reddit.getSubmissionWithComments(postId);
 }
 
 async function self(host, originalPost, { reddit }) {
-    const post = await getFullPost(originalPost.id, reddit) || originalPost;
-    const curatedComments = curateComments(post.comments);
+    const { post, comments } = await getFullPost(originalPost.id, reddit);
+    const curatedComments = curateComments(comments);
 
     return {
         album: null,
@@ -41,9 +35,9 @@ async function self(host, originalPost, { reddit }) {
             id: post.id,
             url: post.url,
             title: post.title,
-            text: post.text,
-            author: post.author.name,
-            datetime: post.datetime,
+            text: post.selftext,
+            author: post.author,
+            datetime: new Date(post.created_utc * 1000),
             comments: curatedComments,
             type: 'text/plain',
             self: true,

src/methods/tube.js

@@ -25,7 +25,7 @@ async function tube(host, post) {
                     title: data.fulltitle || data.title,
                     description: data.description,
                     type: `video/${data.ext}`,
-                    datetime: dateFns.parse(data.upload_date, 'YYYYMMDD'),
+                    datetime: dateFns.parse(data.upload_date, 'yyyyMMdd', new Date()),
                     original: data,
                 },
             ],

src/methods/vidbleAlbum.js

@@ -1,11 +1,11 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 const UrlPattern = require('url-pattern');
 const cheerio = require('cheerio');
 const mime = require('mime-types');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 
 const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');

src/methods/vidbleImage.js

@@ -1,9 +1,10 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const $ = require('cheerio');
 const mime = require('mime-types');
 
+const { fetch } = require('../proxy');
+
 async function vidbleImage(host, post) {
     const res = await fetch(`https://vidble.com/${host.id}`);
 

src/methods/vidbleVideo.js

@@ -1,8 +1,9 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 async function vidbleVideo(host, post) {
     const res = await fetch(`https://www.vidble.com/watch?v=${host.id}`);
 

src/proxy.js

@@ -0,0 +1,146 @@
+'use strict';
+
+const config = require('config');
+const { ProxyAgent } = require('undici');
+const { HttpProxyAgent } = require('http-proxy-agent');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { SocksProxyAgent } = require('socks-proxy-agent');
+
+const logger = require('./logger')(__filename);
+
+const dispatcherCache = new Map();
+const nodeAgentCache = new Map();
+const warnedNoSocksForFetch = new Set();
+
+// same suffix matching as config.reddit.cookies' domain filter (src/reddit/client.js): a
+// bare hostname also matches its subdomains, so 'reddit.com' catches 'www.reddit.com' but
+// not 'v.redd.it' or 'redd.it' - those need their own entry, or fall back to `default`
+function hostMatches(hostname, pattern) {
+    const normalized = pattern.replace(/^\./, '').toLowerCase();
+
+    return hostname === normalized || hostname.endsWith(`.${normalized}`);
+}
+
+function resolveProxyUrl(hostname, { flip = false } = {}) {
+    if (config.proxy?.use === false) {
+        return null;
+    }
+
+    const hosts = config.proxy?.hosts || {};
+    const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
+
+    const normal = (() => {
+        if (!matchedPattern) {
+            return config.proxy?.default || null;
+        }
+
+        const matched = hosts[matchedPattern];
+
+        // `true` explicitly opts this host into `default`, `null`/`false`/'' explicitly opts it
+        // out (rather than falling through to `default`), anything else is a per-host override
+        if (matched === true) {
+            return config.proxy?.default || null;
+        }
+
+        return matched || null;
+    })();
+
+    if (!flip) {
+        return normal;
+    }
+
+    // used for config.proxy.retryFlipped: retry via whichever of proxied/direct is NOT what
+    // this host would normally get. A request that would normally be proxied (whether via
+    // `default` or a per-host override) retries direct; a request that would normally go
+    // direct (no proxy configured for this host, or explicitly excluded from `default`)
+    // retries through `default`, if one is set.
+    return normal ? null : (config.proxy?.default || null);
+}
+
+// 'socks4a'/'socks5h' (as opposed to plain 'socks4'/'socks5') tell socks-proxy-agent to
+// resolve the target hostname on the proxy side rather than locally - the same convention
+// curl uses, and worth keeping since it's the whole point of routing through SOCKS/Tor for
+// most people
+function isSocks(proxyUrl) {
+    return /^socks(4a?|5h?):/i.test(proxyUrl);
+}
+
+// undici (which node's built-in fetch() is implemented with) only tunnels through http(s)
+// proxies (CONNECT) via ProxyAgent - it has no built-in SOCKS support. agentFor() below can
+// still do SOCKS, since bhttp goes through node's plain http/https modules instead.
+function dispatcherFor(targetUrl, { flip = false } = {}) {
+    const { hostname } = new URL(targetUrl);
+    const proxyUrl = resolveProxyUrl(hostname, { flip });
+
+    if (!proxyUrl) {
+        return undefined;
+    }
+
+    if (isSocks(proxyUrl)) {
+        if (!warnedNoSocksForFetch.has(proxyUrl)) {
+            warnedNoSocksForFetch.add(proxyUrl);
+            logger.warn(`SOCKS proxy '${proxyUrl}' configured for '${hostname}' can't be used for fetch()-based requests (only http(s) proxies can) - these will go out directly instead. Item downloads (which use bhttp) can still use it.`);
+        }
+
+        return undefined;
+    }
+
+    if (!dispatcherCache.has(proxyUrl)) {
+        dispatcherCache.set(proxyUrl, new ProxyAgent(proxyUrl));
+    }
+
+    return dispatcherCache.get(proxyUrl);
+}
+
+// bhttp uses node's raw http/https modules under the hood, which take a plain Agent
+// instance - unlike fetch() above, this does support SOCKS
+function agentFor(targetUrl, { flip = false } = {}) {
+    const url = new URL(targetUrl);
+    const proxyUrl = resolveProxyUrl(url.hostname, { flip });
+
+    if (!proxyUrl) {
+        return undefined;
+    }
+
+    const cacheKey = `${proxyUrl}|${isSocks(proxyUrl) ? 'socks' : url.protocol}`;
+
+    if (!nodeAgentCache.has(cacheKey)) {
+        if (isSocks(proxyUrl)) {
+            nodeAgentCache.set(cacheKey, new SocksProxyAgent(proxyUrl));
+        } else if (url.protocol === 'https:') {
+            nodeAgentCache.set(cacheKey, new HttpsProxyAgent(proxyUrl));
+        } else {
+            nodeAgentCache.set(cacheKey, new HttpProxyAgent(proxyUrl));
+        }
+    }
+
+    return nodeAgentCache.get(cacheKey);
+}
+
+// drop-in replacement for the global fetch() - importing this as `fetch` transparently
+// proxies requests per config.proxy, if one is configured for the request's host. Also
+// applies a hard wall-clock ceiling (config.fetch.timeout) to every request: unlike bhttp,
+// fetch() doesn't hang *forever* on a stalled/black-holed connection (undici defaults to a
+// 300s headers/body timeout), but that default is high enough - and, per live testing, a
+// proxy CONNECT tunnel that never completes can still take well over 20s to surface any
+// error - to look like a freeze, especially since reddit/client.js's requestJson() serializes
+// all reddit API calls behind a single-concurrency limiter. Bounding it here covers every
+// fetch()-based call site (reddit's API, and every hosting site's info-lookup) at once.
+// Pass `proxyFlip: true` in options (stripped before reaching fetch()) to use whichever of
+// proxied/direct is NOT what this host would normally get - see resolveProxyUrl().
+async function proxiedFetch(url, options = {}) {
+    const { proxyFlip, signal, ...fetchOptions } = options;
+    const dispatcher = dispatcherFor(String(url), { flip: proxyFlip });
+    const timeoutSignal = AbortSignal.timeout(config.fetch.timeout);
+
+    return fetch(url, {
+        ...fetchOptions,
+        ...(dispatcher ? { dispatcher } : {}),
+        signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal,
+    });
+}
+
+module.exports = {
+    fetch: proxiedFetch,
+    agentFor,
+};

src/reddit/client.js

@@ -0,0 +1,373 @@
+'use strict';
+
+const Bottleneck = require('bottleneck');
+const cheerio = require('cheerio');
+const config = require('config');
+
+const { fetch } = require('../proxy');
+const { version } = require('../../package.json');
+const logger = require('../logger')(__filename);
+
+const BASE_URL = 'https://www.reddit.com';
+const MAX_RETRIES = 5;
+
+// deliberately much heavier throttling than the old OAuth limiter (src/limiter.js reddit: 30/min) -
+// this is unauthenticated/public access and getting flagged as a bot defeats the point
+const limiter = new Bottleneck({
+    reservoir: config.reddit.throttle?.reservoir ?? 6,
+    reservoirRefreshAmount: config.reddit.throttle?.reservoir ?? 6,
+    reservoirRefreshInterval: 60000,
+    maxConcurrent: 1,
+    minTime: config.reddit.throttle?.minTime ?? 1500,
+});
+
+// parses a Netscape/Mozilla-format cookie jar dump (domain, includeSubdomains, path,
+// secure, expiry, name, value - tab separated) into a `name=value; name2=value2` header
+function parseCookieJar(raw) {
+    const now = Date.now() / 1000;
+
+    return raw
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith('#'))
+        .map((line) => line.split('\t'))
+        .filter((fields) => fields.length === 7)
+        .filter(([, , , , expiry]) => expiry === '0' || Number(expiry) > now)
+        .filter(([domain]) => domain.replace(/^\./, '').endsWith('reddit.com'))
+        .map(([, , , , , name, value]) => `${name}=${value}`)
+        .join('; ');
+}
+
+function soonestExpiry(raw) {
+    const expiries = raw
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith('#'))
+        .map((line) => line.split('\t'))
+        .filter((fields) => fields.length === 7)
+        .map(([, , , , expiry]) => Number(expiry))
+        .filter((expiry) => expiry > 0);
+
+    return expiries.length ? Math.min(...expiries) : null;
+}
+
+let cachedCookieHeader;
+
+function cookieHeader() {
+    if (cachedCookieHeader === undefined) {
+        cachedCookieHeader = config.reddit.cookies
+            ? parseCookieJar(config.reddit.cookies)
+            : (config.reddit.cookie || null);
+
+        logger.info(cachedCookieHeader
+            ? `Using ${cachedCookieHeader.split(';').length} reddit cookie(s) for authenticated requests`
+            : 'No reddit cookies configured, requesting unauthenticated');
+
+        if (config.reddit.cookies) {
+            const expiry = soonestExpiry(config.reddit.cookies);
+            const hoursLeft = expiry ? (expiry - Date.now() / 1000) / 3600 : null;
+
+            if (hoursLeft !== null && hoursLeft < 48) {
+                logger.warn(`Soonest-expiring reddit cookie expires in ${hoursLeft.toFixed(1)}h (${new Date(expiry * 1000).toISOString()}) - refresh config/cookies.js before then`);
+            }
+        }
+    }
+
+    return cachedCookieHeader;
+}
+
+function headers(accept = 'application/json') {
+    const requestHeaders = {
+        'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
+        accept,
+    };
+
+    const cookie = cookieHeader();
+
+    if (cookie) {
+        requestHeaders.cookie = cookie;
+    }
+
+    return requestHeaders;
+}
+
+function buildUrl(path, params) {
+    const url = new URL(`${BASE_URL}${path}`);
+
+    Object.entries({ raw_json: 1, ...params }).forEach(([key, value]) => {
+        if (value !== undefined && value !== null) {
+            url.searchParams.set(key, value);
+        }
+    });
+
+    return url;
+}
+
+async function wait(seconds) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, seconds * 1000);
+    });
+}
+
+// shared by requestJson() and requestHtml() below - handles the actual throttled network
+// call plus retrying on a network error or a 429/503, regardless of what kind of response
+// the caller ultimately wants back.
+//
+// Only the actual throttled network call belongs inside limiter.schedule() - this has
+// maxConcurrent: 1, and every retry path below used to recurse back into this function
+// (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't
+// start a second job while the first is still running, and the first can't finish until
+// its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever
+// needed (e.g. a single 429), freezing every reddit request behind it for good. Retrying
+// now happens here, after the scheduled call has actually returned and freed its slot.
+async function fetchWithRetry(path, params, attempt, accept) {
+    const url = buildUrl(path, params);
+    let res;
+
+    try {
+        // config.proxy.retryFlipped: once this request has already failed once, retry it via
+        // whichever of proxied/direct is NOT what it'd normally get - see proxy.js
+        res = await limiter.schedule(async () => fetch(url.toString(), {
+            headers: headers(accept),
+            proxyFlip: config.proxy.retryFlipped && attempt > 0,
+        }));
+    } catch (error) {
+        // fetch() has no timeout of its own beyond undici's fairly generous defaults (see
+        // proxy.js) - this catches both that and genuine network errors (DNS, connection
+        // refused/reset, etc.), neither of which was retried before
+        if (attempt >= MAX_RETRIES) {
+            throw new Error(`Failed to reach reddit after ${attempt} retries for '${path}': ${error.message}`);
+        }
+
+        logger.warn(`Failed to reach reddit for '${path}', retrying: ${error.message}`);
+
+        return fetchWithRetry(path, params, attempt + 1, accept);
+    }
+
+    if (res.status === 429 || res.status === 503) {
+        if (attempt >= MAX_RETRIES) {
+            throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
+        }
+
+        const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;
+
+        logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);
+
+        await wait(retryAfter);
+
+        return fetchWithRetry(path, params, attempt + 1, accept);
+    }
+
+    return res;
+}
+
+async function requestJson(path, params = {}, attempt = 0) {
+    const res = await fetchWithRetry(path, params, attempt, 'application/json');
+    const contentType = res.headers.get('content-type') || '';
+
+    if (!contentType.includes('json')) {
+        // reddit serves an HTML interstitial/block page instead of JSON when it decides
+        // the request looks like a bot - this is a distinct failure mode from a genuine
+        // 4xx and should be logged/handled separately rather than treated as "bad request"
+        const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
+
+        throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
+    }
+
+    const data = await res.json();
+
+    if (!res.ok) {
+        const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);
+
+        throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
+    }
+
+    return data;
+}
+
+// unlike requestJson(), this is for reddit's rendered profile pages - there's no public JSON
+// endpoint for a user's social links (the "Social Links" widget on their profile), they're
+// only present in the page's HTML (see getSocialLinks() below)
+async function requestHtml(path, params = {}, attempt = 0) {
+    const res = await fetchWithRetry(path, params, attempt, 'text/html');
+    const contentType = res.headers.get('content-type') || '';
+
+    if (!contentType.includes('html')) {
+        const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
+
+        throw new Error(`Reddit returned non-HTML response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
+    }
+
+    if (!res.ok) {
+        throw new Error(`Reddit returned ${res.status} for '${path}'`);
+    }
+
+    return res.text();
+}
+
+// reddit's "Social Links" profile widget (the row of Twitter/Instagram/OnlyFans/etc. buttons
+// under a user's bio) has no public JSON endpoint - the structured API for it
+// (/r/u_{username}/api/widgets.json) requires OAuth, which this project deliberately doesn't
+// use (see reddit.throttle above). The rendered profile page embeds the same data anyway, as
+// JSON inside each link's `data-faceplate-tracking-context` attribute, and that page is
+// accessible with the same cookies as everything else here.
+function parseSocialLinks(html) {
+    const $ = cheerio.load(html);
+    const links = [];
+
+    $('[data-faceplate-tracking-context]').each((index, el) => {
+        try {
+            const { social_link: link } = JSON.parse($(el).attr('data-faceplate-tracking-context'));
+
+            if (link) {
+                links.push(link);
+            }
+        } catch (error) {
+            // not a social-link tracker, or a shape we don't recognize - ignore
+        }
+    });
+
+    return links
+        .sort((a, b) => a.position - b.position)
+        .map(({ type, name, url }) => ({ type, name, url }));
+}
+
+async function getSocialLinks(username) {
+    const html = await requestHtml(`/user/${username}/`);
+
+    return parseSocialLinks(html);
+}
+
+async function getUser(username) {
+    const data = await requestJson(`/user/${username}/about.json`);
+
+    return data.data;
+}
+
+async function getSubmissions(username, { sort = 'new', limit = Infinity } = {}) {
+    const submissions = [];
+    let after = null;
+
+    do {
+        // eslint-disable-next-line no-await-in-loop
+        const data = await requestJson(`/user/${username}/submitted.json`, {
+            sort,
+            limit: Math.min(100, Number.isFinite(limit) ? limit - submissions.length : 100),
+            after,
+        });
+
+        const page = data.data.children.map((child) => child.data);
+
+        submissions.push(...page);
+        after = data.data.after;
+    } while (after && submissions.length < limit);
+
+    return submissions;
+}
+
+// reddit's raw comment listing is a tree of {kind, data} "things": kind 't1' is a comment
+// (whose own `replies` field is either '' or a nested Listing of more things), kind 'more'
+// is a stub meaning "N additional comment IDs exist here, fetch them separately".
+function flattenThings(things) {
+    const comments = [];
+    const moreIds = [];
+
+    things.forEach((thing) => {
+        if (thing.kind === 't1') {
+            comments.push(thing.data);
+
+            if (thing.data.replies?.data?.children) {
+                const nested = flattenThings(thing.data.replies.data.children);
+
+                comments.push(...nested.comments);
+                moreIds.push(...nested.moreIds);
+            }
+        } else if (thing.kind === 'more' && thing.data.id !== '_') {
+            moreIds.push(...thing.data.children);
+        }
+    });
+
+    return { comments, moreIds };
+}
+
+async function moreChildren(linkId, ids) {
+    const unique = [...new Set(ids)];
+    const results = [];
+
+    for (let offset = 0; offset < unique.length; offset += 100) {
+        const chunk = unique.slice(offset, offset + 100);
+
+        // eslint-disable-next-line no-await-in-loop
+        const data = await requestJson('/api/morechildren.json', {
+            api_type: 'json',
+            link_id: linkId,
+            children: chunk.join(','),
+            limit_children: false,
+        });
+
+        results.push(...(data.json?.data?.things || []));
+    }
+
+    return results;
+}
+
+// re-fetches 'more' stubs and re-attaches them by parent_id; morechildren can itself return
+// further 'more' stubs on very deep threads, so this recurses with a depth cap rather than
+// looping forever
+async function resolveMore(moreIds, linkId, depth = 0) {
+    if (!moreIds.length) {
+        return [];
+    }
+
+    if (depth >= 4) {
+        logger.warn(`Gave up resolving ${moreIds.length} deeply-nested 'more comments' stubs for '${linkId}' after ${depth} rounds`);
+
+        return [];
+    }
+
+    const more = await moreChildren(linkId, moreIds);
+    const resolvedComments = more.filter((thing) => thing.kind === 't1').map((thing) => thing.data);
+    const nestedMoreIds = more.filter((thing) => thing.kind === 'more').flatMap((thing) => thing.data.children);
+
+    return [...resolvedComments, ...await resolveMore(nestedMoreIds, linkId, depth + 1)];
+}
+
+function buildCommentTree(comments, parentId) {
+    return comments
+        .filter((comment) => comment.parent_id === parentId)
+        .map((comment) => ({
+            ...comment,
+            replies: buildCommentTree(comments, `t1_${comment.id}`),
+        }));
+}
+
+// post metadata only - cheap, no comment-tree resolution (limit: 0 asks reddit to skip
+// returning comments too, but we ignore the second listing element regardless)
+async function getSubmission(id) {
+    const [postListing] = await requestJson(`/comments/${id}.json`, { limit: 0 });
+
+    return postListing.data.children[0].data;
+}
+
+// post + fully-resolved comment tree, incl. 'more comments' stubs - expensive on busy
+// threads (each round of stubs is its own throttled request), only use where the full
+// thread is actually needed
+async function getSubmissionWithComments(id) {
+    const linkId = `t3_${id}`;
+    const [postListing, commentsListing] = await requestJson(`/comments/${id}.json`);
+    const post = postListing.data.children[0].data;
+
+    const { comments, moreIds } = flattenThings(commentsListing.data.children);
+    const resolved = [...comments, ...await resolveMore(moreIds, linkId)];
+
+    return { post, comments: buildCommentTree(resolved, linkId) };
+}
+
+module.exports = {
+    requestJson,
+    getUser,
+    getSocialLinks,
+    getSubmissions,
+    getSubmission,
+    getSubmissionWithComments,
+};

src/save/mux.js

@@ -18,7 +18,7 @@ async function mux(target, sources) {
 
                 resolve(stdout);
             })
-            .on('error', () => reject)
+            .on('error', (error) => reject(error))
             .save(target);
     }).then(() => Promise.all(sources.map((source) => fs.remove(source))).then(() => {
         logger.verbose(`Cleaned up temporary files for '${target}'`);

src/save/profileDetails.js

@@ -3,6 +3,7 @@
 const config = require('config');
 const Promise = require('bluebird');
 const UrlPattern = require('url-pattern');
+const yaml = require('js-yaml');
 
 const interpolate = require('../interpolate');
 const fetchItem = require('../fetch/item');
@@ -62,9 +63,15 @@ async function saveProfileDescription(user, args) {
     }
 
     if (config.library.profile.description && !user.fallback && !user.deleted) {
-        if (user.profile && user.profile.description) {
+        const links = user.profile?.links || [];
+
+        // YAML rather than plain text (like index.js) so the free-form bio and the structured
+        // social links (reddit/client.js's getSocialLinks()) stay separately parseable instead
+        // of just being smushed into one text blob
+        if (user.profile && (user.profile.description || links.length)) {
             const filepath = interpolate(config.library.profile.description, null, null, null, null, user);
-            const stream = Buffer.from(user.profile.description, 'utf8');
+            const data = { description: user.profile.description || null, links };
+            const stream = Buffer.from(yaml.dump(data, { skipInvalid: true }), 'utf8');
 
             try {
                 const targets = await save(filepath, stream);
@@ -77,7 +84,7 @@ async function saveProfileDescription(user, args) {
             }
         }
 
-        logger.verbose(`No profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
+        logger.verbose(`No profile description or social links for '${user.name}' (https://reddit.com/user/${user.name})`);
 
         return null;
     }

src/save/writeToIndex.js

@@ -61,7 +61,7 @@ async function writeToIndex(posts, profilePaths, user, args) {
     }
 
     try {
-        const yamlIndex = yaml.safeDump(data, { skipInvalid: true });
+        const yamlIndex = yaml.dump(data, { skipInvalid: true });
         const saved = await save(filepath, Buffer.from(yamlIndex, 'utf8'));
 
         logger.info(`Saved index with ${posts.length} new posts for ${user.name}`);

src/sources/getIndex.js

@@ -13,7 +13,7 @@ async function getIndex(user) {
     try {
         const indexFile = await fs.readFile(indexFilePath, 'utf8');
 
-        return yaml.safeLoad(indexFile, { json: true }); // allow duplicate keys
+        return yaml.load(indexFile, { json: true }); // allow duplicate keys
     } catch (error) {
         logger.info(`No index file found for '${user.name}' at '${indexFilePath}': ${error.message}`);
 

src/sources/getPosts.js

@@ -6,13 +6,26 @@ const getIndex = require('./getIndex');
 const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').reddit;
+
+async function getSocialLinks(username, reddit) {
+    try {
+        return await reddit.getSocialLinks(username);
+    } catch (error) {
+        // supplementary, not worth failing the whole user fetch over - fall back to none
+        logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
+
+        return [];
+    }
+}
 
 async function getUser(username, reddit) {
     try {
-        const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
+        const [user, socialLinks] = await Promise.all([
+            reddit.getUser(username),
+            getSocialLinks(username, reddit),
+        ]);
 
-        return curateUser(user);
+        return curateUser(user, socialLinks);
     } catch (error) {
         logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
 
@@ -25,22 +38,22 @@ async function getUser(username, reddit) {
 
 const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) {
     return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => {
-        const post = await limiter.schedule(async () => reddit.getSubmission(postId).fetch());
+        const post = await reddit.getSubmission(postId);
 
         post.direct = true;
 
-        if (accUserPosts[post.author.name]) {
+        if (accUserPosts[post.author]) {
             return {
                 ...accUserPosts,
-                [post.author.name]: {
-                    ...accUserPosts[post.author.name],
-                    posts: [...accUserPosts[post.author.name].posts, post],
+                [post.author]: {
+                    ...accUserPosts[post.author],
+                    posts: [...accUserPosts[post.author].posts, post],
                 },
             };
         }
 
         // don't attempt to fetch deleted user
-        if (post.author.name === '[deleted]') {
+        if (post.author === '[deleted]') {
             return {
                 ...accUserPosts,
                 '[deleted]': {
@@ -56,12 +69,12 @@ const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) {
             };
         }
 
-        const user = await getUser(post.author.name, reddit);
+        const user = await getUser(post.author, reddit);
         const { profile, posts: indexed } = await getIndex(user);
 
         return {
             ...accUserPosts,
-            [post.author.name]: {
+            [post.author]: {
                 ...user,
                 posts: [post],
                 indexed: {

src/sources/getUserPosts.js

@@ -7,15 +7,28 @@ const getArchivePostIds = require('../archives/getArchivePostIds');
 const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').reddit;
+
+async function getSocialLinks(username, reddit) {
+    try {
+        return await reddit.getSocialLinks(username);
+    } catch (error) {
+        // supplementary, not worth failing the whole user fetch over - fall back to none
+        logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
+
+        return [];
+    }
+}
 
 async function getUser(username, reddit) {
     try {
-        const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
+        const [user, socialLinks] = await Promise.all([
+            reddit.getUser(username),
+            getSocialLinks(username, reddit),
+        ]);
 
         logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
 
-        return curateUser(user);
+        return curateUser(user, socialLinks);
     } catch (error) {
         logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
 
@@ -28,12 +41,10 @@ async function getUser(username, reddit) {
 
 async function getPosts(username, reddit, args) {
     try {
-        const submissions = await limiter.schedule(async () => reddit
-            .getUser(username)
-            .getSubmissions({
+        const submissions = await reddit.getSubmissions(username, {
             sort: args.sort,
             limit: Infinity,
-            }));
+        });
 
         logger.info(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);
 
@@ -48,7 +59,7 @@ async function getPosts(username, reddit, args) {
 async function getArchivedPosts(username, posts, reddit) {
     const postIds = await getArchivePostIds(username, posts.map((post) => post.id));
 
-    return Promise.all(postIds.map((postId) => limiter.schedule(async () => reddit.getSubmission(postId).fetch())));
+    return Promise.all(postIds.map((postId) => reddit.getSubmission(postId)));
 }
 
 function getUserPostsWrap(reddit, args) {