Storing social links in profile file, changed format to YAML.

config/default.js

@@ -60,6 +60,13 @@ module.exports = {
         avoidDuplicates: true,
         retries: 3,
         concurrency: 10,
+        // wall-clock ceiling (ms) for a single item-download attempt (src/fetch/item.js).
+        // bhttp has no timeout of its own - a request that connects but then just stops
+        // responding (a stalled/misbehaving proxy tunnel is a common cause) would otherwise
+        // hang forever, and since the limiter's concurrency is capped, that one stuck request
+        // blocks every other download queued behind it indefinitely. This bounds each attempt
+        // so a stall fails (and retries, per `retries` above) instead of hanging forever.
+        timeout: 60000,
         watch: {
             schedule: '*/30 * * * *',
         },
@@ -74,8 +81,19 @@ module.exports = {
         level: 'info',
     },
     limiter: {
+        // default concurrency/interval for item/media downloads (src/fetch/item.js) - applies
+        // to any host without its own entry in `hosts` below
         concurrency: 10,
         interval: 100,
+        hosts: {
+            // per-host overrides of `concurrency`/`interval` above, for hosts that can safely
+            // be hit harder (most media CDNs) or need to be treated more cautiously than the
+            // default. Hostnames are matched by suffix, same as proxy.hosts above - a bare
+            // 'redgifs.com' also covers 'api.redgifs.com' and 'media.redgifs.com'. Unlike a
+            // single shared limiter, each entry here runs independently: a host that's
+            // currently stalling or being throttled doesn't block any other host's downloads.
+            // 'redgifs.com': { concurrency: 20, interval: 50 },
+        },
     },
     reddit: {
         // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to
@@ -99,4 +117,26 @@ module.exports = {
             clientId: null,
         },
     },
+    proxy: {
+        use: true,
+        default: null,
+        // if a request fails (including timing out - see fetch.timeout above) and is about to
+        // be retried, retry it using whichever of proxied/direct is NOT what would normally
+        // apply for that host - e.g. the proxy itself is down or being blocked by the target
+        // site, or (the reverse) a host reachable directly is refusing unproxied traffic.
+        // Off by default: for a privacy-sensitive proxy (e.g. Tor) silently falling back to a
+        // direct connection on failure is very much not what you want.
+        retryFlipped: false,
+        hosts: {
+            // hostnames are matched by suffix, so e.g. 'reddit.com' also covers
+            // 'www.reddit.com', but not 'v.redd.it' or 'redd.it' - those need their own
+            // entry, or fall back to `default`. A host entry can be:
+            // - a proxy URL, to override `default` for that host
+            // - `true`, to explicitly use `default` for that host
+            // - `null`/`false`, to explicitly exclude that host from `default`
+            // 'reddit.com': true,
+            // 'v.redd.it': null,
+            // 'imgur.com': 'socks5://127.0.0.1:9050',
+        },
+    },
 };

package-lock.json

@@ -20,6 +20,8 @@
         "dist-exiftool": "^10.53.0",
         "fluent-ffmpeg": "^2.1.3",
         "fs-extra": "^11.4.0",
+        "http-proxy-agent": "^7.0.2",
+        "https-proxy-agent": "^7.0.6",
         "js-yaml": "^5.2.3",
         "mime": "^3.0.0",
         "mime-types": "^3.0.2",
@@ -28,7 +30,9 @@
         "object.omit": "^3.0.0",
         "object.pick": "^1.3.0",
         "sharp-phash": "^2.2.0",
+        "socks-proxy-agent": "^8.0.5",
         "template-format": "^1.2.5",
+        "undici": "^7.29.0",
         "unprint": "^0.19.20",
         "url-pattern": "^1.0.3",
         "winston": "^3.19.0",
@@ -597,9 +601,6 @@
       "cpu": [
         "arm"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -617,9 +618,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -637,9 +635,6 @@
       "cpu": [
         "ppc64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -657,9 +652,6 @@
       "cpu": [
         "riscv64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -677,9 +669,6 @@
       "cpu": [
         "s390x"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -697,9 +686,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -717,9 +703,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -737,9 +720,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -757,9 +737,6 @@
       "cpu": [
         "arm"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -783,9 +760,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -809,9 +783,6 @@
       "cpu": [
         "ppc64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -835,9 +806,6 @@
       "cpu": [
         "riscv64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -861,9 +829,6 @@
       "cpu": [
         "s390x"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -887,9 +852,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -913,9 +875,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -939,9 +898,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -1279,6 +1235,15 @@
         "node": ">=0.4.0"
       }
     },
+    "node_modules/agent-base": {
+      "version": "7.1.4",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
+      "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/ajv": {
       "version": "6.15.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz",
@@ -4223,6 +4188,32 @@
         "url": "https://opencollective.com/express"
       }
     },
+    "node_modules/http-proxy-agent": {
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
+      "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "^7.1.0",
+        "debug": "^4.3.4"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/https-proxy-agent": {
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
+      "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "^7.1.2",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/human-readable-ids": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/human-readable-ids/-/human-readable-ids-1.0.4.tgz",
@@ -4318,6 +4309,15 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/ip-address": {
+      "version": "10.5.0",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz",
+      "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 12"
+      }
+    },
     "node_modules/ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -6601,6 +6601,44 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/smart-buffer": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
+      "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 6.0.0",
+        "npm": ">= 3.0.0"
+      }
+    },
+    "node_modules/socks": {
+      "version": "2.8.9",
+      "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz",
+      "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==",
+      "license": "MIT",
+      "dependencies": {
+        "ip-address": "^10.1.1",
+        "smart-buffer": "^4.2.0"
+      },
+      "engines": {
+        "node": ">= 10.0.0",
+        "npm": ">= 3.0.0"
+      }
+    },
+    "node_modules/socks-proxy-agent": {
+      "version": "8.0.5",
+      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz",
+      "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==",
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "^7.1.2",
+        "debug": "^4.3.4",
+        "socks": "^2.8.3"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/source-map": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",

package.json

@@ -42,6 +42,8 @@
     "dist-exiftool": "^10.53.0",
     "fluent-ffmpeg": "^2.1.3",
     "fs-extra": "^11.4.0",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.6",
     "js-yaml": "^5.2.3",
     "mime": "^3.0.0",
     "mime-types": "^3.0.2",
@@ -50,7 +52,9 @@
     "object.omit": "^3.0.0",
     "object.pick": "^1.3.0",
     "sharp-phash": "^2.2.0",
+    "socks-proxy-agent": "^8.0.5",
     "template-format": "^1.2.5",
+    "undici": "^7.29.0",
     "unprint": "^0.19.20",
     "url-pattern": "^1.0.3",
     "winston": "^3.19.0",
@@ -63,5 +67,10 @@
     "eslint": "^8.57.1",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-import": "^2.32.0"
+  },
+  "allowScripts": {
+    "blake2@5.0.1": true,
+    "dist-exiftool@10.53.0": true,
+    "youtube-dl-exec@3.1.12": true
   }
 }

src/archives/ip.js

@@ -2,6 +2,8 @@
 
 const $ = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 function findOnIp(username, page = 1, acc = []) {
     return Promise.resolve().then(() => {
         return fetch(`https://www.imageporn.net/user/${username}/all/${page}`);

src/curate/posts.js

@@ -62,6 +62,12 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
         preview: post.preview ? post.preview.images.map((image) => image.source) : null,
         host,
         direct: post.direct,
+        // reddit's submission listing (src/reddit/client.js's getSubmissions, fetched once up
+        // front) already embeds these for gallery posts - carrying them through lets
+        // methods/redditAlbum.js use them directly instead of re-fetching the same post
+        // individually through reddit's own separately-throttled client (see there)
+        mediaMetadata: post.media_metadata || null,
+        galleryData: post.gallery_data || null,
         // count only - the full thread (when needed) is fetched separately via methods/self.js
         comments: post.num_comments,
         hash: hashPost(post),

src/curate/user.js

@@ -1,6 +1,6 @@
 'use strict';
 
-function curateUser(user) {
+function curateUser(user, socialLinks = []) {
     const curatedUser = {
         id: user.id,
         name: user.name,
@@ -21,6 +21,9 @@ function curateUser(user) {
                 banner: user.subreddit.banner_img,
                 description: user.subreddit.public_description,
                 over18: user.subreddit.over_18,
+                // the "Social Links" widget on their profile (Twitter/Instagram/OnlyFans/etc.) -
+                // see reddit/client.js's getSocialLinks()
+                links: socialLinks,
             },
         });
     }

src/fetch/item.js

@@ -6,8 +6,9 @@ const blake2 = require('blake2');
 const phash = require('sharp-phash');
 // const phashDistance = require('sharp-phash/distance');
 
+const { agentFor } = require('../proxy');
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').items;
+const limiterFor = require('../limiter');
 
 async function fetchItem(url, attempt, context) {
     async function retry(error) {
@@ -22,7 +23,25 @@ async function fetchItem(url, attempt, context) {
 
     try {
         // throw new Error('Failed it!');
-        const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers }));
+        // config.proxy.retryFlipped: once this url has already failed once, retry it via
+        // whichever of proxied/direct is NOT what it'd normally get - see resolveProxyUrl()
+        const flip = config.proxy.retryFlipped && attempt > 0;
+
+        const res = await limiterFor(url).schedule(() => bhttp.get(url, {
+            headers: context.headers,
+            agent: agentFor(url, { flip }),
+            // bhttp has no timeout at all by default - a server (or, more likely, a proxy
+            // tunnel) that connects fine but then just stops responding would otherwise hang
+            // forever. This isn't just a wall-clock ceiling on the *promise* (an external
+            // Promise.race achieves that, but leaves the real connection dangling) - bhttp
+            // calls req.abort() itself when this fires, actually releasing the socket. Without
+            // that, every stalled attempt leaks one connection onto the shared agent for that
+            // host; enough of those piling up (e.g. after several retries against a host that's
+            // currently black-holing everything, as seen with i.redd.it) exhausts it, and every
+            // *future* request to that host silently queues forever waiting for a socket that's
+            // never coming back - a freeze with no further log lines, not just a slow retry.
+            responseTimeout: config.fetch.timeout,
+        }));
 
         if (res.statusCode !== 200) {
             throw new Error(`Response not OK for ${url} (${res.statusCode})`);

src/interpolate.js

@@ -27,6 +27,17 @@ function safeFormat(date, dateFormat, label) {
     return dateFns.format(date, dateFormat);
 }
 
+// mime.extension() returns the boolean `false` (not undefined/null) for a mime type it
+// doesn't recognize - reddit's gallery API is a known offender, sending the non-standard
+// 'image/jpg' instead of 'image/jpeg' - which without this check ends up interpolated
+// verbatim as the literal filename extension '.false'. Fall back to whatever extension the
+// URL itself has instead, same as when there's no type at all.
+function resolveExt(item) {
+    const extFromType = item.type && mime.extension(item.type);
+
+    return extFromType ? `.${extFromType}` : path.extname(url.parse(item.url).pathname);
+}
+
 function interpolate(pattern, item = null, content = null, host = null, post = null, user = null, strip = true, dateFormat = config.library.dateFormat) {
     const data = {
         tags: {},
@@ -41,9 +52,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 date: safeFormat(item.datetime, dateFormat, 'item'),
                 index: item.index + config.library.indexOffset,
             },
-            ext: item.type
+            ext: resolveExt(item),
-                ? `.${mime.extension(item.type)}`
-                : path.extname(url.parse(item.url).pathname),
             tags: {
                 ...data.tags,
                 extracted: item.extracted && config.library.tags.extracted,

src/limiter.js

@@ -1,13 +1,50 @@
 'use strict';
 
 const Bottleneck = require('bottleneck');
+const config = require('config');
 
 const args = require('./cli')();
 
-module.exports = {
+// same suffix matching as config.proxy.hosts (src/proxy.js): a bare hostname also matches its
-    // reddit requests are throttled inside src/reddit/client.js itself (config.reddit.throttle)
+// subdomains, so 'redgifs.com' catches 'media.redgifs.com' but not the reverse
-    items: new Bottleneck({
+function hostMatches(hostname, pattern) {
-        maxConcurrent: args.concurrency,
+    const normalized = pattern.replace(/^\./, '').toLowerCase();
-        minTime: args.interval,
+
-    }),
+    return hostname === normalized || hostname.endsWith(`.${normalized}`);
-};
+}
+
+// reddit's own API requests are throttled separately, inside src/reddit/client.js
+// (config.reddit.throttle) - this only governs item/media downloads (src/fetch/item.js)
+const defaultLimiter = new Bottleneck({
+    maxConcurrent: args.concurrency,
+    minTime: args.interval,
+});
+
+const hostLimiters = new Map();
+
+// config.limiter.hosts: most media CDNs (redgifs, imgur, i.redd.it, ...) don't need to be
+// treated as cautiously as reddit's own API, but a single shared limiter forces every host
+// down to whatever pace the most conservative one requires - worse, it means one host that's
+// currently stalling/throttling (see src/fetch/item.js's timeout handling) occupies the one
+// shared slot and blocks every *other* host's downloads behind it too, even though they have
+// nothing to do with each other. Giving a host its own limiter here lets it run at its own
+// concurrency/interval, independently of every other host's.
+function limiterFor(url) {
+    const { hostname } = new URL(url);
+    const hosts = config.limiter?.hosts || {};
+    const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
+
+    if (!matchedPattern) {
+        return defaultLimiter;
+    }
+
+    if (!hostLimiters.has(matchedPattern)) {
+        const { concurrency = args.concurrency, interval = args.interval } = hosts[matchedPattern];
+
+        hostLimiters.set(matchedPattern, new Bottleneck({ maxConcurrent: concurrency, minTime: interval }));
+    }
+
+    return hostLimiters.get(matchedPattern);
+}
+
+module.exports = limiterFor;

src/methods/erome.js

@@ -3,6 +3,7 @@
 const config = require('config');
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 
 const base = 'https://www.erome.com/';

src/methods/eroshareAlbum.js

@@ -2,6 +2,8 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
+
 async function eroshareAlbum(host) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);
 

src/methods/eroshareItem.js

@@ -2,6 +2,8 @@
 
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 async function eroshareItem(host, post) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${host.id}`);
     if (!res.ok) {

src/methods/gfycat.js

@@ -1,15 +1,19 @@
 'use strict';
 
 const bhttp = require('bhttp');
+
+const { agentFor } = require('../proxy');
 const redgifs = require('./redgifs');
 
 async function gfycat(host) {
-    const res = await bhttp.get(`https://api.gfycat.com/v1/gfycats/${host.id}`);
+    const infoUrl = `https://api.gfycat.com/v1/gfycats/${host.id}`;
+    const res = await bhttp.get(infoUrl, { agent: agentFor(infoUrl) });
     const data = await res.body;
 
     if (data.errorMessage) {
         const redirectRes = await bhttp.head(host.url, {
             followRedirects: false,
+            agent: agentFor(host.url),
         });
 
         if (redirectRes.statusCode === 301) {

src/methods/imgurAlbum.js

@@ -2,6 +2,7 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 const { fetchPredata } = require('./imgurImage');
 

src/methods/imgurImage.js

@@ -2,6 +2,8 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
+
 async function fetchPredata() {
     const data = {
         limit: 0,

src/methods/redditAlbum.js

@@ -8,22 +8,47 @@ function resolveMediaUrl(media) {
     return url ? url.replace(/&/g, '&') : null;
 }
 
-async function redditAlbum(host, post, { reddit }) {
+// reddit sends the non-standard 'image/jpg' instead of the real IANA 'image/jpeg' for
-    // fetched through the authenticated/throttled client, not a raw fetch()/bhttp request -
+// gallery items - left as-is, this silently breaks anything matching against real mime
-    // reddit's anonymous-traffic bot challenge blocks plain unauthenticated .json requests
+// types downstream (interpolate.js's extension lookup ends up with a '.false' extension,
-    const data = await reddit.getSubmission(post.id);
+// fetch/content.js's addMeta() skips embedding metadata since it only matches 'image/jpeg')
+function normalizeType(type) {
+    return type === 'image/jpg' ? 'image/jpeg' : type;
+}
 
-    const items = (data.gallery_data?.items || [])
+async function redditAlbum(host, post, { reddit }) {
+    // reddit's submission listing already embeds media_metadata/gallery_data for gallery posts,
+    // and curate/posts.js carries them through as post.mediaMetadata/post.galleryData - reuse
+    // those instead of re-fetching the same post individually through reddit's own client
+    // (reservoir: 6/min, maxConcurrent: 1 - see src/reddit/client.js), which would otherwise cost
+    // one full throttled round trip per gallery post. Only fall back to a fresh fetch if the
+    // listing didn't have it for some reason (e.g. gallery_data null on the listing too).
+    const data = post.mediaMetadata
+        ? { media_metadata: post.mediaMetadata, gallery_data: post.galleryData }
+        : await reddit.getSubmission(post.id);
+    const mediaMetadata = data.media_metadata || {};
+
+    // gallery_data (the ordered {media_id, is_deleted, ...} list for this gallery) has been
+    // seen coming back null on a freshly-posted gallery even though media_metadata itself
+    // was already fully populated - fall back to media_metadata's own keys, in the order the
+    // JSON provided them (reddit sends them in gallery order, and object key order is
+    // preserved through JSON.parse), skipping anything not (yet, or any longer) valid
+    const galleryItems = data.gallery_data?.items
+        || Object.entries(mediaMetadata)
+            .filter(([, media]) => media.status === 'valid')
+            .map(([mediaId]) => ({ media_id: mediaId }));
+
+    const items = galleryItems
         .filter((item) => !item.is_deleted)
         .map((item) => {
-            const media = data.media_metadata?.[item.media_id];
+            const media = mediaMetadata[item.media_id];
             const url = resolveMediaUrl(media);
 
             return url && {
                 id: item.media_id,
                 url,
                 datetime: post.datetime,
-                type: media.m || 'image/jpeg',
+                type: normalizeType(media.m) || 'image/jpeg',
             };
         })
         .filter(Boolean);

src/methods/redditVideo.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { fetch } = require('../proxy');
+
 // the DASH manifest lists the actual audio track filename - reddit has changed this naming
 // scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the
 // manifest rather than guessing a hardcoded suffix off the video URL again

src/methods/redgifs.js

@@ -3,6 +3,7 @@
 const mime = require('mime');
 // const unprint = require('unprint');
 
+const { fetch } = require('../proxy');
 const { version } = require('../../package.json');
 
 async function fetchPredata() {

src/methods/vidbleAlbum.js

@@ -5,6 +5,7 @@ const UrlPattern = require('url-pattern');
 const cheerio = require('cheerio');
 const mime = require('mime-types');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 
 const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');

src/methods/vidbleImage.js

@@ -3,6 +3,8 @@
 const $ = require('cheerio');
 const mime = require('mime-types');
 
+const { fetch } = require('../proxy');
+
 async function vidbleImage(host, post) {
     const res = await fetch(`https://vidble.com/${host.id}`);
 

src/methods/vidbleVideo.js

@@ -2,6 +2,8 @@
 
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 async function vidbleVideo(host, post) {
     const res = await fetch(`https://www.vidble.com/watch?v=${host.id}`);
 

src/proxy.js

@@ -0,0 +1,146 @@
+'use strict';
+
+const config = require('config');
+const { ProxyAgent } = require('undici');
+const { HttpProxyAgent } = require('http-proxy-agent');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { SocksProxyAgent } = require('socks-proxy-agent');
+
+const logger = require('./logger')(__filename);
+
+const dispatcherCache = new Map();
+const nodeAgentCache = new Map();
+const warnedNoSocksForFetch = new Set();
+
+// same suffix matching as config.reddit.cookies' domain filter (src/reddit/client.js): a
+// bare hostname also matches its subdomains, so 'reddit.com' catches 'www.reddit.com' but
+// not 'v.redd.it' or 'redd.it' - those need their own entry, or fall back to `default`
+function hostMatches(hostname, pattern) {
+    const normalized = pattern.replace(/^\./, '').toLowerCase();
+
+    return hostname === normalized || hostname.endsWith(`.${normalized}`);
+}
+
+function resolveProxyUrl(hostname, { flip = false } = {}) {
+    if (config.proxy?.use === false) {
+        return null;
+    }
+
+    const hosts = config.proxy?.hosts || {};
+    const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
+
+    const normal = (() => {
+        if (!matchedPattern) {
+            return config.proxy?.default || null;
+        }
+
+        const matched = hosts[matchedPattern];
+
+        // `true` explicitly opts this host into `default`, `null`/`false`/'' explicitly opts it
+        // out (rather than falling through to `default`), anything else is a per-host override
+        if (matched === true) {
+            return config.proxy?.default || null;
+        }
+
+        return matched || null;
+    })();
+
+    if (!flip) {
+        return normal;
+    }
+
+    // used for config.proxy.retryFlipped: retry via whichever of proxied/direct is NOT what
+    // this host would normally get. A request that would normally be proxied (whether via
+    // `default` or a per-host override) retries direct; a request that would normally go
+    // direct (no proxy configured for this host, or explicitly excluded from `default`)
+    // retries through `default`, if one is set.
+    return normal ? null : (config.proxy?.default || null);
+}
+
+// 'socks4a'/'socks5h' (as opposed to plain 'socks4'/'socks5') tell socks-proxy-agent to
+// resolve the target hostname on the proxy side rather than locally - the same convention
+// curl uses, and worth keeping since it's the whole point of routing through SOCKS/Tor for
+// most people
+function isSocks(proxyUrl) {
+    return /^socks(4a?|5h?):/i.test(proxyUrl);
+}
+
+// undici (which node's built-in fetch() is implemented with) only tunnels through http(s)
+// proxies (CONNECT) via ProxyAgent - it has no built-in SOCKS support. agentFor() below can
+// still do SOCKS, since bhttp goes through node's plain http/https modules instead.
+function dispatcherFor(targetUrl, { flip = false } = {}) {
+    const { hostname } = new URL(targetUrl);
+    const proxyUrl = resolveProxyUrl(hostname, { flip });
+
+    if (!proxyUrl) {
+        return undefined;
+    }
+
+    if (isSocks(proxyUrl)) {
+        if (!warnedNoSocksForFetch.has(proxyUrl)) {
+            warnedNoSocksForFetch.add(proxyUrl);
+            logger.warn(`SOCKS proxy '${proxyUrl}' configured for '${hostname}' can't be used for fetch()-based requests (only http(s) proxies can) - these will go out directly instead. Item downloads (which use bhttp) can still use it.`);
+        }
+
+        return undefined;
+    }
+
+    if (!dispatcherCache.has(proxyUrl)) {
+        dispatcherCache.set(proxyUrl, new ProxyAgent(proxyUrl));
+    }
+
+    return dispatcherCache.get(proxyUrl);
+}
+
+// bhttp uses node's raw http/https modules under the hood, which take a plain Agent
+// instance - unlike fetch() above, this does support SOCKS
+function agentFor(targetUrl, { flip = false } = {}) {
+    const url = new URL(targetUrl);
+    const proxyUrl = resolveProxyUrl(url.hostname, { flip });
+
+    if (!proxyUrl) {
+        return undefined;
+    }
+
+    const cacheKey = `${proxyUrl}|${isSocks(proxyUrl) ? 'socks' : url.protocol}`;
+
+    if (!nodeAgentCache.has(cacheKey)) {
+        if (isSocks(proxyUrl)) {
+            nodeAgentCache.set(cacheKey, new SocksProxyAgent(proxyUrl));
+        } else if (url.protocol === 'https:') {
+            nodeAgentCache.set(cacheKey, new HttpsProxyAgent(proxyUrl));
+        } else {
+            nodeAgentCache.set(cacheKey, new HttpProxyAgent(proxyUrl));
+        }
+    }
+
+    return nodeAgentCache.get(cacheKey);
+}
+
+// drop-in replacement for the global fetch() - importing this as `fetch` transparently
+// proxies requests per config.proxy, if one is configured for the request's host. Also
+// applies a hard wall-clock ceiling (config.fetch.timeout) to every request: unlike bhttp,
+// fetch() doesn't hang *forever* on a stalled/black-holed connection (undici defaults to a
+// 300s headers/body timeout), but that default is high enough - and, per live testing, a
+// proxy CONNECT tunnel that never completes can still take well over 20s to surface any
+// error - to look like a freeze, especially since reddit/client.js's requestJson() serializes
+// all reddit API calls behind a single-concurrency limiter. Bounding it here covers every
+// fetch()-based call site (reddit's API, and every hosting site's info-lookup) at once.
+// Pass `proxyFlip: true` in options (stripped before reaching fetch()) to use whichever of
+// proxied/direct is NOT what this host would normally get - see resolveProxyUrl().
+async function proxiedFetch(url, options = {}) {
+    const { proxyFlip, signal, ...fetchOptions } = options;
+    const dispatcher = dispatcherFor(String(url), { flip: proxyFlip });
+    const timeoutSignal = AbortSignal.timeout(config.fetch.timeout);
+
+    return fetch(url, {
+        ...fetchOptions,
+        ...(dispatcher ? { dispatcher } : {}),
+        signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal,
+    });
+}
+
+module.exports = {
+    fetch: proxiedFetch,
+    agentFor,
+};

src/reddit/client.js

@@ -1,8 +1,10 @@
 'use strict';
 
 const Bottleneck = require('bottleneck');
+const cheerio = require('cheerio');
 const config = require('config');
 
+const { fetch } = require('../proxy');
 const { version } = require('../../package.json');
 const logger = require('../logger')(__filename);
 
@@ -74,10 +76,10 @@ function cookieHeader() {
     return cachedCookieHeader;
 }
 
-function headers() {
+function headers(accept = 'application/json') {
     const requestHeaders = {
         'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
-        accept: 'application/json',
+        accept,
     };
 
     const cookie = cookieHeader();
@@ -107,46 +109,133 @@ async function wait(seconds) {
     });
 }
 
-async function requestJson(path, params = {}, attempt = 0) {
+// shared by requestJson() and requestHtml() below - handles the actual throttled network
+// call plus retrying on a network error or a 429/503, regardless of what kind of response
+// the caller ultimately wants back.
+//
+// Only the actual throttled network call belongs inside limiter.schedule() - this has
+// maxConcurrent: 1, and every retry path below used to recurse back into this function
+// (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't
+// start a second job while the first is still running, and the first can't finish until
+// its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever
+// needed (e.g. a single 429), freezing every reddit request behind it for good. Retrying
+// now happens here, after the scheduled call has actually returned and freed its slot.
+async function fetchWithRetry(path, params, attempt, accept) {
     const url = buildUrl(path, params);
+    let res;
 
-    return limiter.schedule(async () => {
+    try {
-        const res = await fetch(url.toString(), { headers: headers() });
+        // config.proxy.retryFlipped: once this request has already failed once, retry it via
-        const contentType = res.headers.get('content-type') || '';
+        // whichever of proxied/direct is NOT what it'd normally get - see proxy.js
+        res = await limiter.schedule(async () => fetch(url.toString(), {
+            headers: headers(accept),
+            proxyFlip: config.proxy.retryFlipped && attempt > 0,
+        }));
+    } catch (error) {
+        // fetch() has no timeout of its own beyond undici's fairly generous defaults (see
+        // proxy.js) - this catches both that and genuine network errors (DNS, connection
+        // refused/reset, etc.), neither of which was retried before
+        if (attempt >= MAX_RETRIES) {
+            throw new Error(`Failed to reach reddit after ${attempt} retries for '${path}': ${error.message}`);
+        }
 
-        if (res.status === 429 || res.status === 503) {
+        logger.warn(`Failed to reach reddit for '${path}', retrying: ${error.message}`);
-            if (attempt >= MAX_RETRIES) {
+
-                throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
+        return fetchWithRetry(path, params, attempt + 1, accept);
+    }
+
+    if (res.status === 429 || res.status === 503) {
+        if (attempt >= MAX_RETRIES) {
+            throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
+        }
+
+        const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;
+
+        logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);
+
+        await wait(retryAfter);
+
+        return fetchWithRetry(path, params, attempt + 1, accept);
+    }
+
+    return res;
+}
+
+async function requestJson(path, params = {}, attempt = 0) {
+    const res = await fetchWithRetry(path, params, attempt, 'application/json');
+    const contentType = res.headers.get('content-type') || '';
+
+    if (!contentType.includes('json')) {
+        // reddit serves an HTML interstitial/block page instead of JSON when it decides
+        // the request looks like a bot - this is a distinct failure mode from a genuine
+        // 4xx and should be logged/handled separately rather than treated as "bad request"
+        const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
+
+        throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
+    }
+
+    const data = await res.json();
+
+    if (!res.ok) {
+        const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);
+
+        throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
+    }
+
+    return data;
+}
+
+// unlike requestJson(), this is for reddit's rendered profile pages - there's no public JSON
+// endpoint for a user's social links (the "Social Links" widget on their profile), they're
+// only present in the page's HTML (see getSocialLinks() below)
+async function requestHtml(path, params = {}, attempt = 0) {
+    const res = await fetchWithRetry(path, params, attempt, 'text/html');
+    const contentType = res.headers.get('content-type') || '';
+
+    if (!contentType.includes('html')) {
+        const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
+
+        throw new Error(`Reddit returned non-HTML response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
+    }
+
+    if (!res.ok) {
+        throw new Error(`Reddit returned ${res.status} for '${path}'`);
+    }
+
+    return res.text();
+}
+
+// reddit's "Social Links" profile widget (the row of Twitter/Instagram/OnlyFans/etc. buttons
+// under a user's bio) has no public JSON endpoint - the structured API for it
+// (/r/u_{username}/api/widgets.json) requires OAuth, which this project deliberately doesn't
+// use (see reddit.throttle above). The rendered profile page embeds the same data anyway, as
+// JSON inside each link's `data-faceplate-tracking-context` attribute, and that page is
+// accessible with the same cookies as everything else here.
+function parseSocialLinks(html) {
+    const $ = cheerio.load(html);
+    const links = [];
+
+    $('[data-faceplate-tracking-context]').each((index, el) => {
+        try {
+            const { social_link: link } = JSON.parse($(el).attr('data-faceplate-tracking-context'));
+
+            if (link) {
+                links.push(link);
             }
-
+        } catch (error) {
-            const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;
+            // not a social-link tracker, or a shape we don't recognize - ignore
-
-            logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);
-
-            await wait(retryAfter);
-
-            return requestJson(path, params, attempt + 1);
         }
-
-        if (!contentType.includes('json')) {
-            // reddit serves an HTML interstitial/block page instead of JSON when it decides
-            // the request looks like a bot - this is a distinct failure mode from a genuine
-            // 4xx and should be logged/handled separately rather than treated as "bad request"
-            const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
-
-            throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
-        }
-
-        const data = await res.json();
-
-        if (!res.ok) {
-            const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);
-
-            throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
-        }
-
-        return data;
     });
+
+    return links
+        .sort((a, b) => a.position - b.position)
+        .map(({ type, name, url }) => ({ type, name, url }));
+}
+
+async function getSocialLinks(username) {
+    const html = await requestHtml(`/user/${username}/`);
+
+    return parseSocialLinks(html);
 }
 
 async function getUser(username) {
@@ -277,6 +366,7 @@ async function getSubmissionWithComments(id) {
 module.exports = {
     requestJson,
     getUser,
+    getSocialLinks,
     getSubmissions,
     getSubmission,
     getSubmissionWithComments,

src/save/profileDetails.js

@@ -3,6 +3,7 @@
 const config = require('config');
 const Promise = require('bluebird');
 const UrlPattern = require('url-pattern');
+const yaml = require('js-yaml');
 
 const interpolate = require('../interpolate');
 const fetchItem = require('../fetch/item');
@@ -62,9 +63,15 @@ async function saveProfileDescription(user, args) {
     }
 
     if (config.library.profile.description && !user.fallback && !user.deleted) {
-        if (user.profile && user.profile.description) {
+        const links = user.profile?.links || [];
+
+        // YAML rather than plain text (like index.js) so the free-form bio and the structured
+        // social links (reddit/client.js's getSocialLinks()) stay separately parseable instead
+        // of just being smushed into one text blob
+        if (user.profile && (user.profile.description || links.length)) {
             const filepath = interpolate(config.library.profile.description, null, null, null, null, user);
-            const stream = Buffer.from(user.profile.description, 'utf8');
+            const data = { description: user.profile.description || null, links };
+            const stream = Buffer.from(yaml.dump(data, { skipInvalid: true }), 'utf8');
 
             try {
                 const targets = await save(filepath, stream);
@@ -77,7 +84,7 @@ async function saveProfileDescription(user, args) {
             }
         }
 
-        logger.verbose(`No profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
+        logger.verbose(`No profile description or social links for '${user.name}' (https://reddit.com/user/${user.name})`);
 
         return null;
     }

src/sources/getPosts.js

@@ -7,11 +7,25 @@ const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
 
+async function getSocialLinks(username, reddit) {
+    try {
+        return await reddit.getSocialLinks(username);
+    } catch (error) {
+        // supplementary, not worth failing the whole user fetch over - fall back to none
+        logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
+
+        return [];
+    }
+}
+
 async function getUser(username, reddit) {
     try {
-        const user = await reddit.getUser(username);
+        const [user, socialLinks] = await Promise.all([
+            reddit.getUser(username),
+            getSocialLinks(username, reddit),
+        ]);
 
-        return curateUser(user);
+        return curateUser(user, socialLinks);
     } catch (error) {
         logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
 

src/sources/getUserPosts.js

@@ -8,13 +8,27 @@ const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
 
+async function getSocialLinks(username, reddit) {
+    try {
+        return await reddit.getSocialLinks(username);
+    } catch (error) {
+        // supplementary, not worth failing the whole user fetch over - fall back to none
+        logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
+
+        return [];
+    }
+}
+
 async function getUser(username, reddit) {
     try {
-        const user = await reddit.getUser(username);
+        const [user, socialLinks] = await Promise.all([
+            reddit.getUser(username),
+            getSocialLinks(username, reddit),
+        ]);
 
         logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
 
-        return curateUser(user);
+        return curateUser(user, socialLinks);
     } catch (error) {
         logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);