Added proxy.

config/default.js

@@ -99,4 +99,19 @@ module.exports = {
             clientId: null,
         },
     },
+    proxy: {
+        use: true,
+        default: null,
+        hosts: {
+            // hostnames are matched by suffix, so e.g. 'reddit.com' also covers
+            // 'www.reddit.com', but not 'v.redd.it' or 'redd.it' - those need their own
+            // entry, or fall back to `default`. A host entry can be:
+            // - a proxy URL, to override `default` for that host
+            // - `true`, to explicitly use `default` for that host
+            // - `null`/`false`, to explicitly exclude that host from `default`
+            // 'reddit.com': true,
+            // 'v.redd.it': null,
+            // 'imgur.com': 'socks5://127.0.0.1:9050',
+        },
+    },
 };

package-lock.json

@@ -20,6 +20,8 @@
         "dist-exiftool": "^10.53.0",
         "fluent-ffmpeg": "^2.1.3",
         "fs-extra": "^11.4.0",
+        "http-proxy-agent": "^7.0.2",
+        "https-proxy-agent": "^7.0.6",
         "js-yaml": "^5.2.3",
         "mime": "^3.0.0",
         "mime-types": "^3.0.2",
@@ -28,7 +30,9 @@
         "object.omit": "^3.0.0",
         "object.pick": "^1.3.0",
         "sharp-phash": "^2.2.0",
+        "socks-proxy-agent": "^8.0.5",
         "template-format": "^1.2.5",
+        "undici": "^7.29.0",
         "unprint": "^0.19.20",
         "url-pattern": "^1.0.3",
         "winston": "^3.19.0",
@@ -597,9 +601,6 @@
       "cpu": [
         "arm"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -617,9 +618,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -637,9 +635,6 @@
       "cpu": [
         "ppc64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -657,9 +652,6 @@
       "cpu": [
         "riscv64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -677,9 +669,6 @@
       "cpu": [
         "s390x"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -697,9 +686,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -717,9 +703,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -737,9 +720,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "LGPL-3.0-or-later",
       "optional": true,
       "os": [
@@ -757,9 +737,6 @@
       "cpu": [
         "arm"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -783,9 +760,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -809,9 +783,6 @@
       "cpu": [
         "ppc64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -835,9 +806,6 @@
       "cpu": [
         "riscv64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -861,9 +829,6 @@
       "cpu": [
         "s390x"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -887,9 +852,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -913,9 +875,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -939,9 +898,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "Apache-2.0",
       "optional": true,
       "os": [
@@ -1279,6 +1235,15 @@
         "node": ">=0.4.0"
       }
     },
+    "node_modules/agent-base": {
+      "version": "7.1.4",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
+      "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/ajv": {
       "version": "6.15.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz",
@@ -4223,6 +4188,32 @@
         "url": "https://opencollective.com/express"
       }
     },
+    "node_modules/http-proxy-agent": {
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
+      "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "^7.1.0",
+        "debug": "^4.3.4"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/https-proxy-agent": {
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
+      "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "^7.1.2",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/human-readable-ids": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/human-readable-ids/-/human-readable-ids-1.0.4.tgz",
@@ -4318,6 +4309,15 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/ip-address": {
+      "version": "10.5.0",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz",
+      "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 12"
+      }
+    },
     "node_modules/ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -6601,6 +6601,44 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/smart-buffer": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
+      "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 6.0.0",
+        "npm": ">= 3.0.0"
+      }
+    },
+    "node_modules/socks": {
+      "version": "2.8.9",
+      "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz",
+      "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==",
+      "license": "MIT",
+      "dependencies": {
+        "ip-address": "^10.1.1",
+        "smart-buffer": "^4.2.0"
+      },
+      "engines": {
+        "node": ">= 10.0.0",
+        "npm": ">= 3.0.0"
+      }
+    },
+    "node_modules/socks-proxy-agent": {
+      "version": "8.0.5",
+      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz",
+      "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==",
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "^7.1.2",
+        "debug": "^4.3.4",
+        "socks": "^2.8.3"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/source-map": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",

package.json

@@ -42,6 +42,8 @@
     "dist-exiftool": "^10.53.0",
     "fluent-ffmpeg": "^2.1.3",
     "fs-extra": "^11.4.0",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.6",
     "js-yaml": "^5.2.3",
     "mime": "^3.0.0",
     "mime-types": "^3.0.2",
@@ -50,7 +52,9 @@
     "object.omit": "^3.0.0",
     "object.pick": "^1.3.0",
     "sharp-phash": "^2.2.0",
+    "socks-proxy-agent": "^8.0.5",
     "template-format": "^1.2.5",
+    "undici": "^7.29.0",
     "unprint": "^0.19.20",
     "url-pattern": "^1.0.3",
     "winston": "^3.19.0",
@@ -63,5 +67,10 @@
     "eslint": "^8.57.1",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-import": "^2.32.0"
+  },
+  "allowScripts": {
+    "blake2@5.0.1": true,
+    "dist-exiftool@10.53.0": true,
+    "youtube-dl-exec@3.1.12": true
   }
 }

src/archives/ip.js

@@ -2,6 +2,8 @@
 
 const $ = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 function findOnIp(username, page = 1, acc = []) {
     return Promise.resolve().then(() => {
         return fetch(`https://www.imageporn.net/user/${username}/all/${page}`);

src/fetch/item.js

@@ -6,6 +6,7 @@ const blake2 = require('blake2');
 const phash = require('sharp-phash');
 // const phashDistance = require('sharp-phash/distance');
 
+const { agentFor } = require('../proxy');
 const logger = require('../logger')(__filename);
 const limiter = require('../limiter').items;
 
@@ -22,7 +23,7 @@ async function fetchItem(url, attempt, context) {
 
     try {
         // throw new Error('Failed it!');
-        const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers }));
+        const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers, agent: agentFor(url) }));
 
         if (res.statusCode !== 200) {
             throw new Error(`Response not OK for ${url} (${res.statusCode})`);

src/methods/erome.js

@@ -3,6 +3,7 @@
 const config = require('config');
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 
 const base = 'https://www.erome.com/';

src/methods/eroshareAlbum.js

@@ -2,6 +2,8 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
+
 async function eroshareAlbum(host) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);
 

src/methods/eroshareItem.js

@@ -2,6 +2,8 @@
 
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 async function eroshareItem(host, post) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${host.id}`);
     if (!res.ok) {

src/methods/gfycat.js

@@ -1,15 +1,19 @@
 'use strict';
 
 const bhttp = require('bhttp');
+
+const { agentFor } = require('../proxy');
 const redgifs = require('./redgifs');
 
 async function gfycat(host) {
-    const res = await bhttp.get(`https://api.gfycat.com/v1/gfycats/${host.id}`);
+    const infoUrl = `https://api.gfycat.com/v1/gfycats/${host.id}`;
+    const res = await bhttp.get(infoUrl, { agent: agentFor(infoUrl) });
     const data = await res.body;
 
     if (data.errorMessage) {
         const redirectRes = await bhttp.head(host.url, {
             followRedirects: false,
+            agent: agentFor(host.url),
         });
 
         if (redirectRes.statusCode === 301) {

src/methods/imgurAlbum.js

@@ -2,6 +2,7 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 const { fetchPredata } = require('./imgurImage');
 

src/methods/imgurImage.js

@@ -2,6 +2,8 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
+
 async function fetchPredata() {
     const data = {
         limit: 0,

src/methods/redditVideo.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { fetch } = require('../proxy');
+
 // the DASH manifest lists the actual audio track filename - reddit has changed this naming
 // scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the
 // manifest rather than guessing a hardcoded suffix off the video URL again

src/methods/redgifs.js

@@ -3,6 +3,7 @@
 const mime = require('mime');
 // const unprint = require('unprint');
 
+const { fetch } = require('../proxy');
 const { version } = require('../../package.json');
 
 async function fetchPredata() {

src/methods/vidbleAlbum.js

@@ -5,6 +5,7 @@ const UrlPattern = require('url-pattern');
 const cheerio = require('cheerio');
 const mime = require('mime-types');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 
 const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');

src/methods/vidbleImage.js

@@ -3,6 +3,8 @@
 const $ = require('cheerio');
 const mime = require('mime-types');
 
+const { fetch } = require('../proxy');
+
 async function vidbleImage(host, post) {
     const res = await fetch(`https://vidble.com/${host.id}`);
 

src/methods/vidbleVideo.js

@@ -2,6 +2,8 @@
 
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 async function vidbleVideo(host, post) {
     const res = await fetch(`https://www.vidble.com/watch?v=${host.id}`);
 

src/proxy.js

@@ -0,0 +1,118 @@
+'use strict';
+
+const config = require('config');
+const { ProxyAgent } = require('undici');
+const { HttpProxyAgent } = require('http-proxy-agent');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { SocksProxyAgent } = require('socks-proxy-agent');
+
+const logger = require('./logger')(__filename);
+
+const dispatcherCache = new Map();
+const nodeAgentCache = new Map();
+const warnedNoSocksForFetch = new Set();
+
+// same suffix matching as config.reddit.cookies' domain filter (src/reddit/client.js): a
+// bare hostname also matches its subdomains, so 'reddit.com' catches 'www.reddit.com' but
+// not 'v.redd.it' or 'redd.it' - those need their own entry, or fall back to `default`
+function hostMatches(hostname, pattern) {
+    const normalized = pattern.replace(/^\./, '').toLowerCase();
+
+    return hostname === normalized || hostname.endsWith(`.${normalized}`);
+}
+
+function resolveProxyUrl(hostname) {
+    if (config.proxy?.use === false) {
+        return null;
+    }
+
+    const hosts = config.proxy?.hosts || {};
+    const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
+
+    if (!matchedPattern) {
+        return config.proxy?.default || null;
+    }
+
+    const matched = hosts[matchedPattern];
+
+    // `true` explicitly opts this host into `default`, `null`/`false`/'' explicitly opts it
+    // out (rather than falling through to `default`), anything else is a per-host override
+    if (matched === true) {
+        return config.proxy?.default || null;
+    }
+
+    return matched || null;
+}
+
+// 'socks4a'/'socks5h' (as opposed to plain 'socks4'/'socks5') tell socks-proxy-agent to
+// resolve the target hostname on the proxy side rather than locally - the same convention
+// curl uses, and worth keeping since it's the whole point of routing through SOCKS/Tor for
+// most people
+function isSocks(proxyUrl) {
+    return /^socks(4a?|5h?):/i.test(proxyUrl);
+}
+
+// undici (which node's built-in fetch() is implemented with) only tunnels through http(s)
+// proxies (CONNECT) via ProxyAgent - it has no built-in SOCKS support. agentFor() below can
+// still do SOCKS, since bhttp goes through node's plain http/https modules instead.
+function dispatcherFor(targetUrl) {
+    const { hostname } = new URL(targetUrl);
+    const proxyUrl = resolveProxyUrl(hostname);
+
+    if (!proxyUrl) {
+        return undefined;
+    }
+
+    if (isSocks(proxyUrl)) {
+        if (!warnedNoSocksForFetch.has(proxyUrl)) {
+            warnedNoSocksForFetch.add(proxyUrl);
+            logger.warn(`SOCKS proxy '${proxyUrl}' configured for '${hostname}' can't be used for fetch()-based requests (only http(s) proxies can) - these will go out directly instead. Item downloads (which use bhttp) can still use it.`);
+        }
+
+        return undefined;
+    }
+
+    if (!dispatcherCache.has(proxyUrl)) {
+        dispatcherCache.set(proxyUrl, new ProxyAgent(proxyUrl));
+    }
+
+    return dispatcherCache.get(proxyUrl);
+}
+
+// bhttp uses node's raw http/https modules under the hood, which take a plain Agent
+// instance - unlike fetch() above, this does support SOCKS
+function agentFor(targetUrl) {
+    const url = new URL(targetUrl);
+    const proxyUrl = resolveProxyUrl(url.hostname);
+
+    if (!proxyUrl) {
+        return undefined;
+    }
+
+    const cacheKey = `${proxyUrl}|${isSocks(proxyUrl) ? 'socks' : url.protocol}`;
+
+    if (!nodeAgentCache.has(cacheKey)) {
+        if (isSocks(proxyUrl)) {
+            nodeAgentCache.set(cacheKey, new SocksProxyAgent(proxyUrl));
+        } else if (url.protocol === 'https:') {
+            nodeAgentCache.set(cacheKey, new HttpsProxyAgent(proxyUrl));
+        } else {
+            nodeAgentCache.set(cacheKey, new HttpProxyAgent(proxyUrl));
+        }
+    }
+
+    return nodeAgentCache.get(cacheKey);
+}
+
+// drop-in replacement for the global fetch() - importing this as `fetch` transparently
+// proxies requests per config.proxy, if one is configured for the request's host
+async function proxiedFetch(url, options = {}) {
+    const dispatcher = dispatcherFor(String(url));
+
+    return fetch(url, dispatcher ? { ...options, dispatcher } : options);
+}
+
+module.exports = {
+    fetch: proxiedFetch,
+    agentFor,
+};

src/reddit/client.js

@@ -3,6 +3,7 @@
 const Bottleneck = require('bottleneck');
 const config = require('config');
 
+const { fetch } = require('../proxy');
 const { version } = require('../../package.json');
 const logger = require('../logger')(__filename);