Added proxy.

src/archives/ip.js

@@ -2,6 +2,8 @@
 
 const $ = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 function findOnIp(username, page = 1, acc = []) {
     return Promise.resolve().then(() => {
         return fetch(`https://www.imageporn.net/user/${username}/all/${page}`);

src/fetch/item.js

@@ -6,6 +6,7 @@ const blake2 = require('blake2');
 const phash = require('sharp-phash');
 // const phashDistance = require('sharp-phash/distance');
 
+const { agentFor } = require('../proxy');
 const logger = require('../logger')(__filename);
 const limiter = require('../limiter').items;
 
@@ -22,7 +23,7 @@ async function fetchItem(url, attempt, context) {
 
     try {
         // throw new Error('Failed it!');
-        const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers }));
+        const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers, agent: agentFor(url) }));
 
         if (res.statusCode !== 200) {
             throw new Error(`Response not OK for ${url} (${res.statusCode})`);

src/methods/erome.js

@@ -3,6 +3,7 @@
 const config = require('config');
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 
 const base = 'https://www.erome.com/';

src/methods/eroshareAlbum.js

@@ -2,6 +2,8 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
+
 async function eroshareAlbum(host) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);
 

src/methods/eroshareItem.js

@@ -2,6 +2,8 @@
 
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 async function eroshareItem(host, post) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${host.id}`);
     if (!res.ok) {

src/methods/gfycat.js

@@ -1,15 +1,19 @@
 'use strict';
 
 const bhttp = require('bhttp');
+
+const { agentFor } = require('../proxy');
 const redgifs = require('./redgifs');
 
 async function gfycat(host) {
-    const res = await bhttp.get(`https://api.gfycat.com/v1/gfycats/${host.id}`);
+    const infoUrl = `https://api.gfycat.com/v1/gfycats/${host.id}`;
+    const res = await bhttp.get(infoUrl, { agent: agentFor(infoUrl) });
     const data = await res.body;
 
     if (data.errorMessage) {
         const redirectRes = await bhttp.head(host.url, {
             followRedirects: false,
+            agent: agentFor(host.url),
         });
 
         if (redirectRes.statusCode === 301) {

src/methods/imgurAlbum.js

@@ -2,6 +2,7 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 const { fetchPredata } = require('./imgurImage');
 

src/methods/imgurImage.js

@@ -2,6 +2,8 @@
 
 const config = require('config');
 
+const { fetch } = require('../proxy');
+
 async function fetchPredata() {
     const data = {
         limit: 0,

src/methods/redditVideo.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { fetch } = require('../proxy');
+
 // the DASH manifest lists the actual audio track filename - reddit has changed this naming
 // scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the
 // manifest rather than guessing a hardcoded suffix off the video URL again

src/methods/redgifs.js

@@ -3,6 +3,7 @@
 const mime = require('mime');
 // const unprint = require('unprint');
 
+const { fetch } = require('../proxy');
 const { version } = require('../../package.json');
 
 async function fetchPredata() {

src/methods/vidbleAlbum.js

@@ -5,6 +5,7 @@ const UrlPattern = require('url-pattern');
 const cheerio = require('cheerio');
 const mime = require('mime-types');
 
+const { fetch } = require('../proxy');
 const logger = require('../logger')(__filename);
 
 const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');

src/methods/vidbleImage.js

@@ -3,6 +3,8 @@
 const $ = require('cheerio');
 const mime = require('mime-types');
 
+const { fetch } = require('../proxy');
+
 async function vidbleImage(host, post) {
     const res = await fetch(`https://vidble.com/${host.id}`);
 

src/methods/vidbleVideo.js

@@ -2,6 +2,8 @@
 
 const cheerio = require('cheerio');
 
+const { fetch } = require('../proxy');
+
 async function vidbleVideo(host, post) {
     const res = await fetch(`https://www.vidble.com/watch?v=${host.id}`);
 

src/proxy.js

@@ -0,0 +1,118 @@
+'use strict';
+
+const config = require('config');
+const { ProxyAgent } = require('undici');
+const { HttpProxyAgent } = require('http-proxy-agent');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { SocksProxyAgent } = require('socks-proxy-agent');
+
+const logger = require('./logger')(__filename);
+
+const dispatcherCache = new Map();
+const nodeAgentCache = new Map();
+const warnedNoSocksForFetch = new Set();
+
+// same suffix matching as config.reddit.cookies' domain filter (src/reddit/client.js): a
+// bare hostname also matches its subdomains, so 'reddit.com' catches 'www.reddit.com' but
+// not 'v.redd.it' or 'redd.it' - those need their own entry, or fall back to `default`
+function hostMatches(hostname, pattern) {
+    const normalized = pattern.replace(/^\./, '').toLowerCase();
+
+    return hostname === normalized || hostname.endsWith(`.${normalized}`);
+}
+
+function resolveProxyUrl(hostname) {
+    if (config.proxy?.use === false) {
+        return null;
+    }
+
+    const hosts = config.proxy?.hosts || {};
+    const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
+
+    if (!matchedPattern) {
+        return config.proxy?.default || null;
+    }
+
+    const matched = hosts[matchedPattern];
+
+    // `true` explicitly opts this host into `default`, `null`/`false`/'' explicitly opts it
+    // out (rather than falling through to `default`), anything else is a per-host override
+    if (matched === true) {
+        return config.proxy?.default || null;
+    }
+
+    return matched || null;
+}
+
+// 'socks4a'/'socks5h' (as opposed to plain 'socks4'/'socks5') tell socks-proxy-agent to
+// resolve the target hostname on the proxy side rather than locally - the same convention
+// curl uses, and worth keeping since it's the whole point of routing through SOCKS/Tor for
+// most people
+function isSocks(proxyUrl) {
+    return /^socks(4a?|5h?):/i.test(proxyUrl);
+}
+
+// undici (which node's built-in fetch() is implemented with) only tunnels through http(s)
+// proxies (CONNECT) via ProxyAgent - it has no built-in SOCKS support. agentFor() below can
+// still do SOCKS, since bhttp goes through node's plain http/https modules instead.
+function dispatcherFor(targetUrl) {
+    const { hostname } = new URL(targetUrl);
+    const proxyUrl = resolveProxyUrl(hostname);
+
+    if (!proxyUrl) {
+        return undefined;
+    }
+
+    if (isSocks(proxyUrl)) {
+        if (!warnedNoSocksForFetch.has(proxyUrl)) {
+            warnedNoSocksForFetch.add(proxyUrl);
+            logger.warn(`SOCKS proxy '${proxyUrl}' configured for '${hostname}' can't be used for fetch()-based requests (only http(s) proxies can) - these will go out directly instead. Item downloads (which use bhttp) can still use it.`);
+        }
+
+        return undefined;
+    }
+
+    if (!dispatcherCache.has(proxyUrl)) {
+        dispatcherCache.set(proxyUrl, new ProxyAgent(proxyUrl));
+    }
+
+    return dispatcherCache.get(proxyUrl);
+}
+
+// bhttp uses node's raw http/https modules under the hood, which take a plain Agent
+// instance - unlike fetch() above, this does support SOCKS
+function agentFor(targetUrl) {
+    const url = new URL(targetUrl);
+    const proxyUrl = resolveProxyUrl(url.hostname);
+
+    if (!proxyUrl) {
+        return undefined;
+    }
+
+    const cacheKey = `${proxyUrl}|${isSocks(proxyUrl) ? 'socks' : url.protocol}`;
+
+    if (!nodeAgentCache.has(cacheKey)) {
+        if (isSocks(proxyUrl)) {
+            nodeAgentCache.set(cacheKey, new SocksProxyAgent(proxyUrl));
+        } else if (url.protocol === 'https:') {
+            nodeAgentCache.set(cacheKey, new HttpsProxyAgent(proxyUrl));
+        } else {
+            nodeAgentCache.set(cacheKey, new HttpProxyAgent(proxyUrl));
+        }
+    }
+
+    return nodeAgentCache.get(cacheKey);
+}
+
+// drop-in replacement for the global fetch() - importing this as `fetch` transparently
+// proxies requests per config.proxy, if one is configured for the request's host
+async function proxiedFetch(url, options = {}) {
+    const dispatcher = dispatcherFor(String(url));
+
+    return fetch(url, dispatcher ? { ...options, dispatcher } : options);
+}
+
+module.exports = {
+    fetch: proxiedFetch,
+    agentFor,
+};

src/reddit/client.js

@@ -3,6 +3,7 @@
 const Bottleneck = require('bottleneck');
 const config = require('config');
 
+const { fetch } = require('../proxy');
 const { version } = require('../../package.json');
 const logger = require('../logger')(__filename);