From 941cb280beebd43e0233eb2b28f5b29945e56e11 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Mon, 17 Aug 2026 22:48:02 +0200 Subject: [PATCH] Added proxy. --- config/default.js | 15 ++++ package-lock.json | 134 ++++++++++++++++++++++------------- package.json | 9 +++ src/archives/ip.js | 2 + src/fetch/item.js | 3 +- src/methods/erome.js | 1 + src/methods/eroshareAlbum.js | 2 + src/methods/eroshareItem.js | 2 + src/methods/gfycat.js | 6 +- src/methods/imgurAlbum.js | 1 + src/methods/imgurImage.js | 2 + src/methods/redditVideo.js | 2 + src/methods/redgifs.js | 1 + src/methods/vidbleAlbum.js | 1 + src/methods/vidbleImage.js | 2 + src/methods/vidbleVideo.js | 2 + src/proxy.js | 118 ++++++++++++++++++++++++++++++ src/reddit/client.js | 1 + 18 files changed, 254 insertions(+), 50 deletions(-) create mode 100644 src/proxy.js diff --git a/config/default.js b/config/default.js index 7ca5f94..c4fecc5 100644 --- a/config/default.js +++ b/config/default.js @@ -99,4 +99,19 @@ module.exports = { clientId: null, }, }, + proxy: { + use: true, + default: null, + hosts: { + // hostnames are matched by suffix, so e.g. 'reddit.com' also covers + // 'www.reddit.com', but not 'v.redd.it' or 'redd.it' - those need their own + // entry, or fall back to `default`. A host entry can be: + // - a proxy URL, to override `default` for that host + // - `true`, to explicitly use `default` for that host + // - `null`/`false`, to explicitly exclude that host from `default` + // 'reddit.com': true, + // 'v.redd.it': null, + // 'imgur.com': 'socks5://127.0.0.1:9050', + }, + }, }; diff --git a/package-lock.json b/package-lock.json index aae3dda..3e6555f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,8 @@ "dist-exiftool": "^10.53.0", "fluent-ffmpeg": "^2.1.3", "fs-extra": "^11.4.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", "js-yaml": "^5.2.3", "mime": "^3.0.0", "mime-types": "^3.0.2", @@ -28,7 +30,9 @@ "object.omit": "^3.0.0", "object.pick": "^1.3.0", "sharp-phash": "^2.2.0", + "socks-proxy-agent": "^8.0.5", "template-format": "^1.2.5", + "undici": "^7.29.0", "unprint": "^0.19.20", "url-pattern": "^1.0.3", "winston": "^3.19.0", @@ -597,9 +601,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -617,9 +618,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -637,9 +635,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -657,9 +652,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -677,9 +669,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -697,9 +686,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -717,9 +703,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -737,9 +720,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -757,9 +737,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -783,9 +760,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -809,9 +783,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -835,9 +806,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -861,9 +829,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -887,9 +852,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -913,9 +875,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -939,9 +898,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1279,6 +1235,15 @@ "node": ">=0.4.0" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", @@ -4223,6 +4188,32 @@ "url": "https://opencollective.com/express" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/human-readable-ids": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/human-readable-ids/-/human-readable-ids-1.0.4.tgz", @@ -4318,6 +4309,15 @@ "node": ">= 0.4" } }, + "node_modules/ip-address": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", + "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -6601,6 +6601,44 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz", + "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.1.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index a0b9be6..486213c 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,8 @@ "dist-exiftool": "^10.53.0", "fluent-ffmpeg": "^2.1.3", "fs-extra": "^11.4.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", "js-yaml": "^5.2.3", "mime": "^3.0.0", "mime-types": "^3.0.2", @@ -50,7 +52,9 @@ "object.omit": "^3.0.0", "object.pick": "^1.3.0", "sharp-phash": "^2.2.0", + "socks-proxy-agent": "^8.0.5", "template-format": "^1.2.5", + "undici": "^7.29.0", "unprint": "^0.19.20", "url-pattern": "^1.0.3", "winston": "^3.19.0", @@ -63,5 +67,10 @@ "eslint": "^8.57.1", "eslint-config-airbnb-base": "^15.0.0", "eslint-plugin-import": "^2.32.0" + }, + "allowScripts": { + "blake2@5.0.1": true, + "dist-exiftool@10.53.0": true, + "youtube-dl-exec@3.1.12": true } } diff --git a/src/archives/ip.js b/src/archives/ip.js index 645bf1a..8a0ebb0 100644 --- a/src/archives/ip.js +++ b/src/archives/ip.js @@ -2,6 +2,8 @@ const $ = require('cheerio'); +const { fetch } = require('../proxy'); + function findOnIp(username, page = 1, acc = []) { return Promise.resolve().then(() => { return fetch(`https://www.imageporn.net/user/${username}/all/${page}`); diff --git a/src/fetch/item.js b/src/fetch/item.js index fe7982c..808ea08 100644 --- a/src/fetch/item.js +++ b/src/fetch/item.js @@ -6,6 +6,7 @@ const blake2 = require('blake2'); const phash = require('sharp-phash'); // const phashDistance = require('sharp-phash/distance'); +const { agentFor } = require('../proxy'); const logger = require('../logger')(__filename); const limiter = require('../limiter').items; @@ -22,7 +23,7 @@ async function fetchItem(url, attempt, context) { try { // throw new Error('Failed it!'); - const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers })); + const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers, agent: agentFor(url) })); if (res.statusCode !== 200) { throw new Error(`Response not OK for ${url} (${res.statusCode})`); diff --git a/src/methods/erome.js b/src/methods/erome.js index a23a887..211a85d 100644 --- a/src/methods/erome.js +++ b/src/methods/erome.js @@ -3,6 +3,7 @@ const config = require('config'); const cheerio = require('cheerio'); +const { fetch } = require('../proxy'); const logger = require('../logger')(__filename); const base = 'https://www.erome.com/'; diff --git a/src/methods/eroshareAlbum.js b/src/methods/eroshareAlbum.js index 680c81f..e8dbd32 100644 --- a/src/methods/eroshareAlbum.js +++ b/src/methods/eroshareAlbum.js @@ -2,6 +2,8 @@ const config = require('config'); +const { fetch } = require('../proxy'); + async function eroshareAlbum(host) { const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`); diff --git a/src/methods/eroshareItem.js b/src/methods/eroshareItem.js index 61ffd2c..1bd2372 100644 --- a/src/methods/eroshareItem.js +++ b/src/methods/eroshareItem.js @@ -2,6 +2,8 @@ const cheerio = require('cheerio'); +const { fetch } = require('../proxy'); + async function eroshareItem(host, post) { const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${host.id}`); if (!res.ok) { diff --git a/src/methods/gfycat.js b/src/methods/gfycat.js index 3d826e4..d622557 100644 --- a/src/methods/gfycat.js +++ b/src/methods/gfycat.js @@ -1,15 +1,19 @@ 'use strict'; const bhttp = require('bhttp'); + +const { agentFor } = require('../proxy'); const redgifs = require('./redgifs'); async function gfycat(host) { - const res = await bhttp.get(`https://api.gfycat.com/v1/gfycats/${host.id}`); + const infoUrl = `https://api.gfycat.com/v1/gfycats/${host.id}`; + const res = await bhttp.get(infoUrl, { agent: agentFor(infoUrl) }); const data = await res.body; if (data.errorMessage) { const redirectRes = await bhttp.head(host.url, { followRedirects: false, + agent: agentFor(host.url), }); if (redirectRes.statusCode === 301) { diff --git a/src/methods/imgurAlbum.js b/src/methods/imgurAlbum.js index e62356a..563e23d 100644 --- a/src/methods/imgurAlbum.js +++ b/src/methods/imgurAlbum.js @@ -2,6 +2,7 @@ const config = require('config'); +const { fetch } = require('../proxy'); const logger = require('../logger')(__filename); const { fetchPredata } = require('./imgurImage'); diff --git a/src/methods/imgurImage.js b/src/methods/imgurImage.js index f832c87..74f255e 100644 --- a/src/methods/imgurImage.js +++ b/src/methods/imgurImage.js @@ -2,6 +2,8 @@ const config = require('config'); +const { fetch } = require('../proxy'); + async function fetchPredata() { const data = { limit: 0, diff --git a/src/methods/redditVideo.js b/src/methods/redditVideo.js index 47c0708..b46e910 100644 --- a/src/methods/redditVideo.js +++ b/src/methods/redditVideo.js @@ -1,5 +1,7 @@ 'use strict'; +const { fetch } = require('../proxy'); + // the DASH manifest lists the actual audio track filename - reddit has changed this naming // scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the // manifest rather than guessing a hardcoded suffix off the video URL again diff --git a/src/methods/redgifs.js b/src/methods/redgifs.js index 71b7d0e..f622c45 100644 --- a/src/methods/redgifs.js +++ b/src/methods/redgifs.js @@ -3,6 +3,7 @@ const mime = require('mime'); // const unprint = require('unprint'); +const { fetch } = require('../proxy'); const { version } = require('../../package.json'); async function fetchPredata() { diff --git a/src/methods/vidbleAlbum.js b/src/methods/vidbleAlbum.js index a60f048..bc17a54 100644 --- a/src/methods/vidbleAlbum.js +++ b/src/methods/vidbleAlbum.js @@ -5,6 +5,7 @@ const UrlPattern = require('url-pattern'); const cheerio = require('cheerio'); const mime = require('mime-types'); +const { fetch } = require('../proxy'); const logger = require('../logger')(__filename); const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)'); diff --git a/src/methods/vidbleImage.js b/src/methods/vidbleImage.js index 3a1d65b..2811579 100644 --- a/src/methods/vidbleImage.js +++ b/src/methods/vidbleImage.js @@ -3,6 +3,8 @@ const $ = require('cheerio'); const mime = require('mime-types'); +const { fetch } = require('../proxy'); + async function vidbleImage(host, post) { const res = await fetch(`https://vidble.com/${host.id}`); diff --git a/src/methods/vidbleVideo.js b/src/methods/vidbleVideo.js index 9910494..5ad8248 100644 --- a/src/methods/vidbleVideo.js +++ b/src/methods/vidbleVideo.js @@ -2,6 +2,8 @@ const cheerio = require('cheerio'); +const { fetch } = require('../proxy'); + async function vidbleVideo(host, post) { const res = await fetch(`https://www.vidble.com/watch?v=${host.id}`); diff --git a/src/proxy.js b/src/proxy.js new file mode 100644 index 0000000..15263fd --- /dev/null +++ b/src/proxy.js @@ -0,0 +1,118 @@ +'use strict'; + +const config = require('config'); +const { ProxyAgent } = require('undici'); +const { HttpProxyAgent } = require('http-proxy-agent'); +const { HttpsProxyAgent } = require('https-proxy-agent'); +const { SocksProxyAgent } = require('socks-proxy-agent'); + +const logger = require('./logger')(__filename); + +const dispatcherCache = new Map(); +const nodeAgentCache = new Map(); +const warnedNoSocksForFetch = new Set(); + +// same suffix matching as config.reddit.cookies' domain filter (src/reddit/client.js): a +// bare hostname also matches its subdomains, so 'reddit.com' catches 'www.reddit.com' but +// not 'v.redd.it' or 'redd.it' - those need their own entry, or fall back to `default` +function hostMatches(hostname, pattern) { + const normalized = pattern.replace(/^\./, '').toLowerCase(); + + return hostname === normalized || hostname.endsWith(`.${normalized}`); +} + +function resolveProxyUrl(hostname) { + if (config.proxy?.use === false) { + return null; + } + + const hosts = config.proxy?.hosts || {}; + const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern)); + + if (!matchedPattern) { + return config.proxy?.default || null; + } + + const matched = hosts[matchedPattern]; + + // `true` explicitly opts this host into `default`, `null`/`false`/'' explicitly opts it + // out (rather than falling through to `default`), anything else is a per-host override + if (matched === true) { + return config.proxy?.default || null; + } + + return matched || null; +} + +// 'socks4a'/'socks5h' (as opposed to plain 'socks4'/'socks5') tell socks-proxy-agent to +// resolve the target hostname on the proxy side rather than locally - the same convention +// curl uses, and worth keeping since it's the whole point of routing through SOCKS/Tor for +// most people +function isSocks(proxyUrl) { + return /^socks(4a?|5h?):/i.test(proxyUrl); +} + +// undici (which node's built-in fetch() is implemented with) only tunnels through http(s) +// proxies (CONNECT) via ProxyAgent - it has no built-in SOCKS support. agentFor() below can +// still do SOCKS, since bhttp goes through node's plain http/https modules instead. +function dispatcherFor(targetUrl) { + const { hostname } = new URL(targetUrl); + const proxyUrl = resolveProxyUrl(hostname); + + if (!proxyUrl) { + return undefined; + } + + if (isSocks(proxyUrl)) { + if (!warnedNoSocksForFetch.has(proxyUrl)) { + warnedNoSocksForFetch.add(proxyUrl); + logger.warn(`SOCKS proxy '${proxyUrl}' configured for '${hostname}' can't be used for fetch()-based requests (only http(s) proxies can) - these will go out directly instead. Item downloads (which use bhttp) can still use it.`); + } + + return undefined; + } + + if (!dispatcherCache.has(proxyUrl)) { + dispatcherCache.set(proxyUrl, new ProxyAgent(proxyUrl)); + } + + return dispatcherCache.get(proxyUrl); +} + +// bhttp uses node's raw http/https modules under the hood, which take a plain Agent +// instance - unlike fetch() above, this does support SOCKS +function agentFor(targetUrl) { + const url = new URL(targetUrl); + const proxyUrl = resolveProxyUrl(url.hostname); + + if (!proxyUrl) { + return undefined; + } + + const cacheKey = `${proxyUrl}|${isSocks(proxyUrl) ? 'socks' : url.protocol}`; + + if (!nodeAgentCache.has(cacheKey)) { + if (isSocks(proxyUrl)) { + nodeAgentCache.set(cacheKey, new SocksProxyAgent(proxyUrl)); + } else if (url.protocol === 'https:') { + nodeAgentCache.set(cacheKey, new HttpsProxyAgent(proxyUrl)); + } else { + nodeAgentCache.set(cacheKey, new HttpProxyAgent(proxyUrl)); + } + } + + return nodeAgentCache.get(cacheKey); +} + +// drop-in replacement for the global fetch() - importing this as `fetch` transparently +// proxies requests per config.proxy, if one is configured for the request's host +async function proxiedFetch(url, options = {}) { + const dispatcher = dispatcherFor(String(url)); + + return fetch(url, dispatcher ? { ...options, dispatcher } : options); +} + +module.exports = { + fetch: proxiedFetch, + agentFor, +}; diff --git a/src/reddit/client.js b/src/reddit/client.js index fb9181c..b9e1f6b 100644 --- a/src/reddit/client.js +++ b/src/reddit/client.js @@ -3,6 +3,7 @@ const Bottleneck = require('bottleneck'); const config = require('config'); +const { fetch } = require('../proxy'); const { version } = require('../../package.json'); const logger = require('../logger')(__filename);