Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 DebaucheryLibrarian
|
e23d8becd4 | ||
|
DebaucheryLibrarian
|
1496a48199 | ||
|
DebaucheryLibrarian
|
8b52d73a65 | ||
|
DebaucheryLibrarian
|
3e8ed79ccc | ||
|
DebaucheryLibrarian
|
ba922bb385 | ||
|
DebaucheryLibrarian
|
941cb280be |
@@ -60,6 +60,13 @@ module.exports = {
|
||||
avoidDuplicates: true,
|
||||
retries: 3,
|
||||
concurrency: 10,
|
||||
// wall-clock ceiling (ms) for a single item-download attempt (src/fetch/item.js).
|
||||
// bhttp has no timeout of its own - a request that connects but then just stops
|
||||
// responding (a stalled/misbehaving proxy tunnel is a common cause) would otherwise
|
||||
// hang forever, and since the limiter's concurrency is capped, that one stuck request
|
||||
// blocks every other download queued behind it indefinitely. This bounds each attempt
|
||||
// so a stall fails (and retries, per `retries` above) instead of hanging forever.
|
||||
timeout: 60000,
|
||||
watch: {
|
||||
schedule: '*/30 * * * *',
|
||||
},
|
||||
@@ -74,8 +81,19 @@ module.exports = {
|
||||
level: 'info',
|
||||
},
|
||||
limiter: {
|
||||
// default concurrency/interval for item/media downloads (src/fetch/item.js) - applies
|
||||
// to any host without its own entry in `hosts` below
|
||||
concurrency: 10,
|
||||
interval: 100,
|
||||
hosts: {
|
||||
// per-host overrides of `concurrency`/`interval` above, for hosts that can safely
|
||||
// be hit harder (most media CDNs) or need to be treated more cautiously than the
|
||||
// default. Hostnames are matched by suffix, same as proxy.hosts above - a bare
|
||||
// 'redgifs.com' also covers 'api.redgifs.com' and 'media.redgifs.com'. Unlike a
|
||||
// single shared limiter, each entry here runs independently: a host that's
|
||||
// currently stalling or being throttled doesn't block any other host's downloads.
|
||||
// 'redgifs.com': { concurrency: 20, interval: 50 },
|
||||
},
|
||||
},
|
||||
reddit: {
|
||||
// Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to
|
||||
@@ -99,4 +117,26 @@ module.exports = {
|
||||
clientId: null,
|
||||
},
|
||||
},
|
||||
proxy: {
|
||||
use: true,
|
||||
default: null,
|
||||
// if a request fails (including timing out - see fetch.timeout above) and is about to
|
||||
// be retried, retry it using whichever of proxied/direct is NOT what would normally
|
||||
// apply for that host - e.g. the proxy itself is down or being blocked by the target
|
||||
// site, or (the reverse) a host reachable directly is refusing unproxied traffic.
|
||||
// Off by default: for a privacy-sensitive proxy (e.g. Tor) silently falling back to a
|
||||
// direct connection on failure is very much not what you want.
|
||||
retryFlipped: false,
|
||||
hosts: {
|
||||
// hostnames are matched by suffix, so e.g. 'reddit.com' also covers
|
||||
// 'www.reddit.com', but not 'v.redd.it' or 'redd.it' - those need their own
|
||||
// entry, or fall back to `default`. A host entry can be:
|
||||
// - a proxy URL, to override `default` for that host
|
||||
// - `true`, to explicitly use `default` for that host
|
||||
// - `null`/`false`, to explicitly exclude that host from `default`
|
||||
// 'reddit.com': true,
|
||||
// 'v.redd.it': null,
|
||||
// 'imgur.com': 'socks5://127.0.0.1:9050',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
Generated
+86
-48
@@ -20,6 +20,8 @@
|
||||
"dist-exiftool": "^10.53.0",
|
||||
"fluent-ffmpeg": "^2.1.3",
|
||||
"fs-extra": "^11.4.0",
|
||||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"js-yaml": "^5.2.3",
|
||||
"mime": "^3.0.0",
|
||||
"mime-types": "^3.0.2",
|
||||
@@ -28,7 +30,9 @@
|
||||
"object.omit": "^3.0.0",
|
||||
"object.pick": "^1.3.0",
|
||||
"sharp-phash": "^2.2.0",
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"template-format": "^1.2.5",
|
||||
"undici": "^7.29.0",
|
||||
"unprint": "^0.19.20",
|
||||
"url-pattern": "^1.0.3",
|
||||
"winston": "^3.19.0",
|
||||
@@ -597,9 +601,6 @@
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -617,9 +618,6 @@
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -637,9 +635,6 @@
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -657,9 +652,6 @@
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -677,9 +669,6 @@
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -697,9 +686,6 @@
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -717,9 +703,6 @@
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -737,9 +720,6 @@
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "LGPL-3.0-or-later",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -757,9 +737,6 @@
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -783,9 +760,6 @@
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -809,9 +783,6 @@
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -835,9 +806,6 @@
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -861,9 +829,6 @@
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -887,9 +852,6 @@
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -913,9 +875,6 @@
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -939,9 +898,6 @@
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -1279,6 +1235,15 @@
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "7.1.4",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
|
||||
"integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/ajv": {
|
||||
"version": "6.15.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz",
|
||||
@@ -4223,6 +4188,32 @@
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/http-proxy-agent": {
|
||||
"version": "7.0.2",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
|
||||
"integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.0",
|
||||
"debug": "^4.3.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/https-proxy-agent": {
|
||||
"version": "7.0.6",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
|
||||
"integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.2",
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/human-readable-ids": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/human-readable-ids/-/human-readable-ids-1.0.4.tgz",
|
||||
@@ -4318,6 +4309,15 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/ip-address": {
|
||||
"version": "10.5.0",
|
||||
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz",
|
||||
"integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
}
|
||||
},
|
||||
"node_modules/ipaddr.js": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
|
||||
@@ -6601,6 +6601,44 @@
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/smart-buffer": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
|
||||
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 6.0.0",
|
||||
"npm": ">= 3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/socks": {
|
||||
"version": "2.8.9",
|
||||
"resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz",
|
||||
"integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ip-address": "^10.1.1",
|
||||
"smart-buffer": "^4.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 10.0.0",
|
||||
"npm": ">= 3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/socks-proxy-agent": {
|
||||
"version": "8.0.5",
|
||||
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz",
|
||||
"integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.2",
|
||||
"debug": "^4.3.4",
|
||||
"socks": "^2.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
|
||||
|
||||
@@ -42,6 +42,8 @@
|
||||
"dist-exiftool": "^10.53.0",
|
||||
"fluent-ffmpeg": "^2.1.3",
|
||||
"fs-extra": "^11.4.0",
|
||||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"js-yaml": "^5.2.3",
|
||||
"mime": "^3.0.0",
|
||||
"mime-types": "^3.0.2",
|
||||
@@ -50,7 +52,9 @@
|
||||
"object.omit": "^3.0.0",
|
||||
"object.pick": "^1.3.0",
|
||||
"sharp-phash": "^2.2.0",
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"template-format": "^1.2.5",
|
||||
"undici": "^7.29.0",
|
||||
"unprint": "^0.19.20",
|
||||
"url-pattern": "^1.0.3",
|
||||
"winston": "^3.19.0",
|
||||
@@ -63,5 +67,10 @@
|
||||
"eslint": "^8.57.1",
|
||||
"eslint-config-airbnb-base": "^15.0.0",
|
||||
"eslint-plugin-import": "^2.32.0"
|
||||
},
|
||||
"allowScripts": {
|
||||
"blake2@5.0.1": true,
|
||||
"dist-exiftool@10.53.0": true,
|
||||
"youtube-dl-exec@3.1.12": true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
const $ = require('cheerio');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
|
||||
function findOnIp(username, page = 1, acc = []) {
|
||||
return Promise.resolve().then(() => {
|
||||
return fetch(`https://www.imageporn.net/user/${username}/all/${page}`);
|
||||
|
||||
@@ -62,6 +62,12 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
|
||||
preview: post.preview ? post.preview.images.map((image) => image.source) : null,
|
||||
host,
|
||||
direct: post.direct,
|
||||
// reddit's submission listing (src/reddit/client.js's getSubmissions, fetched once up
|
||||
// front) already embeds these for gallery posts - carrying them through lets
|
||||
// methods/redditAlbum.js use them directly instead of re-fetching the same post
|
||||
// individually through reddit's own separately-throttled client (see there)
|
||||
mediaMetadata: post.media_metadata || null,
|
||||
galleryData: post.gallery_data || null,
|
||||
// count only - the full thread (when needed) is fetched separately via methods/self.js
|
||||
comments: post.num_comments,
|
||||
hash: hashPost(post),
|
||||
|
||||
+4
-1
@@ -1,6 +1,6 @@
|
||||
'use strict';
|
||||
|
||||
function curateUser(user) {
|
||||
function curateUser(user, socialLinks = []) {
|
||||
const curatedUser = {
|
||||
id: user.id,
|
||||
name: user.name,
|
||||
@@ -21,6 +21,9 @@ function curateUser(user) {
|
||||
banner: user.subreddit.banner_img,
|
||||
description: user.subreddit.public_description,
|
||||
over18: user.subreddit.over_18,
|
||||
// the "Social Links" widget on their profile (Twitter/Instagram/OnlyFans/etc.) -
|
||||
// see reddit/client.js's getSocialLinks()
|
||||
links: socialLinks,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
+21
-2
@@ -6,8 +6,9 @@ const blake2 = require('blake2');
|
||||
const phash = require('sharp-phash');
|
||||
// const phashDistance = require('sharp-phash/distance');
|
||||
|
||||
const { agentFor } = require('../proxy');
|
||||
const logger = require('../logger')(__filename);
|
||||
const limiter = require('../limiter').items;
|
||||
const limiterFor = require('../limiter');
|
||||
|
||||
async function fetchItem(url, attempt, context) {
|
||||
async function retry(error) {
|
||||
@@ -22,7 +23,25 @@ async function fetchItem(url, attempt, context) {
|
||||
|
||||
try {
|
||||
// throw new Error('Failed it!');
|
||||
const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers }));
|
||||
// config.proxy.retryFlipped: once this url has already failed once, retry it via
|
||||
// whichever of proxied/direct is NOT what it'd normally get - see resolveProxyUrl()
|
||||
const flip = config.proxy.retryFlipped && attempt > 0;
|
||||
|
||||
const res = await limiterFor(url).schedule(() => bhttp.get(url, {
|
||||
headers: context.headers,
|
||||
agent: agentFor(url, { flip }),
|
||||
// bhttp has no timeout at all by default - a server (or, more likely, a proxy
|
||||
// tunnel) that connects fine but then just stops responding would otherwise hang
|
||||
// forever. This isn't just a wall-clock ceiling on the *promise* (an external
|
||||
// Promise.race achieves that, but leaves the real connection dangling) - bhttp
|
||||
// calls req.abort() itself when this fires, actually releasing the socket. Without
|
||||
// that, every stalled attempt leaks one connection onto the shared agent for that
|
||||
// host; enough of those piling up (e.g. after several retries against a host that's
|
||||
// currently black-holing everything, as seen with i.redd.it) exhausts it, and every
|
||||
// *future* request to that host silently queues forever waiting for a socket that's
|
||||
// never coming back - a freeze with no further log lines, not just a slow retry.
|
||||
responseTimeout: config.fetch.timeout,
|
||||
}));
|
||||
|
||||
if (res.statusCode !== 200) {
|
||||
throw new Error(`Response not OK for ${url} (${res.statusCode})`);
|
||||
|
||||
+12
-3
@@ -27,6 +27,17 @@ function safeFormat(date, dateFormat, label) {
|
||||
return dateFns.format(date, dateFormat);
|
||||
}
|
||||
|
||||
// mime.extension() returns the boolean `false` (not undefined/null) for a mime type it
|
||||
// doesn't recognize - reddit's gallery API is a known offender, sending the non-standard
|
||||
// 'image/jpg' instead of 'image/jpeg' - which without this check ends up interpolated
|
||||
// verbatim as the literal filename extension '.false'. Fall back to whatever extension the
|
||||
// URL itself has instead, same as when there's no type at all.
|
||||
function resolveExt(item) {
|
||||
const extFromType = item.type && mime.extension(item.type);
|
||||
|
||||
return extFromType ? `.${extFromType}` : path.extname(url.parse(item.url).pathname);
|
||||
}
|
||||
|
||||
function interpolate(pattern, item = null, content = null, host = null, post = null, user = null, strip = true, dateFormat = config.library.dateFormat) {
|
||||
const data = {
|
||||
tags: {},
|
||||
@@ -41,9 +52,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
|
||||
date: safeFormat(item.datetime, dateFormat, 'item'),
|
||||
index: item.index + config.library.indexOffset,
|
||||
},
|
||||
ext: item.type
|
||||
? `.${mime.extension(item.type)}`
|
||||
: path.extname(url.parse(item.url).pathname),
|
||||
ext: resolveExt(item),
|
||||
tags: {
|
||||
...data.tags,
|
||||
extracted: item.extracted && config.library.tags.extracted,
|
||||
|
||||
+44
-7
@@ -1,13 +1,50 @@
|
||||
'use strict';
|
||||
|
||||
const Bottleneck = require('bottleneck');
|
||||
const config = require('config');
|
||||
|
||||
const args = require('./cli')();
|
||||
|
||||
module.exports = {
|
||||
// reddit requests are throttled inside src/reddit/client.js itself (config.reddit.throttle)
|
||||
items: new Bottleneck({
|
||||
maxConcurrent: args.concurrency,
|
||||
minTime: args.interval,
|
||||
}),
|
||||
};
|
||||
// same suffix matching as config.proxy.hosts (src/proxy.js): a bare hostname also matches its
|
||||
// subdomains, so 'redgifs.com' catches 'media.redgifs.com' but not the reverse
|
||||
function hostMatches(hostname, pattern) {
|
||||
const normalized = pattern.replace(/^\./, '').toLowerCase();
|
||||
|
||||
return hostname === normalized || hostname.endsWith(`.${normalized}`);
|
||||
}
|
||||
|
||||
// reddit's own API requests are throttled separately, inside src/reddit/client.js
|
||||
// (config.reddit.throttle) - this only governs item/media downloads (src/fetch/item.js)
|
||||
const defaultLimiter = new Bottleneck({
|
||||
maxConcurrent: args.concurrency,
|
||||
minTime: args.interval,
|
||||
});
|
||||
|
||||
const hostLimiters = new Map();
|
||||
|
||||
// config.limiter.hosts: most media CDNs (redgifs, imgur, i.redd.it, ...) don't need to be
|
||||
// treated as cautiously as reddit's own API, but a single shared limiter forces every host
|
||||
// down to whatever pace the most conservative one requires - worse, it means one host that's
|
||||
// currently stalling/throttling (see src/fetch/item.js's timeout handling) occupies the one
|
||||
// shared slot and blocks every *other* host's downloads behind it too, even though they have
|
||||
// nothing to do with each other. Giving a host its own limiter here lets it run at its own
|
||||
// concurrency/interval, independently of every other host's.
|
||||
function limiterFor(url) {
|
||||
const { hostname } = new URL(url);
|
||||
const hosts = config.limiter?.hosts || {};
|
||||
const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
|
||||
|
||||
if (!matchedPattern) {
|
||||
return defaultLimiter;
|
||||
}
|
||||
|
||||
if (!hostLimiters.has(matchedPattern)) {
|
||||
const { concurrency = args.concurrency, interval = args.interval } = hosts[matchedPattern];
|
||||
|
||||
hostLimiters.set(matchedPattern, new Bottleneck({ maxConcurrent: concurrency, minTime: interval }));
|
||||
}
|
||||
|
||||
return hostLimiters.get(matchedPattern);
|
||||
}
|
||||
|
||||
module.exports = limiterFor;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
const config = require('config');
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
const logger = require('../logger')(__filename);
|
||||
|
||||
const base = 'https://www.erome.com/';
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
const config = require('config');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
|
||||
async function eroshareAlbum(host) {
|
||||
const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
|
||||
async function eroshareItem(host, post) {
|
||||
const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${host.id}`);
|
||||
if (!res.ok) {
|
||||
|
||||
@@ -1,15 +1,19 @@
|
||||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
|
||||
const { agentFor } = require('../proxy');
|
||||
const redgifs = require('./redgifs');
|
||||
|
||||
async function gfycat(host) {
|
||||
const res = await bhttp.get(`https://api.gfycat.com/v1/gfycats/${host.id}`);
|
||||
const infoUrl = `https://api.gfycat.com/v1/gfycats/${host.id}`;
|
||||
const res = await bhttp.get(infoUrl, { agent: agentFor(infoUrl) });
|
||||
const data = await res.body;
|
||||
|
||||
if (data.errorMessage) {
|
||||
const redirectRes = await bhttp.head(host.url, {
|
||||
followRedirects: false,
|
||||
agent: agentFor(host.url),
|
||||
});
|
||||
|
||||
if (redirectRes.statusCode === 301) {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
const config = require('config');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
const logger = require('../logger')(__filename);
|
||||
const { fetchPredata } = require('./imgurImage');
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
const config = require('config');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
|
||||
async function fetchPredata() {
|
||||
const data = {
|
||||
limit: 0,
|
||||
|
||||
@@ -8,22 +8,47 @@ function resolveMediaUrl(media) {
|
||||
return url ? url.replace(/&/g, '&') : null;
|
||||
}
|
||||
|
||||
async function redditAlbum(host, post, { reddit }) {
|
||||
// fetched through the authenticated/throttled client, not a raw fetch()/bhttp request -
|
||||
// reddit's anonymous-traffic bot challenge blocks plain unauthenticated .json requests
|
||||
const data = await reddit.getSubmission(post.id);
|
||||
// reddit sends the non-standard 'image/jpg' instead of the real IANA 'image/jpeg' for
|
||||
// gallery items - left as-is, this silently breaks anything matching against real mime
|
||||
// types downstream (interpolate.js's extension lookup ends up with a '.false' extension,
|
||||
// fetch/content.js's addMeta() skips embedding metadata since it only matches 'image/jpeg')
|
||||
function normalizeType(type) {
|
||||
return type === 'image/jpg' ? 'image/jpeg' : type;
|
||||
}
|
||||
|
||||
const items = (data.gallery_data?.items || [])
|
||||
async function redditAlbum(host, post, { reddit }) {
|
||||
// reddit's submission listing already embeds media_metadata/gallery_data for gallery posts,
|
||||
// and curate/posts.js carries them through as post.mediaMetadata/post.galleryData - reuse
|
||||
// those instead of re-fetching the same post individually through reddit's own client
|
||||
// (reservoir: 6/min, maxConcurrent: 1 - see src/reddit/client.js), which would otherwise cost
|
||||
// one full throttled round trip per gallery post. Only fall back to a fresh fetch if the
|
||||
// listing didn't have it for some reason (e.g. gallery_data null on the listing too).
|
||||
const data = post.mediaMetadata
|
||||
? { media_metadata: post.mediaMetadata, gallery_data: post.galleryData }
|
||||
: await reddit.getSubmission(post.id);
|
||||
const mediaMetadata = data.media_metadata || {};
|
||||
|
||||
// gallery_data (the ordered {media_id, is_deleted, ...} list for this gallery) has been
|
||||
// seen coming back null on a freshly-posted gallery even though media_metadata itself
|
||||
// was already fully populated - fall back to media_metadata's own keys, in the order the
|
||||
// JSON provided them (reddit sends them in gallery order, and object key order is
|
||||
// preserved through JSON.parse), skipping anything not (yet, or any longer) valid
|
||||
const galleryItems = data.gallery_data?.items
|
||||
|| Object.entries(mediaMetadata)
|
||||
.filter(([, media]) => media.status === 'valid')
|
||||
.map(([mediaId]) => ({ media_id: mediaId }));
|
||||
|
||||
const items = galleryItems
|
||||
.filter((item) => !item.is_deleted)
|
||||
.map((item) => {
|
||||
const media = data.media_metadata?.[item.media_id];
|
||||
const media = mediaMetadata[item.media_id];
|
||||
const url = resolveMediaUrl(media);
|
||||
|
||||
return url && {
|
||||
id: item.media_id,
|
||||
url,
|
||||
datetime: post.datetime,
|
||||
type: media.m || 'image/jpeg',
|
||||
type: normalizeType(media.m) || 'image/jpeg',
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
|
||||
// the DASH manifest lists the actual audio track filename - reddit has changed this naming
|
||||
// scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the
|
||||
// manifest rather than guessing a hardcoded suffix off the video URL again
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
const mime = require('mime');
|
||||
// const unprint = require('unprint');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
const { version } = require('../../package.json');
|
||||
|
||||
async function fetchPredata() {
|
||||
|
||||
@@ -5,6 +5,7 @@ const UrlPattern = require('url-pattern');
|
||||
const cheerio = require('cheerio');
|
||||
const mime = require('mime-types');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
const logger = require('../logger')(__filename);
|
||||
|
||||
const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
const $ = require('cheerio');
|
||||
const mime = require('mime-types');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
|
||||
async function vidbleImage(host, post) {
|
||||
const res = await fetch(`https://vidble.com/${host.id}`);
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
|
||||
async function vidbleVideo(host, post) {
|
||||
const res = await fetch(`https://www.vidble.com/watch?v=${host.id}`);
|
||||
|
||||
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const { ProxyAgent } = require('undici');
|
||||
const { HttpProxyAgent } = require('http-proxy-agent');
|
||||
const { HttpsProxyAgent } = require('https-proxy-agent');
|
||||
const { SocksProxyAgent } = require('socks-proxy-agent');
|
||||
|
||||
const logger = require('./logger')(__filename);
|
||||
|
||||
const dispatcherCache = new Map();
|
||||
const nodeAgentCache = new Map();
|
||||
const warnedNoSocksForFetch = new Set();
|
||||
|
||||
// same suffix matching as config.reddit.cookies' domain filter (src/reddit/client.js): a
|
||||
// bare hostname also matches its subdomains, so 'reddit.com' catches 'www.reddit.com' but
|
||||
// not 'v.redd.it' or 'redd.it' - those need their own entry, or fall back to `default`
|
||||
function hostMatches(hostname, pattern) {
|
||||
const normalized = pattern.replace(/^\./, '').toLowerCase();
|
||||
|
||||
return hostname === normalized || hostname.endsWith(`.${normalized}`);
|
||||
}
|
||||
|
||||
function resolveProxyUrl(hostname, { flip = false } = {}) {
|
||||
if (config.proxy?.use === false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const hosts = config.proxy?.hosts || {};
|
||||
const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
|
||||
|
||||
const normal = (() => {
|
||||
if (!matchedPattern) {
|
||||
return config.proxy?.default || null;
|
||||
}
|
||||
|
||||
const matched = hosts[matchedPattern];
|
||||
|
||||
// `true` explicitly opts this host into `default`, `null`/`false`/'' explicitly opts it
|
||||
// out (rather than falling through to `default`), anything else is a per-host override
|
||||
if (matched === true) {
|
||||
return config.proxy?.default || null;
|
||||
}
|
||||
|
||||
return matched || null;
|
||||
})();
|
||||
|
||||
if (!flip) {
|
||||
return normal;
|
||||
}
|
||||
|
||||
// used for config.proxy.retryFlipped: retry via whichever of proxied/direct is NOT what
|
||||
// this host would normally get. A request that would normally be proxied (whether via
|
||||
// `default` or a per-host override) retries direct; a request that would normally go
|
||||
// direct (no proxy configured for this host, or explicitly excluded from `default`)
|
||||
// retries through `default`, if one is set.
|
||||
return normal ? null : (config.proxy?.default || null);
|
||||
}
|
||||
|
||||
// 'socks4a'/'socks5h' (as opposed to plain 'socks4'/'socks5') tell socks-proxy-agent to
|
||||
// resolve the target hostname on the proxy side rather than locally - the same convention
|
||||
// curl uses, and worth keeping since it's the whole point of routing through SOCKS/Tor for
|
||||
// most people
|
||||
function isSocks(proxyUrl) {
|
||||
return /^socks(4a?|5h?):/i.test(proxyUrl);
|
||||
}
|
||||
|
||||
// undici (which node's built-in fetch() is implemented with) only tunnels through http(s)
|
||||
// proxies (CONNECT) via ProxyAgent - it has no built-in SOCKS support. agentFor() below can
|
||||
// still do SOCKS, since bhttp goes through node's plain http/https modules instead.
|
||||
function dispatcherFor(targetUrl, { flip = false } = {}) {
|
||||
const { hostname } = new URL(targetUrl);
|
||||
const proxyUrl = resolveProxyUrl(hostname, { flip });
|
||||
|
||||
if (!proxyUrl) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (isSocks(proxyUrl)) {
|
||||
if (!warnedNoSocksForFetch.has(proxyUrl)) {
|
||||
warnedNoSocksForFetch.add(proxyUrl);
|
||||
logger.warn(`SOCKS proxy '${proxyUrl}' configured for '${hostname}' can't be used for fetch()-based requests (only http(s) proxies can) - these will go out directly instead. Item downloads (which use bhttp) can still use it.`);
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!dispatcherCache.has(proxyUrl)) {
|
||||
dispatcherCache.set(proxyUrl, new ProxyAgent(proxyUrl));
|
||||
}
|
||||
|
||||
return dispatcherCache.get(proxyUrl);
|
||||
}
|
||||
|
||||
// bhttp uses node's raw http/https modules under the hood, which take a plain Agent
|
||||
// instance - unlike fetch() above, this does support SOCKS
|
||||
function agentFor(targetUrl, { flip = false } = {}) {
|
||||
const url = new URL(targetUrl);
|
||||
const proxyUrl = resolveProxyUrl(url.hostname, { flip });
|
||||
|
||||
if (!proxyUrl) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const cacheKey = `${proxyUrl}|${isSocks(proxyUrl) ? 'socks' : url.protocol}`;
|
||||
|
||||
if (!nodeAgentCache.has(cacheKey)) {
|
||||
if (isSocks(proxyUrl)) {
|
||||
nodeAgentCache.set(cacheKey, new SocksProxyAgent(proxyUrl));
|
||||
} else if (url.protocol === 'https:') {
|
||||
nodeAgentCache.set(cacheKey, new HttpsProxyAgent(proxyUrl));
|
||||
} else {
|
||||
nodeAgentCache.set(cacheKey, new HttpProxyAgent(proxyUrl));
|
||||
}
|
||||
}
|
||||
|
||||
return nodeAgentCache.get(cacheKey);
|
||||
}
|
||||
|
||||
// drop-in replacement for the global fetch() - importing this as `fetch` transparently
|
||||
// proxies requests per config.proxy, if one is configured for the request's host. Also
|
||||
// applies a hard wall-clock ceiling (config.fetch.timeout) to every request: unlike bhttp,
|
||||
// fetch() doesn't hang *forever* on a stalled/black-holed connection (undici defaults to a
|
||||
// 300s headers/body timeout), but that default is high enough - and, per live testing, a
|
||||
// proxy CONNECT tunnel that never completes can still take well over 20s to surface any
|
||||
// error - to look like a freeze, especially since reddit/client.js's requestJson() serializes
|
||||
// all reddit API calls behind a single-concurrency limiter. Bounding it here covers every
|
||||
// fetch()-based call site (reddit's API, and every hosting site's info-lookup) at once.
|
||||
// Pass `proxyFlip: true` in options (stripped before reaching fetch()) to use whichever of
|
||||
// proxied/direct is NOT what this host would normally get - see resolveProxyUrl().
|
||||
async function proxiedFetch(url, options = {}) {
|
||||
const { proxyFlip, signal, ...fetchOptions } = options;
|
||||
const dispatcher = dispatcherFor(String(url), { flip: proxyFlip });
|
||||
const timeoutSignal = AbortSignal.timeout(config.fetch.timeout);
|
||||
|
||||
return fetch(url, {
|
||||
...fetchOptions,
|
||||
...(dispatcher ? { dispatcher } : {}),
|
||||
signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal,
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetch: proxiedFetch,
|
||||
agentFor,
|
||||
};
|
||||
+126
-36
@@ -1,8 +1,10 @@
|
||||
'use strict';
|
||||
|
||||
const Bottleneck = require('bottleneck');
|
||||
const cheerio = require('cheerio');
|
||||
const config = require('config');
|
||||
|
||||
const { fetch } = require('../proxy');
|
||||
const { version } = require('../../package.json');
|
||||
const logger = require('../logger')(__filename);
|
||||
|
||||
@@ -74,10 +76,10 @@ function cookieHeader() {
|
||||
return cachedCookieHeader;
|
||||
}
|
||||
|
||||
function headers() {
|
||||
function headers(accept = 'application/json') {
|
||||
const requestHeaders = {
|
||||
'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
|
||||
accept: 'application/json',
|
||||
accept,
|
||||
};
|
||||
|
||||
const cookie = cookieHeader();
|
||||
@@ -107,46 +109,133 @@ async function wait(seconds) {
|
||||
});
|
||||
}
|
||||
|
||||
async function requestJson(path, params = {}, attempt = 0) {
|
||||
// shared by requestJson() and requestHtml() below - handles the actual throttled network
|
||||
// call plus retrying on a network error or a 429/503, regardless of what kind of response
|
||||
// the caller ultimately wants back.
|
||||
//
|
||||
// Only the actual throttled network call belongs inside limiter.schedule() - this has
|
||||
// maxConcurrent: 1, and every retry path below used to recurse back into this function
|
||||
// (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't
|
||||
// start a second job while the first is still running, and the first can't finish until
|
||||
// its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever
|
||||
// needed (e.g. a single 429), freezing every reddit request behind it for good. Retrying
|
||||
// now happens here, after the scheduled call has actually returned and freed its slot.
|
||||
async function fetchWithRetry(path, params, attempt, accept) {
|
||||
const url = buildUrl(path, params);
|
||||
let res;
|
||||
|
||||
return limiter.schedule(async () => {
|
||||
const res = await fetch(url.toString(), { headers: headers() });
|
||||
const contentType = res.headers.get('content-type') || '';
|
||||
try {
|
||||
// config.proxy.retryFlipped: once this request has already failed once, retry it via
|
||||
// whichever of proxied/direct is NOT what it'd normally get - see proxy.js
|
||||
res = await limiter.schedule(async () => fetch(url.toString(), {
|
||||
headers: headers(accept),
|
||||
proxyFlip: config.proxy.retryFlipped && attempt > 0,
|
||||
}));
|
||||
} catch (error) {
|
||||
// fetch() has no timeout of its own beyond undici's fairly generous defaults (see
|
||||
// proxy.js) - this catches both that and genuine network errors (DNS, connection
|
||||
// refused/reset, etc.), neither of which was retried before
|
||||
if (attempt >= MAX_RETRIES) {
|
||||
throw new Error(`Failed to reach reddit after ${attempt} retries for '${path}': ${error.message}`);
|
||||
}
|
||||
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
if (attempt >= MAX_RETRIES) {
|
||||
throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
|
||||
logger.warn(`Failed to reach reddit for '${path}', retrying: ${error.message}`);
|
||||
|
||||
return fetchWithRetry(path, params, attempt + 1, accept);
|
||||
}
|
||||
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
if (attempt >= MAX_RETRIES) {
|
||||
throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
|
||||
}
|
||||
|
||||
const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;
|
||||
|
||||
logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);
|
||||
|
||||
await wait(retryAfter);
|
||||
|
||||
return fetchWithRetry(path, params, attempt + 1, accept);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async function requestJson(path, params = {}, attempt = 0) {
|
||||
const res = await fetchWithRetry(path, params, attempt, 'application/json');
|
||||
const contentType = res.headers.get('content-type') || '';
|
||||
|
||||
if (!contentType.includes('json')) {
|
||||
// reddit serves an HTML interstitial/block page instead of JSON when it decides
|
||||
// the request looks like a bot - this is a distinct failure mode from a genuine
|
||||
// 4xx and should be logged/handled separately rather than treated as "bad request"
|
||||
const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
|
||||
|
||||
throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
if (!res.ok) {
|
||||
const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);
|
||||
|
||||
throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
// unlike requestJson(), this is for reddit's rendered profile pages - there's no public JSON
|
||||
// endpoint for a user's social links (the "Social Links" widget on their profile), they're
|
||||
// only present in the page's HTML (see getSocialLinks() below)
|
||||
async function requestHtml(path, params = {}, attempt = 0) {
|
||||
const res = await fetchWithRetry(path, params, attempt, 'text/html');
|
||||
const contentType = res.headers.get('content-type') || '';
|
||||
|
||||
if (!contentType.includes('html')) {
|
||||
const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
|
||||
|
||||
throw new Error(`Reddit returned non-HTML response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Reddit returned ${res.status} for '${path}'`);
|
||||
}
|
||||
|
||||
return res.text();
|
||||
}
|
||||
|
||||
// reddit's "Social Links" profile widget (the row of Twitter/Instagram/OnlyFans/etc. buttons
|
||||
// under a user's bio) has no public JSON endpoint - the structured API for it
|
||||
// (/r/u_{username}/api/widgets.json) requires OAuth, which this project deliberately doesn't
|
||||
// use (see reddit.throttle above). The rendered profile page embeds the same data anyway, as
|
||||
// JSON inside each link's `data-faceplate-tracking-context` attribute, and that page is
|
||||
// accessible with the same cookies as everything else here.
|
||||
function parseSocialLinks(html) {
|
||||
const $ = cheerio.load(html);
|
||||
const links = [];
|
||||
|
||||
$('[data-faceplate-tracking-context]').each((index, el) => {
|
||||
try {
|
||||
const { social_link: link } = JSON.parse($(el).attr('data-faceplate-tracking-context'));
|
||||
|
||||
if (link) {
|
||||
links.push(link);
|
||||
}
|
||||
|
||||
const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;
|
||||
|
||||
logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);
|
||||
|
||||
await wait(retryAfter);
|
||||
|
||||
return requestJson(path, params, attempt + 1);
|
||||
} catch (error) {
|
||||
// not a social-link tracker, or a shape we don't recognize - ignore
|
||||
}
|
||||
|
||||
if (!contentType.includes('json')) {
|
||||
// reddit serves an HTML interstitial/block page instead of JSON when it decides
|
||||
// the request looks like a bot - this is a distinct failure mode from a genuine
|
||||
// 4xx and should be logged/handled separately rather than treated as "bad request"
|
||||
const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
|
||||
|
||||
throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
if (!res.ok) {
|
||||
const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);
|
||||
|
||||
throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
|
||||
}
|
||||
|
||||
return data;
|
||||
});
|
||||
|
||||
return links
|
||||
.sort((a, b) => a.position - b.position)
|
||||
.map(({ type, name, url }) => ({ type, name, url }));
|
||||
}
|
||||
|
||||
async function getSocialLinks(username) {
|
||||
const html = await requestHtml(`/user/${username}/`);
|
||||
|
||||
return parseSocialLinks(html);
|
||||
}
|
||||
|
||||
async function getUser(username) {
|
||||
@@ -277,6 +366,7 @@ async function getSubmissionWithComments(id) {
|
||||
module.exports = {
|
||||
requestJson,
|
||||
getUser,
|
||||
getSocialLinks,
|
||||
getSubmissions,
|
||||
getSubmission,
|
||||
getSubmissionWithComments,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
const config = require('config');
|
||||
const Promise = require('bluebird');
|
||||
const UrlPattern = require('url-pattern');
|
||||
const yaml = require('js-yaml');
|
||||
|
||||
const interpolate = require('../interpolate');
|
||||
const fetchItem = require('../fetch/item');
|
||||
@@ -62,9 +63,15 @@ async function saveProfileDescription(user, args) {
|
||||
}
|
||||
|
||||
if (config.library.profile.description && !user.fallback && !user.deleted) {
|
||||
if (user.profile && user.profile.description) {
|
||||
const links = user.profile?.links || [];
|
||||
|
||||
// YAML rather than plain text (like index.js) so the free-form bio and the structured
|
||||
// social links (reddit/client.js's getSocialLinks()) stay separately parseable instead
|
||||
// of just being smushed into one text blob
|
||||
if (user.profile && (user.profile.description || links.length)) {
|
||||
const filepath = interpolate(config.library.profile.description, null, null, null, null, user);
|
||||
const stream = Buffer.from(user.profile.description, 'utf8');
|
||||
const data = { description: user.profile.description || null, links };
|
||||
const stream = Buffer.from(yaml.dump(data, { skipInvalid: true }), 'utf8');
|
||||
|
||||
try {
|
||||
const targets = await save(filepath, stream);
|
||||
@@ -77,7 +84,7 @@ async function saveProfileDescription(user, args) {
|
||||
}
|
||||
}
|
||||
|
||||
logger.verbose(`No profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
|
||||
logger.verbose(`No profile description or social links for '${user.name}' (https://reddit.com/user/${user.name})`);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
+16
-2
@@ -7,11 +7,25 @@ const curateUser = require('../curate/user');
|
||||
|
||||
const logger = require('../logger')(__filename);
|
||||
|
||||
async function getSocialLinks(username, reddit) {
|
||||
try {
|
||||
return await reddit.getSocialLinks(username);
|
||||
} catch (error) {
|
||||
// supplementary, not worth failing the whole user fetch over - fall back to none
|
||||
logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function getUser(username, reddit) {
|
||||
try {
|
||||
const user = await reddit.getUser(username);
|
||||
const [user, socialLinks] = await Promise.all([
|
||||
reddit.getUser(username),
|
||||
getSocialLinks(username, reddit),
|
||||
]);
|
||||
|
||||
return curateUser(user);
|
||||
return curateUser(user, socialLinks);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
|
||||
|
||||
|
||||
@@ -8,13 +8,27 @@ const curateUser = require('../curate/user');
|
||||
|
||||
const logger = require('../logger')(__filename);
|
||||
|
||||
async function getSocialLinks(username, reddit) {
|
||||
try {
|
||||
return await reddit.getSocialLinks(username);
|
||||
} catch (error) {
|
||||
// supplementary, not worth failing the whole user fetch over - fall back to none
|
||||
logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function getUser(username, reddit) {
|
||||
try {
|
||||
const user = await reddit.getUser(username);
|
||||
const [user, socialLinks] = await Promise.all([
|
||||
reddit.getUser(username),
|
||||
getSocialLinks(username, reddit),
|
||||
]);
|
||||
|
||||
logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
|
||||
|
||||
return curateUser(user);
|
||||
return curateUser(user, socialLinks);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user