From e23d8becd42b613cc91ef77bc773547645f7ad73 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Tue, 29 Sep 2026 17:12:16 +0200 Subject: [PATCH] Storing social links in profile file, changed format to YAML. --- src/curate/user.js | 5 +- src/reddit/client.js | 95 ++++++++++++++++++++++++++++++------- src/save/profileDetails.js | 13 +++-- src/sources/getPosts.js | 18 ++++++- src/sources/getUserPosts.js | 18 ++++++- 5 files changed, 125 insertions(+), 24 deletions(-) diff --git a/src/curate/user.js b/src/curate/user.js index d0cb835..219aa46 100644 --- a/src/curate/user.js +++ b/src/curate/user.js @@ -1,6 +1,6 @@ 'use strict'; -function curateUser(user) { +function curateUser(user, socialLinks = []) { const curatedUser = { id: user.id, name: user.name, @@ -21,6 +21,9 @@ function curateUser(user) { banner: user.subreddit.banner_img, description: user.subreddit.public_description, over18: user.subreddit.over_18, + // the "Social Links" widget on their profile (Twitter/Instagram/OnlyFans/etc.) - + // see reddit/client.js's getSocialLinks() + links: socialLinks, }, }); } diff --git a/src/reddit/client.js b/src/reddit/client.js index 2c612ab..0725757 100644 --- a/src/reddit/client.js +++ b/src/reddit/client.js @@ -1,6 +1,7 @@ 'use strict'; const Bottleneck = require('bottleneck'); +const cheerio = require('cheerio'); const config = require('config'); const { fetch } = require('../proxy'); @@ -75,10 +76,10 @@ function cookieHeader() { return cachedCookieHeader; } -function headers() { +function headers(accept = 'application/json') { const requestHeaders = { 'user-agent': config.reddit.userAgent || `ripunzel/${version}`, - accept: 'application/json', + accept, }; const cookie = cookieHeader(); @@ -108,23 +109,26 @@ async function wait(seconds) { }); } -async function requestJson(path, params = {}, attempt = 0) { +// shared by requestJson() and requestHtml() below - handles the actual throttled network +// call plus retrying on a network error or a 429/503, regardless of what kind of response +// the caller ultimately wants back. +// +// Only the actual throttled network call belongs inside limiter.schedule() - this has +// maxConcurrent: 1, and every retry path below used to recurse back into this function +// (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't +// start a second job while the first is still running, and the first can't finish until +// its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever +// needed (e.g. a single 429), freezing every reddit request behind it for good. Retrying +// now happens here, after the scheduled call has actually returned and freed its slot. +async function fetchWithRetry(path, params, attempt, accept) { const url = buildUrl(path, params); - - // Only the actual throttled network call belongs inside limiter.schedule() - this has - // maxConcurrent: 1, and every retry path below used to recurse back into requestJson() - // (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't - // start a second job while the first is still running, and the first can't finish until - // its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever - // needed (e.g. a single 429), freezing every reddit API call behind it for good. Retrying - // now happens here, after the scheduled call has actually returned and freed its slot. let res; try { // config.proxy.retryFlipped: once this request has already failed once, retry it via // whichever of proxied/direct is NOT what it'd normally get - see proxy.js res = await limiter.schedule(async () => fetch(url.toString(), { - headers: headers(), + headers: headers(accept), proxyFlip: config.proxy.retryFlipped && attempt > 0, })); } catch (error) { @@ -137,11 +141,9 @@ async function requestJson(path, params = {}, attempt = 0) { logger.warn(`Failed to reach reddit for '${path}', retrying: ${error.message}`); - return requestJson(path, params, attempt + 1); + return fetchWithRetry(path, params, attempt + 1, accept); } - const contentType = res.headers.get('content-type') || ''; - if (res.status === 429 || res.status === 503) { if (attempt >= MAX_RETRIES) { throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`); @@ -153,9 +155,16 @@ async function requestJson(path, params = {}, attempt = 0) { await wait(retryAfter); - return requestJson(path, params, attempt + 1); + return fetchWithRetry(path, params, attempt + 1, accept); } + return res; +} + +async function requestJson(path, params = {}, attempt = 0) { + const res = await fetchWithRetry(path, params, attempt, 'application/json'); + const contentType = res.headers.get('content-type') || ''; + if (!contentType.includes('json')) { // reddit serves an HTML interstitial/block page instead of JSON when it decides // the request looks like a bot - this is a distinct failure mode from a genuine @@ -176,6 +185,59 @@ async function requestJson(path, params = {}, attempt = 0) { return data; } +// unlike requestJson(), this is for reddit's rendered profile pages - there's no public JSON +// endpoint for a user's social links (the "Social Links" widget on their profile), they're +// only present in the page's HTML (see getSocialLinks() below) +async function requestHtml(path, params = {}, attempt = 0) { + const res = await fetchWithRetry(path, params, attempt, 'text/html'); + const contentType = res.headers.get('content-type') || ''; + + if (!contentType.includes('html')) { + const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)'; + + throw new Error(`Reddit returned non-HTML response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`); + } + + if (!res.ok) { + throw new Error(`Reddit returned ${res.status} for '${path}'`); + } + + return res.text(); +} + +// reddit's "Social Links" profile widget (the row of Twitter/Instagram/OnlyFans/etc. buttons +// under a user's bio) has no public JSON endpoint - the structured API for it +// (/r/u_{username}/api/widgets.json) requires OAuth, which this project deliberately doesn't +// use (see reddit.throttle above). The rendered profile page embeds the same data anyway, as +// JSON inside each link's `data-faceplate-tracking-context` attribute, and that page is +// accessible with the same cookies as everything else here. +function parseSocialLinks(html) { + const $ = cheerio.load(html); + const links = []; + + $('[data-faceplate-tracking-context]').each((index, el) => { + try { + const { social_link: link } = JSON.parse($(el).attr('data-faceplate-tracking-context')); + + if (link) { + links.push(link); + } + } catch (error) { + // not a social-link tracker, or a shape we don't recognize - ignore + } + }); + + return links + .sort((a, b) => a.position - b.position) + .map(({ type, name, url }) => ({ type, name, url })); +} + +async function getSocialLinks(username) { + const html = await requestHtml(`/user/${username}/`); + + return parseSocialLinks(html); +} + async function getUser(username) { const data = await requestJson(`/user/${username}/about.json`); @@ -304,6 +366,7 @@ async function getSubmissionWithComments(id) { module.exports = { requestJson, getUser, + getSocialLinks, getSubmissions, getSubmission, getSubmissionWithComments, diff --git a/src/save/profileDetails.js b/src/save/profileDetails.js index be8b28f..060306e 100644 --- a/src/save/profileDetails.js +++ b/src/save/profileDetails.js @@ -3,6 +3,7 @@ const config = require('config'); const Promise = require('bluebird'); const UrlPattern = require('url-pattern'); +const yaml = require('js-yaml'); const interpolate = require('../interpolate'); const fetchItem = require('../fetch/item'); @@ -62,9 +63,15 @@ async function saveProfileDescription(user, args) { } if (config.library.profile.description && !user.fallback && !user.deleted) { - if (user.profile && user.profile.description) { + const links = user.profile?.links || []; + + // YAML rather than plain text (like index.js) so the free-form bio and the structured + // social links (reddit/client.js's getSocialLinks()) stay separately parseable instead + // of just being smushed into one text blob + if (user.profile && (user.profile.description || links.length)) { const filepath = interpolate(config.library.profile.description, null, null, null, null, user); - const stream = Buffer.from(user.profile.description, 'utf8'); + const data = { description: user.profile.description || null, links }; + const stream = Buffer.from(yaml.dump(data, { skipInvalid: true }), 'utf8'); try { const targets = await save(filepath, stream); @@ -77,7 +84,7 @@ async function saveProfileDescription(user, args) { } } - logger.verbose(`No profile description for '${user.name}' (https://reddit.com/user/${user.name})`); + logger.verbose(`No profile description or social links for '${user.name}' (https://reddit.com/user/${user.name})`); return null; } diff --git a/src/sources/getPosts.js b/src/sources/getPosts.js index 5403fb4..593abaa 100644 --- a/src/sources/getPosts.js +++ b/src/sources/getPosts.js @@ -7,11 +7,25 @@ const curateUser = require('../curate/user'); const logger = require('../logger')(__filename); +async function getSocialLinks(username, reddit) { + try { + return await reddit.getSocialLinks(username); + } catch (error) { + // supplementary, not worth failing the whole user fetch over - fall back to none + logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`); + + return []; + } +} + async function getUser(username, reddit) { try { - const user = await reddit.getUser(username); + const [user, socialLinks] = await Promise.all([ + reddit.getUser(username), + getSocialLinks(username, reddit), + ]); - return curateUser(user); + return curateUser(user, socialLinks); } catch (error) { logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`); diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index 4451e05..3264a05 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -8,13 +8,27 @@ const curateUser = require('../curate/user'); const logger = require('../logger')(__filename); +async function getSocialLinks(username, reddit) { + try { + return await reddit.getSocialLinks(username); + } catch (error) { + // supplementary, not worth failing the whole user fetch over - fall back to none + logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`); + + return []; + } +} + async function getUser(username, reddit) { try { - const user = await reddit.getUser(username); + const [user, socialLinks] = await Promise.all([ + reddit.getUser(username), + getSocialLinks(username, reddit), + ]); logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`); - return curateUser(user); + return curateUser(user, socialLinks); } catch (error) { logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);