Storing social links in profile file, changed format to YAML.

src/curate/user.js

@@ -1,6 +1,6 @@
 'use strict';
 
-function curateUser(user) {
+function curateUser(user, socialLinks = []) {
     const curatedUser = {
         id: user.id,
         name: user.name,
@@ -21,6 +21,9 @@ function curateUser(user) {
                 banner: user.subreddit.banner_img,
                 description: user.subreddit.public_description,
                 over18: user.subreddit.over_18,
+                // the "Social Links" widget on their profile (Twitter/Instagram/OnlyFans/etc.) -
+                // see reddit/client.js's getSocialLinks()
+                links: socialLinks,
             },
         });
     }

src/reddit/client.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const Bottleneck = require('bottleneck');
+const cheerio = require('cheerio');
 const config = require('config');
 
 const { fetch } = require('../proxy');
@@ -75,10 +76,10 @@ function cookieHeader() {
     return cachedCookieHeader;
 }
 
-function headers() {
+function headers(accept = 'application/json') {
     const requestHeaders = {
         'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
-        accept: 'application/json',
+        accept,
     };
 
     const cookie = cookieHeader();
@@ -108,23 +109,26 @@ async function wait(seconds) {
     });
 }
 
-async function requestJson(path, params = {}, attempt = 0) {
-    const url = buildUrl(path, params);
-
+// shared by requestJson() and requestHtml() below - handles the actual throttled network
+// call plus retrying on a network error or a 429/503, regardless of what kind of response
+// the caller ultimately wants back.
+//
 // Only the actual throttled network call belongs inside limiter.schedule() - this has
-    // maxConcurrent: 1, and every retry path below used to recurse back into requestJson()
+// maxConcurrent: 1, and every retry path below used to recurse back into this function
 // (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't
 // start a second job while the first is still running, and the first can't finish until
 // its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever
-    // needed (e.g. a single 429), freezing every reddit API call behind it for good. Retrying
+// needed (e.g. a single 429), freezing every reddit request behind it for good. Retrying
 // now happens here, after the scheduled call has actually returned and freed its slot.
+async function fetchWithRetry(path, params, attempt, accept) {
+    const url = buildUrl(path, params);
     let res;
 
     try {
         // config.proxy.retryFlipped: once this request has already failed once, retry it via
         // whichever of proxied/direct is NOT what it'd normally get - see proxy.js
         res = await limiter.schedule(async () => fetch(url.toString(), {
-            headers: headers(),
+            headers: headers(accept),
             proxyFlip: config.proxy.retryFlipped && attempt > 0,
         }));
     } catch (error) {
@@ -137,11 +141,9 @@ async function requestJson(path, params = {}, attempt = 0) {
 
         logger.warn(`Failed to reach reddit for '${path}', retrying: ${error.message}`);
 
-        return requestJson(path, params, attempt + 1);
+        return fetchWithRetry(path, params, attempt + 1, accept);
     }
 
-    const contentType = res.headers.get('content-type') || '';
-
     if (res.status === 429 || res.status === 503) {
         if (attempt >= MAX_RETRIES) {
             throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
@@ -153,9 +155,16 @@ async function requestJson(path, params = {}, attempt = 0) {
 
         await wait(retryAfter);
 
-        return requestJson(path, params, attempt + 1);
+        return fetchWithRetry(path, params, attempt + 1, accept);
     }
 
+    return res;
+}
+
+async function requestJson(path, params = {}, attempt = 0) {
+    const res = await fetchWithRetry(path, params, attempt, 'application/json');
+    const contentType = res.headers.get('content-type') || '';
+
     if (!contentType.includes('json')) {
         // reddit serves an HTML interstitial/block page instead of JSON when it decides
         // the request looks like a bot - this is a distinct failure mode from a genuine
@@ -176,6 +185,59 @@ async function requestJson(path, params = {}, attempt = 0) {
     return data;
 }
 
+// unlike requestJson(), this is for reddit's rendered profile pages - there's no public JSON
+// endpoint for a user's social links (the "Social Links" widget on their profile), they're
+// only present in the page's HTML (see getSocialLinks() below)
+async function requestHtml(path, params = {}, attempt = 0) {
+    const res = await fetchWithRetry(path, params, attempt, 'text/html');
+    const contentType = res.headers.get('content-type') || '';
+
+    if (!contentType.includes('html')) {
+        const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
+
+        throw new Error(`Reddit returned non-HTML response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
+    }
+
+    if (!res.ok) {
+        throw new Error(`Reddit returned ${res.status} for '${path}'`);
+    }
+
+    return res.text();
+}
+
+// reddit's "Social Links" profile widget (the row of Twitter/Instagram/OnlyFans/etc. buttons
+// under a user's bio) has no public JSON endpoint - the structured API for it
+// (/r/u_{username}/api/widgets.json) requires OAuth, which this project deliberately doesn't
+// use (see reddit.throttle above). The rendered profile page embeds the same data anyway, as
+// JSON inside each link's `data-faceplate-tracking-context` attribute, and that page is
+// accessible with the same cookies as everything else here.
+function parseSocialLinks(html) {
+    const $ = cheerio.load(html);
+    const links = [];
+
+    $('[data-faceplate-tracking-context]').each((index, el) => {
+        try {
+            const { social_link: link } = JSON.parse($(el).attr('data-faceplate-tracking-context'));
+
+            if (link) {
+                links.push(link);
+            }
+        } catch (error) {
+            // not a social-link tracker, or a shape we don't recognize - ignore
+        }
+    });
+
+    return links
+        .sort((a, b) => a.position - b.position)
+        .map(({ type, name, url }) => ({ type, name, url }));
+}
+
+async function getSocialLinks(username) {
+    const html = await requestHtml(`/user/${username}/`);
+
+    return parseSocialLinks(html);
+}
+
 async function getUser(username) {
     const data = await requestJson(`/user/${username}/about.json`);
 
@@ -304,6 +366,7 @@ async function getSubmissionWithComments(id) {
 module.exports = {
     requestJson,
     getUser,
+    getSocialLinks,
     getSubmissions,
     getSubmission,
     getSubmissionWithComments,

src/save/profileDetails.js

@@ -3,6 +3,7 @@
 const config = require('config');
 const Promise = require('bluebird');
 const UrlPattern = require('url-pattern');
+const yaml = require('js-yaml');
 
 const interpolate = require('../interpolate');
 const fetchItem = require('../fetch/item');
@@ -62,9 +63,15 @@ async function saveProfileDescription(user, args) {
     }
 
     if (config.library.profile.description && !user.fallback && !user.deleted) {
-        if (user.profile && user.profile.description) {
+        const links = user.profile?.links || [];
+
+        // YAML rather than plain text (like index.js) so the free-form bio and the structured
+        // social links (reddit/client.js's getSocialLinks()) stay separately parseable instead
+        // of just being smushed into one text blob
+        if (user.profile && (user.profile.description || links.length)) {
             const filepath = interpolate(config.library.profile.description, null, null, null, null, user);
-            const stream = Buffer.from(user.profile.description, 'utf8');
+            const data = { description: user.profile.description || null, links };
+            const stream = Buffer.from(yaml.dump(data, { skipInvalid: true }), 'utf8');
 
             try {
                 const targets = await save(filepath, stream);
@@ -77,7 +84,7 @@ async function saveProfileDescription(user, args) {
             }
         }
 
-        logger.verbose(`No profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
+        logger.verbose(`No profile description or social links for '${user.name}' (https://reddit.com/user/${user.name})`);
 
         return null;
     }

src/sources/getPosts.js

@@ -7,11 +7,25 @@ const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
 
+async function getSocialLinks(username, reddit) {
+    try {
+        return await reddit.getSocialLinks(username);
+    } catch (error) {
+        // supplementary, not worth failing the whole user fetch over - fall back to none
+        logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
+
+        return [];
+    }
+}
+
 async function getUser(username, reddit) {
     try {
-        const user = await reddit.getUser(username);
+        const [user, socialLinks] = await Promise.all([
+            reddit.getUser(username),
+            getSocialLinks(username, reddit),
+        ]);
 
-        return curateUser(user);
+        return curateUser(user, socialLinks);
     } catch (error) {
         logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
 

src/sources/getUserPosts.js

@@ -8,13 +8,27 @@ const curateUser = require('../curate/user');
 
 const logger = require('../logger')(__filename);
 
+async function getSocialLinks(username, reddit) {
+    try {
+        return await reddit.getSocialLinks(username);
+    } catch (error) {
+        // supplementary, not worth failing the whole user fetch over - fall back to none
+        logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
+
+        return [];
+    }
+}
+
 async function getUser(username, reddit) {
     try {
-        const user = await reddit.getUser(username);
+        const [user, socialLinks] = await Promise.all([
+            reddit.getUser(username),
+            getSocialLinks(username, reddit),
+        ]);
 
         logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
 
-        return curateUser(user);
+        return curateUser(user, socialLinks);
     } catch (error) {
         logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);