Storing social links in profile file, changed format to YAML.

src/reddit/client.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const Bottleneck = require('bottleneck');
+const cheerio = require('cheerio');
 const config = require('config');
 
 const { fetch } = require('../proxy');
@@ -75,10 +76,10 @@ function cookieHeader() {
     return cachedCookieHeader;
 }
 
-function headers() {
+function headers(accept = 'application/json') {
     const requestHeaders = {
         'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
-        accept: 'application/json',
+        accept,
     };
 
     const cookie = cookieHeader();
@@ -108,23 +109,26 @@ async function wait(seconds) {
     });
 }
 
-async function requestJson(path, params = {}, attempt = 0) {
+// shared by requestJson() and requestHtml() below - handles the actual throttled network
+// call plus retrying on a network error or a 429/503, regardless of what kind of response
+// the caller ultimately wants back.
+//
+// Only the actual throttled network call belongs inside limiter.schedule() - this has
+// maxConcurrent: 1, and every retry path below used to recurse back into this function
+// (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't
+// start a second job while the first is still running, and the first can't finish until
+// its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever
+// needed (e.g. a single 429), freezing every reddit request behind it for good. Retrying
+// now happens here, after the scheduled call has actually returned and freed its slot.
+async function fetchWithRetry(path, params, attempt, accept) {
     const url = buildUrl(path, params);
-
-    // Only the actual throttled network call belongs inside limiter.schedule() - this has
-    // maxConcurrent: 1, and every retry path below used to recurse back into requestJson()
-    // (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't
-    // start a second job while the first is still running, and the first can't finish until
-    // its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever
-    // needed (e.g. a single 429), freezing every reddit API call behind it for good. Retrying
-    // now happens here, after the scheduled call has actually returned and freed its slot.
     let res;
 
     try {
         // config.proxy.retryFlipped: once this request has already failed once, retry it via
         // whichever of proxied/direct is NOT what it'd normally get - see proxy.js
         res = await limiter.schedule(async () => fetch(url.toString(), {
-            headers: headers(),
+            headers: headers(accept),
             proxyFlip: config.proxy.retryFlipped && attempt > 0,
         }));
     } catch (error) {
@@ -137,11 +141,9 @@ async function requestJson(path, params = {}, attempt = 0) {
 
         logger.warn(`Failed to reach reddit for '${path}', retrying: ${error.message}`);
 
-        return requestJson(path, params, attempt + 1);
+        return fetchWithRetry(path, params, attempt + 1, accept);
     }
 
-    const contentType = res.headers.get('content-type') || '';
-
     if (res.status === 429 || res.status === 503) {
         if (attempt >= MAX_RETRIES) {
             throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
@@ -153,9 +155,16 @@ async function requestJson(path, params = {}, attempt = 0) {
 
         await wait(retryAfter);
 
-        return requestJson(path, params, attempt + 1);
+        return fetchWithRetry(path, params, attempt + 1, accept);
     }
 
+    return res;
+}
+
+async function requestJson(path, params = {}, attempt = 0) {
+    const res = await fetchWithRetry(path, params, attempt, 'application/json');
+    const contentType = res.headers.get('content-type') || '';
+
     if (!contentType.includes('json')) {
         // reddit serves an HTML interstitial/block page instead of JSON when it decides
         // the request looks like a bot - this is a distinct failure mode from a genuine
@@ -176,6 +185,59 @@ async function requestJson(path, params = {}, attempt = 0) {
     return data;
 }
 
+// unlike requestJson(), this is for reddit's rendered profile pages - there's no public JSON
+// endpoint for a user's social links (the "Social Links" widget on their profile), they're
+// only present in the page's HTML (see getSocialLinks() below)
+async function requestHtml(path, params = {}, attempt = 0) {
+    const res = await fetchWithRetry(path, params, attempt, 'text/html');
+    const contentType = res.headers.get('content-type') || '';
+
+    if (!contentType.includes('html')) {
+        const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';
+
+        throw new Error(`Reddit returned non-HTML response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
+    }
+
+    if (!res.ok) {
+        throw new Error(`Reddit returned ${res.status} for '${path}'`);
+    }
+
+    return res.text();
+}
+
+// reddit's "Social Links" profile widget (the row of Twitter/Instagram/OnlyFans/etc. buttons
+// under a user's bio) has no public JSON endpoint - the structured API for it
+// (/r/u_{username}/api/widgets.json) requires OAuth, which this project deliberately doesn't
+// use (see reddit.throttle above). The rendered profile page embeds the same data anyway, as
+// JSON inside each link's `data-faceplate-tracking-context` attribute, and that page is
+// accessible with the same cookies as everything else here.
+function parseSocialLinks(html) {
+    const $ = cheerio.load(html);
+    const links = [];
+
+    $('[data-faceplate-tracking-context]').each((index, el) => {
+        try {
+            const { social_link: link } = JSON.parse($(el).attr('data-faceplate-tracking-context'));
+
+            if (link) {
+                links.push(link);
+            }
+        } catch (error) {
+            // not a social-link tracker, or a shape we don't recognize - ignore
+        }
+    });
+
+    return links
+        .sort((a, b) => a.position - b.position)
+        .map(({ type, name, url }) => ({ type, name, url }));
+}
+
+async function getSocialLinks(username) {
+    const html = await requestHtml(`/user/${username}/`);
+
+    return parseSocialLinks(html);
+}
+
 async function getUser(username) {
     const data = await requestJson(`/user/${username}/about.json`);
 
@@ -304,6 +366,7 @@ async function getSubmissionWithComments(id) {
 module.exports = {
     requestJson,
     getUser,
+    getSocialLinks,
     getSubmissions,
     getSubmission,
     getSubmissionWithComments,