ripunzel/src/reddit/client.js

'use strict';

const Bottleneck = require('bottleneck');
const cheerio = require('cheerio');
const config = require('config');

const { fetch } = require('../proxy');
const { version } = require('../../package.json');
const logger = require('../logger')(__filename);

const BASE_URL = 'https://www.reddit.com';
const MAX_RETRIES = 5;

// deliberately much heavier throttling than the old OAuth limiter (src/limiter.js reddit: 30/min) -
// this is unauthenticated/public access and getting flagged as a bot defeats the point
const limiter = new Bottleneck({
    reservoir: config.reddit.throttle?.reservoir ?? 6,
    reservoirRefreshAmount: config.reddit.throttle?.reservoir ?? 6,
    reservoirRefreshInterval: 60000,
    maxConcurrent: 1,
    minTime: config.reddit.throttle?.minTime ?? 1500,
});

// parses a Netscape/Mozilla-format cookie jar dump (domain, includeSubdomains, path,
// secure, expiry, name, value - tab separated) into a `name=value; name2=value2` header
function parseCookieJar(raw) {
    const now = Date.now() / 1000;

    return raw
        .split('\n')
        .map((line) => line.trim())
        .filter((line) => line && !line.startsWith('#'))
        .map((line) => line.split('\t'))
        .filter((fields) => fields.length === 7)
        .filter(([, , , , expiry]) => expiry === '0' || Number(expiry) > now)
        .filter(([domain]) => domain.replace(/^\./, '').endsWith('reddit.com'))
        .map(([, , , , , name, value]) => `${name}=${value}`)
        .join('; ');
}

function soonestExpiry(raw) {
    const expiries = raw
        .split('\n')
        .map((line) => line.trim())
        .filter((line) => line && !line.startsWith('#'))
        .map((line) => line.split('\t'))
        .filter((fields) => fields.length === 7)
        .map(([, , , , expiry]) => Number(expiry))
        .filter((expiry) => expiry > 0);

    return expiries.length ? Math.min(...expiries) : null;
}

let cachedCookieHeader;

function cookieHeader() {
    if (cachedCookieHeader === undefined) {
        cachedCookieHeader = config.reddit.cookies
            ? parseCookieJar(config.reddit.cookies)
            : (config.reddit.cookie || null);

        logger.info(cachedCookieHeader
            ? `Using ${cachedCookieHeader.split(';').length} reddit cookie(s) for authenticated requests`
            : 'No reddit cookies configured, requesting unauthenticated');

        if (config.reddit.cookies) {
            const expiry = soonestExpiry(config.reddit.cookies);
            const hoursLeft = expiry ? (expiry - Date.now() / 1000) / 3600 : null;

            if (hoursLeft !== null && hoursLeft < 48) {
                logger.warn(`Soonest-expiring reddit cookie expires in ${hoursLeft.toFixed(1)}h (${new Date(expiry * 1000).toISOString()}) - refresh config/cookies.js before then`);
            }
        }
    }

    return cachedCookieHeader;
}

function headers(accept = 'application/json') {
    const requestHeaders = {
        'user-agent': config.reddit.userAgent || `ripunzel/${version}`,
        accept,
    };

    const cookie = cookieHeader();

    if (cookie) {
        requestHeaders.cookie = cookie;
    }

    return requestHeaders;
}

function buildUrl(path, params) {
    const url = new URL(`${BASE_URL}${path}`);

    Object.entries({ raw_json: 1, ...params }).forEach(([key, value]) => {
        if (value !== undefined && value !== null) {
            url.searchParams.set(key, value);
        }
    });

    return url;
}

async function wait(seconds) {
    return new Promise((resolve) => {
        setTimeout(resolve, seconds * 1000);
    });
}

// shared by requestJson() and requestHtml() below - handles the actual throttled network
// call plus retrying on a network error or a 429/503, regardless of what kind of response
// the caller ultimately wants back.
//
// Only the actual throttled network call belongs inside limiter.schedule() - this has
// maxConcurrent: 1, and every retry path below used to recurse back into this function
// (and so back into limiter.schedule()) from *inside* that same callback. Bottleneck won't
// start a second job while the first is still running, and the first can't finish until
// its awaited recursive call starts - a guaranteed deadlock the instant a retry was ever
// needed (e.g. a single 429), freezing every reddit request behind it for good. Retrying
// now happens here, after the scheduled call has actually returned and freed its slot.
async function fetchWithRetry(path, params, attempt, accept) {
    const url = buildUrl(path, params);
    let res;

    try {
        // config.proxy.retryFlipped: once this request has already failed once, retry it via
        // whichever of proxied/direct is NOT what it'd normally get - see proxy.js
        res = await limiter.schedule(async () => fetch(url.toString(), {
            headers: headers(accept),
            proxyFlip: config.proxy.retryFlipped && attempt > 0,
        }));
    } catch (error) {
        // fetch() has no timeout of its own beyond undici's fairly generous defaults (see
        // proxy.js) - this catches both that and genuine network errors (DNS, connection
        // refused/reset, etc.), neither of which was retried before
        if (attempt >= MAX_RETRIES) {
            throw new Error(`Failed to reach reddit after ${attempt} retries for '${path}': ${error.message}`);
        }

        logger.warn(`Failed to reach reddit for '${path}', retrying: ${error.message}`);

        return fetchWithRetry(path, params, attempt + 1, accept);
    }

    if (res.status === 429 || res.status === 503) {
        if (attempt >= MAX_RETRIES) {
            throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`);
        }

        const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt;

        logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`);

        await wait(retryAfter);

        return fetchWithRetry(path, params, attempt + 1, accept);
    }

    return res;
}

async function requestJson(path, params = {}, attempt = 0) {
    const res = await fetchWithRetry(path, params, attempt, 'application/json');
    const contentType = res.headers.get('content-type') || '';

    if (!contentType.includes('json')) {
        // reddit serves an HTML interstitial/block page instead of JSON when it decides
        // the request looks like a bot - this is a distinct failure mode from a genuine
        // 4xx and should be logged/handled separately rather than treated as "bad request"
        const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';

        throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
    }

    const data = await res.json();

    if (!res.ok) {
        const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200);

        throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`);
    }

    return data;
}

// unlike requestJson(), this is for reddit's rendered profile pages - there's no public JSON
// endpoint for a user's social links (the "Social Links" widget on their profile), they're
// only present in the page's HTML (see getSocialLinks() below)
async function requestHtml(path, params = {}, attempt = 0) {
    const res = await fetchWithRetry(path, params, attempt, 'text/html');
    const contentType = res.headers.get('content-type') || '';

    if (!contentType.includes('html')) {
        const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)';

        throw new Error(`Reddit returned non-HTML response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`);
    }

    if (!res.ok) {
        throw new Error(`Reddit returned ${res.status} for '${path}'`);
    }

    return res.text();
}

// reddit's "Social Links" profile widget (the row of Twitter/Instagram/OnlyFans/etc. buttons
// under a user's bio) has no public JSON endpoint - the structured API for it
// (/r/u_{username}/api/widgets.json) requires OAuth, which this project deliberately doesn't
// use (see reddit.throttle above). The rendered profile page embeds the same data anyway, as
// JSON inside each link's `data-faceplate-tracking-context` attribute, and that page is
// accessible with the same cookies as everything else here.
function parseSocialLinks(html) {
    const $ = cheerio.load(html);
    const links = [];

    $('[data-faceplate-tracking-context]').each((index, el) => {
        try {
            const { social_link: link } = JSON.parse($(el).attr('data-faceplate-tracking-context'));

            if (link) {
                links.push(link);
            }
        } catch (error) {
            // not a social-link tracker, or a shape we don't recognize - ignore
        }
    });

    return links
        .sort((a, b) => a.position - b.position)
        .map(({ type, name, url }) => ({ type, name, url }));
}

async function getSocialLinks(username) {
    const html = await requestHtml(`/user/${username}/`);

    return parseSocialLinks(html);
}

async function getUser(username) {
    const data = await requestJson(`/user/${username}/about.json`);

    return data.data;
}

async function getSubmissions(username, { sort = 'new', limit = Infinity } = {}) {
    const submissions = [];
    let after = null;

    do {
        // eslint-disable-next-line no-await-in-loop
        const data = await requestJson(`/user/${username}/submitted.json`, {
            sort,
            limit: Math.min(100, Number.isFinite(limit) ? limit - submissions.length : 100),
            after,
        });

        const page = data.data.children.map((child) => child.data);

        submissions.push(...page);
        after = data.data.after;
    } while (after && submissions.length < limit);

    return submissions;
}

// reddit's raw comment listing is a tree of {kind, data} "things": kind 't1' is a comment
// (whose own `replies` field is either '' or a nested Listing of more things), kind 'more'
// is a stub meaning "N additional comment IDs exist here, fetch them separately".
function flattenThings(things) {
    const comments = [];
    const moreIds = [];

    things.forEach((thing) => {
        if (thing.kind === 't1') {
            comments.push(thing.data);

            if (thing.data.replies?.data?.children) {
                const nested = flattenThings(thing.data.replies.data.children);

                comments.push(...nested.comments);
                moreIds.push(...nested.moreIds);
            }
        } else if (thing.kind === 'more' && thing.data.id !== '_') {
            moreIds.push(...thing.data.children);
        }
    });

    return { comments, moreIds };
}

async function moreChildren(linkId, ids) {
    const unique = [...new Set(ids)];
    const results = [];

    for (let offset = 0; offset < unique.length; offset += 100) {
        const chunk = unique.slice(offset, offset + 100);

        // eslint-disable-next-line no-await-in-loop
        const data = await requestJson('/api/morechildren.json', {
            api_type: 'json',
            link_id: linkId,
            children: chunk.join(','),
            limit_children: false,
        });

        results.push(...(data.json?.data?.things || []));
    }

    return results;
}

// re-fetches 'more' stubs and re-attaches them by parent_id; morechildren can itself return
// further 'more' stubs on very deep threads, so this recurses with a depth cap rather than
// looping forever
async function resolveMore(moreIds, linkId, depth = 0) {
    if (!moreIds.length) {
        return [];
    }

    if (depth >= 4) {
        logger.warn(`Gave up resolving ${moreIds.length} deeply-nested 'more comments' stubs for '${linkId}' after ${depth} rounds`);

        return [];
    }

    const more = await moreChildren(linkId, moreIds);
    const resolvedComments = more.filter((thing) => thing.kind === 't1').map((thing) => thing.data);
    const nestedMoreIds = more.filter((thing) => thing.kind === 'more').flatMap((thing) => thing.data.children);

    return [...resolvedComments, ...await resolveMore(nestedMoreIds, linkId, depth + 1)];
}

function buildCommentTree(comments, parentId) {
    return comments
        .filter((comment) => comment.parent_id === parentId)
        .map((comment) => ({
            ...comment,
            replies: buildCommentTree(comments, `t1_${comment.id}`),
        }));
}

// post metadata only - cheap, no comment-tree resolution (limit: 0 asks reddit to skip
// returning comments too, but we ignore the second listing element regardless)
async function getSubmission(id) {
    const [postListing] = await requestJson(`/comments/${id}.json`, { limit: 0 });

    return postListing.data.children[0].data;
}

// post + fully-resolved comment tree, incl. 'more comments' stubs - expensive on busy
// threads (each round of stubs is its own throttled request), only use where the full
// thread is actually needed
async function getSubmissionWithComments(id) {
    const linkId = `t3_${id}`;
    const [postListing, commentsListing] = await requestJson(`/comments/${id}.json`);
    const post = postListing.data.children[0].data;

    const { comments, moreIds } = flattenThings(commentsListing.data.children);
    const resolved = [...comments, ...await resolveMore(moreIds, linkId)];

    return { post, comments: buildCommentTree(resolved, linkId) };
}

module.exports = {
    requestJson,
    getUser,
    getSocialLinks,
    getSubmissions,
    getSubmission,
    getSubmissionWithComments,
};