diff --git a/config/default.js b/config/default.js index 962ab49..4f513dc 100644 --- a/config/default.js +++ b/config/default.js @@ -77,14 +77,20 @@ module.exports = { interval: 100, }, reddit: { - api: { - userAgent: 'ripunzel', - clientId: '1234567abcdefg', - access_token: 'abcD123eFg45Hi6J7klmnop8qr9', - token_type: 'bearer', - expires_in: 3600, - refresh_token: '1234567-A-Bc-defg8912hij-klm345opqr', - scope: 'history identity mysubreddits read subscribe', + // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to + // get past reddit's anonymous-traffic bot challenge on www.reddit.com. Needs periodic + // refreshing - src/reddit/client.js logs a warning once the soonest-expiring cookie + // in here is within 48h of expiring. + cookies: null, + // fallback/simpler alternative to `cookies` above: a single pre-built `name=value; ...` + // Cookie header string. Only used if `cookies` is not set. + cookie: null, + userAgent: 'ripunzel', + throttle: { + // deliberately conservative - this is unauthenticated-looking public access, + // not an approved API key, getting flagged as a bot defeats the point + reservoir: 6, + minTime: 1500, }, }, methods: { diff --git a/package.json b/package.json index 784da5b..2fa7918 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,6 @@ "object.omit": "^3.0.0", "object.pick": "^1.3.0", "sharp-phash": "^2.1.0", - "snoowrap": "^1.23.0", "template-format": "^1.2.4", "unprint": "^0.8.2", "url-pattern": "^1.0.3", diff --git a/src/app.js b/src/app.js index aebcad4..4cc361e 100644 --- a/src/app.js +++ b/src/app.js @@ -1,7 +1,6 @@ 'use strict'; const config = require('config'); -const Snoowrap = require('snoowrap'); const fs = require('fs-extra'); const Promise = require('bluebird'); const exiftool = require('node-exiftool'); @@ -10,7 +9,7 @@ const cron = require('node-cron'); require('array.prototype.flatten').shim(); -const reddit = new Snoowrap(config.reddit.api); +const reddit = require('./reddit/client'); const args = require('./cli')(); const logger = require('./logger')(__filename); diff --git a/src/curate/posts.js b/src/curate/posts.js index 07ff6c8..02ddc96 100644 --- a/src/curate/posts.js +++ b/src/curate/posts.js @@ -57,12 +57,13 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) permalink, url: post.url, datetime: new Date(post.created_utc * 1000), - subreddit: post.subreddit.display_name, + subreddit: post.subreddit, score: post.score, preview: post.preview ? post.preview.images.map((image) => image.source) : null, host, direct: post.direct, - comments: post.comments, + // count only - the full thread (when needed) is fetched separately via methods/self.js + comments: post.num_comments, hash: hashPost(post), }; diff --git a/src/curate/user.js b/src/curate/user.js index b51b512..d0cb835 100644 --- a/src/curate/user.js +++ b/src/curate/user.js @@ -9,23 +9,23 @@ function curateUser(user) { gold: user.is_gold, verified: user.verified, verifiedEmail: user.has_verified_email, - fallback: false + fallback: false, }; - if(user.subreddit) { + if (user.subreddit) { Object.assign(curatedUser, { profile: { - id: user.subreddit.display_name.name, - title: user.subreddit.display_name.title, - image: user.subreddit.display_name.icon_img, - banner: user.subreddit.display_name.banner_img, - description: user.subreddit.display_name.public_description, - over18: user.subreddit.display_name.over_18 - } + id: user.subreddit.display_name, + title: user.subreddit.title, + image: user.subreddit.icon_img, + banner: user.subreddit.banner_img, + description: user.subreddit.public_description, + over18: user.subreddit.over_18, + }, }); } return curatedUser; -}; +} module.exports = curateUser; diff --git a/src/limiter.js b/src/limiter.js index 1c4a22e..ba21479 100644 --- a/src/limiter.js +++ b/src/limiter.js @@ -5,13 +5,7 @@ const Bottleneck = require('bottleneck'); const args = require('./cli')(); module.exports = { - reddit: new Bottleneck({ - reservoir: 30, - reservoirRefreshAmount: 30, - reservoirRefreshInterval: 60000, - maxConcurrent: 1, - minTime: 100, - }), + // reddit requests are throttled inside src/reddit/client.js itself (config.reddit.throttle) items: new Bottleneck({ maxConcurrent: args.concurrency, minTime: args.interval, diff --git a/src/methods/self.js b/src/methods/self.js index 31895dd..4e45934 100644 --- a/src/methods/self.js +++ b/src/methods/self.js @@ -4,11 +4,11 @@ function curateComments(comments) { return comments.map((comment) => ({ id: comment.id, url: `https://reddit.com${comment.permalink}`, - author: comment.author.name, + author: comment.author, body: comment.body, html: comment.body_html, score: comment.score, - datetime: new Date(comment.created * 1000), + datetime: new Date(comment.created_utc * 1000), edited: comment.edited, controversiality: comment.controversiality, gilded: comment.gilded, @@ -22,18 +22,12 @@ function curateComments(comments) { } async function getFullPost(postId, reddit) { - return reddit - .getSubmission(postId) - .expandReplies({ - limit: Infinity, - depth: Infinity, - }) - .fetch(); + return reddit.getSubmissionWithComments(postId); } async function self(host, originalPost, { reddit }) { - const post = await getFullPost(originalPost.id, reddit) || originalPost; - const curatedComments = curateComments(post.comments); + const { post, comments } = await getFullPost(originalPost.id, reddit); + const curatedComments = curateComments(comments); return { album: null, @@ -41,9 +35,9 @@ async function self(host, originalPost, { reddit }) { id: post.id, url: post.url, title: post.title, - text: post.text, - author: post.author.name, - datetime: post.datetime, + text: post.selftext, + author: post.author, + datetime: new Date(post.created_utc * 1000), comments: curatedComments, type: 'text/plain', self: true, diff --git a/src/reddit/client.js b/src/reddit/client.js new file mode 100644 index 0000000..b326dc4 --- /dev/null +++ b/src/reddit/client.js @@ -0,0 +1,284 @@ +'use strict'; + +const fetch = require('node-fetch'); +const Bottleneck = require('bottleneck'); +const config = require('config'); + +const { version } = require('../../package.json'); +const logger = require('../logger')(__filename); + +const BASE_URL = 'https://www.reddit.com'; +const MAX_RETRIES = 5; + +// deliberately much heavier throttling than the old OAuth limiter (src/limiter.js reddit: 30/min) - +// this is unauthenticated/public access and getting flagged as a bot defeats the point +const limiter = new Bottleneck({ + reservoir: config.reddit.throttle?.reservoir ?? 6, + reservoirRefreshAmount: config.reddit.throttle?.reservoir ?? 6, + reservoirRefreshInterval: 60000, + maxConcurrent: 1, + minTime: config.reddit.throttle?.minTime ?? 1500, +}); + +// parses a Netscape/Mozilla-format cookie jar dump (domain, includeSubdomains, path, +// secure, expiry, name, value - tab separated) into a `name=value; name2=value2` header +function parseCookieJar(raw) { + const now = Date.now() / 1000; + + return raw + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .map((line) => line.split('\t')) + .filter((fields) => fields.length === 7) + .filter(([, , , , expiry]) => expiry === '0' || Number(expiry) > now) + .filter(([domain]) => domain.replace(/^\./, '').endsWith('reddit.com')) + .map(([, , , , , name, value]) => `${name}=${value}`) + .join('; '); +} + +function soonestExpiry(raw) { + const expiries = raw + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .map((line) => line.split('\t')) + .filter((fields) => fields.length === 7) + .map(([, , , , expiry]) => Number(expiry)) + .filter((expiry) => expiry > 0); + + return expiries.length ? Math.min(...expiries) : null; +} + +let cachedCookieHeader; + +function cookieHeader() { + if (cachedCookieHeader === undefined) { + cachedCookieHeader = config.reddit.cookies + ? parseCookieJar(config.reddit.cookies) + : (config.reddit.cookie || null); + + logger.info(cachedCookieHeader + ? `Using ${cachedCookieHeader.split(';').length} reddit cookie(s) for authenticated requests` + : 'No reddit cookies configured, requesting unauthenticated'); + + if (config.reddit.cookies) { + const expiry = soonestExpiry(config.reddit.cookies); + const hoursLeft = expiry ? (expiry - Date.now() / 1000) / 3600 : null; + + if (hoursLeft !== null && hoursLeft < 48) { + logger.warn(`Soonest-expiring reddit cookie expires in ${hoursLeft.toFixed(1)}h (${new Date(expiry * 1000).toISOString()}) - refresh config/cookies.js before then`); + } + } + } + + return cachedCookieHeader; +} + +function headers() { + const requestHeaders = { + 'user-agent': config.reddit.userAgent || `ripunzel/${version}`, + accept: 'application/json', + }; + + const cookie = cookieHeader(); + + if (cookie) { + requestHeaders.cookie = cookie; + } + + return requestHeaders; +} + +function buildUrl(path, params) { + const url = new URL(`${BASE_URL}${path}`); + + Object.entries({ raw_json: 1, ...params }).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, value); + } + }); + + return url; +} + +async function wait(seconds) { + return new Promise((resolve) => { + setTimeout(resolve, seconds * 1000); + }); +} + +async function requestJson(path, params = {}, attempt = 0) { + const url = buildUrl(path, params); + + return limiter.schedule(async () => { + const res = await fetch(url.toString(), { headers: headers() }); + const contentType = res.headers.get('content-type') || ''; + + if (res.status === 429 || res.status === 503) { + if (attempt >= MAX_RETRIES) { + throw new Error(`Reddit rate-limited request after ${attempt} retries (${res.status}) for '${path}'`); + } + + const retryAfter = Number(res.headers.get('retry-after')) || 2 ** attempt; + + logger.warn(`Rate limited by reddit (${res.status}) for '${path}', retrying in ${retryAfter}s`); + + await wait(retryAfter); + + return requestJson(path, params, attempt + 1); + } + + if (!contentType.includes('json')) { + // reddit serves an HTML interstitial/block page instead of JSON when it decides + // the request looks like a bot - this is a distinct failure mode from a genuine + // 4xx and should be logged/handled separately rather than treated as "bad request" + const hint = cookieHeader() ? ' - session cookie may have expired, refresh config/cookies.js' : ' - likely blocked or interstitial page (consider setting config.reddit.cookies)'; + + throw new Error(`Reddit returned non-JSON response (${res.status}, content-type '${contentType || 'none'}') for '${path}'${hint}`); + } + + const data = await res.json(); + + if (!res.ok) { + const message = data?.message || data?.reason || JSON.stringify(data).slice(0, 200); + + throw new Error(`Reddit returned ${res.status} for '${path}': ${message}`); + } + + return data; + }); +} + +async function getUser(username) { + const data = await requestJson(`/user/${username}/about.json`); + + return data.data; +} + +async function getSubmissions(username, { sort = 'new', limit = Infinity } = {}) { + const submissions = []; + let after = null; + + do { + // eslint-disable-next-line no-await-in-loop + const data = await requestJson(`/user/${username}/submitted.json`, { + sort, + limit: Math.min(100, Number.isFinite(limit) ? limit - submissions.length : 100), + after, + }); + + const page = data.data.children.map((child) => child.data); + + submissions.push(...page); + after = data.data.after; + } while (after && submissions.length < limit); + + return submissions; +} + +// reddit's raw comment listing is a tree of {kind, data} "things": kind 't1' is a comment +// (whose own `replies` field is either '' or a nested Listing of more things), kind 'more' +// is a stub meaning "N additional comment IDs exist here, fetch them separately". +function flattenThings(things) { + const comments = []; + const moreIds = []; + + things.forEach((thing) => { + if (thing.kind === 't1') { + comments.push(thing.data); + + if (thing.data.replies?.data?.children) { + const nested = flattenThings(thing.data.replies.data.children); + + comments.push(...nested.comments); + moreIds.push(...nested.moreIds); + } + } else if (thing.kind === 'more' && thing.data.id !== '_') { + moreIds.push(...thing.data.children); + } + }); + + return { comments, moreIds }; +} + +async function moreChildren(linkId, ids) { + const unique = [...new Set(ids)]; + const results = []; + + for (let offset = 0; offset < unique.length; offset += 100) { + const chunk = unique.slice(offset, offset + 100); + + // eslint-disable-next-line no-await-in-loop + const data = await requestJson('/api/morechildren.json', { + api_type: 'json', + link_id: linkId, + children: chunk.join(','), + limit_children: false, + }); + + results.push(...(data.json?.data?.things || [])); + } + + return results; +} + +// re-fetches 'more' stubs and re-attaches them by parent_id; morechildren can itself return +// further 'more' stubs on very deep threads, so this recurses with a depth cap rather than +// looping forever +async function resolveMore(moreIds, linkId, depth = 0) { + if (!moreIds.length) { + return []; + } + + if (depth >= 4) { + logger.warn(`Gave up resolving ${moreIds.length} deeply-nested 'more comments' stubs for '${linkId}' after ${depth} rounds`); + + return []; + } + + const more = await moreChildren(linkId, moreIds); + const resolvedComments = more.filter((thing) => thing.kind === 't1').map((thing) => thing.data); + const nestedMoreIds = more.filter((thing) => thing.kind === 'more').flatMap((thing) => thing.data.children); + + return [...resolvedComments, ...await resolveMore(nestedMoreIds, linkId, depth + 1)]; +} + +function buildCommentTree(comments, parentId) { + return comments + .filter((comment) => comment.parent_id === parentId) + .map((comment) => ({ + ...comment, + replies: buildCommentTree(comments, `t1_${comment.id}`), + })); +} + +// post metadata only - cheap, no comment-tree resolution (limit: 0 asks reddit to skip +// returning comments too, but we ignore the second listing element regardless) +async function getSubmission(id) { + const [postListing] = await requestJson(`/comments/${id}.json`, { limit: 0 }); + + return postListing.data.children[0].data; +} + +// post + fully-resolved comment tree, incl. 'more comments' stubs - expensive on busy +// threads (each round of stubs is its own throttled request), only use where the full +// thread is actually needed +async function getSubmissionWithComments(id) { + const linkId = `t3_${id}`; + const [postListing, commentsListing] = await requestJson(`/comments/${id}.json`); + const post = postListing.data.children[0].data; + + const { comments, moreIds } = flattenThings(commentsListing.data.children); + const resolved = [...comments, ...await resolveMore(moreIds, linkId)]; + + return { post, comments: buildCommentTree(resolved, linkId) }; +} + +module.exports = { + requestJson, + getUser, + getSubmissions, + getSubmission, + getSubmissionWithComments, +}; diff --git a/src/sources/getPosts.js b/src/sources/getPosts.js index e5dd7f4..5403fb4 100644 --- a/src/sources/getPosts.js +++ b/src/sources/getPosts.js @@ -6,11 +6,10 @@ const getIndex = require('./getIndex'); const curateUser = require('../curate/user'); const logger = require('../logger')(__filename); -const limiter = require('../limiter').reddit; async function getUser(username, reddit) { try { - const user = await limiter.schedule(async () => reddit.getUser(username).fetch()); + const user = await reddit.getUser(username); return curateUser(user); } catch (error) { @@ -25,22 +24,22 @@ async function getUser(username, reddit) { const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) { return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => { - const post = await limiter.schedule(async () => reddit.getSubmission(postId).fetch()); + const post = await reddit.getSubmission(postId); post.direct = true; - if (accUserPosts[post.author.name]) { + if (accUserPosts[post.author]) { return { ...accUserPosts, - [post.author.name]: { - ...accUserPosts[post.author.name], - posts: [...accUserPosts[post.author.name].posts, post], + [post.author]: { + ...accUserPosts[post.author], + posts: [...accUserPosts[post.author].posts, post], }, }; } // don't attempt to fetch deleted user - if (post.author.name === '[deleted]') { + if (post.author === '[deleted]') { return { ...accUserPosts, '[deleted]': { @@ -56,12 +55,12 @@ const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) { }; } - const user = await getUser(post.author.name, reddit); + const user = await getUser(post.author, reddit); const { profile, posts: indexed } = await getIndex(user); return { ...accUserPosts, - [post.author.name]: { + [post.author]: { ...user, posts: [post], indexed: { diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index 0d27200..4451e05 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -7,11 +7,10 @@ const getArchivePostIds = require('../archives/getArchivePostIds'); const curateUser = require('../curate/user'); const logger = require('../logger')(__filename); -const limiter = require('../limiter').reddit; async function getUser(username, reddit) { try { - const user = await limiter.schedule(async () => reddit.getUser(username).fetch()); + const user = await reddit.getUser(username); logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`); @@ -28,12 +27,10 @@ async function getUser(username, reddit) { async function getPosts(username, reddit, args) { try { - const submissions = await limiter.schedule(async () => reddit - .getUser(username) - .getSubmissions({ - sort: args.sort, - limit: Infinity, - })); + const submissions = await reddit.getSubmissions(username, { + sort: args.sort, + limit: Infinity, + }); logger.info(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`); @@ -48,7 +45,7 @@ async function getPosts(username, reddit, args) { async function getArchivedPosts(username, posts, reddit) { const postIds = await getArchivePostIds(username, posts.map((post) => post.id)); - return Promise.all(postIds.map((postId) => limiter.schedule(async () => reddit.getSubmission(postId).fetch()))); + return Promise.all(postIds.map((postId) => reddit.getSubmission(postId))); } function getUserPostsWrap(reddit, args) {