DebaucheryLibrarian/ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Track suspended/deleted accounts and prune them from the users file after a grace period, also as standalone --cleanup. Only fetch social links when the profile description is saved, raised reddit throttle to 15/min.

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DebaucheryLibrarian
2026-11-06 00:05:11 +01:00
co-authored by Claude Opus 5.5
parent 6582bac049
commit 6905fb9b4a
10 changed files with 352 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/sources/accountStatus.js
+137
View File
@@ -0,0 +1,137 @@
'use strict';
const config = require('config');
const fs = require('fs-extra');
const yaml = require('js-yaml');
const logger = require('../logger')(__filename);
const DAY = 24 * 60 * 60 * 1000;
// Keeps a YAML record (config.users.dead.file) of accounts found suspended or deleted, keyed by
// username:
//
// SomeUser:
// status: suspended # or 'deleted'
// firstSeen: 2026-11-05T… # first run that observed this status - reddit exposes no date
// lastSeen: 2026-11-05T…
// removed: 2026-02-03T… # when it was dropped from the users file, if it was
//
// An account stays in the users file (and keeps being re-checked, at the cost of one request
// per run) until it's been dead for longer than config.users.dead.retainDays[status], so a
// lifted suspension or shadowban is noticed in time. After that it's removed from the users
// file - but only if it came from one (--file-users), never from --users on the command line.
// An account that turns out active again has its record dropped. Nothing is written until
// save(), at the end of the run, since users are processed concurrently.
async function loadRecords(file) {
try {
return yaml.load(await fs.readFile(file, 'utf8')) || {};
} catch (error) {
if (error.code !== 'ENOENT') {
throw new Error(`Could not read dead account record '${file}': ${error.message}`);
}
return {};
}
}
function findKey(records, username) {
const lower = username.toLowerCase();
return Object.keys(records).find((key) => key.toLowerCase() === lower);
}
async function removeFromUsersFile(usersFile, usernames) {
const lower = new Set([...usernames].map((name) => name.toLowerCase()));
const contents = await fs.readFile(usersFile, 'utf8');
// keeps comments, blank lines and order intact - only drops the exact matching entries
const kept = contents.split('\n').filter((line) => !lower.has(line.trim().toLowerCase()));
const tmpFile = `${usersFile}.tmp`;
await fs.writeFile(tmpFile, kept.join('\n'));
await fs.rename(tmpFile, usersFile);
}
async function createAccountTracker({ usersFile = null, fileUsernames = [], enabled = true } = {}) {
const deadConfig = config.users?.dead || {};
const file = deadConfig.file || 'users_dead.yml';
const retainDays = deadConfig.retainDays || {};
const fromFile = new Set(fileUsernames.map((name) => name.toLowerCase()));
const records = enabled ? await loadRecords(file) : {};
const removals = new Set();
const counts = {
suspended: 0, deleted: 0, restored: 0, removed: 0,
};
let changed = false;
function record(username, status, date = new Date()) {
const now = date.toISOString();
if (!enabled || status === 'error') {
return;
}
const key = findKey(records, username);
if (status === 'active') {
if (key) {
logger.info(`Account '${username}' is active again (was ${records[key].status} since ${new Date(records[key].firstSeen).toISOString()}), dropped from dead account record`);
delete records[key];
counts.restored += 1;
changed = true;
}
return;
}
counts[status] += 1;
const entry = key && records[key].status === status
? { ...records[key], lastSeen: now }
// new, or changed status (e.g. suspended -> deleted): the clock restarts
: { status, firstSeen: now, lastSeen: now };
if (key && key !== username) {
delete records[key];
}
records[username] = entry;
changed = true;
const deadDays = (date - new Date(entry.firstSeen)) / DAY;
const retain = retainDays[status] ?? 90;
if (deadDays >= retain && fromFile.has(username.toLowerCase())) {
removals.add(username);
entry.removed = now;
logger.info(`Account '${username}' has been ${status} since ${new Date(entry.firstSeen).toISOString()} (${Math.floor(deadDays)} days, retaining ${retain}), removing it from '${usersFile}'`);
} else {
// removed earlier, but (manually) put back in the users file since
if (fromFile.has(username.toLowerCase())) {
delete entry.removed;
}
logger.info(`Account '${username}' is ${status} (first seen ${new Date(entry.firstSeen).toISOString()}, ${Math.floor(deadDays)} of ${retain} days before removal${fromFile.has(username.toLowerCase()) ? '' : ', not from a users file so never removed automatically'})`);
}
}
async function save() {
if (!enabled || !changed) {
return;
}
if (removals.size && usersFile) {
await removeFromUsersFile(usersFile, removals);
counts.removed = removals.size;
}
await fs.writeFile(file, yaml.dump(records, { sortKeys: true }));
logger.info(`Dead account record '${file}' updated: ${counts.suspended} suspended, ${counts.deleted} deleted, ${counts.restored} active again, ${counts.removed} removed from users file`);
}
return { record, save };
}
module.exports = createAccountTracker;
src/sources/getPosts.js
+6 -35
View File
@@ -3,40 +3,9 @@
const Promise = require('bluebird');
const getIndex = require('./getIndex');
const curateUser = require('../curate/user');
const { fetchUser, indexUser, resolveUser } = require('./user');
const logger = require('../logger')(__filename);
async function getSocialLinks(username, reddit) {
try {
return await reddit.getSocialLinks(username);
} catch (error) {
// supplementary, not worth failing the whole user fetch over - fall back to none
logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
return [];
}
}
async function getUser(username, reddit) {
try {
const [user, socialLinks] = await Promise.all([
reddit.getUser(username),
getSocialLinks(username, reddit),
]);
return curateUser(user, socialLinks);
} catch (error) {
logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return {
name: username,
fallback: true,
};
}
}
const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) {
const getPostsWrap = (reddit, args) => function getPosts(postIds, userPosts = {}) {
return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => {
const post = await reddit.getSubmission(postId);
@@ -69,8 +38,10 @@ const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) {
};
}
const user = await getUser(post.author, reddit);
const { profile, posts: indexed } = await getIndex(user);
// a suspended/deleted author still gets their post saved, under the fallback user
const fetched = await fetchUser(post.author, reddit);
const { profile, posts: indexed } = await getIndex(indexUser(post.author, fetched));
const user = await resolveUser(post.author, fetched, profile, reddit, args);
return {
...accUserPosts,
src/sources/getUserPosts.js
+24 -38
View File
@@ -4,41 +4,10 @@ const Promise = require('bluebird');
const getIndex = require('./getIndex');
const getArchivePostIds = require('../archives/getArchivePostIds');
const curateUser = require('../curate/user');
const { fetchUser, indexUser, resolveUser } = require('./user');
const logger = require('../logger')(__filename);
async function getSocialLinks(username, reddit) {
try {
return await reddit.getSocialLinks(username);
} catch (error) {
// supplementary, not worth failing the whole user fetch over - fall back to none
logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
return [];
}
}
async function getUser(username, reddit) {
try {
const [user, socialLinks] = await Promise.all([
reddit.getUser(username),
getSocialLinks(username, reddit),
]);
logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
return curateUser(user, socialLinks);
} catch (error) {
logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return {
name: username,
fallback: true,
};
}
}
// the moment before which posts will be discarded by curate/posts.js anyway (--after, or
// --after-indexed against the existing index) - lets getSubmissions() stop paging there
// instead of always walking reddit's entire (up to ~1000 post) listing. Without either flag,
@@ -87,15 +56,32 @@ async function getArchivedPosts(username, posts, reddit) {
}
function getUserPostsWrap(reddit, args) {
return async function getUserPosts(usernames) {
// tracker: see accountStatus.js - records suspended/deleted accounts (and drops them from the
// users file once they've been dead long enough), optional
return async function getUserPosts(usernames, tracker = null) {
try {
const users = await Promise.map(usernames, async (username) => {
try {
// sequential rather than parallel: the index (whose path may depend on the
// user's details) decides how far back to page, and every reddit request
// goes through the same single-concurrency limiter anyway
const user = await getUser(username, reddit);
const { profile, posts: indexed } = await getIndex(user);
// sequential rather than parallel: the user decides whether there's anything to
// fetch at all, the index (whose path may depend on the user's details)
// decides how far back to page and whether social links are needed, and
// every reddit request goes through the same single-concurrency limiter anyway
const fetched = await fetchUser(username, reddit);
tracker?.record(username, fetched.status);
// nothing more to get from a suspended or deleted account - its listing is a
// 403/404 too, and skipping it saves the rest of this user's requests
if (fetched.status === 'suspended' || fetched.status === 'deleted') {
return null;
}
if (fetched.status === 'active') {
logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
}
const { profile, posts: indexed } = await getIndex(indexUser(username, fetched));
const user = await resolveUser(username, fetched, profile, reddit, args);
const posts = await getPosts(username, reddit, args, getCutoff(indexed, args));
if (args.archives) {
src/sources/user.js
+79
View File
@@ -0,0 +1,79 @@
'use strict';
const config = require('config');
const curateUser = require('../curate/user');
const logger = require('../logger')(__filename);
// reddit tells deleted and suspended accounts apart (verified live, 2026-11):
// - deleted: /user/x/about.json returns a JSON 404 (a shadowbanned account reportedly looks
// the same, so a 404 isn't necessarily permanent - see accountStatus.js's grace period)
// - suspended: about.json returns 200 with only { name, is_suspended: true, ... } - no date of
// suspension is exposed anywhere
// Anything else (network errors, 5xx, or the HTML bot-challenge page, which carries no
// `status` - see reddit/client.js's requestJson()) is 'error', which must never be treated as
// dead: a blocked or expired session would otherwise get the entire users file cleaned out.
async function fetchUser(username, reddit) {
try {
const user = await reddit.getUser(username);
if (user.is_suspended) {
return { status: 'suspended', user: null };
}
return { status: 'active', user };
} catch (error) {
if (error.status === 404) {
return { status: 'deleted', user: null };
}
logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return { status: 'error', user: null, error };
}
}
async function getSocialLinks(username, reddit) {
try {
return await reddit.getSocialLinks(username);
} catch (error) {
// supplementary, not worth failing the whole user fetch over - fall back to none
logger.warn(`Failed to fetch social links for '${username}': ${error.message} (https://reddit.com/user/${username})`);
return [];
}
}
// social links are only ever used for the profile description file (save/profileDetails.js),
// which is skipped once it's in the index (unless --redownload-profile) - scraping the profile
// page for them regardless costs one throttled reddit request per user per run for nothing
function needsSocialLinks(indexedProfile, args) {
return Boolean(config.library.profile.description && (args.redownloadProfile || !indexedProfile?.description));
}
// the user as getIndex() needs it to resolve the index path - the full curated user (minus social
// links, which depend on the index) in case a custom library.index.file pattern uses more than
// the name, otherwise the same fallback as resolveUser() below
function indexUser(username, fetched) {
return fetched.status === 'active' ? curateUser(fetched.user) : { name: username, fallback: true };
}
// curated user, or the same minimal fallback as before for an account that couldn't be fetched
// (or is suspended/deleted, when the caller still wants to proceed - e.g. a --post by them)
async function resolveUser(username, fetched, indexedProfile, reddit, args) {
if (fetched.status !== 'active') {
return { name: username, fallback: true };
}
const socialLinks = fetched.user.subreddit && needsSocialLinks(indexedProfile, args)
? await getSocialLinks(username, reddit)
: [];
return curateUser(fetched.user, socialLinks);
}
module.exports = {
fetchUser,
indexUser,
resolveUser,
};