Reduced reddit requests: stop paging at --after(-indexed) cutoff, reuse listing media for videos, SOCKS5 support for reddit/fetch requests, routed defunct gfycat links to RedGifs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/sources/getUserPosts.js

@@ -39,11 +39,35 @@ async function getUser(username, reddit) {
     }
 }
 
-async function getPosts(username, reddit, args) {
+// the moment before which posts will be discarded by curate/posts.js anyway (--after, or
+// --after-indexed against the existing index) - lets getSubmissions() stop paging there
+// instead of always walking reddit's entire (up to ~1000 post) listing. Without either flag,
+// every page is still fetched, so gaps anywhere in the archive keep getting backfilled.
+function getCutoff(indexedPosts, args) {
+    const cutoffs = [];
+
+    if (args.after) {
+        cutoffs.push(args.after);
+    }
+
+    if (args.afterIndexed && indexedPosts?.length) {
+        const dates = indexedPosts.map((entry) => new Date(entry.date).getTime()).filter((time) => !Number.isNaN(time));
+
+        if (dates.length) {
+            cutoffs.push(new Date(args.afterIndexed === 'latest' ? Math.max(...dates) : Math.min(...dates)));
+        }
+    }
+
+    // with both set, the later of the two discards the most, so stop there
+    return cutoffs.length ? new Date(Math.max(...cutoffs.map((date) => date.getTime()))) : null;
+}
+
+async function getPosts(username, reddit, args, until) {
     try {
         const submissions = await reddit.getSubmissions(username, {
             sort: args.sort,
             limit: Infinity,
+            until,
         });
 
         logger.info(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);
@@ -67,12 +91,12 @@ function getUserPostsWrap(reddit, args) {
         try {
             const users = await Promise.map(usernames, async (username) => {
                 try {
-                    const [user, posts] = await Promise.all([
-                        getUser(username, reddit),
-                        getPosts(username, reddit, args),
-                    ]);
-
+                    // sequential rather than parallel: the index (whose path may depend on the
+                    // user's details) decides how far back to page, and every reddit request
+                    // goes through the same single-concurrency limiter anyway
+                    const user = await getUser(username, reddit);
                     const { profile, posts: indexed } = await getIndex(user);
+                    const posts = await getPosts(username, reddit, args, getCutoff(indexed, args));
 
                     if (args.archives) {
                         posts.push(...await getArchivedPosts(username, posts, reddit));
@@ -92,7 +116,7 @@ function getUserPostsWrap(reddit, args) {
 
                     return null;
                 } catch (error) {
-                    logger.error(`Failed to fetch posts from 'username': ${error.message}`);
+                    logger.error(`Failed to fetch posts from '${username}': ${error.message}`);
 
                     return null;
                 }