Using Imgur API rate limit feedback to prevent exceeding it.

src/app.js

@@ -106,10 +106,12 @@ async function getDirectContent(links, ep) {
         };
     });
 
-    // const predata = await fetchPredata(hosts.map(({ host }) => host));
+    const predata = await fetchPredata(hosts.map(({ host }) => host));
+
+    console.log('app predata', predata);
 
     return Promise.map(hosts, async ({ link, host }) => {
-        const info = await getInfo(host, reddit, link);
+        const info = await getInfo(host, { reddit, link, predata });
 
         if (info) {
             return fetchSaveDirectContent(info, host, ep);

src/cli.js

@@ -102,10 +102,12 @@ function getArgs() {
             choices: ['oldest', 'latest'],
         })
         .option('redownload', {
+            alias: 'force',
             describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed',
             type: 'boolean',
         })
         .option('redownload-profile', {
+            alias: 'force-profile',
             describe: 'Ignore index file and force a redownload of the profile image and description',
             type: 'boolean',
         })

src/fetch/info.js

@@ -6,54 +6,58 @@ const Promise = require('bluebird');
 const logger = require('../logger')(__filename);
 const methods = require('../methods/methods');
 
-const attachContentInfo = (users, { reddit, predata }) => Promise.reduce(Object.values(users), async (accUsers, user) => ({
+async function attachContentInfo(users, { reddit, predata }) {
-    ...accUsers,
+    return Promise.reduce(Object.values(users), async (accUsers, user) => ({
-    [user.name]: {
+        ...accUsers,
-        ...user,
+        [user.name]: {
-        posts: await Promise.reduce(user.posts, async (accPosts, post) => {
+            ...user,
-            if (!post.host || !methods[post.host.method]) {
+            posts: await Promise.reduce(user.posts, async (accPosts, post) => {
-                logger.warn(`Ignoring unsupported content '${post.url}' (${post.permalink})`);
+                if (!post.host || !methods[post.host.method]) {
+                    logger.warn(`Ignoring unsupported content '${post.url}' (${post.permalink})`);
 
-                return accPosts;
+                    return accPosts;
-            }
+                }
 
-            try {
+                console.log('attach predata', predata[post.host.method]);
-                return [
-                    ...accPosts,
-                    {
-                        ...post,
-                        content: await (methods[post.host.method].fetchInfo || methods[post.host.method])(post.host, post, {
-                            predata: predata[post.host.method],
-                            reddit,
-                        }),
-                    },
-                ];
-            } catch (error) {
-                logger.warn(`${error.message} (${post.permalink})`);
-
-                if (config.fetch.archives.preview && post.preview) {
-                    logger.info(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);
 
+                try {
                     return [
                         ...accPosts,
                         {
                             ...post,
-                            previewFallback: true,
+                            content: await (methods[post.host.method].fetchInfo || methods[post.host.method])(post.host, post, {
-                            content: await methods.redditPreview(post.host, post, {
+                                predata: predata[post.host.method],
-                                predata: predata.redditPreview,
                                 reddit,
                             }),
                         },
                     ];
+                } catch (error) {
+                    logger.warn(`${error.message} (${post.permalink})`);
+
+                    if (config.fetch.archives.preview && post.preview) {
+                        logger.info(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);
+
+                        return [
+                            ...accPosts,
+                            {
+                                ...post,
+                                previewFallback: true,
+                                content: await methods.redditPreview(post.host, post, {
+                                    predata: predata.redditPreview,
+                                    reddit,
+                                }),
+                            },
+                        ];
+                    }
+
+                    return accPosts;
                 }
+            }, []),
+        },
+    }), {});
+}
 
-                return accPosts;
+async function getInfo(host, { reddit, url, predata }) {
-            }
-        }, []),
-    },
-}), {});
-
-async function getInfo(host, reddit, url) {
     if (host === null) {
         try {
             const info = await methods.tube(host, null, reddit);
@@ -66,7 +70,7 @@ async function getInfo(host, reddit, url) {
         }
     }
 
-    return (methods[host.method].fetchInfo || methods[host.method])(host, null, reddit);
+    return (methods[host.method].fetchInfo || methods[host.method])(host, null, { reddit, predata });
 }
 
 module.exports = {

src/methods/imgurAlbum.js

@@ -4,14 +4,25 @@ const config = require('config');
 const fetch = require('node-fetch');
 
 const logger = require('../logger')(__filename);
+const { fetchPredata } = require('./imgurImage');
+
+async function imgurAlbumApi(host, post, { predata }) {
+    if (predata.remaining === 10) { // keep a buffer
+        throw new Error(`Reached Imgur API rate limit with source '${host.url}'`);
+    }
 
-async function imgurAlbumApi(host, post) {
     const res = await fetch(`https://api.imgur.com/3/album/${host.id}`, {
         headers: {
             Authorization: `Client-ID ${config.methods.imgur.clientId}`,
         },
     });
 
+    const rateRemaining = Number(res.headers.get('x-ratelimit-userremaining'));
+
+    if (rateRemaining) {
+        predata.setRemaining(rateRemaining);
+    }
+
     const { data } = await res.json();
 
     if (res.status !== 200) {
@@ -33,7 +44,7 @@ async function imgurAlbumApi(host, post) {
             datetime: new Date(data.datetime * 1000),
             original: data,
         },
-        items: data.images.map(item => ({
+        items: data.images.map((item) => ({
             extracted: extract,
             id: item.id,
             url: item.animated ? item.mp4 : item.link,
@@ -46,4 +57,7 @@ async function imgurAlbumApi(host, post) {
     };
 }
 
-module.exports = imgurAlbumApi;
+module.exports = {
+    fetchInfo: imgurAlbumApi,
+    fetchPredata,
+};

src/methods/imgurImage.js

@@ -3,16 +3,52 @@
 const config = require('config');
 const fetch = require('node-fetch');
 
-async function imgurImageApi(host) {
+async function fetchPredata() {
+    const data = {
+        limit: 0,
+        remaining: 0,
+    };
+
+    data.setRemaining = (remaining) => {
+        data.remaining = remaining;
+    };
+
+    const res = await fetch('https://api.imgur.com/3/credits', {
+        headers: {
+            Authorization: `Client-ID ${config.methods.imgur.clientId}`,
+        },
+    });
+
+    if (res.ok) {
+        const body = await res.json();
+
+        if (body.success) {
+            data.limit = body.data.UserLimit;
+            data.remaining = body.data.UserRemaining;
+        }
+    }
+
+    return data;
+}
+
+async function imgurImageApi(host, post, { predata } = {}) {
+    if (predata.remaining === 10) { // keep a buffer
+        throw new Error(`Reached Imgur API rate limit with source '${host.url}'`);
+    }
+
     const res = await fetch(`https://api.imgur.com/3/image/${host.id}`, {
         headers: {
             Authorization: `Client-ID ${config.methods.imgur.clientId}`,
         },
     });
 
-    console.log('imgur headers', res.headers);
+    const rateRemaining = Number(res.headers.get('x-ratelimit-userremaining'));
 
-    if (res.status !== 200) {
+    if (rateRemaining) {
+        predata.setRemaining(rateRemaining);
+    }
+
+    if (!res.ok) {
         throw new Error(`Imgur API returned HTTP ${res.status} for source '${host.url}'`);
     }
 
@@ -36,8 +72,11 @@ async function imgurImageApi(host) {
     };
 }
 
-async function imgurImage(host, post) {
+async function imgurImage(host, post, context) {
-    return imgurImageApi(host, post);
+    return imgurImageApi(host, post, context);
 }
 
-module.exports = imgurImage;
+module.exports = {
+    fetchInfo: imgurImage,
+    fetchPredata,
+};

src/methods/redgifs.js

@@ -6,6 +6,28 @@ const mime = require('mime');
 
 const { version } = require('../../package.json');
 
+async function fetchPredata() {
+    const userAgent = `ripunzel/${version}`;
+    const res = await fetch('https://api.redgifs.com/v2/auth/temporary', {
+        headers: {
+            'user-agent': userAgent,
+        },
+    });
+
+    const data = await res.json();
+
+    if (res.ok) {
+        return {
+            address: data.addr,
+            agent: data.agent,
+            token: data.token,
+            userAgent,
+        };
+    }
+
+    return null;
+}
+
 function scrapeGallery(data) {
     const oldestDate = Math.min(...data.gifs.map((gif) => gif.createDate));
 
@@ -125,28 +147,6 @@ async function redgifs(host, post, { predata }) {
     */
 }
 
-async function fetchPredata() {
-    const userAgent = `ripunzel/${version}`;
-    const res = await fetch('https://api.redgifs.com/v2/auth/temporary', {
-        headers: {
-            'user-agent': userAgent,
-        },
-    });
-
-    const data = await res.json();
-
-    if (res.ok) {
-        return {
-            address: data.addr,
-            agent: data.agent,
-            token: data.token,
-            userAgent,
-        };
-    }
-
-    return null;
-}
-
 module.exports = {
     fetchInfo: redgifs,
     fetchPredata,