Added support for RedGifs and Reddit albums. Improved command line logger. Added rate limiters for reddit and host URLs.

src/fetch/item.js

@@ -4,13 +4,14 @@ const config = require('config');
 const bhttp = require('bhttp');
 const blake2 = require('blake2');
 
+const logger = require('../logger')(__filename);
+const limiter = require('../limiter').items;
+
 async function fetchItem(url, attempt, post, host) {
     async function retry(error) {
-        console.log('\x1b[31m%s\x1b[0m', `Failed to fetch '${url}': ${error.message} (${post ? post.permalink : 'no post'})`);
+        logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${post ? post.permalink : 'no post'})`);
 
         if (attempt < config.fetch.retries) {
-            console.log('Retrying...');
-
             return fetchItem(url, attempt + 1, post);
         }
 
@@ -18,13 +19,17 @@ async function fetchItem(url, attempt, post, host) {
     }
 
     try {
-        const res = await bhttp.get(url);
+        const res = await limiter.schedule(async () => bhttp.get(url));
 
         if (!res.statusCode === 200) {
             throw new Error(`Response not OK for '${url}', HTTP code '${res.status}'`);
         }
 
-        console.log(`Fetched '${host.url}' (${post ? post.permalink : 'no post'})`);
+        if (!Buffer.isBuffer(res.body)) {
+            throw new Error(`Unexpected response for '${url}' (${res.status}): ${res.body}`);
+        }
+
+        logger.debug(`Fetched '${host ? host.url : url}' (${post ? post.permalink : 'no post'})`);
 
         const hash = blake2.createHash('blake2b', { digestLength: 24 });
         hash.update(res.body);
@@ -32,7 +37,6 @@ async function fetchItem(url, attempt, post, host) {
 
         return Object.assign(res.body, { hash: contentHash });
     } catch (error) {
-        console.log(error);
         return retry(error);
     }
 }