Storing pHash. Fixed RedGIFs user agent.

src/fetch/item.js

@@ -3,23 +3,26 @@
 const config = require('config');
 const bhttp = require('bhttp');
 const blake2 = require('blake2');
+const phash = require('sharp-phash');
+// const phashDistance = require('sharp-phash/distance');
 
 const logger = require('../logger')(__filename);
 const limiter = require('../limiter').items;
 
-async function fetchItem(url, attempt, { post, host, headers }) {
+async function fetchItem(url, attempt, context) {
     async function retry(error) {
-        logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${post ? post.permalink : 'no post'})`);
+        logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${context.post ? context.post.permalink : 'no post'})`);
 
         if (attempt < config.fetch.retries) {
-            return fetchItem(url, attempt + 1, { post, host, headers });
+            return fetchItem(url, attempt + 1, context);
         }
 
         return null;
     }
 
     try {
-        const res = await limiter.schedule(async () => bhttp.get(url, { headers }));
+        // throw new Error('Failed it!');
+        const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers }));
 
         if (res.statusCode !== 200) {
             throw new Error(`Response not OK for ${url} (${res.statusCode}): ${res.body.toString()}`);
@@ -29,13 +32,21 @@ async function fetchItem(url, attempt, { post, host, headers }) {
             throw new Error(`Unexpected response for ${url} (${res.statusCode}): ${res.body}`);
         }
 
-        logger.debug(`Fetched '${host ? host.url : url}' (${post ? post.permalink : 'no post'})`);
+        logger.debug(`Fetched '${context.host?.url || url}' (${context.post?.permalink || 'no post'})`);
 
         const hash = blake2.createHash('blake2b', { digestLength: 24 });
         hash.update(res.body);
         const contentHash = hash.digest('hex');
 
-        return Object.assign(res.body, { hash: contentHash });
+        const phashResult = context.item?.type?.includes('image/')
+            ? await phash(res.body)
+            : null;
+
+        return {
+            buffer: res.body,
+            hash: contentHash,
+            phash: phashResult,
+        };
     } catch (error) {
         return retry(error);
     }