Added generic photo page extract method to media module, to allow pre-filtering sources and relief Dogfart scraper. Added 'transsexual' site tag to Trans Angels.

src/media.js

@@ -12,6 +12,7 @@ const blake2 = require('blake2');
 const logger = require('./logger');
 const knex = require('./knex');
 const upsert = require('./utils/upsert');
+const { ex } = require('./utils/q');
 
 function getHash(buffer) {
     const hash = blake2.createHash('blake2b', { digestLength: 24 });
@@ -81,12 +82,22 @@ function curatePhotoEntries(files) {
 
 async function findDuplicates(photos, identifier, prop = null, label) {
     const duplicates = await knex('media')
-        .whereIn(identifier, photos.flat().map(photo => (prop ? photo[prop] : photo)));
+        .whereIn(identifier, photos.flat().map((photo) => {
+            if (prop) return photo[prop];
+            if (photo.src) return photo.src;
+
+            return photo;
+        }));
 
     const duplicateLookup = new Set(duplicates.map(photo => photo[prop || identifier]));
-    const originals = photos.filter(source => (Array.isArray(source) // fallbacks provided?
-        ? !source.some(sourceX => duplicateLookup.has(prop ? sourceX[prop] : sourceX)) // ensure none of the sources match
-        : !duplicateLookup.has(prop ? source[prop] : source)));
+
+    const originals = photos.filter((source) => {
+        if (Array.isArray(source)) {
+            return !source.some(sourceX => !duplicateLookup.has((prop && sourceX[prop]) || (sourceX.src && sourceX)));
+        }
+
+        return !duplicateLookup.has((prop && source[prop]) || (source.src && source));
+    });
 
     if (duplicates.length > 0) {
         logger.info(`${duplicates.length} media items already present by ${identifier} for ${label}`);
@@ -99,7 +110,26 @@ async function findDuplicates(photos, identifier, prop = null, label) {
     return [duplicates, originals];
 }
 
+async function extractPhoto(source) {
+    const res = await bhttp.get(source.src);
+
+    if (res.statusCode === 200) {
+        const { q } = ex(res.body.toString());
+
+        return source.extract(q);
+    }
+
+    return null;
+}
+
 async function fetchPhoto(photoUrl, index, label, attempt = 1) {
+    if (photoUrl.src && photoUrl.extract) {
+        // source links to page containing a (presumably) tokenized photo
+        const photo = await extractPhoto(photoUrl);
+
+        return fetchPhoto(photo, index, label);
+    }
+
     if (Array.isArray(photoUrl)) {
         return photoUrl.reduce(async (outcome, url) => outcome.catch(async () => {
             const photo = await fetchPhoto(url, index, label);