Refactored media module. Returning 320p and 720p videos from MindGeek as teasers instead of trailers.

assets/components/actors/actors.vue

@@ -116,7 +116,7 @@ export default {
 
 .tiles {
     display: grid;
-    grid-template-columns: repeat(auto-fit, minmax(10rem, .5fr));
+    grid-template-columns: repeat(auto-fit, 10rem);
     grid-gap: 0 .5rem;
     padding: 1rem;
     flex-grow: 1;
@@ -127,10 +127,11 @@ export default {
     justify-content: center;
     align-items: center;
     padding: 0 1rem;
-    margin: 1rem 0 0 0;
+    margin: 1rem 0;
 }
 
 .genders {
+    flex-shrink: 0;
     padding: 0 .5rem 0 0;
     border-right: solid 1px $shadow-hint;
     margin: 0 1rem 0 0;
@@ -186,7 +187,7 @@ export default {
 }
 
 @media(max-width: $breakpoint) {
-    .actors {
+    .tiles {
         grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr));
     }
 }

assets/components/tags/tags.vue

@@ -147,7 +147,19 @@ export default {
 
 .tiles {
     display: grid;
-    grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr));
+    grid-template-columns: repeat(auto-fit, minmax(20rem, .25fr));
     grid-gap: .5rem;
 }
+
+@media(max-width: $breakpoint3) {
+    .tiles {
+        grid-template-columns: repeat(auto-fit, minmax(20rem, .5fr));
+    }
+}
+
+@media(max-width: $breakpoint) {
+    .tiles {
+        grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr));
+    }
+}
 </style>

assets/components/tile/release.vue

@@ -48,8 +48,10 @@
                 >{{ `(${formatDate(release.dateAdded, 'MMM D, YYYY')})` }}</a>
             </span>
 
-            <router-link
+            <a
-                :to="{ name: release.type || 'scene', params: { releaseId: release.id, releaseSlug: release.slug }, ...(referer && { hash: `#${referer}` }) }"
+                :href="`/${release.type || 'scene'}/${release.id}/${release.slug}`"
+                target="_blank"
+                rel="noopener noreferrer"
                 class="link"
             >
                 <img
@@ -71,12 +73,14 @@
                     :title="release.title"
                     class="thumbnail"
                 >No thumbnail available</div>
-            </router-link>
+            </a>
         </span>
 
         <div class="info">
-            <router-link
+            <a
-                :to="{ name: release.type || 'scene', params: { releaseId: release.id, releaseSlug: release.slug }, ...(referer && { hash: `#${referer}` }) }"
+                :href="`/${release.type || 'scene'}/${release.id}/${release.slug}`"
+                target="_blank"
+                rel="noopener noreferrer"
                 class="row link"
             >
                 <h3
@@ -89,7 +93,7 @@
                         icon="film"
                     />{{ release.title }}
                 </h3>
-            </router-link>
+            </a>
 
             <span class="row">
                 <ul class="actors nolist">

config/default.js

@@ -98,7 +98,7 @@ module.exports = {
         path: './media',
         thumbnailSize: 320, // width for 16:9 will be exactly 576px
         thumbnailQuality: 100,
-        trailerQuality: [480, 540],
+        videoQuality: [480, 360, 320, 540, 720, 1080, 2160],
         limit: 25, // max number of photos per release
     },
     titleSlugLength: 50,

seeds/04_media.js

@@ -1,39 +1,39 @@
 const upsert = require('../src/utils/upsert');
 
-const tagPosters = Object.entries({
+const tagPosters = [
-    'anal-creampie': [0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'],
+    ['anal-creampie', 0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'],
-    'ass-to-mouth': ['poster', 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
+    ['ass-to-mouth', 'poster', 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
-    'da-tp': [0, 'Natasha Teen in LegalPorno SZ2164'],
+    ['da-tp', 0, 'Natasha Teen in LegalPorno SZ2164'],
-    'double-anal': [5, 'Riley Reid in "The Gangbang of Riley Reid" for Jules Jordan'],
+    ['double-anal', 5, 'Riley Reid in "The Gangbang of Riley Reid" for Jules Jordan'],
-    'double-penetration': ['poster', 'Mia Malkova in "DP!" for HardX'],
+    ['double-penetration', 'poster', 'Mia Malkova in "DP!" for HardX'],
-    'double-vaginal': ['poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
+    ['double-vaginal', 'poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
-    'dv-tp': ['poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'],
+    ['dv-tp', 'poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'],
-    'oral-creampie': [1, 'Keisha Grey in Brazzers House'],
+    ['oral-creampie', 1, 'Keisha Grey in Brazzers House'],
-    'triple-anal': ['poster', 'Kristy Black in SZ1986 for LegalPorno'],
+    ['triple-anal', 'poster', 'Kristy Black in SZ1986 for LegalPorno'],
-    airtight: [1, 'Jynx Maze in "Pump My Ass Full of Cum 3" for Jules Jordan'],
+    ['airtight', 1, 'Jynx Maze in "Pump My Ass Full of Cum 3" for Jules Jordan'],
-    anal: ['poster', 'Jynx Maze in "Anal Buffet 6" for Evil Angel'],
+    ['anal', 'poster', 'Jynx Maze in "Anal Buffet 6" for Evil Angel'],
-    asian: ['poster', 'Vina Sky in "Young and Glamorous 10" for Jules Jordan'],
+    ['asian', 'poster', 'Vina Sky in "Young and Glamorous 10" for Jules Jordan'],
-    blowjob: [0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
+    ['blowjob', 0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
-    blowbang: ['poster'],
+    ['blowbang', 'poster'],
-    bukkake: ['poster'],
+    ['bukkake', 'poster'],
-    caucasian: ['poster'],
+    ['caucasian', 'poster'],
-    creampie: ['poster'],
+    ['creampie', 'poster'],
-    ebony: [1, 'Sarah Banks for Brazzers'],
+    ['ebony', 1, 'Sarah Banks for Brazzers'],
-    facial: ['poster'],
+    ['facial', 'poster'],
-    facefucking: ['1', 'Carrie for Young Throats'],
+    ['facefucking', '1', 'Carrie for Young Throats'],
-    gangbang: ['poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'],
+    ['gangbang', 'poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'],
-    gaping: [0, 'McKenzee Miles in "Anal Buffet 4" for Evil Angel'],
+    ['gaping', 1, 'Vina Sky in "Vina Sky Does Anal" for HardX'],
-    interracial: ['poster'],
+    ['interracial', 'poster'],
-    latina: ['poster'],
+    ['latina', 'poster'],
-    mff: ['poster'],
+    ['mff', 'poster'],
-    mfm: ['poster'],
+    ['mfm', 'poster'],
-    orgy: ['poster'],
+    ['orgy', 'poster'],
-    schoolgirl: [1, 'Eliza Ibarra for Brazzers'],
+    ['schoolgirl', 1, 'Eliza Ibarra for Brazzers'],
-    swallowing: ['poster'],
+    ['swallowing', 'poster'],
-    tattoo: ['poster', 'Kali Roses in "Goes All In For Anal" for Hussie Pass'],
+    ['tattoo', 'poster', 'Kali Roses in "Goes All In For Anal" for Hussie Pass'],
-    trainbang: ['poster', 'Kali Roses in "Passing Me Around" for Blacked'],
+    ['trainbang', 'poster', 'Kali Roses in "Passing Me Around" for Blacked'],
-})
+]
-    .map(([slug, [filename, comment]], index) => ({
+    .map(([slug, filename, comment], index) => ({
         tagSlug: slug,
         path: `tags/${slug}/${filename}.jpeg`,
         thumbnail: `tags/${slug}/${filename}_thumb.jpeg`,
@@ -63,6 +63,7 @@ const tagPhotos = [
     ['gangbang', 1, 'Ginger Lynn in "Gangbang Mystique", a photoset shot by Suze Randall for Puritan No. 10, 1984. This photo pushed the boundaries of pornography at the time, as depicting a woman \'fully occupied\' was unheard of.'],
     ['gangbang', 2, 'Riley Reid\'s double anal in "The Gangbang of Riley Reid" for Jules Jordan'],
     ['gaping', 'poster', 'Paulina in "Anal Buffet 4" for Evil Angel'],
+    ['gaping', 0, 'McKenzee Miles in "Anal Buffet 4" for Evil Angel'],
     ['trainbang', 0, 'Nicole Black in GIO971 for LegalPorno'],
     ['triple-anal', 1, 'Natasha Teen in SZ2098 for LegalPorno'],
     ['triple-anal', 2, 'Kira Thorn in GIO1018 for LegalPorno'],

src/actors.js

@@ -12,7 +12,8 @@ const scrapers = require('./scrapers/scrapers');
 const whereOr = require('./utils/where-or');
 const resolvePlace = require('./utils/resolve-place');
 const slugify = require('./utils/slugify');
-const { createMediaDirectory, storePhotos } = require('./media_legacy');
+// const { createMediaDirectory, storePhotos } = require('./media_legacy');
+const { storeMedia, associateMedia } = require('./media');
 
 async function curateActor(actor) {
     const [aliases, avatar, photos, social] = await Promise.all([
@@ -250,6 +251,17 @@ async function storeSocialLinks(urls, actorId) {
     await knex('actors_social').insert(curatedSocialEntries);
 }
 
+async function storeAvatars(avatars, actorId) {
+    if (!avatars || avatars.length === 0) {
+        return [];
+    }
+
+    const avatarsBySource = await storeMedia(avatars, 'actor', 'avatar');
+    await associateMedia({ [actorId]: avatars }, avatarsBySource, 'actor', 'photo', 'avatar');
+
+    return avatarsBySource;
+}
+
 async function storeActor(actor, scraped = false, scrapeSuccess = false) {
     const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
 
@@ -260,15 +272,7 @@ async function storeActor(actor, scraped = false, scrapeSuccess = false) {
     await storeSocialLinks(actor.social, actorEntry.id);
 
     if (actor.avatars) {
-        await createMediaDirectory('actors', `${actorEntry.slug}/`);
+        await storeAvatars(actor.avatars, actorEntry.id);
-        await storePhotos(actor.avatars, {
-            domain: 'actor',
-            role: 'photo',
-            primaryRole: 'avatar',
-            targetId: actorEntry.id,
-            subpath: `${actorEntry.slug}/`,
-            naming: 'timestamp',
-        }, actorEntry.name);
     }
 
     logger.info(`Added new entry for actor '${actor.name}'`);
@@ -421,19 +425,9 @@ async function scrapeActors(actorNames) {
 
             if (argv.save) {
                 if (actorEntry && profile) {
-                    await createMediaDirectory('actors', `${actorEntry.slug}/`);
-
                     await Promise.all([
                         updateActor(profile, true, true),
-                        // storeAvatars(profile, actorEntry),
+                        storeAvatars(profile.avatars, actorEntry.id),
-                        storePhotos(profile.avatars, {
-                            domain: 'actor',
-                            role: 'photo',
-                            primaryRole: 'avatar',
-                            targetId: actorEntry.id,
-                            subpath: `${actorEntry.slug}/`,
-                            naming: 'timestamp',
-                        }, actorEntry.name),
                     ]);
 
                     return profile;

src/media.js

@@ -4,11 +4,14 @@ const config = require('config');
 const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const mime = require('mime');
+const fs = require('fs-extra');
 const sharp = require('sharp');
+const path = require('path');
 const blake2 = require('blake2');
 
-const logger = require('./logger');
+const logger = require('./logger')(__filename);
 const knex = require('./knex');
+const { ex } = require('./utils/q');
 
 function getHash(buffer) {
     const hash = blake2.createHash('blake2b', { digestLength: 24 });
@@ -17,6 +20,26 @@ function getHash(buffer) {
     return hash.digest('hex');
 }
 
+async function createThumbnail(buffer) {
+    try {
+        const thumbnail = sharp(buffer)
+            .resize({
+                height: config.media.thumbnailSize,
+                withoutEnlargement: true,
+            })
+            .jpeg({
+                quality: config.media.thumbnailQuality,
+            })
+            .toBuffer();
+
+        return thumbnail;
+    } catch (error) {
+        logger.error(`Failed to create thumbnail: ${error.message}`);
+    }
+
+    return null;
+}
+
 function pluckItems(items, specifiedLimit) {
     const limit = specifiedLimit || config.media.limit;
 
@@ -30,6 +53,13 @@ function pluckItems(items, specifiedLimit) {
     return Array.from(new Set(plucked)).map(itemIndex => items[itemIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close
 }
 
+function pickQuality(items) {
+    const itemsByQuality = items.reduce((acc, item) => ({ ...acc, [item.quality]: item }), {});
+    const item = config.media.videoQuality.reduce((acc, quality) => acc || itemsByQuality[quality], null);
+
+    return item || items[0];
+}
+
 async function getEntropy(buffer) {
     try {
         const { entropy } = await sharp(buffer).stats();
@@ -42,33 +72,58 @@ async function getEntropy(buffer) {
     }
 }
 
-async function fetchItem(source, index, existingItemsBySource, attempt = 1) {
+async function extractItem(source) {
+    const res = await bhttp.get(source.src);
+
+    if (res.statusCode === 200) {
+        const { q } = ex(res.body.toString());
+
+        return source.extract(q);
+    }
+
+    return null;
+}
+
+async function fetchItem(source, index, existingItemsBySource, domain, role, attempt = 1, originalSource = null) {
     try {
         if (Array.isArray(source)) {
+            if (source.every(sourceX => !!sourceX.quality)) {
+                // various video qualities provided
+                const selectedSource = pickQuality(source);
+                return fetchItem(selectedSource, index, existingItemsBySource, domain, role, attempt, originalSource);
+            }
+
             // fallbacks provided
-            return source.reduce((outcome, sourceX) => outcome.catch(async () => {
+            return source.reduce(
-                const item = await fetchItem(sourceX, index, existingItemsBySource);
+                (outcome, sourceX) => outcome.catch(async () => fetchItem(sourceX, index, existingItemsBySource, domain, role, attempt, originalSource)),
-
+                Promise.reject(new Error()),
-                if (item) {
+            );
-                    return item;
-                }
-
-                throw new Error(`Item not available: ${source}`);
-            }), Promise.reject(new Error()));
         }
 
+        if (source.src && source.extract) {
+            // source links to page containing a (presumably) tokenized photo
+            const itemSource = await extractItem(source);
+
+            return fetchItem(itemSource, index, existingItemsBySource, domain, role, attempt, source);
+        }
+
+
         if (existingItemsBySource[source]) {
-            return existingItemsBySource[source];
+            return null;
         }
 
-        const res = await bhttp.get(source);
+        logger.verbose(`Fetching media item from ${source.src || source}`);
+
+        const res = await bhttp.get(source.src || source);
 
         if (res.statusCode === 200) {
-            const { pathname } = new URL(source);
+            const { pathname } = new URL(source.src || source);
             const mimetype = mime.getType(pathname);
             const extension = mime.getExtension(mimetype);
             const hash = getHash(res.body);
-            const entropy = await getEntropy(res.body);
+            const entropy = /image/.test(mimetype) ? await getEntropy(res.body) : null;
+
+            logger.verbose(`Fetched media item from ${source.src || source}`);
 
             return {
                 file: res.body,
@@ -76,40 +131,193 @@ async function fetchItem(source, index, existingItemsBySource, attempt = 1) {
                 extension,
                 hash,
                 entropy,
-                source,
+                quality: source.quality || null,
+                source: originalSource?.src || originalSource || source.src || source,
             };
         }
 
         throw new Error(`Response ${res.statusCode} not OK`);
     } catch (error) {
-        if (attempt <= 3) {
+        logger.warn(`Failed attempt ${attempt}/3 to fetch ${domain} ${role} ${index + 1} (${source.src || source}): ${error}`);
-            return fetchItem(source, index, existingItemsBySource, attempt + 1);
+
+        if (attempt < 3) {
+            await Promise.delay(5000);
+            return fetchItem(source, index, existingItemsBySource, domain, role, attempt + 1);
         }
 
-        throw new Error(`Failed to fetch media from ${source}: ${error}`);
+        return null;
     }
 }
 
-async function fetchItems(itemSources, existingItemsBySource) {
+async function fetchItems(itemSources, existingItemsBySource, domain, role) {
-    return Promise.map(itemSources, async (source, index) => fetchItem(source, index, existingItemsBySource));
+    return Promise.map(itemSources, async (source, index) => fetchItem(source, index, existingItemsBySource, domain, role), {
+        concurrency: 10,
+    }).filter(Boolean);
 }
 
-async function storeReleaseMedia(releases, {
+async function saveItems(items, domain, role) {
-    type = 'poster',
+    return Promise.map(items, async (item) => {
-} = {}) {
+        try {
-    const pluckedSources = releases.map(release => pluckItems(release[type]));
+            const dir = item.hash.slice(0, 2);
-    const existingSourceItems = await knex('media').whereIn('source', pluckedSources.flat());
+            const subdir = item.hash.slice(2, 4);
-    const existingItemsBySource = existingSourceItems.reduce((acc, item) => ({ ...acc, [item.source]: item }), {});
+            const filename = item.quality
+                ? `${item.hash.slice(4)}_${item.quality}.${item.extension}`
+                : `${item.hash.slice(4)}.${item.extension}`;
 
-    const fetchedItems = await fetchItems(pluckedSources, existingItemsBySource);
+            const filedir = path.join(`${domain}s`, `${role}s`, dir, subdir);
-    const existingHashItems = await knex('media').whereIn('hash', fetchedItems.map(item => item.hash));
+            const filepath = path.join(filedir, filename);
-    const existingItemsByHash = existingHashItems.reduce((acc, item) => ({ ...acc, [item.hash]: item }), {});
 
-    const newItems = fetchedItems.filter(item => !existingItemsByHash[item.hash]);
+            await fs.mkdir(path.join(config.media.path, filedir), { recursive: true });
+            await fs.writeFile(path.join(config.media.path, filepath), item.file);
 
-    console.log(fetchedItems, existingHashItems, existingItemsByHash, newItems);
+            if (/image/.test(item.mimetype)) {
+               const thumbnail = await createThumbnail(item.file);
+
+                const thumbdir = path.join(`${domain}s`, `${role}s`, 'thumbs', dir, subdir);
+                const thumbpath = path.join(thumbdir, filename);
+
+                await fs.mkdir(path.join(config.media.path, thumbdir), { recursive: true });
+                await fs.writeFile(path.join(config.media.path, thumbpath), thumbnail);
+
+                logger.verbose(`Saved ${domain} ${role} with thumbnail to ${filepath}`);
+
+                return {
+                    ...item,
+                    thumbnail,
+                    filepath,
+                    thumbpath,
+                };
+            }
+
+            logger.verbose(`Saved ${domain} ${role} to ${filepath}`);
+
+            return {
+                ...item,
+                filepath,
+            };
+        } catch (error) {
+            logger.error(`Failed to store ${domain} ${role} from ${item.source}: ${error.message}`);
+            return null;
+        }
+    });
+}
+
+function curateItemEntries(items) {
+    return items.filter(Boolean).map((item, index) => ({
+        path: item.filepath,
+        thumbnail: item.thumbpath,
+        mime: item.mimetype,
+        hash: item.hash,
+        source: item.source,
+        entropy: item.entropy,
+        index,
+    }));
+}
+
+function groupItems(items) {
+    return items.reduce((acc, item) => ({
+        source: { ...acc.source, [item.source]: item },
+        hash: { ...acc.hash, [item.hash]: item },
+    }), {
+        source: {},
+        hash: {},
+    });
+}
+
+async function storeMedia(sources, domain, role) {
+    const presentSources = sources.filter(Boolean);
+
+    if (presentSources.length === 0) {
+        return {};
+    }
+
+    // find source duplicates that don't need to be re-downloaded or re-saved
+    const existingSourceItems = await knex('media').whereIn('source', presentSources.flat().map(source => source.src || source));
+    const { source: existingSourceItemsBySource, hash: existingSourceItemsByHash } = groupItems(existingSourceItems);
+
+    // download media items from new sources
+    const fetchedItems = await fetchItems(presentSources, existingSourceItemsBySource, domain, role);
+    const { hash: fetchedItemsByHash } = groupItems(fetchedItems);
+
+    // find hash duplicates that don't need to be re-saved
+    const uniqueFetchedItems = Object.values(fetchedItemsByHash);
+    const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash));
+    const { hash: existingHashItemsByHash } = groupItems(existingHashItems);
+
+    // save new items to disk
+    const newItems = uniqueFetchedItems.filter(item => !existingHashItemsByHash[item.hash]);
+    const savedItems = await saveItems(newItems, domain, role);
+
+    // store new items in database
+    const curatedItemEntries = curateItemEntries(savedItems);
+    const storedItems = await knex('media').insert(curatedItemEntries).returning('*');
+    const { hash: storedItemsByHash } = groupItems(Array.isArray(storedItems) ? storedItems : []);
+
+    // accumulate existing and new items by source to be mapped onto releases
+    const itemsByHash = { ...existingSourceItemsByHash, ...existingHashItemsByHash, ...storedItemsByHash };
+    const itemsBySource = {
+        ...existingSourceItemsBySource,
+        ...fetchedItems.reduce((acc, item) => ({ ...acc, [item.source]: itemsByHash[item.hash] }), {}),
+    };
+
+    logger.info(`Stored ${fetchedItems.length} new ${domain} ${role}s`);
+
+    return itemsBySource;
+}
+
+function extractPrimaryItem(associations, targetId, role, primaryRole, primaryItemsByTargetId) {
+    if (!primaryRole) {
+        return { [role]: associations, [primaryRole]: null };
+    }
+
+    if (primaryItemsByTargetId[targetId]) {
+        const remainingAssociations = associations.filter(association => association.media_id !== primaryItemsByTargetId[targetId].media_id);
+
+        return { [role]: remainingAssociations, [primaryRole]: null };
+    }
+
+    return {
+        [role]: associations.slice(1),
+        [primaryRole]: associations.slice(0, 1)[0],
+    };
+}
+
+function associateTargetMedia(targetId, sources, mediaBySource, domain, role, primaryRole, primaryItemsByTargetId) {
+    if (!sources) return { [role]: null, [primaryRole]: null };
+
+    const associations = sources
+        .filter(Boolean)
+        .map((source) => {
+            const mediaItem = Array.isArray(source)
+                ? source.reduce((acc, sourceX) => acc || mediaBySource[sourceX.src || sourceX], null)
+                : mediaBySource[source.src || source];
+
+            return mediaItem && { [`${domain}_id`]: targetId, media_id: mediaItem.id };
+        })
+        .filter(Boolean);
+
+    logger.info(`Associating ${associations.length} ${role}s to ${domain} ${targetId}`);
+
+    return extractPrimaryItem(associations, targetId, role, primaryRole, primaryItemsByTargetId);
+}
+
+async function associateMedia(sourcesByTargetId, mediaBySource, domain, role, primaryRole) {
+    const primaryItems = primaryRole ? await knex(`${domain}s_${primaryRole}s`).whereIn(`${domain}_id`, Object.keys(sourcesByTargetId)) : [];
+    const primaryItemsByTargetId = primaryItems.reduce((acc, item) => ({ ...acc, [item[`${domain}_id`]]: item }), {});
+
+    const associationsPerTarget = await Promise.map(Object.entries(sourcesByTargetId), ([targetId, sources]) => associateTargetMedia(targetId, sources, mediaBySource, domain, role, primaryRole, primaryItemsByTargetId));
+
+    const associations = associationsPerTarget.map(association => association[role]).flat().filter(Boolean);
+    const primaryAssociations = associationsPerTarget.map(association => association[primaryRole]).filter(Boolean);
+
+    return Promise.all([
+        (associations.length > 0 && knex.raw(`${knex(`${domain}s_${role}s`).insert(associations).toString()} ON CONFLICT DO NOTHING`)),
+        (primaryAssociations.length > 0 && knex.raw(`${knex(`${domain}s_${primaryRole}s`).insert(primaryAssociations).toString()} ON CONFLICT DO NOTHING`)),
+    ]);
 }
 
 module.exports = {
-    storeReleaseMedia,
+    pluckItems,
+    storeMedia,
+    associateMedia,
 };

src/releases.js

@@ -10,20 +10,11 @@ const argv = require('./argv');
 const whereOr = require('./utils/where-or');
 const { associateTags } = require('./tags');
 const { associateActors, scrapeBasicActors } = require('./actors');
-/*
 const {
-    createMediaDirectory,
+    pluckItems,
-    storePhotos,
+    storeMedia,
-    // storeReleasePhotos,
+    associateMedia,
-    storeTrailer,
-    storeReleaseMedia,
 } = require('./media');
-*/
-const {
-    createMediaDirectory,
-    storePhotos,
-    storeTrailer,
-} = require('./media_legacy');
 const { fetchSites, findSiteByUrl } = require('./sites');
 const slugify = require('./utils/slugify');
 
@@ -337,55 +328,39 @@ function accumulateMovies(releases) {
 }
 
 async function storeReleaseAssets(releases) {
-    // await storeReleasePhotos(releases);
+    const releasePostersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.poster] }), {});
+    const releaseCoversById = releases.reduce((acc, release) => ({ ...acc, [release.id]: release.covers }), {});
+    const releaseTrailersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.trailer] }), {});
+    const releaseTeasersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.teaser] }), {});
+    const releasePhotosById = releases.reduce((acc, release) => ({
+        ...acc,
+        [release.id]: pluckItems(release.photos),
+    }), {});
 
-    // return storeReleaseMedia(releases);
+    const [posters, covers] = await Promise.all([
+        storeMedia(Object.values(releasePostersById).flat(), 'release', 'poster'),
+        storeMedia(Object.values(releaseCoversById).flat(), 'release', 'cover'),
+    ]);
 
-    await Promise.map(releases, async (release) => {
+    // ensure posters are available before fetching supplementary media
-        const subpath = `${release.site.network.slug}/${release.site.slug}/${release.id}/`;
+    await Promise.all([
-        const identifier = `"${release.title}" (${release.id})`;
+        associateMedia(releasePostersById, posters, 'release', 'poster'),
+        associateMedia(releaseCoversById, covers, 'release', 'cover'),
+    ]);
 
-        try {
+    const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
-            await createMediaDirectory('releases', subpath);
+    await associateMedia(releasePhotosById, photos, 'release', 'photo');
 
-            // don't use Promise.all to prevent concurrency issues with duplicate detection
+    // videos take a long time, fetch last
-            if (release.poster) {
+    const [trailers, teasers] = await Promise.all([
-                await storePhotos([release.poster], {
+        storeMedia(Object.values(releaseTrailersById).flat(), 'release', 'trailer'),
-                    role: 'poster',
+        storeMedia(Object.values(releaseTeasersById).flat(), 'release', 'teaser'),
-                    targetId: release.id,
+    ]);
-                    subpath,
-                }, identifier);
-            }
 
-            await storePhotos(release.photos, {
+    await Promise.all([
-                targetId: release.id,
+        associateMedia(releaseTrailersById, trailers, 'release', 'trailer'),
-                subpath,
+        associateMedia(releaseTeasersById, teasers, 'release', 'teaser'),
-                primaryRole: release.poster ? null : 'poster',
+    ]);
-            }, identifier);
-
-            await storePhotos(release.covers, {
-                role: 'cover',
-                targetId: release.id,
-                subpath,
-            }, identifier);
-
-            await storeTrailer(release.trailer, {
-                targetId: release.id,
-                subpath,
-                role: 'trailer',
-            }, identifier);
-
-            await storeTrailer(release.teaser, {
-                targetId: release.id,
-                subpath,
-                role: 'teaser',
-            }, identifier);
-        } catch (error) {
-            logger.error(error.message);
-        }
-    }, {
-        concurrency: 10,
-    });
 }
 
 async function storeRelease(release) {
@@ -453,6 +428,8 @@ async function storeReleases(releases) {
         concurrency: 10,
     }).filter(release => release);
 
+    logger.info(`Stored ${storedReleases.length} new releases`);
+
     const actors = accumulateActors(storedReleases);
     const movies = accumulateMovies(storedReleases);
 

src/scrapers/dogfart.js

@@ -47,7 +47,7 @@ function scrapeLatest(html, site) {
         const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim());
 
         const poster = `https:${element.querySelector('img').src}`;
-        const trailer = sceneLinkElement.dataset.preview_clip_url;
+        const teaser = sceneLinkElement.dataset.preview_clip_url;
 
         return [
             ...acc,
@@ -57,8 +57,8 @@ function scrapeLatest(html, site) {
                 title,
                 actors,
                 poster,
-                trailer: {
+                teaser: {
-                    src: trailer,
+                    src: teaser,
                 },
                 site,
             },
@@ -99,7 +99,7 @@ async function scrapeScene(html, url, site) {
     const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href;
     const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
 
-    const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]').textContent) / 2);
+    const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2);
     const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
 
     return {

src/scrapers/mindgeek.js

@@ -36,35 +36,33 @@ function scrapeLatestX(data, site) {
         return null;
     }
 
-    const { id: entryId, title, description } = data;
+    const release = {
-    const hostname = site.parameters?.native ? site.url : site.network.url;
+        entryId: data.id,
-    const url = `${hostname}/scene/${entryId}/`;
+        title: data.title,
-    const date = new Date(data.dateReleased);
+        description: data.description,
-    const actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
-
-    const tags = data.tags.map(tag => tag.name);
-
-    const [poster, ...photos] = getThumbs(data);
-    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
-    const duration = data.videos.mediabook && data.videos.mediabook.length;
-
-    return {
-        url,
-        entryId,
-        title,
-        description,
-        actors,
-        tags,
-        duration,
-        poster,
-        photos,
-        trailer: trailer && {
-            src: trailer.urls.view,
-            quality: parseInt(trailer.format, 10),
-        },
-        date,
-        site,
     };
+
+    const hostname = site.parameters?.native ? site.url : site.network.url;
+
+    release.url = `${hostname}/scene/${release.entryId}/`;
+    release.date = new Date(data.dateReleased);
+    release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
+
+    release.tags = data.tags.map(tag => tag.name);
+
+    release.duration = data.videos.mediabook?.length;
+    [release.poster, ...release.photos] = getThumbs(data);
+
+    const teaserSources = data.videos.mediabook?.files;
+
+    if (teaserSources) {
+        release.teaser = Object.values(teaserSources).map(teaser => ({
+            src: teaser.urls.view,
+            quality: parseInt(teaser.format, 10),
+        }));
+    }
+
+    return release;
 }
 
 async function scrapeLatest(items, site) {
@@ -89,12 +87,13 @@ function scrapeScene(data, url, _site, networkName) {
 
     [release.poster, ...release.photos] = getThumbs(data);
 
-    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
+    const teaserSources = data.videos.mediabook?.files;
-    if (trailer) {
+
-        release.trailer = {
+    if (teaserSources) {
-            src: trailer.urls.view,
+        release.teaser = Object.values(teaserSources).map(teaser => ({
-            quality: parseInt(trailer.format, 10),
+            src: teaser.urls.view,
-        };
+            quality: parseInt(teaser.format, 10),
+        }));
     }
 
     const siteName = data.collections[0]?.name || data.brand;