Added self hash filtering to media module. Moved Girl Girl back to Jules Jordan.

assets/components/releases/banner.vue

@@ -117,12 +117,12 @@ export default {
 @import 'theme';
 
 .banner {
-    background: $empty;
+    background: var(--empty);
     flex-shrink: 0;
     white-space: nowrap;
     overflow-x: auto;
     scrollbar-width: none;
-    box-shadow: 0 0 3px $shadow;
+    box-shadow: 0 0 3px var(--shadow);
     font-size: 0;
 
     &::-webkit-scrollbar {
@@ -150,7 +150,7 @@ export default {
     .icon {
         width: 1.5rem;
         height: 1.5rem;
-        fill: $highlight;
+        fill: var(--lighten);
     }
 
     &.playing {
@@ -162,7 +162,7 @@ export default {
         opacity: 1;
 
         .icon {
-            fill: $highlight-strong;
+            fill: var(--lighten-strong);
         }
     }
 }

assets/components/tile/tag.vue

@@ -42,13 +42,12 @@ export default {
     background: var(--profile);
     display: flex;
     flex-direction: column;
-    align-items: left;
     justify-content: flex-end;
     box-sizing: border-box;
     position: relative;
     text-align: center;
     text-decoration: none;
-    box-shadow: inset 0 0 3px var(--darken);
+    box-shadow: 0 0 3px var(--darken-weak);
 }
 
 .poster {
@@ -56,7 +55,6 @@ export default {
     height: 16rem;
     object-fit: cover;
     object-position: 50% 100%;
-    box-shadow: 0 0 1px var(--darken);
 }
 
 .title {

migrations/20190325001339_releases.js

@@ -296,6 +296,9 @@ exports.up = knex => Promise.resolve()
         table.string('piercings');
         table.string('tattoos');
 
+        table.datetime('updated_at')
+            .defaultTo(knex.fn.now());
+
         table.datetime('created_at')
             .defaultTo(knex.fn.now());
     }))

seeds/01_networks.js

@@ -196,11 +196,6 @@ const networks = [
         description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!',
         parent: 'gamma',
     },
-    {
-        slug: 'girlgirl',
-        name: 'Girl Girl',
-        url: 'https://www.girlgirl.com',
-    },
     {
         slug: 'hussiepass',
         name: 'Hussie Pass',

seeds/02_sites.js

@@ -2115,15 +2115,6 @@ const sites = [
         tags: ['facefucking', 'blowjob'],
         network: 'fullpornnetwork',
     },
-    // GIRL GIRL
-    {
-        slug: 'girlgirl',
-        name: 'Girl Girl',
-        url: 'https://www.girlgirl.com',
-        tags: ['lesbian'],
-        parameters: { independent: true },
-        network: 'girlgirl',
-    },
     // GIRLSWAY
     {
         slug: 'girlsway',
@@ -2635,6 +2626,13 @@ const sites = [
         url: 'https://www.manuelferrara.com',
         network: 'julesjordan',
     },
+    {
+        slug: 'girlgirl',
+        name: 'Girl Girl',
+        url: 'https://www.girlgirl.com',
+        tags: ['lesbian'],
+        network: 'julesjordan',
+    },
     // KELLY MADISON MEDIA
     {
         slug: 'teenfidelity',

src/media.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const config = require('config');
-const util = require('util');
+// const util = require('util');
 const Promise = require('bluebird');
 const fs = require('fs').promises;
 const path = require('path');
@@ -177,10 +177,7 @@ async function findSourceDuplicates(baseMedias) {
     const existingSourceMediaByUrl = itemsByKey(existingSourceMedia, 'source');
     const existingExtractMediaByUrl = itemsByKey(existingExtractMedia, 'source_page');
 
-    return {
-        existingSourceMediaByUrl,
-        existingExtractMediaByUrl,
-    };
+    return [existingSourceMediaByUrl, existingExtractMediaByUrl];
 }
 
 async function findHashDuplicates(medias) {
@@ -189,16 +186,37 @@ async function findHashDuplicates(medias) {
     const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
     const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
 
-    const uniqueHashMedia = medias.filter(media => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
+    const uniqueHashMedias = medias.filter(media => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
 
-    const existingHashMedia = medias
+    const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
+        if (acc.selfUniqueMediasByHash[media.meta.hash]) {
+            acc.selfDuplicateMedias.push({
+                ...media,
+                use: acc.selfUniqueMediasByHash[media.meta.hash].id,
+            });
+
+            return acc;
+        }
+
+        acc.selfUniqueMediasByHash[media.meta.hash] = media;
+
+        return acc;
+    }, {
+        selfDuplicateMedias: [],
+        selfUniqueMediasByHash: {},
+    });
+
+    const selfUniqueHashMedias = Object.values(selfUniqueMediasByHash);
+
+    const existingHashMedias = medias
         .filter(media => existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash])
         .map(media => ({
             ...media,
             entry: existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash],
-        }));
+        }))
+        .concat(selfDuplicateMedias);
 
-    return { uniqueHashMedia, existingHashMedia };
+    return [selfUniqueHashMedias, existingHashMedias];
 }
 
 async function extractSource(baseSource, { existingExtractMediaByUrl }) {
@@ -337,7 +355,7 @@ async function fetchSource(source) {
     return attempt(1);
 }
 
-async function trySource(baseSource, existingMedias, baseMedia, baseSourceIndex) {
+async function trySource(baseSource, existingMedias) {
     // catch error and try the next source
     const extractedSource = await extractSource(baseSource, existingMedias);
     const existingSourceMedia = existingMedias.existingSourceMediaByUrl[extractedSource.src];
@@ -350,7 +368,7 @@ async function trySource(baseSource, existingMedias, baseMedia, baseSourceIndex)
     }
 
     if (existingSourceMedia) {
-        logger.silly(`Media source URL already in database, skipping ${baseSource.url}`);
+        logger.silly(`Media source URL already in database, skipping ${baseSource.src}`);
 
         // media entry found by source URL, don't fetch
         return {
@@ -359,7 +377,7 @@ async function trySource(baseSource, existingMedias, baseMedia, baseSourceIndex)
         };
     }
 
-    return fetchSource(extractedSource, baseMedia, baseSourceIndex, 1);
+    return fetchSource(extractedSource);
 }
 
 async function fetchMedia(baseMedia, existingMedias) {
@@ -420,21 +438,21 @@ function curateMediaEntry(media, index) {
 }
 
 async function storeMedias(baseMedias) {
-    const { existingSourceMediaByUrl, existingExtractMediaByUrl } = await findSourceDuplicates(baseMedias);
+    const [existingSourceMediaByUrl, existingExtractMediaByUrl] = await findSourceDuplicates(baseMedias);
 
     const savedMedias = await Promise.map(
         baseMedias,
         async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
     );
 
-    const { uniqueHashMedia, existingHashMedia } = await findHashDuplicates(savedMedias);
+    const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(savedMedias);
 
-    const newMediaWithEntries = uniqueHashMedia.map((media, index) => curateMediaEntry(media, index));
-    const newMediaEntries = newMediaWithEntries.filter(media => !media.newEntry).map(media => media.entry);
+    const newMediaWithEntries = uniqueHashMedias.map((media, index) => curateMediaEntry(media, index));
+    const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry);
 
     await knex('media').insert(newMediaEntries);
 
-    return [...newMediaWithEntries, ...existingHashMedia];
+    return [...newMediaWithEntries, ...existingHashMedias];
 }
 
 async function associateReleaseMedia(releases) {
@@ -445,12 +463,13 @@ async function associateReleaseMedia(releases) {
     // TODO: internal duplicate filtering
     // TODO: media count limits
     // TODO: catch errors
+    // TODO: stage by role
 
     const baseMediasByReleaseId = releases.reduce((acc, release) => ({
         ...acc,
         [release.id]: [
             ...(argv.images && argv.poster ? toBaseMedias([release.poster], 'posters') : []),
-            ...(argv.images && argv.photos ? toBaseMedias(release.photos, 'photos').slice(0, 5) : []),
+            ...(argv.images && argv.photos ? toBaseMedias(release.photos, 'photos') : []),
             ...(argv.videos && argv.trailer ? toBaseMedias([release.trailer], 'trailers') : []),
             ...(argv.videos && argv.teaser ? toBaseMedias([release.teaser], 'teasers') : []),
         ],
@@ -473,17 +492,15 @@ async function associateReleaseMedia(releases) {
 
             acc[media.role].push({
                 release_id: releaseId,
-                media_id: media.entry.id,
+                media_id: media.use || media.entry.id,
             });
         });
 
         return acc;
     }, {});
 
-    console.log(util.inspect(associationsByRole, null, null));
-
     await Promise.all(Object.entries(associationsByRole)
-        .map(async ([role, associations]) => knex(`releases_${role}`).insert(associations)));
+        .map(async ([role, associations]) => knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`)));
 }
 
 module.exports = {