Improved duplicate filtering, now also applying to upcoming updates. Updated Gamma fetchLatest method parameters. Added shortcut for SFW-mode.

assets/components/tile/tag.vue

@@ -9,6 +9,7 @@
         <img
             v-if="tag.poster"
             :src="sfw ? `/img/${tag.poster.sfw.thumbnail}` : `/img/${tag.poster.thumbnail}`"
+            :title="sfw ? tag.poster.sfw.comment : tag.poster.comment"
             :alt="tag.name"
             class="poster"
         >

assets/js/main.js

@@ -5,6 +5,8 @@ import VTooltip from 'v-tooltip';
 import router from './router';
 import initStore from './store';
 
+import initUiObservers from './ui/observers';
+
 import '../css/style.scss';
 
 import Container from '../components/container/container.vue';
@@ -13,6 +15,8 @@ import Icon from '../components/icon/icon.vue';
 function init() {
     const store = initStore(router);
 
+    initUiObservers(store, router);
+
     if (window.env.sfw) {
         store.dispatch('setSfw', true);
     }

assets/js/tags/actions.js

@@ -122,8 +122,10 @@ function initTagsActions(store, _router) {
                     poster: tagsPosterByTagId {
                         media {
                             thumbnail
+                            comment
                             sfw: sfwMedia {
                                 thumbnail
+                                comment
                             }
                         }
                     }

assets/js/ui/observers.js

@@ -0,0 +1,30 @@
+function initUiObservers(store, _router) {
+    let keyHistory = [];
+
+    document.addEventListener('keypress', (event) => {
+        if (event.target.tagName === 'INPUT') {
+            return;
+        }
+
+        keyHistory = keyHistory
+            .concat({
+                key: event.key,
+                time: new Date().getTime(),
+            })
+            .slice(-3);
+
+        if (keyHistory.length === 3
+            && keyHistory.every(input => input.key === 's')
+            && new Date().getTime() - keyHistory[0].time < 1000
+        ) {
+            store.dispatch('setSfw', false);
+            return;
+        }
+
+        if (event.key === 's') {
+            store.dispatch('setSfw', true);
+        }
+    });
+}
+
+export default initUiObservers;

seeds/04_media.js

@@ -15,7 +15,7 @@ const tagPosters = [
     ['brunette', 0, 'Nicole Black in GIO971 for LegalPorno'],
     ['bukkake', 'poster', 'Mia Malkova in "Facialized 2" for HardX'],
     ['caucasian', 0, 'Remy Lacroix for HardX'],
-    ['creampie', 'poster'],
+    ['creampie', 'poster', 'ALina Lopez in "Making Yourself Unforgettable" for Blacked'],
     ['da-tp', 0, 'Natasha Teen in LegalPorno SZ2164'],
     ['deepthroat', 0, 'Chanel Grey in "Deepthroating Is Fun" for Throated'],
     ['double-anal', 7, 'Adriana Chechik in "DP Masters 6" for Jules Jordan'],
@@ -34,9 +34,9 @@ const tagPosters = [
     ['maid', 0, 'Whitney Wright in "Dredd Up Your Ass 2" for Jules Jordan'],
     ['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'],
     ['mff', 0, 'Madison Ivy, Adriana Chechik and Keiran Lee in "Day With A Pornstar" for Brazzers'],
-    ['mfm', 5, 'Vina Sky, Jules Jordan and Steve Holmes in "Slut Puppies 15" for Jules Jordan'],
+    ['mfm', 5, 'Vina Sky in "Slut Puppies 15" for Jules Jordan'],
     ['cum-in-mouth', 1, 'Keisha Grey in Brazzers House'],
-    ['orgy', 1, 'Megan Rain (DP), Morgan Lee (anal), Jessa Rhodes, Melissa Moore, Kimmy Granger, Mick Blue and Markus Dupree in "Orgy Masters 8" for Jules Jordan'],
+    ['orgy', 1, 'Megan Rain (DP), Morgan Lee (anal), Jessa Rhodes, Melissa Moore and Kimmy Granger in "Orgy Masters 8" for Jules Jordan'],
     ['pussy-eating', 0, 'Elena Kosha and Ivy Wolfe in "Bare" for Jules Jordan'],
     ['redhead', 0, 'Penny Pax in "The Submission of Emma Marx: Boundaries" for New Sensations'],
     ['schoolgirl', 1, 'Eliza Ibarra for Brazzers'],
@@ -96,7 +96,7 @@ const tagPhotos = [
     ['gaping', 2, 'Alex Grey in "DP Masters 5" for Jules Jordan'],
     // ['mfm', 0, 'Vina Sky in "Jules Jordan\'s Three Ways" for Jules Jordan'],
     ['mfm', 1, 'Jynx Maze in "Don\'t Make Me Beg 4" for Evil Angel'],
-    ['orgy', 'poster', 'Zoey Mornoe (DP), Jillian Janson (sex), Frida Sante, Katerina Kay, Natasha Starr, Mick Blue and Ramon Nomar in "Orgy Masters 6" for Jules Jordan'],
+    ['orgy', 'poster', 'Zoey Mornoe (DP), Jillian Janson (sex), Frida Sante, Katerina Kay and  Natasha Starr in "Orgy Masters 6" for Jules Jordan'],
     ['trainbang', 0, 'Nicole Black in GIO971 for LegalPorno'],
     ['triple-anal', 1, 'Natasha Teen in SZ2098 for LegalPorno'],
     ['triple-anal', 2, 'Kira Thorn in GIO1018 for LegalPorno'],

src/deep.js

@@ -112,6 +112,8 @@ async function scrapeRelease(baseRelease, sites, type = 'scene') {
     }
 
     try {
+        logger.verbose(`Fetching ${type} ${baseRelease.url}`);
+
         const scrapedRelease = type === 'scene'
             ? await scraper.fetchScene(baseRelease.url, site, baseRelease, null, include)
             : await scraper.fetchMovie(baseRelease.url, site, baseRelease, null, include);

src/scrapers/gamma.js

@@ -136,7 +136,7 @@ async function scrapeApiReleases(json, site) {
             ];
         }
 
-        release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
+        // release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
 
         return release;
     }).filter(Boolean);
@@ -408,7 +408,7 @@ async function fetchApiCredentials(referer, site) {
     return getApiUrl(appId, apiKey);
 }
 
-async function fetchApiLatest(site, page = 1, upcoming = false) {
+async function fetchApiLatest(site, page = 1, preData, include, upcoming = false) {
     const referer = site.parameters?.referer || `${site.parameters?.networkReferer ? site.network.url : site.url}/en/videos`;
     const { apiUrl } = await fetchApiCredentials(referer, site);
 
@@ -433,8 +433,8 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
     return [];
 }
 
-async function fetchApiUpcoming(site) {
+async function fetchApiUpcoming(site, page = 1, preData, include) {
-    return fetchApiLatest(site, 1, true);
+    return fetchApiLatest(site, page, preData, include, true);
 }
 
 function getLatestUrl(site, page) {

src/store-releases.js

@@ -110,9 +110,27 @@ function attachReleaseIds(releases, storedReleases) {
     return releasesWithId;
 }
 
-async function extractUniqueReleases(releases) {
+function filterInternalDuplicateReleases(releases) {
+    const releasesBySiteIdAndEntryId = releases.reduce((acc, release) => {
+        if (!acc[release.site.id]) {
+            acc[release.site.id] = {};
+        }
+
+        acc[release.site.id][release.entryId] = release;
+
+        return acc;
+    }, {});
+
+    return Object.values(releasesBySiteIdAndEntryId)
+        .map(siteReleases => Object.values(siteReleases))
+        .flat();
+}
+
+async function filterDuplicateReleases(releases) {
+    const internalUniqueReleases = filterInternalDuplicateReleases(releases);
+
     const duplicateReleaseEntries = await knex('releases')
-        .whereIn(['entry_id', 'site_id'], releases.map(release => [release.entryId, release.site.id]));
+        .whereIn(['entry_id', 'site_id'], internalUniqueReleases.map(release => [release.entryId, release.site.id]));
 
     const duplicateReleasesBySiteIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
         if (!acc[release.site_id]) acc[release.site_id] = {};
@@ -121,8 +139,8 @@ async function extractUniqueReleases(releases) {
         return acc;
     }, {});
 
-    const duplicateReleases = releases.filter(release => duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
+    const duplicateReleases = internalUniqueReleases.filter(release => duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
-    const uniqueReleases = releases.filter(release => !duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
+    const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
 
     return {
         uniqueReleases,
@@ -138,21 +156,25 @@ async function storeReleases(releases) {
     const releasesWithStudios = await attachStudios(releasesWithSites);
 
     // uniqueness is site ID + entry ID, filter uniques after adding sites
-    const { uniqueReleases, duplicateReleaseEntries } = await extractUniqueReleases(releasesWithStudios);
+    const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
 
     const curatedNewReleaseEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId));
 
+    // console.log(curatedNewReleaseEntries);
+
     const storedReleases = await knex('releases').insert(curatedNewReleaseEntries).returning('*');
     // TODO: update duplicate releases
 
     const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
-    const releasesWithId = attachReleaseIds(releases, [].concat(storedReleaseEntries, duplicateReleaseEntries));
+    const releasesWithId = attachReleaseIds([].concat(uniqueReleases, duplicateReleases), [].concat(storedReleaseEntries, duplicateReleaseEntries));
 
     await Promise.all([
         associateTags(releasesWithId),
         associateActors(releasesWithId),
     ]);
 
+    logger.info(`Stored ${storedReleaseEntries.length} releases`);
+
     return releasesWithId;
 }
 

src/tags.js

@@ -54,7 +54,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId)
                 ? releaseTags // obsolete scraper returned pre-matched tags
                 : releaseTags.map(tag => tagIdsBySlug[slugify(tag)]);
 
-            return [...new Set(
+            const tags = [...new Set(
                 // filter duplicates and empties
                 releaseTagIds
                     .concat(siteTagIds)
@@ -64,14 +64,17 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId)
                     release_id: release.id,
                     tag_id: tagId,
                 }));
+
+            return tags;
         })
         .flat();
 
     return tagAssociations;
 }
 
-async function extractUniqueAssociations(tagAssociations) {
+async function filterUniqueAssociations(tagAssociations) {
-    const duplicateAssociations = await knex('releases_tags').whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id]));
+    const duplicateAssociations = await knex('releases_tags')
+        .whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id]));
 
     const duplicateAssociationsByReleaseIdAndTagId = duplicateAssociations.reduce((acc, association) => {
         if (!acc[association.release_id]) {
@@ -94,7 +97,7 @@ async function associateTags(releases) {
     const siteTagIdsBySiteId = await getSiteTags(releases);
 
     const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId);
-    const uniqueAssociations = await extractUniqueAssociations(tagAssociations);
+    const uniqueAssociations = await filterUniqueAssociations(tagAssociations);
 
     await knex('releases_tags').insert(uniqueAssociations);
 }

src/updates.js

@@ -25,7 +25,7 @@ const afterDate = (() => {
         .toDate();
 })();
 
-async function extractUniqueReleases(latestReleases, accReleases) {
+async function filterUniqueReleases(latestReleases, accReleases) {
     const latestReleaseIdentifiers = latestReleases
         .map(release => [release.site.id, release.entryId]);
 
@@ -75,13 +75,11 @@ function needNextPage(uniqueReleases, pageAccReleases) {
     return pageAccReleases.length <= argv.nullDateLimit;
 }
 
-async function scrapeLatestReleases(scraper, site, preData) {
+async function scrapeReleases(scraper, site, preData, upcoming = false) {
-    if (!scraper.fetchLatest) {
-        return [];
-    }
-
     const scrapePage = async (page = 1, accReleases = []) => {
-        const latestReleases = await scraper.fetchLatest(site, page, preData, include);
+        const latestReleases = upcoming
+            ? await scraper.fetchUpcoming(site, page, preData, include)
+            : await scraper.fetchLatest(site, page, preData, include);
 
         if (!Array.isArray(latestReleases)) {
             // scraper is unable to fetch the releases and returned a HTTP code or null
@@ -98,11 +96,11 @@ async function scrapeLatestReleases(scraper, site, preData) {
 
         const uniqueReleases = argv.redownload
             ? latestReleasesWithSite
-            : await extractUniqueReleases(latestReleasesWithSite, accReleases);
+            : await filterUniqueReleases(latestReleasesWithSite, accReleases);
 
         const pageAccReleases = accReleases.concat(uniqueReleases);
 
-        logger.verbose(`Scraped '${site.name}' (${site.network.name}) page ${page}, found ${uniqueReleases.length} unique releases`);
+        logger.verbose(`Scraped '${site.name}' (${site.network.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`);
 
         if (needNextPage(uniqueReleases, pageAccReleases)) {
             return scrapePage(page + 1, pageAccReleases);
@@ -111,37 +109,45 @@ async function scrapeLatestReleases(scraper, site, preData) {
         return pageAccReleases;
     };
 
-    const releases = await scrapePage(1, []);
+    const rawReleases = await scrapePage(1, []);
+    const releases = upcoming
+        ? rawReleases.map(rawRelease => ({ ...rawRelease, upcoming: true }))
+        : rawReleases;
 
     if (argv.last) {
         return releases.slice(0, argv.last);
     }
 
     if (releases.every(release => release.date)) {
-        return releases
+        return releases.filter(release => moment(release.date).isAfter(afterDate));
-            .filter(release => moment(release.date).isAfter(afterDate));
     }
 
     return releases.slice(0, argv.nullDateLimit);
 }
 
+async function scrapeLatestReleases(scraper, site, preData) {
+    if (!scraper.fetchLatest) {
+        return [];
+    }
+
+    try {
+        return await scrapeReleases(scraper, site, preData, false);
+    } catch (error) {
+        logger.warn(`Failed to scrape latest updates for '${site.slug}' (${site.network.slug})`);
+    }
+
+    return [];
+}
+
 async function scrapeUpcomingReleases(scraper, site, preData) {
     if (!scraper.fetchUpcoming) {
         return [];
     }
 
     try {
-        const upcomingReleases = await scraper.fetchUpcoming(site, 1, preData, include);
+        return await scrapeReleases(scraper, site, preData, true);
-
-        if (upcomingReleases) {
-            return upcomingReleases.map(release => ({
-                ...release,
-                site,
-                upcoming: true,
-            }));
-        }
     } catch (error) {
-        logger.warn(`Failed to scrape upcoming releases for '${site.slug}' (${site.network.slug})`);
+        logger.warn(`Failed to scrape upcoming updates for '${site.slug}' (${site.network.slug})`);
     }
 
     return [];
@@ -157,6 +163,8 @@ async function scrapeSiteReleases(scraper, site, preData) {
             : [],
     ]);
 
+    logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${site.name}' (${site.network.name})`);
+
     return [...latestReleases, ...upcomingReleases];
 }