From 6d9f96c5d55c1748ca71b85abd9f875d0f5fc88c Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Sat, 28 Mar 2020 04:37:04 +0100 Subject: [PATCH] Improved duplicate filtering, now also applying to upcoming updates. Updated Gamma fetchLatest method parameters. Added shortcut for SFW-mode. --- assets/components/tile/tag.vue | 1 + assets/js/main.js | 4 +++ assets/js/tags/actions.js | 2 ++ assets/js/ui/observers.js | 30 ++++++++++++++++++++ seeds/04_media.js | 8 +++--- src/deep.js | 2 ++ src/scrapers/gamma.js | 8 +++--- src/store-releases.js | 34 ++++++++++++++++++---- src/tags.js | 11 ++++--- src/updates.js | 52 ++++++++++++++++++++-------------- 10 files changed, 112 insertions(+), 40 deletions(-) create mode 100644 assets/js/ui/observers.js diff --git a/assets/components/tile/tag.vue b/assets/components/tile/tag.vue index 2e094c7d..ece05ebd 100644 --- a/assets/components/tile/tag.vue +++ b/assets/components/tile/tag.vue @@ -9,6 +9,7 @@ diff --git a/assets/js/main.js b/assets/js/main.js index 0f34be82..74aa0d0a 100644 --- a/assets/js/main.js +++ b/assets/js/main.js @@ -5,6 +5,8 @@ import VTooltip from 'v-tooltip'; import router from './router'; import initStore from './store'; +import initUiObservers from './ui/observers'; + import '../css/style.scss'; import Container from '../components/container/container.vue'; @@ -13,6 +15,8 @@ import Icon from '../components/icon/icon.vue'; function init() { const store = initStore(router); + initUiObservers(store, router); + if (window.env.sfw) { store.dispatch('setSfw', true); } diff --git a/assets/js/tags/actions.js b/assets/js/tags/actions.js index aba57b76..341eded6 100644 --- a/assets/js/tags/actions.js +++ b/assets/js/tags/actions.js @@ -122,8 +122,10 @@ function initTagsActions(store, _router) { poster: tagsPosterByTagId { media { thumbnail + comment sfw: sfwMedia { thumbnail + comment } } } diff --git a/assets/js/ui/observers.js b/assets/js/ui/observers.js new file mode 100644 index 00000000..06f046df --- /dev/null +++ b/assets/js/ui/observers.js @@ -0,0 +1,30 @@ +function initUiObservers(store, _router) { + let keyHistory = []; + + document.addEventListener('keypress', (event) => { + if (event.target.tagName === 'INPUT') { + return; + } + + keyHistory = keyHistory + .concat({ + key: event.key, + time: new Date().getTime(), + }) + .slice(-3); + + if (keyHistory.length === 3 + && keyHistory.every(input => input.key === 's') + && new Date().getTime() - keyHistory[0].time < 1000 + ) { + store.dispatch('setSfw', false); + return; + } + + if (event.key === 's') { + store.dispatch('setSfw', true); + } + }); +} + +export default initUiObservers; diff --git a/seeds/04_media.js b/seeds/04_media.js index 5c56d1d2..f1fd7692 100644 --- a/seeds/04_media.js +++ b/seeds/04_media.js @@ -15,7 +15,7 @@ const tagPosters = [ ['brunette', 0, 'Nicole Black in GIO971 for LegalPorno'], ['bukkake', 'poster', 'Mia Malkova in "Facialized 2" for HardX'], ['caucasian', 0, 'Remy Lacroix for HardX'], - ['creampie', 'poster'], + ['creampie', 'poster', 'ALina Lopez in "Making Yourself Unforgettable" for Blacked'], ['da-tp', 0, 'Natasha Teen in LegalPorno SZ2164'], ['deepthroat', 0, 'Chanel Grey in "Deepthroating Is Fun" for Throated'], ['double-anal', 7, 'Adriana Chechik in "DP Masters 6" for Jules Jordan'], @@ -34,9 +34,9 @@ const tagPosters = [ ['maid', 0, 'Whitney Wright in "Dredd Up Your Ass 2" for Jules Jordan'], ['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'], ['mff', 0, 'Madison Ivy, Adriana Chechik and Keiran Lee in "Day With A Pornstar" for Brazzers'], - ['mfm', 5, 'Vina Sky, Jules Jordan and Steve Holmes in "Slut Puppies 15" for Jules Jordan'], + ['mfm', 5, 'Vina Sky in "Slut Puppies 15" for Jules Jordan'], ['cum-in-mouth', 1, 'Keisha Grey in Brazzers House'], - ['orgy', 1, 'Megan Rain (DP), Morgan Lee (anal), Jessa Rhodes, Melissa Moore, Kimmy Granger, Mick Blue and Markus Dupree in "Orgy Masters 8" for Jules Jordan'], + ['orgy', 1, 'Megan Rain (DP), Morgan Lee (anal), Jessa Rhodes, Melissa Moore and Kimmy Granger in "Orgy Masters 8" for Jules Jordan'], ['pussy-eating', 0, 'Elena Kosha and Ivy Wolfe in "Bare" for Jules Jordan'], ['redhead', 0, 'Penny Pax in "The Submission of Emma Marx: Boundaries" for New Sensations'], ['schoolgirl', 1, 'Eliza Ibarra for Brazzers'], @@ -96,7 +96,7 @@ const tagPhotos = [ ['gaping', 2, 'Alex Grey in "DP Masters 5" for Jules Jordan'], // ['mfm', 0, 'Vina Sky in "Jules Jordan\'s Three Ways" for Jules Jordan'], ['mfm', 1, 'Jynx Maze in "Don\'t Make Me Beg 4" for Evil Angel'], - ['orgy', 'poster', 'Zoey Mornoe (DP), Jillian Janson (sex), Frida Sante, Katerina Kay, Natasha Starr, Mick Blue and Ramon Nomar in "Orgy Masters 6" for Jules Jordan'], + ['orgy', 'poster', 'Zoey Mornoe (DP), Jillian Janson (sex), Frida Sante, Katerina Kay and Natasha Starr in "Orgy Masters 6" for Jules Jordan'], ['trainbang', 0, 'Nicole Black in GIO971 for LegalPorno'], ['triple-anal', 1, 'Natasha Teen in SZ2098 for LegalPorno'], ['triple-anal', 2, 'Kira Thorn in GIO1018 for LegalPorno'], diff --git a/src/deep.js b/src/deep.js index 9189b422..fcfd84b3 100644 --- a/src/deep.js +++ b/src/deep.js @@ -112,6 +112,8 @@ async function scrapeRelease(baseRelease, sites, type = 'scene') { } try { + logger.verbose(`Fetching ${type} ${baseRelease.url}`); + const scrapedRelease = type === 'scene' ? await scraper.fetchScene(baseRelease.url, site, baseRelease, null, include) : await scraper.fetchMovie(baseRelease.url, site, baseRelease, null, include); diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index 44fd3b31..c86b5f23 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -136,7 +136,7 @@ async function scrapeApiReleases(json, site) { ]; } - release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`; + // release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`; return release; }).filter(Boolean); @@ -408,7 +408,7 @@ async function fetchApiCredentials(referer, site) { return getApiUrl(appId, apiKey); } -async function fetchApiLatest(site, page = 1, upcoming = false) { +async function fetchApiLatest(site, page = 1, preData, include, upcoming = false) { const referer = site.parameters?.referer || `${site.parameters?.networkReferer ? site.network.url : site.url}/en/videos`; const { apiUrl } = await fetchApiCredentials(referer, site); @@ -433,8 +433,8 @@ async function fetchApiLatest(site, page = 1, upcoming = false) { return []; } -async function fetchApiUpcoming(site) { - return fetchApiLatest(site, 1, true); +async function fetchApiUpcoming(site, page = 1, preData, include) { + return fetchApiLatest(site, page, preData, include, true); } function getLatestUrl(site, page) { diff --git a/src/store-releases.js b/src/store-releases.js index 1601088d..1c84f8c6 100644 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -110,9 +110,27 @@ function attachReleaseIds(releases, storedReleases) { return releasesWithId; } -async function extractUniqueReleases(releases) { +function filterInternalDuplicateReleases(releases) { + const releasesBySiteIdAndEntryId = releases.reduce((acc, release) => { + if (!acc[release.site.id]) { + acc[release.site.id] = {}; + } + + acc[release.site.id][release.entryId] = release; + + return acc; + }, {}); + + return Object.values(releasesBySiteIdAndEntryId) + .map(siteReleases => Object.values(siteReleases)) + .flat(); +} + +async function filterDuplicateReleases(releases) { + const internalUniqueReleases = filterInternalDuplicateReleases(releases); + const duplicateReleaseEntries = await knex('releases') - .whereIn(['entry_id', 'site_id'], releases.map(release => [release.entryId, release.site.id])); + .whereIn(['entry_id', 'site_id'], internalUniqueReleases.map(release => [release.entryId, release.site.id])); const duplicateReleasesBySiteIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => { if (!acc[release.site_id]) acc[release.site_id] = {}; @@ -121,8 +139,8 @@ async function extractUniqueReleases(releases) { return acc; }, {}); - const duplicateReleases = releases.filter(release => duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]); - const uniqueReleases = releases.filter(release => !duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]); + const duplicateReleases = internalUniqueReleases.filter(release => duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]); + const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]); return { uniqueReleases, @@ -138,21 +156,25 @@ async function storeReleases(releases) { const releasesWithStudios = await attachStudios(releasesWithSites); // uniqueness is site ID + entry ID, filter uniques after adding sites - const { uniqueReleases, duplicateReleaseEntries } = await extractUniqueReleases(releasesWithStudios); + const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios); const curatedNewReleaseEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId)); + // console.log(curatedNewReleaseEntries); + const storedReleases = await knex('releases').insert(curatedNewReleaseEntries).returning('*'); // TODO: update duplicate releases const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : []; - const releasesWithId = attachReleaseIds(releases, [].concat(storedReleaseEntries, duplicateReleaseEntries)); + const releasesWithId = attachReleaseIds([].concat(uniqueReleases, duplicateReleases), [].concat(storedReleaseEntries, duplicateReleaseEntries)); await Promise.all([ associateTags(releasesWithId), associateActors(releasesWithId), ]); + logger.info(`Stored ${storedReleaseEntries.length} releases`); + return releasesWithId; } diff --git a/src/tags.js b/src/tags.js index 260094ca..fa8d91a7 100644 --- a/src/tags.js +++ b/src/tags.js @@ -54,7 +54,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId) ? releaseTags // obsolete scraper returned pre-matched tags : releaseTags.map(tag => tagIdsBySlug[slugify(tag)]); - return [...new Set( + const tags = [...new Set( // filter duplicates and empties releaseTagIds .concat(siteTagIds) @@ -64,14 +64,17 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId) release_id: release.id, tag_id: tagId, })); + + return tags; }) .flat(); return tagAssociations; } -async function extractUniqueAssociations(tagAssociations) { - const duplicateAssociations = await knex('releases_tags').whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id])); +async function filterUniqueAssociations(tagAssociations) { + const duplicateAssociations = await knex('releases_tags') + .whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id])); const duplicateAssociationsByReleaseIdAndTagId = duplicateAssociations.reduce((acc, association) => { if (!acc[association.release_id]) { @@ -94,7 +97,7 @@ async function associateTags(releases) { const siteTagIdsBySiteId = await getSiteTags(releases); const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId); - const uniqueAssociations = await extractUniqueAssociations(tagAssociations); + const uniqueAssociations = await filterUniqueAssociations(tagAssociations); await knex('releases_tags').insert(uniqueAssociations); } diff --git a/src/updates.js b/src/updates.js index 413d72b1..2004f04e 100644 --- a/src/updates.js +++ b/src/updates.js @@ -25,7 +25,7 @@ const afterDate = (() => { .toDate(); })(); -async function extractUniqueReleases(latestReleases, accReleases) { +async function filterUniqueReleases(latestReleases, accReleases) { const latestReleaseIdentifiers = latestReleases .map(release => [release.site.id, release.entryId]); @@ -75,13 +75,11 @@ function needNextPage(uniqueReleases, pageAccReleases) { return pageAccReleases.length <= argv.nullDateLimit; } -async function scrapeLatestReleases(scraper, site, preData) { - if (!scraper.fetchLatest) { - return []; - } - +async function scrapeReleases(scraper, site, preData, upcoming = false) { const scrapePage = async (page = 1, accReleases = []) => { - const latestReleases = await scraper.fetchLatest(site, page, preData, include); + const latestReleases = upcoming + ? await scraper.fetchUpcoming(site, page, preData, include) + : await scraper.fetchLatest(site, page, preData, include); if (!Array.isArray(latestReleases)) { // scraper is unable to fetch the releases and returned a HTTP code or null @@ -98,11 +96,11 @@ async function scrapeLatestReleases(scraper, site, preData) { const uniqueReleases = argv.redownload ? latestReleasesWithSite - : await extractUniqueReleases(latestReleasesWithSite, accReleases); + : await filterUniqueReleases(latestReleasesWithSite, accReleases); const pageAccReleases = accReleases.concat(uniqueReleases); - logger.verbose(`Scraped '${site.name}' (${site.network.name}) page ${page}, found ${uniqueReleases.length} unique releases`); + logger.verbose(`Scraped '${site.name}' (${site.network.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`); if (needNextPage(uniqueReleases, pageAccReleases)) { return scrapePage(page + 1, pageAccReleases); @@ -111,37 +109,45 @@ async function scrapeLatestReleases(scraper, site, preData) { return pageAccReleases; }; - const releases = await scrapePage(1, []); + const rawReleases = await scrapePage(1, []); + const releases = upcoming + ? rawReleases.map(rawRelease => ({ ...rawRelease, upcoming: true })) + : rawReleases; if (argv.last) { return releases.slice(0, argv.last); } if (releases.every(release => release.date)) { - return releases - .filter(release => moment(release.date).isAfter(afterDate)); + return releases.filter(release => moment(release.date).isAfter(afterDate)); } return releases.slice(0, argv.nullDateLimit); } +async function scrapeLatestReleases(scraper, site, preData) { + if (!scraper.fetchLatest) { + return []; + } + + try { + return await scrapeReleases(scraper, site, preData, false); + } catch (error) { + logger.warn(`Failed to scrape latest updates for '${site.slug}' (${site.network.slug})`); + } + + return []; +} + async function scrapeUpcomingReleases(scraper, site, preData) { if (!scraper.fetchUpcoming) { return []; } try { - const upcomingReleases = await scraper.fetchUpcoming(site, 1, preData, include); - - if (upcomingReleases) { - return upcomingReleases.map(release => ({ - ...release, - site, - upcoming: true, - })); - } + return await scrapeReleases(scraper, site, preData, true); } catch (error) { - logger.warn(`Failed to scrape upcoming releases for '${site.slug}' (${site.network.slug})`); + logger.warn(`Failed to scrape upcoming updates for '${site.slug}' (${site.network.slug})`); } return []; @@ -157,6 +163,8 @@ async function scrapeSiteReleases(scraper, site, preData) { : [], ]); + logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${site.name}' (${site.network.name})`); + return [...latestReleases, ...upcomingReleases]; }