Improved duplicate filtering, now also applying to upcoming updates. Updated Gamma fetchLatest method parameters. Added shortcut for SFW-mode.
This commit is contained in:
parent
95d115b585
commit
6d9f96c5d5
|
|
@ -9,6 +9,7 @@
|
|||
<img
|
||||
v-if="tag.poster"
|
||||
:src="sfw ? `/img/${tag.poster.sfw.thumbnail}` : `/img/${tag.poster.thumbnail}`"
|
||||
:title="sfw ? tag.poster.sfw.comment : tag.poster.comment"
|
||||
:alt="tag.name"
|
||||
class="poster"
|
||||
>
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import VTooltip from 'v-tooltip';
|
|||
import router from './router';
|
||||
import initStore from './store';
|
||||
|
||||
import initUiObservers from './ui/observers';
|
||||
|
||||
import '../css/style.scss';
|
||||
|
||||
import Container from '../components/container/container.vue';
|
||||
|
|
@ -13,6 +15,8 @@ import Icon from '../components/icon/icon.vue';
|
|||
function init() {
|
||||
const store = initStore(router);
|
||||
|
||||
initUiObservers(store, router);
|
||||
|
||||
if (window.env.sfw) {
|
||||
store.dispatch('setSfw', true);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -122,8 +122,10 @@ function initTagsActions(store, _router) {
|
|||
poster: tagsPosterByTagId {
|
||||
media {
|
||||
thumbnail
|
||||
comment
|
||||
sfw: sfwMedia {
|
||||
thumbnail
|
||||
comment
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
function initUiObservers(store, _router) {
|
||||
let keyHistory = [];
|
||||
|
||||
document.addEventListener('keypress', (event) => {
|
||||
if (event.target.tagName === 'INPUT') {
|
||||
return;
|
||||
}
|
||||
|
||||
keyHistory = keyHistory
|
||||
.concat({
|
||||
key: event.key,
|
||||
time: new Date().getTime(),
|
||||
})
|
||||
.slice(-3);
|
||||
|
||||
if (keyHistory.length === 3
|
||||
&& keyHistory.every(input => input.key === 's')
|
||||
&& new Date().getTime() - keyHistory[0].time < 1000
|
||||
) {
|
||||
store.dispatch('setSfw', false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.key === 's') {
|
||||
store.dispatch('setSfw', true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export default initUiObservers;
|
||||
|
|
@ -15,7 +15,7 @@ const tagPosters = [
|
|||
['brunette', 0, 'Nicole Black in GIO971 for LegalPorno'],
|
||||
['bukkake', 'poster', 'Mia Malkova in "Facialized 2" for HardX'],
|
||||
['caucasian', 0, 'Remy Lacroix for HardX'],
|
||||
['creampie', 'poster'],
|
||||
['creampie', 'poster', 'ALina Lopez in "Making Yourself Unforgettable" for Blacked'],
|
||||
['da-tp', 0, 'Natasha Teen in LegalPorno SZ2164'],
|
||||
['deepthroat', 0, 'Chanel Grey in "Deepthroating Is Fun" for Throated'],
|
||||
['double-anal', 7, 'Adriana Chechik in "DP Masters 6" for Jules Jordan'],
|
||||
|
|
@ -34,9 +34,9 @@ const tagPosters = [
|
|||
['maid', 0, 'Whitney Wright in "Dredd Up Your Ass 2" for Jules Jordan'],
|
||||
['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'],
|
||||
['mff', 0, 'Madison Ivy, Adriana Chechik and Keiran Lee in "Day With A Pornstar" for Brazzers'],
|
||||
['mfm', 5, 'Vina Sky, Jules Jordan and Steve Holmes in "Slut Puppies 15" for Jules Jordan'],
|
||||
['mfm', 5, 'Vina Sky in "Slut Puppies 15" for Jules Jordan'],
|
||||
['cum-in-mouth', 1, 'Keisha Grey in Brazzers House'],
|
||||
['orgy', 1, 'Megan Rain (DP), Morgan Lee (anal), Jessa Rhodes, Melissa Moore, Kimmy Granger, Mick Blue and Markus Dupree in "Orgy Masters 8" for Jules Jordan'],
|
||||
['orgy', 1, 'Megan Rain (DP), Morgan Lee (anal), Jessa Rhodes, Melissa Moore and Kimmy Granger in "Orgy Masters 8" for Jules Jordan'],
|
||||
['pussy-eating', 0, 'Elena Kosha and Ivy Wolfe in "Bare" for Jules Jordan'],
|
||||
['redhead', 0, 'Penny Pax in "The Submission of Emma Marx: Boundaries" for New Sensations'],
|
||||
['schoolgirl', 1, 'Eliza Ibarra for Brazzers'],
|
||||
|
|
@ -96,7 +96,7 @@ const tagPhotos = [
|
|||
['gaping', 2, 'Alex Grey in "DP Masters 5" for Jules Jordan'],
|
||||
// ['mfm', 0, 'Vina Sky in "Jules Jordan\'s Three Ways" for Jules Jordan'],
|
||||
['mfm', 1, 'Jynx Maze in "Don\'t Make Me Beg 4" for Evil Angel'],
|
||||
['orgy', 'poster', 'Zoey Mornoe (DP), Jillian Janson (sex), Frida Sante, Katerina Kay, Natasha Starr, Mick Blue and Ramon Nomar in "Orgy Masters 6" for Jules Jordan'],
|
||||
['orgy', 'poster', 'Zoey Mornoe (DP), Jillian Janson (sex), Frida Sante, Katerina Kay and Natasha Starr in "Orgy Masters 6" for Jules Jordan'],
|
||||
['trainbang', 0, 'Nicole Black in GIO971 for LegalPorno'],
|
||||
['triple-anal', 1, 'Natasha Teen in SZ2098 for LegalPorno'],
|
||||
['triple-anal', 2, 'Kira Thorn in GIO1018 for LegalPorno'],
|
||||
|
|
|
|||
|
|
@ -112,6 +112,8 @@ async function scrapeRelease(baseRelease, sites, type = 'scene') {
|
|||
}
|
||||
|
||||
try {
|
||||
logger.verbose(`Fetching ${type} ${baseRelease.url}`);
|
||||
|
||||
const scrapedRelease = type === 'scene'
|
||||
? await scraper.fetchScene(baseRelease.url, site, baseRelease, null, include)
|
||||
: await scraper.fetchMovie(baseRelease.url, site, baseRelease, null, include);
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ async function scrapeApiReleases(json, site) {
|
|||
];
|
||||
}
|
||||
|
||||
release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
|
||||
// release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
|
||||
|
||||
return release;
|
||||
}).filter(Boolean);
|
||||
|
|
@ -408,7 +408,7 @@ async function fetchApiCredentials(referer, site) {
|
|||
return getApiUrl(appId, apiKey);
|
||||
}
|
||||
|
||||
async function fetchApiLatest(site, page = 1, upcoming = false) {
|
||||
async function fetchApiLatest(site, page = 1, preData, include, upcoming = false) {
|
||||
const referer = site.parameters?.referer || `${site.parameters?.networkReferer ? site.network.url : site.url}/en/videos`;
|
||||
const { apiUrl } = await fetchApiCredentials(referer, site);
|
||||
|
||||
|
|
@ -433,8 +433,8 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
|
|||
return [];
|
||||
}
|
||||
|
||||
async function fetchApiUpcoming(site) {
|
||||
return fetchApiLatest(site, 1, true);
|
||||
async function fetchApiUpcoming(site, page = 1, preData, include) {
|
||||
return fetchApiLatest(site, page, preData, include, true);
|
||||
}
|
||||
|
||||
function getLatestUrl(site, page) {
|
||||
|
|
|
|||
|
|
@ -110,9 +110,27 @@ function attachReleaseIds(releases, storedReleases) {
|
|||
return releasesWithId;
|
||||
}
|
||||
|
||||
async function extractUniqueReleases(releases) {
|
||||
function filterInternalDuplicateReleases(releases) {
|
||||
const releasesBySiteIdAndEntryId = releases.reduce((acc, release) => {
|
||||
if (!acc[release.site.id]) {
|
||||
acc[release.site.id] = {};
|
||||
}
|
||||
|
||||
acc[release.site.id][release.entryId] = release;
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
return Object.values(releasesBySiteIdAndEntryId)
|
||||
.map(siteReleases => Object.values(siteReleases))
|
||||
.flat();
|
||||
}
|
||||
|
||||
async function filterDuplicateReleases(releases) {
|
||||
const internalUniqueReleases = filterInternalDuplicateReleases(releases);
|
||||
|
||||
const duplicateReleaseEntries = await knex('releases')
|
||||
.whereIn(['entry_id', 'site_id'], releases.map(release => [release.entryId, release.site.id]));
|
||||
.whereIn(['entry_id', 'site_id'], internalUniqueReleases.map(release => [release.entryId, release.site.id]));
|
||||
|
||||
const duplicateReleasesBySiteIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
|
||||
if (!acc[release.site_id]) acc[release.site_id] = {};
|
||||
|
|
@ -121,8 +139,8 @@ async function extractUniqueReleases(releases) {
|
|||
return acc;
|
||||
}, {});
|
||||
|
||||
const duplicateReleases = releases.filter(release => duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
|
||||
const uniqueReleases = releases.filter(release => !duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
|
||||
const duplicateReleases = internalUniqueReleases.filter(release => duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
|
||||
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
|
||||
|
||||
return {
|
||||
uniqueReleases,
|
||||
|
|
@ -138,21 +156,25 @@ async function storeReleases(releases) {
|
|||
const releasesWithStudios = await attachStudios(releasesWithSites);
|
||||
|
||||
// uniqueness is site ID + entry ID, filter uniques after adding sites
|
||||
const { uniqueReleases, duplicateReleaseEntries } = await extractUniqueReleases(releasesWithStudios);
|
||||
const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
|
||||
|
||||
const curatedNewReleaseEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId));
|
||||
|
||||
// console.log(curatedNewReleaseEntries);
|
||||
|
||||
const storedReleases = await knex('releases').insert(curatedNewReleaseEntries).returning('*');
|
||||
// TODO: update duplicate releases
|
||||
|
||||
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
|
||||
const releasesWithId = attachReleaseIds(releases, [].concat(storedReleaseEntries, duplicateReleaseEntries));
|
||||
const releasesWithId = attachReleaseIds([].concat(uniqueReleases, duplicateReleases), [].concat(storedReleaseEntries, duplicateReleaseEntries));
|
||||
|
||||
await Promise.all([
|
||||
associateTags(releasesWithId),
|
||||
associateActors(releasesWithId),
|
||||
]);
|
||||
|
||||
logger.info(`Stored ${storedReleaseEntries.length} releases`);
|
||||
|
||||
return releasesWithId;
|
||||
}
|
||||
|
||||
|
|
|
|||
11
src/tags.js
11
src/tags.js
|
|
@ -54,7 +54,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId)
|
|||
? releaseTags // obsolete scraper returned pre-matched tags
|
||||
: releaseTags.map(tag => tagIdsBySlug[slugify(tag)]);
|
||||
|
||||
return [...new Set(
|
||||
const tags = [...new Set(
|
||||
// filter duplicates and empties
|
||||
releaseTagIds
|
||||
.concat(siteTagIds)
|
||||
|
|
@ -64,14 +64,17 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId)
|
|||
release_id: release.id,
|
||||
tag_id: tagId,
|
||||
}));
|
||||
|
||||
return tags;
|
||||
})
|
||||
.flat();
|
||||
|
||||
return tagAssociations;
|
||||
}
|
||||
|
||||
async function extractUniqueAssociations(tagAssociations) {
|
||||
const duplicateAssociations = await knex('releases_tags').whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id]));
|
||||
async function filterUniqueAssociations(tagAssociations) {
|
||||
const duplicateAssociations = await knex('releases_tags')
|
||||
.whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id]));
|
||||
|
||||
const duplicateAssociationsByReleaseIdAndTagId = duplicateAssociations.reduce((acc, association) => {
|
||||
if (!acc[association.release_id]) {
|
||||
|
|
@ -94,7 +97,7 @@ async function associateTags(releases) {
|
|||
const siteTagIdsBySiteId = await getSiteTags(releases);
|
||||
|
||||
const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, siteTagIdsBySiteId);
|
||||
const uniqueAssociations = await extractUniqueAssociations(tagAssociations);
|
||||
const uniqueAssociations = await filterUniqueAssociations(tagAssociations);
|
||||
|
||||
await knex('releases_tags').insert(uniqueAssociations);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ const afterDate = (() => {
|
|||
.toDate();
|
||||
})();
|
||||
|
||||
async function extractUniqueReleases(latestReleases, accReleases) {
|
||||
async function filterUniqueReleases(latestReleases, accReleases) {
|
||||
const latestReleaseIdentifiers = latestReleases
|
||||
.map(release => [release.site.id, release.entryId]);
|
||||
|
||||
|
|
@ -75,13 +75,11 @@ function needNextPage(uniqueReleases, pageAccReleases) {
|
|||
return pageAccReleases.length <= argv.nullDateLimit;
|
||||
}
|
||||
|
||||
async function scrapeLatestReleases(scraper, site, preData) {
|
||||
if (!scraper.fetchLatest) {
|
||||
return [];
|
||||
}
|
||||
|
||||
async function scrapeReleases(scraper, site, preData, upcoming = false) {
|
||||
const scrapePage = async (page = 1, accReleases = []) => {
|
||||
const latestReleases = await scraper.fetchLatest(site, page, preData, include);
|
||||
const latestReleases = upcoming
|
||||
? await scraper.fetchUpcoming(site, page, preData, include)
|
||||
: await scraper.fetchLatest(site, page, preData, include);
|
||||
|
||||
if (!Array.isArray(latestReleases)) {
|
||||
// scraper is unable to fetch the releases and returned a HTTP code or null
|
||||
|
|
@ -98,11 +96,11 @@ async function scrapeLatestReleases(scraper, site, preData) {
|
|||
|
||||
const uniqueReleases = argv.redownload
|
||||
? latestReleasesWithSite
|
||||
: await extractUniqueReleases(latestReleasesWithSite, accReleases);
|
||||
: await filterUniqueReleases(latestReleasesWithSite, accReleases);
|
||||
|
||||
const pageAccReleases = accReleases.concat(uniqueReleases);
|
||||
|
||||
logger.verbose(`Scraped '${site.name}' (${site.network.name}) page ${page}, found ${uniqueReleases.length} unique releases`);
|
||||
logger.verbose(`Scraped '${site.name}' (${site.network.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`);
|
||||
|
||||
if (needNextPage(uniqueReleases, pageAccReleases)) {
|
||||
return scrapePage(page + 1, pageAccReleases);
|
||||
|
|
@ -111,37 +109,45 @@ async function scrapeLatestReleases(scraper, site, preData) {
|
|||
return pageAccReleases;
|
||||
};
|
||||
|
||||
const releases = await scrapePage(1, []);
|
||||
const rawReleases = await scrapePage(1, []);
|
||||
const releases = upcoming
|
||||
? rawReleases.map(rawRelease => ({ ...rawRelease, upcoming: true }))
|
||||
: rawReleases;
|
||||
|
||||
if (argv.last) {
|
||||
return releases.slice(0, argv.last);
|
||||
}
|
||||
|
||||
if (releases.every(release => release.date)) {
|
||||
return releases
|
||||
.filter(release => moment(release.date).isAfter(afterDate));
|
||||
return releases.filter(release => moment(release.date).isAfter(afterDate));
|
||||
}
|
||||
|
||||
return releases.slice(0, argv.nullDateLimit);
|
||||
}
|
||||
|
||||
async function scrapeLatestReleases(scraper, site, preData) {
|
||||
if (!scraper.fetchLatest) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
return await scrapeReleases(scraper, site, preData, false);
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to scrape latest updates for '${site.slug}' (${site.network.slug})`);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function scrapeUpcomingReleases(scraper, site, preData) {
|
||||
if (!scraper.fetchUpcoming) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const upcomingReleases = await scraper.fetchUpcoming(site, 1, preData, include);
|
||||
|
||||
if (upcomingReleases) {
|
||||
return upcomingReleases.map(release => ({
|
||||
...release,
|
||||
site,
|
||||
upcoming: true,
|
||||
}));
|
||||
}
|
||||
return await scrapeReleases(scraper, site, preData, true);
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to scrape upcoming releases for '${site.slug}' (${site.network.slug})`);
|
||||
logger.warn(`Failed to scrape upcoming updates for '${site.slug}' (${site.network.slug})`);
|
||||
}
|
||||
|
||||
return [];
|
||||
|
|
@ -157,6 +163,8 @@ async function scrapeSiteReleases(scraper, site, preData) {
|
|||
: [],
|
||||
]);
|
||||
|
||||
logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${site.name}' (${site.network.name})`);
|
||||
|
||||
return [...latestReleases, ...upcomingReleases];
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue