DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 36 Pull Requests 1 Releases Wiki Activity

Including unextracted scenes in date determination.

This commit is contained in:
DebaucheryLibrarian 2021-10-28 02:10:30 +02:00
parent 83dd233991
commit 29b8c5e38e
2 changed files with 16 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/scrapers/mindgeek.js
View File

@ -45,21 +45,6 @@ function getVideos(data) {
} }
function scrapeLatestX(data, site, filterChannel) { function scrapeLatestX(data, site, filterChannel) {
if (site.parameters?.extract === true && data.collections.length > 0) {
// release should not belong to any channel
return null;
}
if (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) {
// release should belong to specific channel
return null;
}
if (filterChannel && !data.collections?.some(collection => collection.id === site.parameters?.siteId)) {
// used to separate upcoming Brazzers scenes
return null;
}
const release = { const release = {
entryId: data.id, entryId: data.id,
title: data.title, title: data.title,
@ -90,16 +75,24 @@ function scrapeLatestX(data, site, filterChannel) {
tags: [chapter.name], tags: [chapter.name],
})); }));
if ((site.parameters?.extract === true && data.collections.length > 0) // release should not belong to any channel
|| (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) // release should belong to specific channel
|| (filterChannel && !data.collections?.some(collection => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes
return {
...release,
exclude: true,
};
}
return release; return release;
} }
async function scrapeLatest(items, site, filterChannel) { async function scrapeLatest(items, site, filterChannel) {
const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel)); const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel));
const extractedScenes = latestReleases.filter(Boolean);
return { return {
scenes: extractedScenes, scenes: latestReleases.filter(scene => !scene.exclude),
unextracted: latestReleases.length - extractedScenes.length, unextracted: latestReleases.filter(scene => scene.exclude),
}; };
} }

9
src/updates.js
View File

@ -66,10 +66,10 @@ async function filterUniqueReleases(releases) {
return { uniqueReleases, duplicateReleases }; return { uniqueReleases, duplicateReleases };
} }
function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = 0) { function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases); const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases);
if (uniquePageReleases.length + unextracted === 0) { if (uniquePageReleases.length + unextracted.length === 0) {
// page is empty, or only contains scenes from previous page // page is empty, or only contains scenes from previous page
return false; return false;
} }
@ -78,17 +78,18 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = 0) {
return uniquePageReleases.length > 0 && argv.paginateUpcoming; return uniquePageReleases.length > 0 && argv.paginateUpcoming;
} }
if (uniquePageReleases.length + unextracted > 0) { if (uniquePageReleases.length + unextracted.length > 0) {
if (argv.last) { if (argv.last) {
return accReleases.length + pageReleases.length < argv.last; return accReleases.length + pageReleases.length < argv.last;
} }
if (!pageReleases.every(release => !!release.date)) { // some scenes don't have dates if (!pageReleases.concat(unextracted).every(release => !!release.date)) { // some scenes don't have dates
return accReleases.length + pageReleases.length < argv.missingDateLimit; return accReleases.length + pageReleases.length < argv.missingDateLimit;
} }
if (argv.after) { if (argv.after) {
const oldestReleaseOnPage = pageReleases const oldestReleaseOnPage = pageReleases
.concat(unextracted)
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) .sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0]; .slice(-1)[0];