DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added unextracted property to keep paginating when extracting scenes.

This commit is contained in:
DebaucheryLibrarian 2021-10-28 01:59:53 +02:00
parent 53357d4bd2
commit 0864154a0e
3 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
seeds/02_sites.js
View File

@ -1531,6 +1531,16 @@ const sites = [
url: 'https://www.bradmontana.com', url: 'https://www.bradmontana.com',
}, },
// BRAZZERS // BRAZZERS
{
slug: 'brazzers',
name: 'Brazzers',
url: 'https://www.brazzers.com/videos',
parent: 'brazzers',
parameters: {
extract: true,
scene: 'https://www.brazzers.com/video',
},
},
{ {
slug: 'momsincontrol', slug: 'momsincontrol',
name: 'Moms in Control', name: 'Moms in Control',

8
src/scrapers/mindgeek.js
View File

@ -95,8 +95,12 @@ function scrapeLatestX(data, site, filterChannel) {
async function scrapeLatest(items, site, filterChannel) { async function scrapeLatest(items, site, filterChannel) {
const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel)); const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel));
const extractedScenes = latestReleases.filter(Boolean);
return latestReleases.filter(Boolean); return {
scenes: extractedScenes,
unextracted: latestReleases.length - extractedScenes.length,
};
} }
function scrapeScene(data, url, _site, networkName) { function scrapeScene(data, url, _site, networkName) {
@ -240,7 +244,7 @@ async function fetchLatest(site, page = 1, options) {
const { session, instanceToken } = options.beforeNetwork || await getSession(site, options.parameters); const { session, instanceToken } = options.beforeNetwork || await getSession(site, options.parameters);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD'); const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 10; const limit = 24;
const apiUrl = site.parameters?.native || site.parameters?.extract const apiUrl = site.parameters?.native || site.parameters?.extract
? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene` ? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`; : `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;

12
src/updates.js
View File

@ -66,10 +66,10 @@ async function filterUniqueReleases(releases) {
return { uniqueReleases, duplicateReleases }; return { uniqueReleases, duplicateReleases };
} }
function needNextPage(pageReleases, accReleases, isUpcoming) { function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = 0) {
const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases); const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases);
if (uniquePageReleases.length === 0) { if (uniquePageReleases.length + unextracted === 0) {
// page is empty, or only contains scenes from previous page // page is empty, or only contains scenes from previous page
return false; return false;
} }
@ -78,7 +78,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming) {
return uniquePageReleases.length > 0 && argv.paginateUpcoming; return uniquePageReleases.length > 0 && argv.paginateUpcoming;
} }
if (uniquePageReleases.length > 0) { if (uniquePageReleases.length + unextracted > 0) {
if (argv.last) { if (argv.last) {
return accReleases.length + pageReleases.length < argv.last; return accReleases.length + pageReleases.length < argv.last;
} }
@ -111,10 +111,12 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
parameters: getRecursiveParameters(entity), parameters: getRecursiveParameters(entity),
}; };
const pageReleases = isUpcoming const rawPageReleases = isUpcoming
? await scraper.fetchUpcoming(entity, page, options, preData) ? await scraper.fetchUpcoming(entity, page, options, preData)
: await scraper.fetchLatest(entity, page, options, preData); : await scraper.fetchLatest(entity, page, options, preData);
const pageReleases = rawPageReleases.scenes || rawPageReleases;
if (!Array.isArray(pageReleases)) { if (!Array.isArray(pageReleases)) {
// scraper is unable to fetch the releases and returned a HTTP code or null // scraper is unable to fetch the releases and returned a HTTP code or null
logger.warn(`Scraper returned ${pageReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`); logger.warn(`Scraper returned ${pageReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
@ -128,7 +130,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`); logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
} }
if (needNextPage(pageReleasesWithEntity, accReleases, isUpcoming)) { if (needNextPage(pageReleasesWithEntity, accReleases, isUpcoming, rawPageReleases.unextracted)) {
return scrapeReleasesPage(page + 1, accReleases.concat(pageReleasesWithEntity), isUpcoming); return scrapeReleasesPage(page + 1, accReleases.concat(pageReleasesWithEntity), isUpcoming);
} }