DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added unextracted property to keep paginating when extracting scenes.

This commit is contained in:
DebaucheryLibrarian 2021-10-28 01:59:53 +02:00
parent 53357d4bd2
commit 0864154a0e
3 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
seeds/02_sites.js
View File

@ -1531,6 +1531,16 @@ const sites = [
url: 'https://www.bradmontana.com',
},
// BRAZZERS
{
slug: 'brazzers',
name: 'Brazzers',
url: 'https://www.brazzers.com/videos',
parent: 'brazzers',
parameters: {
extract: true,
scene: 'https://www.brazzers.com/video',
},
},
{
slug: 'momsincontrol',
name: 'Moms in Control',

8
src/scrapers/mindgeek.js
View File

@ -95,8 +95,12 @@ function scrapeLatestX(data, site, filterChannel) {
async function scrapeLatest(items, site, filterChannel) {
const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel));
const extractedScenes = latestReleases.filter(Boolean);
return latestReleases.filter(Boolean);
return {
scenes: extractedScenes,
unextracted: latestReleases.length - extractedScenes.length,
};
}
function scrapeScene(data, url, _site, networkName) {
@ -240,7 +244,7 @@ async function fetchLatest(site, page = 1, options) {
const { session, instanceToken } = options.beforeNetwork || await getSession(site, options.parameters);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 10;
const limit = 24;
const apiUrl = site.parameters?.native || site.parameters?.extract
? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;

12
src/updates.js
View File

@ -66,10 +66,10 @@ async function filterUniqueReleases(releases) {
return { uniqueReleases, duplicateReleases };
}
function needNextPage(pageReleases, accReleases, isUpcoming) {
function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = 0) {
const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases);
if (uniquePageReleases.length === 0) {
if (uniquePageReleases.length + unextracted === 0) {
// page is empty, or only contains scenes from previous page
return false;
}
@ -78,7 +78,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming) {
return uniquePageReleases.length > 0 && argv.paginateUpcoming;
}
if (uniquePageReleases.length > 0) {
if (uniquePageReleases.length + unextracted > 0) {
if (argv.last) {
return accReleases.length + pageReleases.length < argv.last;
}
@ -111,10 +111,12 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
parameters: getRecursiveParameters(entity),
};
const pageReleases = isUpcoming
const rawPageReleases = isUpcoming
? await scraper.fetchUpcoming(entity, page, options, preData)
: await scraper.fetchLatest(entity, page, options, preData);
const pageReleases = rawPageReleases.scenes || rawPageReleases;
if (!Array.isArray(pageReleases)) {
// scraper is unable to fetch the releases and returned a HTTP code or null
logger.warn(`Scraper returned ${pageReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
@ -128,7 +130,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
}
if (needNextPage(pageReleasesWithEntity, accReleases, isUpcoming)) {
if (needNextPage(pageReleasesWithEntity, accReleases, isUpcoming, rawPageReleases.unextracted)) {
return scrapeReleasesPage(page + 1, accReleases.concat(pageReleasesWithEntity), isUpcoming);
}