Returning duplicate releases from pagination.
This commit is contained in:
DebaucheryLibrarian
parent
5f5c48ea05
commit
3a5ea3dd9a
|
|
@ -2681,6 +2681,7 @@ const sites = [
|
|||
name: 'Booty Annihilation',
|
||||
tags: ['interracial'],
|
||||
parent: 'interracialpass',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/BootyAnnihilation_%d_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
|
|
|
|||
|
|
@ -33,9 +33,11 @@ async function findEntities(baseReleases) {
|
|||
));
|
||||
|
||||
const entities = await knex('entities')
|
||||
.select(knex.raw('entities.*, row_to_json(parents) as parent'))
|
||||
.select(knex.raw('entities.*, row_to_json(parents) as parent, json_agg(children) as children'))
|
||||
.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
|
||||
.leftJoin('entities as children', 'children.parent_id', 'entities.id')
|
||||
.whereIn('entities.slug', entitySlugs)
|
||||
.groupBy('entities.id', 'parents.id')
|
||||
.orderBy('entities.type', 'asc');
|
||||
|
||||
// channel entity will overwrite network entity
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ async function fetchIncludedEntities() {
|
|||
)
|
||||
/* select recursive channels as children of networks */
|
||||
SELECT
|
||||
entities.*, json_agg(channels) as children
|
||||
entities.*, json_agg(channels ORDER BY channels.id) as children
|
||||
FROM
|
||||
channels
|
||||
LEFT JOIN
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ const { feetInchesToCm } = require('../utils/convert');
|
|||
async function getChannelRegExp(site) {
|
||||
if (!['hushpass', 'interracialpass'].includes(site.parent.slug)) return null;
|
||||
|
||||
const sites = await knex('sites').where('network_id', site.parent.id);
|
||||
const sites = await knex('entities').where('parent_id', site.parent.id);
|
||||
|
||||
return new RegExp(sites.map(channel => channel.parameters?.match || channel.name).join('|'), 'i');
|
||||
}
|
||||
|
|
@ -80,7 +80,7 @@ function scrapeAll(scenes, site) {
|
|||
});
|
||||
}
|
||||
|
||||
function scrapeAllT1(scenes, site, accSiteReleases) {
|
||||
function scrapeAllT1(scenes, site, accNetworkReleases) {
|
||||
return scenes.map(({ qu }) => {
|
||||
const release = {};
|
||||
|
||||
|
|
@ -107,7 +107,9 @@ function scrapeAllT1(scenes, site, accSiteReleases) {
|
|||
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
|
||||
release.entryId = deriveEntryId(release);
|
||||
|
||||
if (site.parameters?.accFilter && accSiteReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
|
||||
console.log(site.name, accNetworkReleases.map(accRelease => accRelease.entryId), release.entryId, accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId));
|
||||
|
||||
if (site.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
|
||||
// filter out releases that were already scraped from a categorized site, requires sequential site scraping
|
||||
return null;
|
||||
}
|
||||
|
|
@ -187,6 +189,8 @@ function scrapeSceneT1({ html, qu }, site, url, baseRelease, channelRegExp) {
|
|||
const stars = qu.q('.update-rating', true).match(/\d.\d/)?.[0];
|
||||
if (stars) release.stars = Number(stars);
|
||||
|
||||
console.log(channelRegExp, site);
|
||||
|
||||
if (channelRegExp) {
|
||||
const channel = release.tags.find(tag => channelRegExp.test(tag));
|
||||
|
||||
|
|
@ -356,7 +360,7 @@ function scrapeProfileTour({ el, qu }, site) {
|
|||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1, include, preflight, accSiteReleases) {
|
||||
async function fetchLatest(site, page = 1, include, { accNetworkReleases }) {
|
||||
const url = (site.parameters?.latest && util.format(site.parameters.latest, page))
|
||||
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|
||||
|| `${site.url}/categories/movies_${page}_d.html`;
|
||||
|
|
@ -364,13 +368,13 @@ async function fetchLatest(site, page = 1, include, preflight, accSiteReleases)
|
|||
const res = await geta(url, '.modelfeature, .item-video, .updateItem');
|
||||
|
||||
if (!res.ok) return res.status;
|
||||
if (site.parameters?.t1) return scrapeAllT1(res.items, site, accSiteReleases);
|
||||
if (site.parameters?.tour) return scrapeAllTour(res.items, site, accSiteReleases);
|
||||
if (site.parameters?.t1) return scrapeAllT1(res.items, site, accNetworkReleases);
|
||||
if (site.parameters?.tour) return scrapeAllTour(res.items, site, accNetworkReleases);
|
||||
|
||||
return scrapeAll(res.items, site, accSiteReleases);
|
||||
return scrapeAll(res.items, site, accNetworkReleases);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site, baseRelease, beforeFetchLatest) {
|
||||
async function fetchScene(url, site, baseRelease, include, beforeFetchLatest) {
|
||||
const channelRegExp = beforeFetchLatest || await getChannelRegExp(site);
|
||||
const res = await get(url);
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ const include = require('./utils/argv-include')(argv);
|
|||
const scrapers = require('./scrapers/scrapers');
|
||||
const { fetchIncludedEntities } = require('./entities');
|
||||
|
||||
const emptyReleases = { uniqueReleases: [], duplicateReleases: [] };
|
||||
|
||||
async function filterUniqueReleases(latestReleases, accReleases) {
|
||||
const latestReleaseIdentifiers = latestReleases
|
||||
.map(release => [release.entity.id, release.entryId]);
|
||||
|
|
@ -31,10 +33,9 @@ async function filterUniqueReleases(latestReleases, accReleases) {
|
|||
return acc;
|
||||
}, {});
|
||||
|
||||
const uniqueReleases = latestReleases
|
||||
.filter(release => !duplicateReleasesSiteIdAndEntryIds[release.entity.id]?.[release.entryId]);
|
||||
const uniqueReleases = latestReleases.filter(release => !duplicateReleasesSiteIdAndEntryIds[release.entity.id]?.[release.entryId]);
|
||||
|
||||
return uniqueReleases;
|
||||
return { uniqueReleases, duplicateReleases };
|
||||
}
|
||||
|
||||
function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) {
|
||||
|
|
@ -66,7 +67,7 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcomin
|
|||
return false;
|
||||
}
|
||||
|
||||
async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, accReleases = [], totalReleases = 0) {
|
||||
async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, acc = emptyReleases, totalReleases = 0) {
|
||||
const releases = upcoming
|
||||
? await scraper.fetchUpcoming(entity, page, include, preData)
|
||||
: await scraper.fetchLatest(entity, page, include, preData);
|
||||
|
|
@ -74,7 +75,7 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
|
|||
if (!Array.isArray(releases)) {
|
||||
// scraper is unable to fetch the releases and returned a HTTP code or null
|
||||
logger.warn(`Scraper returned ${releases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
|
||||
return accReleases;
|
||||
return acc;
|
||||
}
|
||||
|
||||
const releasesWithEntity = releases.map(release => ({
|
||||
|
|
@ -88,20 +89,25 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
|
|||
|| (hasDates && releasesWithEntity.filter(release => moment(release.date).isAfter(argv.after)))
|
||||
|| releasesWithEntity.slice(0, Math.max(argv.nullDateLimit - totalReleases, 0));
|
||||
|
||||
const uniqueReleases = argv.force
|
||||
? limitedReleases
|
||||
: await filterUniqueReleases(limitedReleases, accReleases);
|
||||
const { uniqueReleases, duplicateReleases } = argv.force
|
||||
? { uniqueReleases: limitedReleases, duplicateReleases: [] }
|
||||
: await filterUniqueReleases(limitedReleases, acc.uniqueReleases);
|
||||
|
||||
const accReleases = {
|
||||
uniqueReleases: acc.uniqueReleases.concat(uniqueReleases),
|
||||
duplicateReleases: acc.duplicateReleases.concat(duplicateReleases),
|
||||
};
|
||||
|
||||
if (needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming)) {
|
||||
return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length);
|
||||
return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases, totalReleases + releases.length);
|
||||
}
|
||||
|
||||
return accReleases.concat(uniqueReleases);
|
||||
return accReleases;
|
||||
}
|
||||
|
||||
async function scrapeLatestReleases(scraper, entity, preData) {
|
||||
if (!scraper.fetchLatest) {
|
||||
return [];
|
||||
if (!argv.latest || !scraper.fetchLatest) {
|
||||
return emptyReleases;
|
||||
}
|
||||
|
||||
try {
|
||||
|
|
@ -114,12 +120,12 @@ async function scrapeLatestReleases(scraper, entity, preData) {
|
|||
logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
|
||||
}
|
||||
|
||||
return [];
|
||||
return emptyReleases;
|
||||
}
|
||||
|
||||
async function scrapeUpcomingReleases(scraper, entity, preData) {
|
||||
if (!scraper.fetchUpcoming) {
|
||||
return [];
|
||||
if (!argv.upcoming || !scraper.fetchUpcoming) {
|
||||
return emptyReleases;
|
||||
}
|
||||
|
||||
try {
|
||||
|
|
@ -132,11 +138,11 @@ async function scrapeUpcomingReleases(scraper, entity, preData) {
|
|||
logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
|
||||
}
|
||||
|
||||
return [];
|
||||
return emptyReleases;
|
||||
}
|
||||
|
||||
async function scrapeMovies(scraper, entity) {
|
||||
if (!scraper.fetchMovies) {
|
||||
if (!argv.movies || !scraper.fetchMovies) {
|
||||
return [];
|
||||
}
|
||||
|
||||
|
|
@ -152,20 +158,16 @@ async function scrapeMovies(scraper, entity) {
|
|||
|
||||
async function scrapeChannelReleases(scraper, channelEntity, preData) {
|
||||
const [latestReleases, upcomingReleases] = await Promise.all([
|
||||
argv.latest
|
||||
? scrapeLatestReleases(scraper, channelEntity, preData)
|
||||
: [],
|
||||
argv.upcoming
|
||||
? scrapeUpcomingReleases(scraper, channelEntity, preData)
|
||||
: [],
|
||||
argv.movies
|
||||
? scrapeMovies(scraper, channelEntity, preData)
|
||||
: [],
|
||||
scrapeLatestReleases(scraper, channelEntity, preData),
|
||||
scrapeUpcomingReleases(scraper, channelEntity, preData),
|
||||
scrapeMovies(scraper, channelEntity, preData),
|
||||
]);
|
||||
|
||||
console.log(latestReleases);
|
||||
|
||||
logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
|
||||
|
||||
return [...latestReleases, ...upcomingReleases];
|
||||
return [...latestReleases.uniqueReleases, ...upcomingReleases.uniqueReleases];
|
||||
}
|
||||
|
||||
async function scrapeChannel(channelEntity, accNetworkReleases) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue