DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Providing duplicate releases in predata. Using duplicates for filtering scenes without channel in Hush scraper.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-08-23 02:43:10 +02:00
parent 3a5ea3dd9a
commit 278b74e78c
3 changed files with 34 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/updates.js
View File

@@ -6,6 +6,7 @@ const moment = require('moment');
const argv = require('./argv');
const logger = require('./logger')(__filename);
const knex = require('./knex');
const { curateRelease } = require('./releases');
const include = require('./utils/argv-include')(argv);
const scrapers = require('./scrapers/scrapers');
const { fetchIncludedEntities } = require('./entities');
@@ -16,16 +17,20 @@ async function filterUniqueReleases(latestReleases, accReleases) {
const latestReleaseIdentifiers = latestReleases
.map(release => [release.entity.id, release.entryId]);
const duplicateReleases = await knex('releases')
const duplicateReleaseEntries = await knex('releases')
.select(knex.raw('releases.*, row_to_json(entities) as entity'))
.leftJoin('entities', 'entities.id', 'releases.entity_id')
.whereIn(['entity_id', 'entry_id'], latestReleaseIdentifiers);
const duplicateReleases = duplicateReleaseEntries.map(release => curateRelease(release));
// add entry IDs of accumulated releases to prevent an infinite scrape loop
// when one page contains the same release as the previous
const duplicateReleasesSiteIdAndEntryIds = duplicateReleases
.concat(accReleases)
.reduce((acc, release) => {
const entityId = release.entity_id || release.entity.id;
const entryId = release.entry_id || release.entryId;
const entityId = release.entityId || release.entity.id;
const entryId = release.entryId || release.entryId;
if (!acc[entityId]) acc[entityId] = {};
acc[entityId][entryId] = true;
@@ -163,11 +168,12 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
scrapeMovies(scraper, channelEntity, preData),
]);
console.log(latestReleases);
logger.info(`Fetching ${latestReleases.uniqueReleases.length} latest and ${upcomingReleases.uniqueReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
return [...latestReleases.uniqueReleases, ...upcomingReleases.uniqueReleases];
return {
uniqueReleases: [...latestReleases.uniqueReleases, ...upcomingReleases.uniqueReleases],
duplicateReleases: [...latestReleases.duplicateReleases, ...upcomingReleases.duplicateReleases],
};
}
async function scrapeChannel(channelEntity, accNetworkReleases) {
@@ -183,12 +189,10 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
try {
const beforeFetchLatest = await scraper.beforeFetchLatest?.(channelEntity);
const channelEntityReleases = await scrapeChannelReleases(scraper, channelEntity, {
accNetworkReleases,
return await scrapeChannelReleases(scraper, channelEntity, {
...accNetworkReleases,
beforeFetchLatest,
});
return channelEntityReleases.map(release => ({ ...release, channelEntity }));
} catch (error) {
logger.error(`Failed to scrape releases from ${channelEntity.name} using ${scraper.slug}: ${error.message}`);
@@ -197,22 +201,31 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
}
async function scrapeNetworkSequential(networkEntity) {
return Promise.reduce(
const releases = await Promise.reduce(
networkEntity.children,
async (chain, channelEntity) => {
const accNetworkReleases = await chain;
const channelReleases = await scrapeChannel(channelEntity, accNetworkReleases);
const { uniqueReleases, duplicateReleases } = await scrapeChannel(channelEntity, accNetworkReleases);
return accNetworkReleases.concat(channelReleases);
return {
uniqueReleases: accNetworkReleases.uniqueReleases.concat(uniqueReleases),
duplicateReleases: accNetworkReleases.duplicateReleases.concat(duplicateReleases),
};
},
Promise.resolve([]),
Promise.resolve(emptyReleases),
);
return releases.uniqueReleases;
}
async function scrapeNetworkParallel(networkEntity) {
return Promise.map(
networkEntity.children,
async channelEntity => scrapeChannel(channelEntity, networkEntity),
async (channelEntity) => {
const { uniqueReleases } = await scrapeChannel(channelEntity, networkEntity);
return uniqueReleases;
},
{ concurrency: 3 },
);
}