DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Improved update runner. Improved HTTP module API, added default user agent. Added PornCZ and Czechav logos.

Browse Source
This commit is contained in:
ThePendulum
2020-03-21 02:48:24 +01:00
parent 4b310e9dfa
commit d765543b30
140 changed files with 2454 additions and 577 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/updates.js
View File

@@ -26,23 +26,33 @@ const afterDate = (() => {
})();
async function extractUniqueReleases(latestReleases, accReleases) {
const latestReleaseEntryIds = latestReleases.map(release => release.entryId);
const duplicateReleases = await knex('releases')
.whereIn('entry_id', latestReleaseEntryIds);
const latestReleaseIdentifiers = latestReleases
.map(release => [release.site.id, release.entryId]);
// add entry IDs of accumulated releases to prevent an infinite loop
const duplicateReleases = await knex('releases')
.whereIn(['site_id', 'entry_id'], latestReleaseIdentifiers);
// add entry IDs of accumulated releases to prevent an infinite scrape loop
// when one page contains the same release as the previous
const duplicateReleaseEntryIds = new Set(duplicateReleases
.map(release => String(release.entry_id))
.concat(accReleases.map(release => String(release.entryId))));
const duplicateReleaseIdentifiers = duplicateReleases
.concat(accReleases)
.reduce((acc, release) => {
const siteId = release.site_id || release.site.id;
const entryId = release.entry_id || release.entryId;
if (!acc[siteId]) acc[siteId] = {};
acc[siteId][entryId] = true;
return acc;
}, {});
const uniqueReleases = latestReleases
.filter(release => !duplicateReleaseEntryIds.has(String(release.entryId)));
.filter(release => !duplicateReleaseIdentifiers[release.site.id]?.[release.entryId]);
return uniqueReleases;
}
function getNextPage(uniqueReleases, pageAccReleases, oldestReleaseOnPage) {
function needNextPage(uniqueReleases, pageAccReleases) {
if (uniqueReleases === 0) {
return false;
}
@@ -52,9 +62,13 @@ function getNextPage(uniqueReleases, pageAccReleases, oldestReleaseOnPage) {
return true;
}
if (oldestReleaseOnPage && moment(oldestReleaseOnPage.date).isAfter(afterDate)) {
// oldest release on page is newer than the specified date cut-off
return true;
const oldestReleaseOnPage = uniqueReleases
.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0];
if (oldestReleaseOnPage && moment(oldestReleaseOnPage.date).isAfter(afterDate)) {
// oldest release on page is newer than the specified date cut-off
return true;
}
// dates missing, and limit for scenes without dates not yet reached
@@ -81,7 +95,6 @@ async function scrapeLatestReleases(scraper, site, preData) {
}
const latestReleasesWithSite = latestReleases.map(release => ({ ...release, site: release.site || site })); // attach site release is assigned to when stored
const oldestReleaseOnPage = latestReleases.sort((releaseA, releaseB) => releaseB.date - releaseA.date).slice(-1)[0];
const uniqueReleases = argv.redownload
? latestReleasesWithSite
@@ -91,25 +104,25 @@ async function scrapeLatestReleases(scraper, site, preData) {
logger.verbose(`Scraped '${site.name}' (${site.network.name}) page ${page}, found ${uniqueReleases.length} unique releases`);
if (getNextPage(uniqueReleases, pageAccReleases, oldestReleaseOnPage)) {
return scrapePage(page + 1, accReleases.concat(uniqueReleases));
if (needNextPage(uniqueReleases, pageAccReleases)) {
return scrapePage(page + 1, pageAccReleases);
}
if (argv.last) {
return pageAccReleases.slice(0, argv.last);
}
if (oldestReleaseOnPage) {
const recentReleases = uniqueReleases
.filter(release => moment(release.date).isAfter(afterDate));
return accReleases.concat(recentReleases);
}
return pageAccReleases.slice(0, argv.nullDateLimit);
return pageAccReleases;
};
return scrapePage(1, []);
const releases = await scrapePage(1, []);
if (argv.last) {
return releases.slice(0, argv.last);
}
if (releases.every(release => release.date)) {
return releases
.filter(release => moment(release.date).isAfter(afterDate));
}
return releases.slice(0, argv.nullDateLimit);
}
async function scrapeUpcomingReleases(scraper, site, preData) {