DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updated dependencies. Added periodic memory logger.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-11-20 23:59:15 +01:00
parent a867817dc1
commit 26539b74a5
109 changed files with 10238 additions and 10833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/updates.js
View File

@@ -28,8 +28,8 @@ function mapReleasesToEntityIdAndEntryId(acc, release) {
function filterLocalUniqueReleases(releases, accReleases) {
const localDuplicateReleasesBySiteIdAndEntryId = accReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
const localUniqueReleases = releases.filter(release => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
const localDuplicateReleases = releases.filter(release => localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
const localUniqueReleases = releases.filter((release) => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
const localDuplicateReleases = releases.filter((release) => localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
return {
localUniqueReleases,
@@ -39,7 +39,7 @@ function filterLocalUniqueReleases(releases, accReleases) {
async function filterUniqueReleases(releases) {
const releaseIdentifiers = releases
.map(release => [release.entity.id, release.entryId]);
.map((release) => [release.entity.id, release.entryId]);
const duplicateReleaseEntries = await knex('releases')
.select(knex.raw('releases.*, row_to_json(entities) as entity'))
@@ -55,13 +55,13 @@ async function filterUniqueReleases(releases) {
.orWhere(knex.raw('updated_at - date > INTERVAL \'1 day\'')); // scene was updated after the release date, no updates expected
});
const duplicateReleases = duplicateReleaseEntries.map(release => curateRelease(release));
const duplicateReleases = duplicateReleaseEntries.map((release) => curateRelease(release));
const duplicateReleasesByEntityIdAndEntryId = duplicateReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
const internalUniqueReleasesByEntityIdAndEntryId = releases.reduce((acc, release) => mapReleasesToEntityIdAndEntryId(acc, release), {});
const internalUniqueReleases = Object.values(internalUniqueReleasesByEntityIdAndEntryId).map(releasesByEntryId => Object.values(releasesByEntryId)).flat();
const internalUniqueReleases = Object.values(internalUniqueReleasesByEntityIdAndEntryId).map((releasesByEntryId) => Object.values(releasesByEntryId)).flat();
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
const uniqueReleases = internalUniqueReleases.filter((release) => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
return { uniqueReleases, duplicateReleases };
}
@@ -83,7 +83,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
return accReleases.length + pageReleases.length < argv.last;
}
if (!pageReleases.concat(unextracted).every(release => !!release.date)) { // some scenes don't have dates
if (!pageReleases.concat(unextracted).every((release) => !!release.date)) { // some scenes don't have dates
return accReleases.length + pageReleases.length < argv.missingDateLimit;
}
@@ -124,8 +124,8 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
return accReleases;
}
const validPageReleases = pageReleases.filter(release => release?.entryId); // filter out empty and unidentified releases
const pageReleasesWithEntity = validPageReleases.map(release => ({ ...release, entity: release.entity || entity }));
const validPageReleases = pageReleases.filter((release) => release?.entryId); // filter out empty and unidentified releases
const pageReleasesWithEntity = validPageReleases.map((release) => ({ ...release, entity: release.entity || entity }));
if (pageReleases.length > validPageReleases.length) {
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
@@ -140,10 +140,10 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const releases = await scrapeReleasesPage(argv.page || 1, []);
const hasDates = releases.every(release => !!release.date);
const hasDates = releases.every((release) => !!release.date);
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|| (hasDates && releases.filter(release => moment(release.date).isAfter(argv.after)))
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|| releases.slice(0, Math.max(argv.missingDateLimit, 0));
const { uniqueReleases, duplicateReleases } = argv.force
@@ -280,7 +280,7 @@ async function fetchUpdates() {
const scrapedNetworks = await Promise.map(
includedNetworks,
async networkEntity => (networkEntity.parameters?.sequential
async (networkEntity) => (networkEntity.parameters?.sequential
? scrapeNetworkSequential(networkEntity)
: scrapeNetworkParallel(networkEntity)),
{ concurrency: 5 },