boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed ID number/string incompatability in duplicate detection. Expanded Reality Kings scraper to handle older scenes and fix URLs.

Browse Source
This commit is contained in:
ThePendulum
2019-11-17 00:45:31 +01:00
parent 88c8bb1ced
commit bd041c528d
4 changed files with 32 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/scrape-sites.js
View File

@@ -24,8 +24,8 @@ async function findDuplicateReleaseIds(latestReleases, accReleases) {
// include accumulated releases as duplicates to prevent an infinite
// loop when the next page contains the same releases as the previous
return new Set(duplicateReleases
.map(release => release.entry_id)
.concat(accReleases.map(release => release.entryId)));
.map(release => String(release.entry_id))
.concat(accReleases.map(release => String(release.entryId))));
}
async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), accReleases = [], page = 1) {
@@ -122,11 +122,11 @@ async function scrapeReleases() {
}
} catch (error) {
if (argv.debug) {
console.error(`${site.id}: Failed to fetch releases`, error);
console.error(`${site.id}: Failed to scrape releases`, error);
return;
}
console.warn(`${site.id}: Failed to fetch releases`);
console.warn(`${site.id}: Failed to scrape releases`);
}
}, {
concurrency: 2,