boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Scraping actor profiles from FreeOnes.

Browse Source
This commit is contained in:
ThePendulum
2019-11-17 03:56:45 +01:00
parent abcdb52335
commit e8130c3634
268 changed files with 19161 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/scrape-sites.js
View File

@@ -36,7 +36,7 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a
return [];
}
const duplicateReleaseIds = await findDuplicateReleaseIds(latestReleases, accReleases);
const duplicateReleaseIds = argv.redownload ? new Set() : await findDuplicateReleaseIds(latestReleases, accReleases);
const uniqueReleases = latestReleases
.filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
@@ -107,7 +107,7 @@ async function scrapeReleases() {
console.log(`Found ${sites.length} sites in database`);
await Promise.map(sites, async (site) => {
const scraper = scrapers[site.slug] || scrapers[site.network.slug];
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
if (!scraper) {
console.warn(`No scraper found for '${site.name}' (${site.slug})`);