Refactored update scraper into new module.

This commit is contained in:
2020-03-14 02:56:28 +01:00
parent 4fc0053bd9
commit c8ebe7892a
6 changed files with 263 additions and 35 deletions

View File

@@ -113,7 +113,6 @@ async function fetchSitesFromArgv() {
)
.whereIn('sites.slug', argv.sites || [])
.orWhereIn('networks.slug', argv.networks || [])
.where('sites.scrape', true)
.leftJoin('networks', 'sites.network_id', 'networks.id');
const curatedSites = await curateSites(rawSites, true);
@@ -132,7 +131,6 @@ async function fetchSitesFromConfig() {
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.leftJoin('networks', 'sites.network_id', 'networks.id')
.where('sites.scrape', true)
.where((builder) => {
if (config.include) {
builder