Changed sites from argument query to group by network.
This commit is contained in:
@@ -8,7 +8,7 @@ const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
const include = require('./utils/argv-include')(argv);
|
||||
const scrapers = require('./scrapers/scrapers');
|
||||
const { fetchEntitiesFromArgv, fetchEntitiesFromConfig } = require('./entities');
|
||||
const { fetchSitesFromArgv, fetchSitesFromConfig } = require('./entities');
|
||||
|
||||
const afterDate = (() => {
|
||||
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
|
||||
@@ -219,8 +219,10 @@ async function scrapeNetworkParallel(network) {
|
||||
|
||||
async function fetchUpdates() {
|
||||
const includedNetworks = argv.sites || argv.networks || argv.from
|
||||
? await fetchEntitiesFromArgv()
|
||||
: await fetchEntitiesFromConfig();
|
||||
? await fetchSitesFromArgv()
|
||||
: await fetchSitesFromConfig();
|
||||
|
||||
// console.log('included', includedNetworks);
|
||||
|
||||
const scrapedNetworks = await Promise.map(
|
||||
includedNetworks,
|
||||
|
||||
Reference in New Issue
Block a user