Adding networks and sites as entities,

This commit is contained in:
2020-06-04 01:03:02 +02:00
parent 8abcc7194a
commit 09d849eb9d
9 changed files with 283 additions and 221 deletions

View File

@@ -22,7 +22,7 @@ const { argv } = yargs
.option('sites', {
describe: 'Sites to scrape (overrides configuration)',
type: 'array',
alias: 'site',
alias: 'entities',
})
.option('actors', {
describe: 'Scrape actors by name or slug',

View File

@@ -8,7 +8,7 @@ const logger = require('./logger')(__filename);
const knex = require('./knex');
const include = require('./utils/argv-include')(argv);
const scrapers = require('./scrapers/scrapers');
const { fetchSitesFromArgv, fetchSitesFromConfig } = require('./sites');
const { fetchEntitiesFromArgv, fetchEntitiesFromConfig } = require('./entities');
const afterDate = (() => {
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
@@ -218,9 +218,9 @@ async function scrapeNetworkParallel(network) {
}
async function fetchUpdates() {
const includedNetworks = argv.sites || argv.networks
? await fetchSitesFromArgv()
: await fetchSitesFromConfig();
const includedNetworks = argv.sites || argv.networks || argv.from
? await fetchEntitiesFromArgv()
: await fetchEntitiesFromConfig();
const scrapedNetworks = await Promise.map(
includedNetworks,