forked from DebaucheryLibrarian/traxxx
Adding networks and sites as entities,
This commit is contained in:
@@ -22,7 +22,7 @@ const { argv } = yargs
|
||||
.option('sites', {
|
||||
describe: 'Sites to scrape (overrides configuration)',
|
||||
type: 'array',
|
||||
alias: 'site',
|
||||
alias: 'entities',
|
||||
})
|
||||
.option('actors', {
|
||||
describe: 'Scrape actors by name or slug',
|
||||
|
||||
@@ -8,7 +8,7 @@ const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
const include = require('./utils/argv-include')(argv);
|
||||
const scrapers = require('./scrapers/scrapers');
|
||||
const { fetchSitesFromArgv, fetchSitesFromConfig } = require('./sites');
|
||||
const { fetchEntitiesFromArgv, fetchEntitiesFromConfig } = require('./entities');
|
||||
|
||||
const afterDate = (() => {
|
||||
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
|
||||
@@ -218,9 +218,9 @@ async function scrapeNetworkParallel(network) {
|
||||
}
|
||||
|
||||
async function fetchUpdates() {
|
||||
const includedNetworks = argv.sites || argv.networks
|
||||
? await fetchSitesFromArgv()
|
||||
: await fetchSitesFromConfig();
|
||||
const includedNetworks = argv.sites || argv.networks || argv.from
|
||||
? await fetchEntitiesFromArgv()
|
||||
: await fetchEntitiesFromConfig();
|
||||
|
||||
const scrapedNetworks = await Promise.map(
|
||||
includedNetworks,
|
||||
|
||||
Reference in New Issue
Block a user