2019-03-03 03:18:33 +00:00
|
|
|
'use strict';
|
|
|
|
|
2019-03-04 01:46:33 +00:00
|
|
|
const util = require('util');
|
|
|
|
const config = require('config');
|
|
|
|
const networks = require('../networks.js');
|
|
|
|
const scrapers = require('./scrapers');
|
|
|
|
|
|
|
|
function accumulateIncludedSites() {
|
|
|
|
return config.include.reduce((acc, network) => {
|
|
|
|
// network included with specific sites, only include specified sites
|
|
|
|
if (Array.isArray(network)) {
|
|
|
|
const [networkId, siteIds] = network;
|
|
|
|
|
|
|
|
return [
|
|
|
|
...acc,
|
|
|
|
...siteIds.map(siteId => ({
|
|
|
|
id: siteId,
|
|
|
|
network: networkId,
|
|
|
|
...networks[networkId].sites[siteId],
|
|
|
|
})),
|
|
|
|
];
|
|
|
|
}
|
|
|
|
|
|
|
|
// network included without further specification, include all sites
|
|
|
|
return [
|
|
|
|
...acc,
|
|
|
|
...Object.entries(networks[network].sites).map(([siteId, site]) => ({
|
|
|
|
id: siteId,
|
|
|
|
network,
|
|
|
|
...site,
|
|
|
|
})),
|
|
|
|
];
|
|
|
|
}, []);
|
|
|
|
}
|
|
|
|
|
|
|
|
function accumulateExcludedSites() {
|
|
|
|
return Object.entries(networks).reduce((acc, [networkId, network]) => {
|
|
|
|
const excludedNetwork = config.exclude.find((excludedNetworkX) => {
|
|
|
|
if (Array.isArray(excludedNetworkX)) {
|
|
|
|
return excludedNetworkX[0] === networkId;
|
|
|
|
}
|
|
|
|
|
|
|
|
return excludedNetworkX === networkId;
|
|
|
|
});
|
|
|
|
|
|
|
|
// network excluded with specific sites, only exclude specified sites
|
|
|
|
if (excludedNetwork && Array.isArray(excludedNetwork)) {
|
|
|
|
const [, excludedSiteIds] = excludedNetwork;
|
|
|
|
|
|
|
|
return [
|
|
|
|
...acc,
|
|
|
|
...Object.entries(network.sites)
|
|
|
|
.filter(([siteId]) => !excludedSiteIds.includes(siteId))
|
|
|
|
.map(([siteId, site]) => ({
|
|
|
|
id: siteId,
|
|
|
|
network: networkId,
|
|
|
|
...site,
|
|
|
|
})),
|
|
|
|
];
|
|
|
|
}
|
|
|
|
|
|
|
|
// network excluded without further specification, exclude all its sites
|
|
|
|
if (excludedNetwork) {
|
|
|
|
return acc;
|
|
|
|
}
|
|
|
|
|
|
|
|
// network not excluded, include all its sites
|
|
|
|
return [
|
|
|
|
...acc,
|
|
|
|
...Object.entries(network.sites).map(([siteId, site]) => ({
|
|
|
|
id: siteId,
|
|
|
|
network: networkId,
|
|
|
|
...site,
|
|
|
|
})),
|
|
|
|
];
|
|
|
|
}, []);
|
|
|
|
}
|
|
|
|
|
|
|
|
function accumulateSites() {
|
|
|
|
return config.include ? accumulateIncludedSites() : accumulateExcludedSites();
|
|
|
|
}
|
2019-03-03 03:18:33 +00:00
|
|
|
|
|
|
|
async function init() {
|
2019-03-04 01:46:33 +00:00
|
|
|
const sites = accumulateSites();
|
|
|
|
|
|
|
|
const scenes = await Promise.all(sites.map(async (site) => {
|
|
|
|
const scraper = scrapers[site.id] || scrapers[site.network];
|
|
|
|
|
|
|
|
if (scraper) {
|
|
|
|
return scraper(site);
|
|
|
|
}
|
|
|
|
|
|
|
|
return [];
|
|
|
|
}));
|
|
|
|
|
|
|
|
console.log(util.inspect(scenes, { depth: 5 }));
|
2019-03-03 03:18:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
init();
|