DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Removed site and tag matching from Dogfart scraper. Filtering duplicate tag associations before insert attempt.

Browse Source
This commit is contained in:
ThePendulum
2020-01-17 01:55:54 +01:00
parent 0b84c977da
commit daee426ba6
5 changed files with 9 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/sites.js
View File

@@ -120,17 +120,14 @@ async function fetchSitesFromArgv() {
async function fetchSitesFromConfig() {
const included = destructConfigNetworks(config.include);
const networks = await knex('networks').select('id').whereIn('slug', included.networks || []);
const networkIds = networks.map(network => network.id);
const rawSites = await knex('sites')
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.leftJoin('networks', 'sites.network_id', 'networks.id')
.whereIn('sites.slug', included.sites || [])
.orWhereIn('network_id', networkIds)
.leftJoin('networks', 'sites.network_id', 'networks.id');
.orWhereIn('networks.slug', included.networks || []);
const curatedSites = await curateSites(rawSites, true);
logger.info(`Found ${curatedSites.length} sites in database`);