From 02e81a8be90c134855d45902cf6d7c626324d99d Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Sun, 8 Sep 2019 04:06:49 +0200 Subject: [PATCH] Refactored site accumulation to work with new database structure --- src/fetch-releases.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/fetch-releases.js b/src/fetch-releases.js index cd9050e2..a43b9471 100644 --- a/src/fetch-releases.js +++ b/src/fetch-releases.js @@ -48,10 +48,13 @@ function curateSites(sites) { async function accumulateIncludedSites() { if (argv.networks || argv.sites) { + const networks = await knex('networks').select('id').whereIn('slug', argv.networks); + const networkIds = networks.map(network => network.id); + const rawSites = await knex('sites') .select('sites.*', 'networks.name as network_name') - .whereIn('sites.id', argv.sites || []) - .orWhereIn('network_id', argv.networks || []) + .whereIn('sites.slug', argv.sites || []) + .orWhereIn('network_id', networkIds) .leftJoin('networks', 'sites.network_id', 'networks.id'); return curateSites(rawSites); @@ -59,10 +62,13 @@ async function accumulateIncludedSites() { const included = destructConfigNetworks(config.include); + const networks = await knex('networks').select('id').whereIn('slug', included.networks); + const networkIds = networks.map(network => network.id); + const rawSites = await knex('sites') .select('sites.*', 'networks.name as network_name') - .whereIn('sites.id', included.sites) - .orWhereIn('network_id', included.networks) + .whereIn('sites.slug', included.sites) + .orWhereIn('network_id', networkIds) .leftJoin('networks', 'sites.network_id', 'networks.id'); return curateSites(rawSites); @@ -178,6 +184,10 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page async function fetchReleases() { const sites = await accumulateIncludedSites(); + console.log(sites); + + return; + const scenesPerSite = await Promise.map(sites, async (site) => { const scraper = scrapers[site.id] || scrapers[site.network.id];