boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactored site accumulation to work with new database structure

This commit is contained in:
ThePendulum 2019-09-08 04:06:49 +02:00
parent 926c227b85
commit 02e81a8be9
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/fetch-releases.js
View File

@ -48,10 +48,13 @@ function curateSites(sites) {
async function accumulateIncludedSites() { async function accumulateIncludedSites() {
if (argv.networks || argv.sites) { if (argv.networks || argv.sites) {
const networks = await knex('networks').select('id').whereIn('slug', argv.networks);
const networkIds = networks.map(network => network.id);
const rawSites = await knex('sites') const rawSites = await knex('sites')
.select('sites.*', 'networks.name as network_name') .select('sites.*', 'networks.name as network_name')
.whereIn('sites.id', argv.sites || []) .whereIn('sites.slug', argv.sites || [])
.orWhereIn('network_id', argv.networks || []) .orWhereIn('network_id', networkIds)
.leftJoin('networks', 'sites.network_id', 'networks.id'); .leftJoin('networks', 'sites.network_id', 'networks.id');
return curateSites(rawSites); return curateSites(rawSites);
@ -59,10 +62,13 @@ async function accumulateIncludedSites() {
const included = destructConfigNetworks(config.include); const included = destructConfigNetworks(config.include);
const networks = await knex('networks').select('id').whereIn('slug', included.networks);
const networkIds = networks.map(network => network.id);
const rawSites = await knex('sites') const rawSites = await knex('sites')
.select('sites.*', 'networks.name as network_name') .select('sites.*', 'networks.name as network_name')
.whereIn('sites.id', included.sites) .whereIn('sites.slug', included.sites)
.orWhereIn('network_id', included.networks) .orWhereIn('network_id', networkIds)
.leftJoin('networks', 'sites.network_id', 'networks.id'); .leftJoin('networks', 'sites.network_id', 'networks.id');
return curateSites(rawSites); return curateSites(rawSites);
@ -178,6 +184,10 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page
async function fetchReleases() { async function fetchReleases() {
const sites = await accumulateIncludedSites(); const sites = await accumulateIncludedSites();
console.log(sites);
return;
const scenesPerSite = await Promise.map(sites, async (site) => { const scenesPerSite = await Promise.map(sites, async (site) => {
const scraper = scrapers[site.id] || scrapers[site.network.id]; const scraper = scrapers[site.id] || scrapers[site.network.id];