'use strict'; const config = require('config'); const moment = require('moment'); const argv = require('./argv'); const knex = require('./knex'); const scrapers = require('./scrapers'); function destructConfigNetworks(networks) { return networks.reduce((acc, network) => { if (Array.isArray(network)) { // network specifies sites return { ...acc, sites: [...acc.sites, ...network[1]], }; } return { ...acc, networks: [...acc.networks, network], }; }, { networks: [], sites: [], }); } function curateSites(sites) { return sites.map(site => ({ id: site.id, name: site.name, description: site.description, url: site.url, networkId: site.network_id, parameters: JSON.parse(site.parameters), })); } async function accumulateIncludedSites() { if (argv.networks || argv.sites) { const rawSites = await knex('sites') .whereIn('id', argv.sites || []) .orWhereIn('network_id', argv.networks || []); return curateSites(rawSites); } const included = destructConfigNetworks(config.include); const rawSites = await knex('sites') .whereIn('id', included.sites) .orWhereIn('network_id', included.networks); return curateSites(rawSites); } async function getStoredReleases(siteId, limit) { return knex('releases') .where({ site_id: siteId }) .orderBy('date', 'desc') .limit(limit); } async function storeReleases(releases) { const curatedReleases = releases.map(release => ({ site_id: release.site.id, shoot_id: release.shootId || null, url: release.url, title: release.title, date: release.date, description: release.description, director: release.director, duration: release.duration, likes: release.rating && release.rating.likes, dislikes: release.rating && release.rating.dislikes, rating: release.rating && release.rating.stars, })); if (curatedReleases.length) { console.log(`Adding ${curatedReleases.length} releases to database (if unique)`); const insertQuery = knex('releases').insert(curatedReleases).toString(); await knex.raw(insertQuery.replace('insert', 'INSERT OR IGNORE')); return curatedReleases; } return []; } async function fetchReleases() { const sites = await accumulateIncludedSites(); // const releases = await getExistingReleases(); const scenesPerSite = await Promise.all(sites.map(async (site) => { const scraper = scrapers[site.id] || scrapers[site.networkId]; if (scraper) { const storedReleases = await getStoredReleases(site.id, 100); const [latest, upcoming] = await Promise.all([ scraper.fetchLatest(site, storedReleases), scraper.fetchUpcoming ? scraper.fetchUpcoming(site) : [], ]); console.log(`${latest.length} published releases and ${upcoming.length} upcoming releases found`); await storeReleases(latest); return [...latest, ...upcoming]; } return []; })); const accumulatedScenes = scenesPerSite.reduce((acc, siteScenes) => ([...acc, ...siteScenes]), []); const sortedScenes = accumulatedScenes.sort(({ date: dateA }, { date: dateB }) => moment(dateB).diff(dateA)); return sortedScenes; } module.exports = fetchReleases;