traxxx/src/scrape-releases.js

'use strict';

const config = require('config');
const Promise = require('bluebird');

const argv = require('./argv');
const scrapers = require('./scrapers/scrapers');
const { storeReleases } = require('./releases');
const { findSiteByUrl } = require('./sites');
const { findNetworkByUrl } = require('./networks');

async function findSite(url, release) {
    const site = (release && release.site) || await findSiteByUrl(url);

    if (site) {
        return site;
    }

    const network = await findNetworkByUrl(url);

    if (network) {
        return {
            ...network,
            network,
            isFallback: true,
        };
    }

    return null;
}

async function scrapeRelease(url, release, type = 'scene') {
    const site = await findSite(url, release);

    if (!site) {
        throw new Error('Could not find site in database');
    }

    const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];

    if (!scraper) {
        throw new Error('Could not find scraper for URL');
    }

    if (type === 'scene' && !scraper.fetchScene) {
        throw new Error(`The '${site.name}'-scraper cannot fetch individual scenes`);
    }

    if (type === 'movie' && !scraper.fetchMovie) {
        throw new Error(`The '${site.name}'-scraper cannot fetch individual movies`);
    }

    const scrapedRelease = type === 'scene'
        ? await scraper.fetchScene(url, site, release)
        : await scraper.fetchMovie(url, site, release);

    return scrapedRelease;
}

async function scrapeReleases(urls, release, type = 'scene') {
    const scrapedReleases = await Promise.map(urls, async url => scrapeRelease(url, release, type), {
        concurrency: 5,
    });

    const curatedReleases = scrapedReleases.map(scrapedRelease => ({ ...scrapedRelease, type }));

    if (argv.save) {
        /*
        const movie = scrapedRelease.movie
            ? await scrapeRelease(scrapedRelease.movie, null, false, 'movie')
            : null;

        if (movie) {
            const { releases: [storedMovie] } = await storeReleases([movie]);
            curatedRelease.parentId = storedMovie.id;
        }
        */

        const { releases: storedReleases } = await storeReleases(curatedReleases);

        if (storedReleases) {
            console.log(storedReleases.map(storedRelease => `http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`).join('\n'));
        }
    }
}

module.exports = {
    scrapeRelease,
    scrapeReleases,
};