traxxx/src/scrape-release.js

'use strict';

const config = require('config');

const argv = require('./argv');
const scrapers = require('./scrapers/scrapers');
const { storeRelease } = require('./releases');
const { findSiteByUrl } = require('./sites');
const { findNetworkByUrl } = require('./networks');

async function findSite(url, release) {
    const site = (release && release.site) || await findSiteByUrl(url);

    if (site) {
        return site;
    }

    const network = await findNetworkByUrl(url);

    if (network) {
        return {
            ...network,
            network,
            isFallback: true,
        };
    }

    return null;
}

async function scrapeRelease(url, release, deep = false) {
    const site = await findSite(url, release);
    const scraper = scrapers[site.slug] || scrapers[site.network.slug];

    if (!site) {
        throw new Error('Could not find site in database');
    }

    if (!scraper) {
        throw new Error('Could not find scraper for URL');
    }

    if (!scraper.fetchScene) {
        throw new Error(`The '${site.name}'-scraper cannot fetch individual releases`);
    }

    const scene = await scraper.fetchScene(url, site);

    if (!deep && argv.save) {
        // don't store release when called by site scraper
        const releaseId = await storeRelease(scene);

        console.log(`http://${config.web.host}:${config.web.port}/scene/${releaseId}`);
    }

    return scene;
}

module.exports = scrapeRelease;