traxxx/src/scrape-releases.js

'use strict';

const config = require('config');
const Promise = require('bluebird');

const logger = require('./logger')(__filename);
const argv = require('./argv');
const scrapers = require('./scrapers/scrapers');
const { findSiteByUrl } = require('./sites');
const { findNetworkByUrl } = require('./networks');
const { storeReleases } = require('./releases');

async function findSite(url, release) {
    if (release?.site) return release.site;
    if (!url) return null;

    const site = await findSiteByUrl(url);

    if (site) {
        return site;
    }

    const network = await findNetworkByUrl(url);

    if (network) {
        return {
            ...network,
            network,
            isFallback: true,
        };
    }

    return null;
}

async function scrapeRelease(source, basicRelease = null, type = 'scene', preflight) {
    // profile scraper may return either URLs or pre-scraped scenes
    const sourceIsUrlOrEmpty = typeof source === 'string' || source === undefined;
    const url = sourceIsUrlOrEmpty ? source : source?.url;
    const release = sourceIsUrlOrEmpty ? basicRelease : source;

    const site = basicRelease?.site || await findSite(url, release);

    if (!site) {
        throw new Error(`Could not find site ${url} in database`);
    }

    if (!argv.deep && release) {
        return {
            ...release,
            site,
        };
    }

    const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];

    if (!scraper) {
        throw new Error('Could not find scraper for URL');
    }

    if (type === 'scene' && !scraper.fetchScene) {
        if (release) {
            logger.warn(`The '${site.name}'-scraper cannot fetch individual scenes`);
            return null;
        }

        throw new Error(`The '${site.name}'-scraper cannot fetch individual scenes`);
    }

    if (type === 'movie' && !scraper.fetchMovie) {
        if (release) {
            logger.warn(`The '${site.name}'-scraper cannot fetch individual movies`);
            return null;
        }

        throw new Error(`The '${site.name}'-scraper cannot fetch individual movies`);
    }

    if (!release) {
        logger.info(`Scraping release from ${url}`);
    }

    const scrapedRelease = type === 'scene'
        ? await scraper.fetchScene(url, site, release, preflight)
        : await scraper.fetchMovie(url, site, release, preflight);

    return {
        ...release,
        ...scrapedRelease,
        ...(scrapedRelease && release?.tags && {
            tags: release.tags.concat(scrapedRelease.tags),
        }),
        site,
    };
}

async function scrapeReleases(sources, release = null, type = 'scene', preflight = null) {
    const scrapedReleases = await Promise.map(sources, async source => scrapeRelease(source, release, type, preflight), {
        concurrency: 5,
    }).filter(Boolean);

    const curatedReleases = scrapedReleases.map(scrapedRelease => ({ ...scrapedRelease, type }));

    if (argv.scene && argv.inspect) {
        // only show when fetching from URL
        console.log(curatedReleases);
    }

    if (argv.save) {
        /*
        const movie = scrapedRelease.movie
            ? await scrapeRelease(scrapedRelease.movie, null, false, 'movie')
            : null;

        if (movie) {
            const { releases: [storedMovie] } = await storeReleases([movie]);
            curatedRelease.parentId = storedMovie.id;
        }
        */

        const { releases: storedReleases } = await storeReleases(curatedReleases);

        if (storedReleases) {
            logger.info(storedReleases.map(storedRelease => `\nhttp://${config.web.host}:${config.web.port}/scene/${storedRelease.id}/${storedRelease.slug}`).join(''));
        }
    }
}

module.exports = {
    scrapeRelease,
    scrapeReleases,
};