traxxx/src/scrape-sites.js

'use strict';

const Promise = require('bluebird');
const moment = require('moment');

const argv = require('./argv');
const logger = require('./logger');
const knex = require('./knex');
const { fetchIncludedSites } = require('./sites');
const scrapers = require('./scrapers/scrapers');
const { scrapeRelease } = require('./scrape-releases');
const { storeReleases } = require('./releases');

function getAfterDate() {
    return moment
        .utc()
        .subtract(...argv.after.split(' '))
        .toDate();
}

async function findDuplicateReleaseIds(latestReleases, accReleases) {
    const duplicateReleases = await knex('releases')
        .whereIn('entry_id', latestReleases.map(({ entryId }) => entryId));

    // include accumulated releases as duplicates to prevent an infinite
    // loop when the next page contains the same releases as the previous
    return new Set(duplicateReleases
        .map(release => String(release.entry_id))
        .concat(accReleases.map(release => String(release.entryId))));
}

async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), accReleases = [], page = 1) {
    const latestReleases = await scraper.fetchLatest(site, page);

    if (latestReleases.length === 0) {
        return accReleases;
    }

    const oldestReleaseOnPage = latestReleases.slice(-1)[0].date;
    const duplicateReleaseIds = argv.redownload ? new Set() : await findDuplicateReleaseIds(latestReleases, accReleases);

    const uniqueReleases = latestReleases
        .filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
            && moment(release.date).isAfter(afterDate)); // release is older than specified date limit

    logger.info(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);

    if (
        uniqueReleases.length > 0
            && (oldestReleaseOnPage || page < argv.pages)
            && moment(oldestReleaseOnPage).isAfter(afterDate)
    ) {
        // oldest release on page is newer that specified limit, fetch next page
        return scrapeUniqueReleases(scraper, site, afterDate, accReleases.concat(uniqueReleases), page + 1);
    }

    return accReleases.concat(uniqueReleases);
}

async function scrapeUpcomingReleases(scraper, site) {
    if (scraper.fetchUpcoming) {
        const upcomingReleases = await scraper.fetchUpcoming(site);

        return upcomingReleases.map(release => ({ ...release, upcoming: true }));
    }

    return [];
}

async function deepFetchReleases(baseReleases) {
    return Promise.map(baseReleases, async (release) => {
        if (release.url) {
            try {
                const fullRelease = await scrapeRelease(release.url, release, 'scene');

                return {
                    ...release,
                    ...fullRelease,
                    deep: true,
                };
            } catch (error) {
                logger.error(error.message);

                return {
                    ...release,
                    deep: false,
                };
            }
        }

        return release;
    }, {
        concurrency: 2,
    });
}

async function scrapeSiteReleases(scraper, site) {
    const [newReleases, upcomingReleases] = await Promise.all([
        scrapeUniqueReleases(scraper, site), // fetch basic release info from scene overview
        scrapeUpcomingReleases(scraper, site), // fetch basic release info from upcoming overview
    ]);

    logger.info(`${site.name}: Found ${newReleases.length} recent releases, ${upcomingReleases.length} upcoming releases`);

    const baseReleases = [...newReleases, ...upcomingReleases];

    if (argv.deep) {
        // follow URL for every release
        return deepFetchReleases(baseReleases);
    }

    return baseReleases;
}

async function scrapeSites() {
    const networks = await fetchIncludedSites();

    const scrapedNetworks = await Promise.map(networks, async network => Promise.map(network.sites, async (site) => {
        const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];

        if (!scraper) {
            logger.warn(`No scraper found for '${site.name}' (${site.slug})`);
            return [];
        }

        try {
            return await scrapeSiteReleases(scraper, site);
        } catch (error) {
            logger.error(`${site.name}: Failed to scrape releases: ${error.message}`);

            return [];
        }
    }, {
        // 2 network sites at a time
        concurrency: 2,
    }),
    {
        // 5 networks at a time
        concurrency: 5,
    });

    if (argv.save) {
        await storeReleases(scrapedNetworks.flat(2));
    }
}

module.exports = scrapeSites;