2019-11-16 02:33:36 +00:00
|
|
|
'use strict';
|
|
|
|
|
|
|
|
const config = require('config');
|
2020-01-02 23:59:02 +00:00
|
|
|
const Promise = require('bluebird');
|
2019-11-16 02:33:36 +00:00
|
|
|
|
2020-01-31 00:55:55 +00:00
|
|
|
const logger = require('./logger');
|
2019-11-16 02:33:36 +00:00
|
|
|
const argv = require('./argv');
|
|
|
|
const scrapers = require('./scrapers/scrapers');
|
|
|
|
const { findSiteByUrl } = require('./sites');
|
|
|
|
const { findNetworkByUrl } = require('./networks');
|
2020-01-31 00:55:55 +00:00
|
|
|
const { storeReleases } = require('./releases');
|
2019-11-16 02:33:36 +00:00
|
|
|
|
|
|
|
async function findSite(url, release) {
|
|
|
|
const site = (release && release.site) || await findSiteByUrl(url);
|
|
|
|
|
|
|
|
if (site) {
|
|
|
|
return site;
|
|
|
|
}
|
|
|
|
|
|
|
|
const network = await findNetworkByUrl(url);
|
|
|
|
|
|
|
|
if (network) {
|
|
|
|
return {
|
|
|
|
...network,
|
2019-11-16 22:37:33 +00:00
|
|
|
network,
|
2019-11-16 02:33:36 +00:00
|
|
|
isFallback: true,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
2020-01-02 23:59:02 +00:00
|
|
|
async function scrapeRelease(url, release, type = 'scene') {
|
2019-11-16 02:33:36 +00:00
|
|
|
const site = await findSite(url, release);
|
2019-12-06 23:42:47 +00:00
|
|
|
|
2019-11-16 02:33:36 +00:00
|
|
|
if (!site) {
|
|
|
|
throw new Error('Could not find site in database');
|
|
|
|
}
|
|
|
|
|
2019-12-06 23:42:47 +00:00
|
|
|
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
|
|
|
|
|
2019-11-16 02:33:36 +00:00
|
|
|
if (!scraper) {
|
|
|
|
throw new Error('Could not find scraper for URL');
|
|
|
|
}
|
|
|
|
|
2019-12-15 04:42:51 +00:00
|
|
|
if (type === 'scene' && !scraper.fetchScene) {
|
2019-12-13 02:28:52 +00:00
|
|
|
throw new Error(`The '${site.name}'-scraper cannot fetch individual scenes`);
|
2019-11-16 02:33:36 +00:00
|
|
|
}
|
|
|
|
|
2019-12-15 04:42:51 +00:00
|
|
|
if (type === 'movie' && !scraper.fetchMovie) {
|
2019-12-13 02:28:52 +00:00
|
|
|
throw new Error(`The '${site.name}'-scraper cannot fetch individual movies`);
|
|
|
|
}
|
|
|
|
|
2020-01-31 00:55:55 +00:00
|
|
|
if (!release) {
|
|
|
|
logger.info(`Scraping release from ${url}`);
|
|
|
|
}
|
|
|
|
|
2019-12-15 04:42:51 +00:00
|
|
|
const scrapedRelease = type === 'scene'
|
|
|
|
? await scraper.fetchScene(url, site, release)
|
|
|
|
: await scraper.fetchMovie(url, site, release);
|
|
|
|
|
2020-01-28 02:05:53 +00:00
|
|
|
return {
|
|
|
|
...scrapedRelease,
|
|
|
|
site,
|
|
|
|
};
|
2020-01-02 23:59:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
async function scrapeReleases(urls, release, type = 'scene') {
|
|
|
|
const scrapedReleases = await Promise.map(urls, async url => scrapeRelease(url, release, type), {
|
|
|
|
concurrency: 5,
|
|
|
|
});
|
2019-11-16 02:33:36 +00:00
|
|
|
|
2020-01-02 23:59:02 +00:00
|
|
|
const curatedReleases = scrapedReleases.map(scrapedRelease => ({ ...scrapedRelease, type }));
|
|
|
|
|
|
|
|
if (argv.save) {
|
2019-12-31 02:12:52 +00:00
|
|
|
/*
|
2019-12-15 04:42:51 +00:00
|
|
|
const movie = scrapedRelease.movie
|
|
|
|
? await scrapeRelease(scrapedRelease.movie, null, false, 'movie')
|
|
|
|
: null;
|
|
|
|
|
|
|
|
if (movie) {
|
|
|
|
const { releases: [storedMovie] } = await storeReleases([movie]);
|
|
|
|
curatedRelease.parentId = storedMovie.id;
|
|
|
|
}
|
2019-12-31 02:12:52 +00:00
|
|
|
*/
|
2019-12-15 04:42:51 +00:00
|
|
|
|
2020-01-02 23:59:02 +00:00
|
|
|
const { releases: storedReleases } = await storeReleases(curatedReleases);
|
2019-11-16 02:33:36 +00:00
|
|
|
|
2020-01-02 23:59:02 +00:00
|
|
|
if (storedReleases) {
|
|
|
|
console.log(storedReleases.map(storedRelease => `http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`).join('\n'));
|
2019-12-09 04:00:49 +00:00
|
|
|
}
|
2019-11-16 02:33:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-02 23:59:02 +00:00
|
|
|
module.exports = {
|
|
|
|
scrapeRelease,
|
|
|
|
scrapeReleases,
|
|
|
|
};
|