2019-11-16 02:33:36 +00:00
|
|
|
'use strict';
|
|
|
|
|
|
|
|
const config = require('config');
|
|
|
|
|
|
|
|
const argv = require('./argv');
|
|
|
|
const scrapers = require('./scrapers/scrapers');
|
2019-11-17 02:56:45 +00:00
|
|
|
const { storeReleases } = require('./releases');
|
2019-11-16 02:33:36 +00:00
|
|
|
const { findSiteByUrl } = require('./sites');
|
|
|
|
const { findNetworkByUrl } = require('./networks');
|
|
|
|
|
|
|
|
async function findSite(url, release) {
|
|
|
|
const site = (release && release.site) || await findSiteByUrl(url);
|
|
|
|
|
|
|
|
if (site) {
|
|
|
|
return site;
|
|
|
|
}
|
|
|
|
|
|
|
|
const network = await findNetworkByUrl(url);
|
|
|
|
|
|
|
|
if (network) {
|
|
|
|
return {
|
|
|
|
...network,
|
2019-11-16 22:37:33 +00:00
|
|
|
network,
|
2019-11-16 02:33:36 +00:00
|
|
|
isFallback: true,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
async function scrapeRelease(url, release, deep = false) {
|
|
|
|
const site = await findSite(url, release);
|
2019-12-06 23:42:47 +00:00
|
|
|
|
2019-11-16 02:33:36 +00:00
|
|
|
if (!site) {
|
|
|
|
throw new Error('Could not find site in database');
|
|
|
|
}
|
|
|
|
|
2019-12-06 23:42:47 +00:00
|
|
|
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
|
|
|
|
|
2019-11-16 02:33:36 +00:00
|
|
|
if (!scraper) {
|
|
|
|
throw new Error('Could not find scraper for URL');
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!scraper.fetchScene) {
|
|
|
|
throw new Error(`The '${site.name}'-scraper cannot fetch individual releases`);
|
|
|
|
}
|
|
|
|
|
|
|
|
const scene = await scraper.fetchScene(url, site);
|
|
|
|
|
|
|
|
if (!deep && argv.save) {
|
|
|
|
// don't store release when called by site scraper
|
2019-12-05 00:26:22 +00:00
|
|
|
const [storedRelease] = await storeReleases([scene]);
|
2019-11-16 02:33:36 +00:00
|
|
|
|
2019-12-05 00:26:22 +00:00
|
|
|
console.log(`http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`);
|
2019-11-16 02:33:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return scene;
|
|
|
|
}
|
|
|
|
|
|
|
|
module.exports = scrapeRelease;
|