Added default deep scrape fetch method. Added Karups scene and profile scraper. Added schoolgirl tag photo.
This commit is contained in:
25
src/deep.js
25
src/deep.js
@@ -7,6 +7,7 @@ const argv = require('./argv');
|
||||
const include = require('./utils/argv-include')(argv);
|
||||
const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
const qu = require('./utils/qu');
|
||||
const scrapers = require('./scrapers/scrapers');
|
||||
|
||||
function urlToSiteSlug(url) {
|
||||
@@ -87,6 +88,24 @@ function toBaseReleases(baseReleasesOrUrls, entity = null) {
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async function fetchScene(scraper, url, entity, baseRelease, options) {
|
||||
if (scraper.fetchScene) {
|
||||
return scraper.fetchScene(baseRelease.url, entity, baseRelease, options, null);
|
||||
}
|
||||
|
||||
if (scraper.scrapeScene) {
|
||||
const res = await qu.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scraper.scrapeScene(res.item, url, entity, baseRelease, options);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrapeRelease(baseRelease, entities, type = 'scene') {
|
||||
const entity = baseRelease.entity || entities[urlToSiteSlug(baseRelease.url)];
|
||||
|
||||
@@ -110,8 +129,8 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
|
||||
return baseRelease;
|
||||
}
|
||||
|
||||
if ((type === 'scene' && !layoutScraper.fetchScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
|
||||
logger.warn(`The '${entity.name}'-scraper cannot fetch individual ${type}s`);
|
||||
if ((type === 'scene' && !layoutScraper.fetchScene && !layoutScraper.scrapeScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
|
||||
logger.warn(`The '${entity.name}'-scraper cannot scrape individual ${type}s`);
|
||||
return baseRelease;
|
||||
}
|
||||
|
||||
@@ -119,7 +138,7 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
|
||||
logger.verbose(`Fetching ${type} ${baseRelease.url}`);
|
||||
|
||||
const scrapedRelease = type === 'scene'
|
||||
? await layoutScraper.fetchScene(baseRelease.url, entity, baseRelease, include, null)
|
||||
? await fetchScene(scraper, baseRelease.url, entity, baseRelease, include, null)
|
||||
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, include, null);
|
||||
|
||||
const mergedRelease = {
|
||||
|
||||
Reference in New Issue
Block a user