Added default deep scrape fetch method. Added Karups scene and profile scraper. Added schoolgirl tag photo.

This commit is contained in:
DebaucheryLibrarian
2021-01-13 21:29:05 +01:00
parent cc64f2911f
commit 39d149c728
28 changed files with 155 additions and 4 deletions

View File

@@ -7,6 +7,7 @@ const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const logger = require('./logger')(__filename);
const knex = require('./knex');
const qu = require('./utils/qu');
const scrapers = require('./scrapers/scrapers');
function urlToSiteSlug(url) {
@@ -87,6 +88,24 @@ function toBaseReleases(baseReleasesOrUrls, entity = null) {
.filter(Boolean);
}
async function fetchScene(scraper, url, entity, baseRelease, options) {
if (scraper.fetchScene) {
return scraper.fetchScene(baseRelease.url, entity, baseRelease, options, null);
}
if (scraper.scrapeScene) {
const res = await qu.get(url);
if (res.ok) {
return scraper.scrapeScene(res.item, url, entity, baseRelease, options);
}
return res.status;
}
return null;
}
async function scrapeRelease(baseRelease, entities, type = 'scene') {
const entity = baseRelease.entity || entities[urlToSiteSlug(baseRelease.url)];
@@ -110,8 +129,8 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
return baseRelease;
}
if ((type === 'scene' && !layoutScraper.fetchScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
logger.warn(`The '${entity.name}'-scraper cannot fetch individual ${type}s`);
if ((type === 'scene' && !layoutScraper.fetchScene && !layoutScraper.scrapeScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
logger.warn(`The '${entity.name}'-scraper cannot scrape individual ${type}s`);
return baseRelease;
}
@@ -119,7 +138,7 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
logger.verbose(`Fetching ${type} ${baseRelease.url}`);
const scrapedRelease = type === 'scene'
? await layoutScraper.fetchScene(baseRelease.url, entity, baseRelease, include, null)
? await fetchScene(scraper, baseRelease.url, entity, baseRelease, include, null)
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, include, null);
const mergedRelease = {