Added default deep scrape fetch method. Added Karups scene and profile scraper. Added schoolgirl tag photo.

src/deep.js

@@ -7,6 +7,7 @@ const argv = require('./argv');
 const include = require('./utils/argv-include')(argv);
 const logger = require('./logger')(__filename);
 const knex = require('./knex');
+const qu = require('./utils/qu');
 const scrapers = require('./scrapers/scrapers');
 
 function urlToSiteSlug(url) {
@@ -87,6 +88,24 @@ function toBaseReleases(baseReleasesOrUrls, entity = null) {
 		.filter(Boolean);
 }
 
+async function fetchScene(scraper, url, entity, baseRelease, options) {
+	if (scraper.fetchScene) {
+		return scraper.fetchScene(baseRelease.url, entity, baseRelease, options, null);
+	}
+
+	if (scraper.scrapeScene) {
+		const res = await qu.get(url);
+
+		if (res.ok) {
+			return scraper.scrapeScene(res.item, url, entity, baseRelease, options);
+		}
+
+		return res.status;
+	}
+
+	return null;
+}
+
 async function scrapeRelease(baseRelease, entities, type = 'scene') {
 	const entity = baseRelease.entity || entities[urlToSiteSlug(baseRelease.url)];
 
@@ -110,8 +129,8 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
 		return baseRelease;
 	}
 
-	if ((type === 'scene' && !layoutScraper.fetchScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
-		logger.warn(`The '${entity.name}'-scraper cannot fetch individual ${type}s`);
+	if ((type === 'scene' && !layoutScraper.fetchScene && !layoutScraper.scrapeScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
+		logger.warn(`The '${entity.name}'-scraper cannot scrape individual ${type}s`);
 		return baseRelease;
 	}
 
@@ -119,7 +138,7 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
 		logger.verbose(`Fetching ${type} ${baseRelease.url}`);
 
 		const scrapedRelease = type === 'scene'
-			? await layoutScraper.fetchScene(baseRelease.url, entity, baseRelease, include, null)
+			? await fetchScene(scraper, baseRelease.url, entity, baseRelease, include, null)
 			: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, include, null);
 
 		const mergedRelease = {

src/scrapers/karups.js

@@ -0,0 +1,98 @@
+'use strict';
+
+const qu = require('../utils/qu');
+const slugify = require('../utils/slugify');
+
+const channelSlugs = {
+	kpc: 'karupsprivatecollection',
+	kha: 'karupshometownamateurs',
+	kow: 'karupsolderwomen',
+};
+
+function scrapeAll(scenes) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.url = query.url('a');
+		release.entryId = new URL(release.url).pathname.match(/(\d+)\.html/)?.[1];
+
+		release.title = query.cnt('.title');
+		release.date = query.date('.date', 'MMM Do, YYYY');
+
+		release.channel = channelSlugs[query.cnt('.site')];
+
+		release.poster = query.img('.thumb img');
+
+		return release;
+	});
+}
+
+function scrapeScene({ query }, url) {
+	const release = {};
+
+	release.entryId = new URL(url).pathname.match(/(\d+)\.html/)?.[1];
+
+	release.title = query.cnt('.title');
+	release.date = query.date('.date .content', 'MMM Do, YYYY');
+
+	release.actors = query.all('.models .content a').map(modelEl => ({
+		name: query.cnt(modelEl),
+		url: query.url(modelEl, null),
+	}));
+
+	release.poster = query.poster();
+	release.photos = query.imgs('.video-thumbs img').slice(1);
+
+	release.trailer = query.video();
+
+	return release;
+}
+
+function scrapeProfile({ query }, entity) {
+	const profile = {};
+
+	profile.gender = 'female';
+
+	profile.avatar = query.img('.model-thumb img[src*=".jpg"]');
+	profile.scenes = scrapeAll(qu.initAll(query.all('.listing-videos .item')), entity);
+
+	return profile;
+}
+
+async function fetchLatest(channel, page) {
+	const res = await qu.getAll(`${channel.url}/videos/page${page}.html`, '.listing-videos .item');
+
+	if (res.ok) {
+		return scrapeAll(res.items, channel);
+	}
+
+	return res.status;
+}
+
+async function fetchProfile(baseActor, entity) {
+	const searchRes = await qu.getAll(`https://www.karups.com/models/search/${baseActor.slug}/`, '.listing-models .item');
+
+	if (!searchRes.ok) {
+		return searchRes.status;
+	}
+
+	const actorUrl = searchRes.items.find(item => slugify(item.query.cnt('.title')) === baseActor.slug)?.query.url('a');
+
+	if (!actorUrl) {
+		return null;
+	}
+
+	const actorRes = await qu.get(actorUrl);
+
+	if (actorRes.ok) {
+		return scrapeProfile(actorRes.item, entity);
+	}
+
+	return actorRes.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchProfile,
+	scrapeScene,
+};

src/scrapers/scrapers.js

@@ -35,6 +35,7 @@ const inthecrack = require('./inthecrack');
 const jayrock = require('./jayrock');
 const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials');
 const julesjordan = require('./julesjordan');
+const karups = require('./karups');
 const kellymadison = require('./kellymadison');
 const killergram = require('./killergram');
 const kink = require('./kink');
@@ -127,6 +128,7 @@ const scrapers = {
 		jayrock,
 		jesseloadsmonsterfacials,
 		julesjordan,
+		karups,
 		kellymadison,
 		killergram,
 		kink,
@@ -226,6 +228,7 @@ const scrapers = {
 		inthecrack,
 		jamesdeen: fullpornnetwork,
 		julesjordan,
+		karups,
 		kellymadison,
 		killergram,
 		kink,