Added series as channels with logos and photo album scraping to Little Caprice. Added various tag photos.

seeds/00_tags.js

@@ -842,6 +842,11 @@ const tags = [
 		group: 'finish',
 		priority: 6,
 	},
+	{
+		name: 'swinging',
+		slug: 'swinging',
+		group: 'group',
+	},
 	{
 		name: 'tattoos',
 		slug: 'tattoos',
@@ -1726,6 +1731,10 @@ const aliases = [
 		name: 'swallow',
 		for: 'swallowing',
 	},
+	{
+		name: 'swingers',
+		for: 'swinging',
+	},
 	{
 		name: 'strap-on',
 		for: 'strap-on-dildo',

seeds/01_networks.js

@@ -314,6 +314,11 @@ const networks = [
 		url: 'https://letsdoeit.com',
 		parent: 'porndoe',
 	},
+	{
+		slug: 'littlecapricedreams',
+		name: 'Little Caprice Dreams',
+		url: 'https://www.littlecaprice-dreams.com',
+	},
 	{
 		slug: 'mamacitaz',
 		name: 'Mamacitaz',

seeds/02_sites.js

@@ -4226,12 +4226,83 @@ const sites = [
 		parent: 'letsdoeit',
 	},
 	// LITTLE CAPRICE
+	/*
 	{
 		name: 'Little Caprice Dreams',
 		slug: 'littlecapricedreams',
 		alias: ['little caprice'],
 		url: 'https://www.littlecaprice-dreams.com',
 	},
+	*/
+	{
+		name: 'Caprice Casting',
+		slug: 'capricecasting',
+		url: 'https://www.littlecaprice-dreams.com/caprice-casting',
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'Caprice Divas',
+		slug: 'capricedivas',
+		url: 'https://www.littlecaprice-dreams.com/caprice-divas',
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'Nassty',
+		slug: 'nassty',
+		url: 'https://www.littlecaprice-dreams.com/nassty',
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'POV Dreams',
+		slug: 'povdreams',
+		url: 'https://www.littlecaprice-dreams.com/pov-dreams',
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'Porn Lifestyle',
+		slug: 'pornlifestyle',
+		url: 'https://www.littlecaprice-dreams.com/porn-lifestyle',
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'Public Sex',
+		slug: 'publicsex',
+		url: 'https://www.littlecaprice-dreams.com/public-sex',
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'Super Private X',
+		slug: 'superprivatex',
+		url: 'https://www.littlecaprice-dreams.com/superprivatex',
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'Sex Lessons',
+		slug: 'sexlessons',
+		url: 'https://www.littlecaprice-dreams.com/sexlessons',
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'Virtual Reality',
+		slug: 'littlecapricevr',
+		url: 'https://www.littlecaprice-dreams.com/virtual-reality-little-caprice',
+		tags: ['virtual-reality'],
+		hasLogo: false,
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'We Cum To You',
+		slug: 'wecumtoyou',
+		url: 'https://www.littlecaprice-dreams.com/wecumtoyou-swingers',
+		tags: ['swinging', 'orgy'],
+		parent: 'littlecapricedreams',
+	},
+	{
+		name: 'Xpervo',
+		slug: 'xpervo',
+		url: 'https://www.littlecaprice-dreams.com/xpervo',
+		parent: 'littlecapricedreams',
+	},
 	// MAMACITAZ
 	{
 		name: 'Her Big Ass',

seeds/04_media.js

@@ -792,16 +792,18 @@ const tagPhotos = [
 	['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'],
 	['fake-boobs', 14, 'Rikki Six for Dream Dolls'],
 	['fake-boobs', 2, 'Gia Milana in "Hot Anal Latina" for HardX'],
+	['fake-boobs', 17, 'Felina in "With Flors On The Floor" for LouisDeMirabert'],
+	['fake-boobs', 18, 'Ebony Godess for Action Girls'],
 	['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
 	['fake-boobs', 16, 'Marsha May in "Once You Go Black 7" for Jules Jordan'],
+	['fake-boobs', 9, 'Putri Cinta for Watch 4 Beauty'],
 	['fake-boobs', 10, 'Tia Cyrus in "Titty-Fucked Yoga Goddess" for Latina Sex Tapes'],
-	['fake-boobs', 9, 'Putri Cinta for StasyQ'],
 	['fake-boobs', 11, 'Jessa Rhodes and Cali Carter in "Busty Anal Workout" for LesbianX'],
-	['fake-boobs', 13, 'Kitana Lure for Asshole Fever'],
 	['fake-boobs', 3, 'Ashly Anderson for Passion HD'],
-	['fake-boobs', 15, 'Amber Jade and Karma Rx in "Amber In The Hills: Part 1" for Brazzers'],
+	['fake-boobs', 13, 'Kitana Lure for Asshole Fever'],
 	['fake-boobs', 8, 'Amber Alena for Score'],
 	['fake-boobs', 4, 'Capri Cavanni for Big Tits in Sports'],
+	['fake-boobs', 15, 'Amber Jade and Karma Rx in "Amber In The Hills: Part 1" for Brazzers'],
 	// ['fake-boobs', 6, 'Cathy Heaven in "Heavenly Ass" for Big Wett Butts'],
 	['fake-boobs', 12, 'Nikki Monroe and Kortney Kane for Big Tits In Uniform'],
 	['fake-cum', 0, 'Jynx Maze for Cumshot Surprise (Porn Pros)'],

src/deep.js

@@ -57,7 +57,7 @@ function toBaseReleases(baseReleasesOrUrls, entity = null) {
 				// base release with URL
 				return {
 					...baseReleaseOrUrl,
-					entity,
+					entity: baseReleaseOrUrl.entity || entity,
 					deep: false,
 				};
 			}
@@ -75,7 +75,7 @@ function toBaseReleases(baseReleasesOrUrls, entity = null) {
 				// base release without URL, prepare for passthrough
 				return {
 					...baseReleaseOrUrl,
-					entity,
+					entity: baseReleaseOrUrl.entity || entity,
 					deep: false,
 				};
 			}

src/entities.js

@@ -39,6 +39,16 @@ function curateEntity(entity, includeParameters = false) {
 		}, includeParameters));
 	}
 
+	if (entity.siblings) {
+		curatedEntity.parent = {
+			...curatedEntity.parent,
+			children: entity.siblings.map(sibling => curateEntity({
+				...sibling,
+				parent: curatedEntity.parent,
+			}, includeParameters)),
+		};
+	}
+
 	if (entity.tags) {
 		curatedEntity.tags = entity.tags.map(tag => ({
 			id: tag.id,
@@ -72,9 +82,10 @@ async function fetchIncludedEntities() {
 		WITH RECURSIVE channels AS (
 			/* select configured channels and networks */
 			SELECT
-				entities.*
+				entities.*, json_agg(siblings) as siblings
 			FROM
 				entities
+			LEFT JOIN entities AS siblings ON siblings.parent_id = entities.parent_id
 			WHERE
 				CASE WHEN :includeAll
 				THEN
@@ -91,12 +102,13 @@ async function fetchIncludedEntities() {
 					AND entities.type = 'network')
 					OR (entities.slug = ANY(:excludedChannels)
 					AND entities.type = 'channel'))
+			GROUP BY entities.id
 
 			UNION ALL
 
 			/* select recursive children of configured networks */
 			SELECT
-				entities.*
+				entities.*, null as siblings
 			FROM
 				entities
 			INNER JOIN
@@ -117,7 +129,7 @@ async function fetchIncludedEntities() {
 		WHERE
 			channels.type = 'channel'
 		GROUP BY
-			entities.id
+			entities.id;
 	`, include);
 
 	const curatedNetworks = rawNetworks.rows.map(entity => curateEntity(entity, true));

src/scrapers/littlecapricedreams.js

@@ -1,8 +1,33 @@
 'use strict';
 
 const qu = require('../utils/qu');
+const slugify = require('../utils/slugify');
 
-function scrapeAll(scenes) {
+function matchChannel(release, channel) {
+	const series = channel.children || channel.parent.children;
+
+	console.log(channel, series);
+
+	const serieNames = series.reduce((acc, serie) => ({
+		...acc,
+		[serie.name]: serie,
+		[serie.slug]: serie,
+	}), {});
+
+	const serieName = release.title.match(new RegExp(Object.keys(serieNames).join('|'), 'i'))?.[0];
+	const serie = serieName && serieNames[slugify(serieName, '')];
+
+	if (serie) {
+		return {
+			channel: serie.slug,
+			title: release.title.replace(new RegExp(`${serieName}[\\s:–-]*`), ''),
+		};
+	}
+
+	return null;
+}
+
+function scrapeAll(scenes, channel) {
 	return scenes.map(({ query, el }) => {
 		const release = {};
 
@@ -14,11 +39,29 @@ function scrapeAll(scenes) {
 
 		release.poster = query.img('img');
 
-		return release;
+		return {
+			...release,
+			...matchChannel(release, channel),
+		};
 	});
 }
 
-function scrapeScene({ query }) {
+async function fetchPhotos(url) {
+	if (url) {
+		const res = await qu.get(url, '.et_post_gallery');
+
+		if (res.ok) {
+			return res.item.query.urls('a').map(imgUrl => ({
+				src: imgUrl,
+				referer: url,
+			}));
+		}
+	}
+
+	return null;
+}
+
+async function scrapeScene({ query }, url, channel, include) {
 	const release = {};
 
 	const script = query.cnt('script.yoast-schema-graph');
@@ -41,20 +84,31 @@ function scrapeScene({ query }) {
 
 	const posterData = data['@graph']?.find(item => item['@type'] === 'ImageObject');
 
-	release.poster = posterData?.url
+	const poster = posterData?.url
 		|| query.q('meta[property="og:image"]', 'content')
 		|| query.q('meta[name="twitter:image"]', 'content');
 
+	release.poster = {
+		src: poster,
+		referer: url,
+	};
+
 	release.stars = Math.min(Number(query.q('.post-ratings-image', 'title')?.match(/average:\s*(\d\.\d+)/)?.[1]), 5) || null; // rating out of 5, yet sometimes 5.07?
 
-	// TODO: photo gallery, find if any video has a trailer
+	if (include.photos) {
-	console.log(release);
+		release.photos = await fetchPhotos(query.url('.vid_buttons a[href*="project/"]'));
-	return release;
+	}
+
+	return {
+		...release,
+		...matchChannel(release, channel),
+	};
 }
 
 async function fetchLatest(channel) {
 	// no apparent pagination, all updates on one page
-	const res = await qu.getAll(`${channel.url}/videos/`, '.project');
+	// using channels in part because main overview contains indistinguishable photo albums
+	const res = await qu.getAll(channel.url, '.project');
 
 	if (res.ok) {
 		return scrapeAll(res.items, channel);
@@ -63,11 +117,11 @@ async function fetchLatest(channel) {
 	return res.status;
 }
 
-async function fetchScene(url, channel) {
+async function fetchScene(url, channel, baseRelease, include) {
 	const res = await qu.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, channel);
+		return scrapeScene(res.item, url, channel, include);
 	}
 
 	return res.status;

src/utils/virtual-console.js

@@ -7,7 +7,7 @@ const logger = require('../logger')(__filename);
 
 function getVirtualConsole(filepath) {
 	const virtualConsole = new VirtualConsole();
-	const context = path.basename(filepath);
+	const context = path.basename(filepath).replace(path.extname(filepath), '');
 
 	virtualConsole.on('error', message => logger.warn(`Error from JSDOM in ${context}: ${message}`));
 	virtualConsole.on('jsdomError', message => logger.warn(`Error from JSDOM in ${context}: ${message}`));