Merge branch 'experimental' into master

assets/components/actors/actor.vue

@@ -249,7 +249,7 @@
 							{{ description.text }}
 							<router-link :to="`/${description.entity.type}/${description.entity.slug}`">
 								<img
-									v-if="description.entity.type === 'network' || description.entity.independent"
+									v-if="description.entity.type === 'network' || !description.entity.parent || description.entity.independent"
 									:src="`/img/logos/${description.entity.slug}/thumbs/network.png`"
 									class="description-logo"
 								>

assets/components/entities/entity.vue

@@ -160,6 +160,7 @@ export default {
 	data() {
 		return {
 			entity: null,
+			pageTitle: null,
 			totalCount: null,
 			limit: Number(this.$route.query.limit) || 20,
 			expanded: false,

config/default.js

@@ -75,7 +75,8 @@ module.exports = {
 			'blowpass',
 		],
 		[
-			// MindGeek; Brazzers and Mile High Media have their own assets
+			// MindGeek; Mile High Media has its own assets
+			'brazzers',
 			'realitykings',
 			'mofos',
 			'digitalplayground',
@@ -90,7 +91,6 @@ module.exports = {
 		],
 		'wicked',
 		'burningangel',
-		'brazzers',
 		'milehighmedia',
 		[
 			'vixen',

migrations/20190325001339_releases.js

@@ -325,6 +325,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('priority', 4)
 			.defaultTo(1);
 
+		table.text('url');
+
 		table.text('real_name');
 		table.text('gender', 18);
 

seeds/00_tags.js

@@ -184,7 +184,7 @@ const tags = [
 	},
 	{
 		name: 'behind the scenes',
-		slug: 'behind-the-scenes',
+		slug: 'bts',
 		priority: 6,
 	},
 	{
@@ -1053,8 +1053,8 @@ const aliases = [
 		for: 'titty-fuck',
 	},
 	{
-		name: 'bts',
+		name: 'behind the scenes',
-		for: 'behind-the-scenes',
+		for: 'bts',
 		secondary: true,
 	},
 	{
@@ -1718,14 +1718,18 @@ const aliases = [
 		for: 'transsexual',
 		secondary: true,
 	},
-	{
-		name: 'trimmed pussy',
-		for: 'trimmed',
-	},
 	{
 		name: 'ts',
 		for: 'transsexual',
 	},
+	{
+		name: 'ts cock',
+		for: 'transsexual',
+	},
+	{
+		name: 'trimmed pussy',
+		for: 'trimmed',
+	},
 	{
 		name: 'vr',
 		for: 'virtual reality',

seeds/02_sites.js

@@ -2557,6 +2557,12 @@ const sites = [
 			siteId: 1,
 		},
 	},
+	// HOOKUP HOTSHOT
+	{
+		slug: 'hookuphotshot',
+		name: 'Hookup Hotshot',
+		url: 'https://www.hookuphotshot.com',
+	},
 	// HUSSIE PASS
 	{
 		slug: 'hussiepass',

seeds/04_media.js

@@ -595,7 +595,7 @@ const tagPosters = [
 	['atm', 2, 'Jureka Del Mar in "Stretched Out" for Her Limit'],
 	['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
 	['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'],
-	['behind-the-scenes', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'],
+	['bts', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'],
 	['blonde', 1, 'Marsha May in "Once You Go Black 7" for Jules Jordan'],
 	['blowbang', 0, 'Lacy Lennon in "Lacy Lennon\'s First Blowbang" for HardX'],
 	['blowjob', 0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
@@ -628,7 +628,7 @@ const tagPosters = [
 	['latina', 2, 'Veronica Leal for Her Limit'],
 	['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'],
 	['maid', 0, 'Whitney Wright in "Dredd Up Your Ass 2" for Jules Jordan'],
-	['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'],
+	['milf', 1, 'Francesca Le for Evil Angel'],
 	['mff', 1, 'Anikka Albrite, Kelsi Monroe and Mick Blue for HardX'],
 	['mfm', 0, 'Vina Sky in "Jules Jordan\'s Three Ways" for Jules Jordan'],
 	['natural-boobs', 1, 'Nia Nacci for First Class POV'],
@@ -676,7 +676,7 @@ const tagPhotos = [
 	['anal', 3, 'Dakota Skye for Brazzers'],
 	// ['anal', 1, 'Veronica Leal and Tina Kay in "Agents On Anal Mission" for Asshole Fever'],
 	// ['anal', 0, 'Veronica Leal'],
-	['behind-the-scenes', 1, 'Madison Ivy in "Day With A Pornstar" for Brazzers'],
+	['bts', 1, 'Madison Ivy in "Day With A Pornstar" for Brazzers'],
 	['blonde', 2, 'Isabelle Deltore for Her Limit'],
 	['blowbang', 'poster', 'Marsha May in "Feeding Frenzy 12" for Jules Jordan'],
 	// ['bukkake', 'poster', 'Mia Malkova in "Facialized 2" for HardX'],
@@ -708,6 +708,7 @@ const tagPhotos = [
 	['facial', 'poster', 'Jynx Maze'],
 	['facefucking', 2, 'Jynx Maze for Throated'],
 	['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'],
+	['fake-boobs', 9, 'Putri Cinta for StasyQ'],
 	['fake-boobs', 8, 'Amber Alena for Score'],
 	['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
 	// ['fake-boobs', 6, 'Cathy Heaven in "Heavenly Ass" for Big Wett Butts'],
@@ -723,6 +724,7 @@ const tagPhotos = [
 	['gaping', 2, 'Alex Grey in "DP Masters 5" for Jules Jordan'],
 	['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'],
 	['latina', 0, 'Vienna Black for Spizoo'],
+	// ['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'],
 	['mff', 0, 'Madison Ivy, Adriana Chechik and Keiran Lee in "Day With A Pornstar" for Brazzers'],
 	['mfm', 6, 'Honey Gold in "Slut Puppies 12" for Jules Jordan'],
 	['natural-boobs', 0, 'Valentina Nappi in "Hypnotic Curves" for LesbianX'],

src/actors.js

@@ -123,8 +123,9 @@ function toBaseActors(actorsOrNames, release) {
 		const baseActor = {
 			name,
 			slug,
-			entryId: entryId || null,
+			entryId: entryId || actorOrName.entryId || null,
-			entity: release?.site?.network || release?.entity?.parent || release?.entity || null,
+			entity: release?.entity?.parent || release?.entity || null,
+			hasProfile: !!actorOrName.name, // actor contains profile information
 		};
 
 		if (actorOrName.name) {
@@ -147,6 +148,7 @@ function curateActor(actor, withDetails = false, isProfile = false) {
 		id: actor.id,
 		name: actor.name,
 		slug: actor.slug,
+		url: actor.url,
 		gender: actor.gender,
 		entityId: actor.entity_id,
 		aliasFor: actor.alias_for,
@@ -227,12 +229,17 @@ function curateActorEntries(baseActors, batchId) {
 }
 
 function curateProfileEntry(profile) {
+	if (!profile.id) {
+		return null;
+	}
+
 	const curatedProfileEntry = {
 		...(profile.update !== false && { id: profile.update }),
 		actor_id: profile.id,
 		entity_id: profile.entity?.id || null,
 		date_of_birth: profile.dateOfBirth,
 		date_of_death: profile.dateOfDeath,
+		url: profile.url,
 		gender: profile.gender,
 		ethnicity: profile.ethnicity,
 		description: profile.description,
@@ -371,10 +378,10 @@ async function curateProfile(profile) {
 	}
 }
 
-async function interpolateProfiles(actors) {
+async function interpolateProfiles(actorIds) {
 	const profiles = await knex('actors_profiles')
 		.select(['actors_profiles.*', 'media.width as avatar_width', 'media.height as avatar_height', 'media.size as avatar_size'])
-		.whereIn('actor_id', actors.map(actor => actor.id))
+		.whereIn('actor_id', actorIds)
 		.leftJoin('media', 'actors_profiles.avatar_media_id', 'media.id');
 
 	const profilesByActorId = profiles.reduce((acc, profile) => ({
@@ -482,8 +489,8 @@ async function interpolateProfiles(actors) {
 }
 
 async function upsertProfiles(profiles) {
-	const newProfileEntries = profiles.filter(profile => !profile.update).map(profile => curateProfileEntry(profile));
+	const newProfileEntries = profiles.filter(profile => !profile.update).map(profile => curateProfileEntry(profile)).filter(Boolean);
-	const updatingProfileEntries = profiles.filter(profile => profile.update).map(profile => curateProfileEntry(profile));
+	const updatingProfileEntries = profiles.filter(profile => profile.update).map(profile => curateProfileEntry(profile)).filter(Boolean);
 
 	if (newProfileEntries.length > 0) {
 		await bulkInsert('actors_profiles', newProfileEntries);
@@ -547,7 +554,10 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
 
 					logger.verbose(`Searching profile for '${actor.name}' on '${label}'`);
 
-					const profile = await scraper.fetchProfile(actor, context, include);
+					const profile = await scraper.fetchProfile(curateActor({
+						...existingProfile,
+						...actor,
+					}), context, include);
 
 					if (!profile || typeof profile === 'number') { // scraper returns HTTP code on request failure
 						logger.verbose(`Profile for '${actor.name}' not available on ${label}, scraper returned ${profile}`);
@@ -602,6 +612,14 @@ async function getActorNames(actorNames) {
 	return actorsWithoutProfiles.rows.map(actor => actor.name);
 }
 
+async function storeProfiles(profiles) {
+	const profilesWithAvatarIds = await associateAvatars(profiles);
+	const actorIds = Array.from(new Set(profiles.map(profile => profile.id)));
+
+	await upsertProfiles(profilesWithAvatarIds);
+	await interpolateProfiles(actorIds);
+}
+
 async function scrapeActors(argNames) {
 	const actorNames = await getActorNames(argNames);
 	const baseActors = toBaseActors(actorNames);
@@ -644,7 +662,11 @@ async function scrapeActors(argNames) {
 
 	const actors = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []);
 
-	const existingProfiles = await knex('actors_profiles').whereIn('actor_id', actors.map(actor => actor.id));
+	const existingProfiles = await knex('actors_profiles')
+		.select(knex.raw('actors_profiles.*, row_to_json(avatars) as avatar'))
+		.whereIn('actor_id', actors.map(actor => actor.id))
+		.leftJoin('media as avatars', 'avatars.id', 'actors_profiles.avatar_media_id');
+
 	const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({
 		...acc,
 		[profile.actor_id]: {
@@ -668,10 +690,7 @@ async function scrapeActors(argNames) {
 	}
 
 	if (argv.save) {
-		const profilesWithAvatarIds = await associateAvatars(profiles);
+		await storeProfiles(profiles);
-
-		await upsertProfiles(profilesWithAvatarIds);
-		await interpolateProfiles(actors);
 	}
 
 	return profiles;
@@ -698,6 +717,26 @@ async function getOrCreateActors(baseActors, batchId) {
 	const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
 	const newActors = await bulkInsert('actors', curatedActorEntries);
 
+	const newActorIdsByEntityIdAndSlug = newActors.reduce((acc, actor) => ({
+		...acc,
+		[actor.entity_id]: {
+			...acc[actor.entity_id],
+			[actor.slug]: actor.id,
+		},
+	}), {});
+
+	const newActorProfiles = baseActors
+		.filter(actor => actor.hasProfile)
+		.map(actor => ({
+			...actor,
+			id: newActorIdsByEntityIdAndSlug[actor.entity?.id]?.[actor.slug] || newActorIdsByEntityIdAndSlug.null?.[actor.slug],
+		}))
+		.filter(actor => !!actor.id);
+
+	console.log(newActorIdsByEntityIdAndSlug, newActorProfiles);
+
+	await storeProfiles(newActorProfiles);
+
 	if (Array.isArray(newActors)) {
 		return newActors.concat(existingActors);
 	}

src/scrapers/elegantangel.js

@@ -2,6 +2,7 @@
 
 const qu = require('../utils/q');
 const slugify = require('../utils/slugify');
+const { feetInchesToCm, lbsToKg } = require('../utils/convert');
 
 function scrapeAll(scenes, channel) {
 	return scenes.map(({ query }) => {
@@ -58,6 +59,7 @@ async function scrapeRelease({ query, html }, url, channel, baseRelease, type =
 
 		return {
 			name: qu.query.cnt(el, 'span'),
+			url: qu.query.url(el, 'a', 'href', { origin: channel.url }),
 			avatar: [
 				avatar.replace(/\/actor\/\d+/, '/actor/1600'),
 				avatar,
@@ -124,6 +126,68 @@ function scrapeMovies(movies, channel) {
 	});
 }
 
+function scrapeActorScenes(scenes, channel) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.url = query.url('a', 'href', { origin: channel.url });
+		release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
+
+		release.title = query.cnt('.grid-item-title');
+
+		const poster = query.img('a img');
+		release.poster = [
+			poster.replace(/\/\d+\//, '/1600/'),
+			poster,
+		];
+
+		return release;
+	});
+}
+
+async function scrapeProfile({ query }, url, channel, include) {
+	const profile = {};
+
+	const bio = query.cnts('.performer-page-header li').reduce((acc, info) => {
+		const [key, value] = info.split(':');
+
+		return {
+			...acc,
+			[slugify(key, '_')]: value.trim(),
+		};
+	}, {});
+
+	const measurements = bio.meas?.match(/(\d+)(\w+)-(\d+)-(\d+)/);
+
+	if (measurements) {
+		[profile.bust, profile.cup, profile.waist, profile.hip] = measurements.slice(1);
+	}
+
+	profile.hair = bio.hair;
+	profile.eyes = bio.eyes;
+	profile.ethnicity = bio.ethnicity;
+
+	profile.height = feetInchesToCm(bio.height);
+	profile.weight = lbsToKg(bio.weight);
+
+	profile.avatar = query.img('picture img');
+
+	if (include) {
+		const actorId = new URL(url).pathname.match(/\/(\d+)/)[1];
+		const res = await qu.getAll(`https://www.elegantangel.com/streaming-video-by-scene.html?cast=${actorId}`, '.grid-item', null, {
+			rejectUnauthorized: false,
+		});
+
+		if (res.ok) {
+			profile.releases = scrapeActorScenes(res.items, channel);
+		}
+	}
+
+	console.log(profile);
+
+	return profile;
+}
+
 async function fetchLatest(channel, page = 1) {
 	const url = `${channel.url}/tour?page=${page}`;
 	const res = await qu.getAll(url, '.scene-update', null, {
@@ -177,9 +241,26 @@ async function fetchMovies(channel, page = 1) {
 	return res.status;
 }
 
+async function fetchProfile(baseActor, channel, include) {
+	if (!baseActor.url) {
+		return null;
+	}
+
+	const res = await qu.get(baseActor.url, '.performer-page', null, {
+		rejectUnauthorized: false,
+	});
+
+	if (res.ok) {
+		return scrapeProfile(res.item, baseActor.url, channel, include);
+	}
+
+	return res.status;
+}
+
 module.exports = {
 	fetchLatest,
 	fetchScene,
 	fetchMovies,
 	fetchMovie,
+	fetchProfile,
 };

src/scrapers/hookuphotshot.js

@@ -0,0 +1,103 @@
+'use strict';
+
+const qu = require('../utils/q');
+const slugify = require('../utils/slugify');
+
+function scrapeAll(scenes) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.url = query.url('.date-title a');
+
+		const avatarEl = query.el('.girl-thumb-container img');
+		release.actors = query.all('.date-starring a').map((actorEl) => {
+			const name = query.cnt(actorEl);
+
+			return {
+				name,
+				gender: 'female',
+				url: query.url(actorEl, null),
+				...(new RegExp(name).test(avatarEl.alt) && {
+					avatar: [
+						avatarEl.src.replace(/-\d+x\d+/, ''),
+						avatarEl.src,
+					].map(src => ({ src, queueMethod: '1s' })),
+				}),
+			};
+		}).concat({
+			name: 'Bryan Gozzling',
+			gender: 'male',
+		});
+
+		release.duration = query.dur('.date-facts');
+		release.stars = query.number('[data-rating]', null, 'data-rating');
+
+		const photoCount = query.number('input[id*=count]', null, 'value');
+		const photoPath = query.url('input[id*=baseurl]', 'value');
+
+		release.poster = {
+			src: query.img('.date-img-swap'),
+			queueMethod: '1s',
+		};
+
+		release.photos = [...Array(photoCount)].map((value, index) => ({
+			src: `${photoPath}/${String(index + 1).padStart(2, '0')}.jpg`,
+			queueMethod: '1s',
+		}));
+
+		// dates appear to be manually curated
+		const fullTitle = query.cnt('.date-title a');
+		const [monthName, date, title] = fullTitle.match(/(\w+)\.? (\d+)\s*-?\s*(.*)/)?.slice(1) || [];
+		const [year, month] = release.poster.src.match(/uploads\/(\d+)\/(\d+)/)?.slice(1) || [];
+
+		release.title = title.replace(/behind the\.\.\./i, 'Behind the Scenes');
+		release.date = qu.extractDate(`${year}-${monthName || month}-${date}`, ['YYYY-MM-DD', 'YYYY-MMM-DD', 'YYYY-MMMM-DD']);
+
+		// release.entryId = new URL(release.url).pathname.split('/')[2];
+		release.entryId = `${release.date.getFullYear()}-${release.date.getMonth() + 1}-${release.date.getDate()}-${slugify(release.actors[0].name)}`;
+
+		release.tags = ['rough', ...release.title.match(/behind the scenes|anal/gi) || []];
+
+		return release;
+	});
+}
+
+function scrapeProfile({ query }) {
+	const profile = {};
+
+	profile.gender = 'female';
+
+	profile.description = query.cnts('.girl-about p:not(.bio-facts)').join(' ');
+	profile.avatar = query.img('.girl-pic');
+
+	// no deep scraping available, and not all scene details available here
+
+	return profile;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const url = `${channel.url}/the-dates/page/${page}`;
+	const res = await qu.getAll(url, '#et-projects li');
+
+	if (res.ok) {
+		return scrapeAll(res.items, channel);
+	}
+
+	return res.status;
+}
+
+async function fetchProfile({ name: actorName }, entity, include) {
+	const url = `${entity.url}/girls/${slugify(actorName)}`;
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		return scrapeProfile(res.item, actorName, entity, include);
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchProfile,
+};

src/scrapers/mindgeek.js

@@ -267,8 +267,8 @@ async function fetchScene(url, site, baseScene) {
 	return null;
 }
 
-async function fetchProfile({ name: actorName }, networkSlug, actorPath = 'model') {
+async function fetchProfile({ name: actorName }, networkOrNetworkSlug, actorPath = 'model') {
-	const url = `https://www.${networkSlug}.com`;
+	const url = `https://www.${networkOrNetworkSlug.slug || networkOrNetworkSlug}.com`;
 	const { session, instanceToken } = await getSession(url);
 
 	const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
@@ -281,7 +281,7 @@ async function fetchProfile({ name: actorName }, networkSlug, actorPath = 'model
 		const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
 
 		if (actorData) {
-			const actorUrl = `https://www.${networkSlug}.com/${actorPath}/${actorData.id}/`;
+			const actorUrl = `https://www.${networkOrNetworkSlug.slug || networkOrNetworkSlug}.com/${actorPath}/${actorData.id}/`;
 			const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`;
 
 			const [actorRes, actorReleasesRes] = await Promise.all([
@@ -294,11 +294,11 @@ async function fetchProfile({ name: actorName }, networkSlug, actorPath = 'model
 			]);
 
 			if (actorRes.statusCode === 200 && actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
-				return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, networkSlug);
+				return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, networkOrNetworkSlug.slug || networkOrNetworkSlug);
 			}
 
 			if (actorRes.statusCode === 200) {
-				return scrapeProfile(actorData, actorRes.body.toString(), null, networkSlug);
+				return scrapeProfile(actorData, actorRes.body.toString(), null, networkOrNetworkSlug.slug || networkOrNetworkSlug);
 			}
 		}
 	}

src/scrapers/scrapers.js

@@ -23,6 +23,7 @@ const fcuk = require('./fcuk');
 const fullpornnetwork = require('./fullpornnetwork');
 const girlsway = require('./girlsway');
 const hitzefrei = require('./hitzefrei');
+const hookuphotshot = require('./hookuphotshot');
 const hush = require('./hush');
 const iconmale = require('./iconmale');
 const insex = require('./insex');
@@ -104,6 +105,7 @@ module.exports = {
 		girlsway,
 		girlgirl: julesjordan,
 		hitzefrei,
+		hookuphotshot,
 		hussiepass: hush,
 		hushpass: hush,
 		insex,
@@ -178,6 +180,7 @@ module.exports = {
 		devilsfilm: famedigital,
 		digitalplayground,
 		dtfsluts: fullpornnetwork,
+		elegantangel,
 		evilangel,
 		eyeontheguy: hush,
 		fakehub,
@@ -190,6 +193,7 @@ module.exports = {
 		hergape: fullpornnetwork,
 		hitzefrei,
 		homemadeanalwhores: fullpornnetwork,
+		hookuphotshot,
 		hotcrazymess: nubiles,
 		hushpass: hush,
 		hussiepass: hush,

src/updates.js

@@ -183,7 +183,7 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
 
 	if (!scraper) {
 		logger.warn(`No scraper found for '${channelEntity.name}' (${channelEntity.parent?.name})`);
-		return [];
+		return emptyReleases;
 	}
 
 	try {
@@ -196,7 +196,7 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
 	} catch (error) {
 		logger.error(`Failed to scrape releases from ${channelEntity.name} using ${scraper.slug}: ${error.message}`);
 
-		return [];
+		return emptyReleases;
 	}
 }