Refactored Whale Member (Porn Pros) scraper.

package.json

@@ -132,6 +132,8 @@
         "object-merge-advanced": "^12.1.0",
         "object.omit": "^3.0.0",
         "pg": "^8.11.3",
+        "playwright": "^1.57.0",
+        "playwright-core": "^1.57.0",
         "postgraphile": "^4.14.0",
         "postgraphile-plugin-connection-filter": "^2.3.0",
         "promise-task-queue": "^1.2.0",
@@ -149,7 +151,7 @@
         "tunnel": "0.0.6",
         "ua-parser-js": "^1.0.37",
         "undici": "^5.28.1",
-        "unprint": "^0.16.3",
+        "unprint": "^0.17.4",
         "url-pattern": "^1.0.3",
         "v-tooltip": "^2.1.3",
         "video.js": "^8.6.1",

seeds/00_tags.js

@@ -746,6 +746,10 @@ const tags = [
 		name: 'leather',
 		slug: 'leather',
 	},
+	{
+		name: 'leg warmers',
+		slug: 'leg-warmers',
+	},
 	{
 		name: 'lesbian',
 		slug: 'lesbian',

seeds/01_networks.js

@@ -638,9 +638,6 @@ const networks = [
 		name: 'Porn+',
 		url: 'https://pornplus.com',
 		parent: 'whalemember',
-		parameters: {
-			layout: 'plus',
-		},
 	},
 	{
 		slug: 'private',

seeds/02_sites.js

@@ -9319,21 +9319,21 @@ const sites = [
 	{
 		name: 'Porn Pros',
 		slug: 'pornpros',
-		url: 'https://pornpros.com/site/pornpros',
+		url: 'https://pornpros.com/sites/pornpros',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Real Ex Girlfriends',
 		slug: 'realexgirlfriends',
 		alias: ['reg'],
-		url: 'https://pornpros.com/site/realexgirlfriends',
+		url: 'https://pornpros.com/sites/realexgirlfriends',
 		parent: 'pornpros',
 	},
 	{
 		name: '18 Years Old',
 		slug: 'eighteenyearsold',
 		alias: ['18yo'],
-		url: 'https://pornpros.com/site/18yearsold',
+		url: 'https://pornpros.com/sites/18yearsold',
 		tags: ['teen'],
 		parent: 'pornpros',
 	},
@@ -9341,14 +9341,14 @@ const sites = [
 		name: 'Massage Creep',
 		slug: 'massagecreep',
 		alias: ['mc'],
-		url: 'https://pornpros.com/site/massagecreep',
+		url: 'https://pornpros.com/sites/massagecreep',
 		tags: ['massage'],
 		parent: 'pornpros',
 	},
 	{
 		name: 'Deep Throat Love',
 		slug: 'deepthroatlove',
-		url: 'https://pornpros.com/site/deepthroatlove',
+		url: 'https://pornpros.com/sites/deepthroatlove',
 		tags: ['blowjob', 'deepthroat'],
 		parent: 'pornpros',
 	},
@@ -9356,74 +9356,74 @@ const sites = [
 		name: 'Teen BFF',
 		slug: 'teenbff',
 		alias: ['tbff'],
-		url: 'https://pornpros.com/site/teenbff',
+		url: 'https://pornpros.com/sites/teenbff',
 		tags: ['mff'],
 		parent: 'pornpros',
 	},
 	{
 		name: 'Shady P.I.',
 		slug: 'shadypi',
-		url: 'https://pornpros.com/site/shadypi',
+		url: 'https://pornpros.com/sites/shadypi',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Cruelty Party',
 		slug: 'crueltyparty',
-		url: 'https://pornpros.com/site/crueltyparty',
+		url: 'https://pornpros.com/sites/crueltyparty',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Disgraced 18',
 		slug: 'disgraced18',
-		url: 'https://pornpros.com/site/disgraced18',
+		url: 'https://pornpros.com/sites/disgraced18',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Cumshot Surprise',
 		slug: 'cumshotsurprise',
-		url: 'https://pornpros.com/site/cumshotsurprise',
+		url: 'https://pornpros.com/sites/cumshotsurprise',
 		parent: 'pornpros',
 	},
 	{
 		name: '40oz Bounce',
 		slug: 'fortyozbounce',
-		url: 'https://pornpros.com/site/40ozbounce',
+		url: 'https://pornpros.com/sites/40ozbounce',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Jurassic Cock',
 		slug: 'jurassiccock',
-		url: 'https://pornpros.com/site/jurassiccock',
+		url: 'https://pornpros.com/sites/jurassiccock',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Freaks Of Cock',
 		slug: 'freaksofcock',
-		url: 'https://pornpros.com/site/freaksofcock',
+		url: 'https://pornpros.com/sites/freaksofcock',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Euro Humpers',
 		slug: 'eurohumpers',
-		url: 'https://pornpros.com/site/eurohumpers',
+		url: 'https://pornpros.com/sites/eurohumpers',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Freaks Of Boobs',
 		slug: 'freaksofboobs',
-		url: 'https://pornpros.com/site/freaksofboobs',
+		url: 'https://pornpros.com/sites/freaksofboobs',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Cock Competition',
 		slug: 'cockcompetition',
-		url: 'https://pornpros.com/site/cockcompetition',
+		url: 'https://pornpros.com/sites/cockcompetition',
 		parent: 'pornpros',
 	},
 	{
 		name: 'Pimp Parade',
 		slug: 'pimpparade',
-		url: 'https://pornpros.com/site/pimpparade',
+		url: 'https://pornpros.com/sites/pimpparade',
 		parent: 'pornpros',
 	},
 	{
@@ -9445,8 +9445,7 @@ const sites = [
 		url: 'https://flexiblepositions.com',
 		parent: 'pornpros',
 		parameters: {
-			parent: true,
+			latest: 'https://pornpros.com/sites/flexiblepositions',
-			latest: 'https://pornpros.com/site/flexiblepositions',
 		},
 	},
 	{
@@ -9455,8 +9454,7 @@ const sites = [
 		url: 'https://publicviolations.com',
 		parent: 'pornpros',
 		parameters: {
-			parent: true,
+			latest: 'https://pornpros.com/sites/publicviolations',
-			latest: 'https://pornpros.com/site/publicviolations',
 		},
 	},
 	{
@@ -9464,6 +9462,9 @@ const sites = [
 		slug: 'amateurviolations',
 		url: 'https://amateurviolations.com',
 		parent: 'pornpros',
+		parameters: {
+			latest: 'https://pornpros.com/sites/amateurviolations',
+		},
 	},
 	{
 		name: 'Squirt Disgrace',
@@ -9471,8 +9472,7 @@ const sites = [
 		url: 'https://squirtdisgrace.com',
 		parent: 'pornpros',
 		parameters: {
-			parent: true,
+			latest: 'https://pornpros.com/sites/squirtdisgrace',
-			latest: 'https://pornpros.com/site/squirtdisgrace',
 		},
 	},
 	{
@@ -9480,12 +9480,18 @@ const sites = [
 		slug: 'cumdisgrace',
 		url: 'https://cumdisgrace.com',
 		parent: 'pornpros',
+		parameters: {
+			latest: 'https://pornpros.com/sites/cumdisgrace',
+		},
 	},
 	{
 		name: 'Webcam Hackers',
 		slug: 'webcamhackers',
 		url: 'https://webcamhackers.com',
 		parent: 'pornpros',
+		parameters: {
+			latest: 'https://pornpros.com/sites/webcamhackers',
+		},
 	},
 	{
 		name: 'College Teens',
@@ -14608,6 +14614,9 @@ const sites = [
 		url: 'https://facials4k.com',
 		tags: ['fake-cum', 'facial', '4k'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://facials4k.com/sites/facials4k',
+		},
 	},
 	{
 		name: 'Cum 4K',
@@ -14615,6 +14624,9 @@ const sites = [
 		url: 'https://cum4k.com',
 		tags: ['fake-cum', 'creampie', '4k'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://cum4k.com/sites/cum4k',
+		},
 	},
 	{
 		name: 'Anal 4K',
@@ -14622,6 +14634,9 @@ const sites = [
 		url: 'https://anal4k.com',
 		tags: ['anal', '4k'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://anal4k.com/sites/anal-4k',
+		},
 	},
 	{
 		name: 'Tiny 4K',
@@ -14629,6 +14644,9 @@ const sites = [
 		url: 'https://tiny4k.com',
 		tags: ['4k'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://tiny4k.com/sites/tiny4k',
+		},
 	},
 	{
 		name: 'POVD',
@@ -14636,6 +14654,9 @@ const sites = [
 		url: 'https://povd.com',
 		tags: ['pov'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://povd.com/sites/povd',
+		},
 	},
 	{
 		name: 'Lubed',
@@ -14643,6 +14664,9 @@ const sites = [
 		url: 'https://lubed.com',
 		tags: ['oil'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://lubed.com/sites/lubed',
+		},
 	},
 	{
 		name: 'Casting Couch X',
@@ -14650,6 +14674,9 @@ const sites = [
 		alias: ['castingcouch x', 'castingcouch-x', 'casting couch-x'],
 		url: 'https://castingcouch-x.com',
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://castingcouch-x.com/sites/casting-couch-x',
+		},
 	},
 	{
 		name: 'Passion HD',
@@ -14657,18 +14684,27 @@ const sites = [
 		alias: ['phd', 'passion-hd'],
 		url: 'https://passion-hd.com',
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://passion-hd.com/sites/passion-hd',
+		},
 	},
 	{
 		name: 'Nanny Spy',
 		slug: 'nannyspy',
 		url: 'https://nannyspy.com',
 		parent: 'whalemember',
+		parameters: {
+			latest: 'http://nannyspy.com/sites/nannyspy',
+		},
 	},
 	{
 		name: 'Girl Cum',
 		slug: 'girlcum',
 		url: 'https://girlcum.com',
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://girlcum.com/sites/girlcum',
+		},
 	},
 	{
 		name: 'Pure Mature',
@@ -14676,6 +14712,9 @@ const sites = [
 		url: 'https://puremature.com',
 		tags: ['milf'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://puremature.com/sites/pure-mature',
+		},
 	},
 	{
 		name: 'Fantasy HD',
@@ -14683,6 +14722,9 @@ const sites = [
 		alias: ['fhd'],
 		url: 'https://fantasyhd.com',
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://fantasyhd.com/sites/fantasy-hd',
+		},
 	},
 	{
 		name: 'Spy Fam',
@@ -14690,6 +14732,9 @@ const sites = [
 		url: 'https://spyfam.com',
 		tags: ['family'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://spyfam.com/sites/spyfam',
+		},
 	},
 	{
 		name: 'Holed',
@@ -14697,6 +14742,9 @@ const sites = [
 		url: 'https://holed.com',
 		tags: ['anal'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://holed.com/sites/holed',
+		},
 	},
 	{
 		name: 'BBC Pie',
@@ -14704,6 +14752,9 @@ const sites = [
 		url: 'https://bbcpie.com',
 		tags: ['bbc', 'interracial'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://bbcpie.com/sites/bbc-pie',
+		},
 	},
 	{
 		name: 'Wet VR',
@@ -14711,6 +14762,9 @@ const sites = [
 		url: 'https://wetvr.com',
 		tags: ['vr'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://wetvr.com/sites/wetvr',
+		},
 	},
 	{
 		name: 'Exotic 4K',
@@ -14718,6 +14772,9 @@ const sites = [
 		url: 'https://exotic4k.com',
 		tags: ['4k'],
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://exotic4k.com/sites/exotic4k',
+		},
 	},
 	{
 		name: 'My Very First Time',
@@ -14725,6 +14782,9 @@ const sites = [
 		alias: ['mvft'],
 		url: 'https://myveryfirsttime.com',
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://myveryfirsttime.com/sites/my-very-first-time',
+		},
 	},
 	{
 		name: 'Baeb',
@@ -14732,6 +14792,9 @@ const sites = [
 		alias: ['baebz'],
 		url: 'https://baeb.com',
 		parent: 'whalemember',
+		parameters: {
+			latest: 'https://baeb.com/sites/baeb',
+		},
 	},
 	// WICKED
 	{

src/app.js

@@ -229,6 +229,7 @@ async function init() {
 
 	await http.destroyBypassSessions();
 	await http.destroyBrowserSessions();
+	await unprint.closeAllBrowsers();
 
 	knex.destroy();
 	redis.disconnect();

src/scrapers/whalemember.js

@@ -5,26 +5,38 @@ const unprint = require('unprint');
 const { stripQuery } = require('../utils/url');
 const slugify = require('../utils/slugify');
 
-function scrapeLatest(scenes, channel) {
+const dateRegex = /\d{4}-\d{2}-\d{2}T/;
-	return scenes.map(({ query, _element }) => {
+
+function scrapeLatest(scenes, fullData, channel) {
+	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.url = query.url('[href*="/video"]');
+		release.url = query.url('[href*="/video"]', { origin: new URL(channel.url).origin });
-		// release.entryId = unprint.query.attribute(element, null, 'data-vid'); // does not match old videos
+		release.title = query.content('a[href*="/video"] strong');
-
-		release.title = query.content('.video-thumbnail-footer a[href*="/video"]');
-		release.date = query.date('.actor-list + span', 'MM/DD/YYYY');
 
 		release.entryId = release.url
 			? new URL(release.url).pathname.split('/').at(-1)
 			: slugify(release.title);
 
-		release.actors = query.all('.actor-list a').map((actorEl) => ({
+		// Nuxt data array does not have a predictable structure, don't rely on it more than necessary
+		const dataIndex = fullData?.indexOf(release.entryId);
+		const data = dataIndex > -1 ? fullData?.slice(dataIndex - 5, dataIndex + 35) : [];
+
+		// older scenes don't have date in html
+		const date = data.find((item) => dateRegex.test(item));
+
+		if (date) {
+			release.date = new Date(date);
+		} else {
+			release.date = query.date('a[href*="/video"] + p + p', 'MM/DD/YYYY');
+		}
+
+		release.actors = query.all('a[href*="/model"]').map((actorEl) => ({
 			name: unprint.query.content(actorEl),
 			url: unprint.query.url(actorEl, null, { origin: channel.url }),
 		}));
 
-		const poster = query.poster() || query.img('a img', { attribute: 'data-src' });
+		const poster = query.img('img[alt]');
 
 		if (poster) {
 			release.poster = [
@@ -33,99 +45,107 @@ function scrapeLatest(scenes, channel) {
 			];
 		}
 
-		release.photos = query.imgs('img[data-index]', { attribute: 'data-src' }).map((src) => [
+		// photos and teasers can't be reliably extracted, MP4s include trailers and FULL SCENES
-			stripQuery(src),
-			src,
-		]);
-
-		release.teaser = query.video('source', { attribute: 'data-src' });
 
 		return release;
 	});
 }
 
-function scrapeLatestPlus(scenes, channel) {
+async function passAgeCheck(ctx) {
-	return scenes.map(({ query, _element }) => {
+	const ageButton = await ctx.getByText('Continue', { exact: true });
-		const release = {};
 
-		release.title = query.content('.title');
+	if (await ageButton.count() > 0) {
-		release.date = query.date('.video-thumbnail-footer', 'MM/DD/YYYY');
+		await ageButton.click();
-
+	}
-		const slug = slugify(release.title);
-
-		release.url = `${new URL(channel.url).origin}/video/${slug}`;
-		release.entryId = slug;
-
-		release.actors = query.all('.actor-list a').map((actorEl) => ({
-			name: unprint.query.content(actorEl),
-			url: unprint.query.url(actorEl, null, { origin: channel.url }),
-		}));
-
-		const poster = query.poster() || query.img('a[data-video-thumbnail-single-image-target] img');
-
-		if (poster) {
-			release.poster = [
-				stripQuery(poster),
-				poster,
-			];
-		}
-
-		release.teaser = query.video('source', { attribute: 'data-src' });
-
-		release.channel = slugify(query.attribute('.series-link img', 'alt'), '');
-
-		return release;
-	});
-}
-
-function scrapeScene({ query }, { url, entity }) {
-	const release = {};
-
-	// release.entryId = query.attribute('div[data-id]', 'data-id');
-	release.entryId = new URL(url).pathname.split('/').at(-1);
-
-	release.title = query.content('.scene-info h1');
-	release.description = query.content('//div[contains(@class, \'scene-info\')]//i[contains(@class, \'fa-quote\')]/following-sibling::span');
-
-	release.duration = (query.number('//div[contains(@class, \'scene-info\')]//span[contains(text(), \'Duration\')]/following-sibling::span[contains(text(), \'minutes\')]') * 60) || null;
-
-	release.actors = query.all('.scene-info a[href*="/models"]').map((actorEl) => ({
-		name: unprint.query.content(actorEl),
-		url: unprint.query.url(actorEl, null, { origin: entity.url }),
-	}));
-
-	release.poster = query.poster('#player-wrapper video');
-
-	release.photos = query.imgs('#trailer_player .hidden > a img').map((src) => [
-		stripQuery(src),
-		src,
-	]);
-
-	release.teaser = query.video('#player-wrapper source');
-	release.qualities = query.contents('#trailer_player .resolution').map((resolution) => Number(resolution.split('x')[1])).filter(Boolean);
-
-	console.log(release);
-
-	return release;
 }
 
 async function fetchLatest(channel, page = 1) {
-	const url = `${channel.parameters?.latest || channel.url}?page=${page}`;
+	// going to e.g. https://holed.com/sites/holed defined by parameter gets rid of 'top rated' section, simplifying query
-	const res = await unprint.get(url, { selectAll: '//*[(starts-with(text(), \'Latest\') and contains(text(), \'Movies\')) or contains(text(), \'Most Recent\')]/following::div[contains(@class, \'video-thumbnail\') and @data-vid]' });
+	const url = `${channel.parameters?.latest || channel.url.replace('/series', '/sites')}?page=${page}`;
+
+	// site uses Nuxt without SSR, easiest to render in browser
+	const res = await unprint.browserRequest(url, {
+		async control(ctx) {
+			await passAgeCheck(ctx);
+		},
+	});
 
 	if (res.status === 200) {
-		return scrapeLatest(res.context, channel);
+		const scenes = unprint.initAll(res.context.query.all('.card-grid > div'));
+		const data = res.context.query.json('#__NUXT_DATA__');
+
+		return scrapeLatest(scenes, data, channel);
 	}
 
 	return res.status;
 }
 
-async function fetchLatestPlus(channel, page = 1) {
+function scrapeScene({ query }, { url, entity }) {
-	const url = `${channel.url}?page=${page}`;
+	const release = {};
-	const res = await unprint.get(url, { selectAll: '.video-thumbnail' });
+	const { query: infoQuery } = unprint.init(query.element('//div[./*/span[contains(text(), \'Featuring\')]]'));
 
-	if (res.status === 200) {
+	// release.entryId = query.attribute('div[data-id]', 'data-id');
-		return scrapeLatestPlus(res.context, channel);
+	release.entryId = new URL(url).pathname.split('/').at(-1);
+
+	release.title = infoQuery.content('h2');
+
+	const description = infoQuery.content('h2 + p + p');
+
+	if (!description.toLowerCase().includes('n/a')) {
+		release.description = description;
+	}
+
+	// Nuxt data array does not have a predictable structure, don't rely on it more than necessary
+	const fullData = query.json('#__NUXT_DATA__');
+	const dataIndex = fullData?.indexOf(release.entryId);
+	const data = dataIndex > -1 ? fullData?.slice(dataIndex - 5, dataIndex + 50) : [];
+
+	const date = data.find((item) => dateRegex.test(item));
+
+	if (date) {
+		release.date = new Date(date);
+	}
+
+	release.actors = infoQuery.all('a[href*="/models"]').map((actorEl) => ({
+		name: unprint.query.content(actorEl),
+		url: unprint.query.url(actorEl, null, { origin: entity.url }),
+	}));
+
+	const poster = query.img('media-poster img');
+
+	if (poster) {
+		release.poster = [
+			stripQuery(poster),
+			poster,
+		];
+	}
+
+	release.photos = Array.from(new Set(query.imgs('a img[src*="content/videos"]'))).map((src) => [
+		stripQuery(src),
+		src,
+	]);
+
+	release.trailer = query.video('media-player video');
+	release.qualities = query.contents('//table[.//span[contains(text(), \'480p\')]]//tr').map((resolution) => Number(resolution.split('x')[1])).filter(Boolean);
+
+	return release;
+}
+
+async function fetchScene(url, entity) {
+	const res = await unprint.browserRequest(url, {
+		async control(ctx) {
+			await passAgeCheck(ctx);
+
+			try {
+				await ctx.locator('media-player video').hover({ trial: true, timeout: 1000 }); // wait for trailer to initialize
+			} catch (__error) {
+				// no trailer, that's fine
+			}
+		},
+	});
+
+	if (res.ok) {
+		return scrapeScene(res.context, { url, entity });
 	}
 
 	return res.status;
@@ -133,9 +153,5 @@ async function fetchLatestPlus(channel, page = 1) {
 
 module.exports = {
 	fetchLatest,
-	scrapeScene,
+	fetchScene,
-	plus: {
-		fetchLatest: fetchLatestPlus,
-		scrapeScene,
-	},
 };