Merged DDF Network with Porn World. Fixed and updated DDF/PW scraper.

assets/components/releases/tile-details.vue

@@ -79,7 +79,7 @@ export default {
 	box-shadow: 0 0 3px var(--darken);
 
     .favicon {
-        height: 1rem;
+		width: 1rem;
         margin: 0 .25rem 0 0;
     }
 

config/default.js

@@ -121,6 +121,8 @@ module.exports = {
 		],
 		'21sextury',
 		'julesjordan',
+		'bang',
+		'pervcity',
 		'peternorth',
 		'naughtyamerica',
 		'cherrypimps',

migrations/20190325001339_releases.js

@@ -174,7 +174,7 @@ exports.up = knex => Promise.resolve()
 
 		table.unique(['slug', 'type']);
 
-		table.text('alias');
+		table.specificType('alias', 'text[]');
 
 		table.text('url');
 		table.text('description');
@@ -820,6 +820,8 @@ exports.up = knex => Promise.resolve()
 				WHERE
 				name ILIKE ('%' || search || '%') OR
 				slug ILIKE ('%' || search || '%') OR
+				array_to_string(alias, '') ILIKE ('%' || search || '%') OR
+				replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || search || '%') OR
 				url ILIKE ('%' || search || '%')
 			$$ LANGUAGE SQL STABLE;
 

seeds/01_networks.js

@@ -136,9 +136,13 @@ const networks = [
 		type: 'info',
 	},
 	{
-		slug: 'ddfnetwork',
+		slug: 'pornworld',
-		name: 'DDF Network',
+		name: 'Porn World',
-		url: 'https://ddfnetwork.com',
+		alias: [
+			'ddf network',
+			'denys defrancesco',
+		],
+		url: 'https://pornworld.com',
 		description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
 		parent: 'wgcz',
 	},
@@ -417,7 +421,7 @@ exports.seed = knex => Promise.resolve()
 			slug: network.slug,
 			name: network.name,
 			type: network.type || 'network',
-			alias: network.alias ? network.alias.join(',') : null,
+			alias: network.alias,
 			url: network.url,
 			description: network.description,
 			parameters: network.parameters,

seeds/02_sites.js

@@ -1455,7 +1455,7 @@ const sites = [
 		alias: ['ddfb'],
 		url: 'https://ddfbusty.com',
 		description: 'Gorgeous Babes with big tits and Euro pornstars with huge natural boobs filmed in Exclusive Full HD, 4K, & VR porn videos.',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
 	},
 	{
 		slug: 'handsonhardcore',
@@ -1463,7 +1463,7 @@ const sites = [
 		alias: ['hoh'],
 		url: 'https://handsonhardcore.com',
 		description: 'Hardcore Sex & Anal Fucking Exclusive XXX Videos in VR, 4K and full HD with Hot European Pornstars',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
 	},
 	{
 		slug: 'houseoftaboo',
@@ -1471,7 +1471,7 @@ const sites = [
 		alias: ['hotb', 'hotab'],
 		url: 'https://houseoftaboo.com',
 		description: 'Exclusive BDSM Porn & Extreme Sex Videos Produced in VR, 4K and full HD with The Hottest European Fetish Pornstars',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
 	},
 	{
 		slug: 'ddfnetworkvr',
@@ -1479,21 +1479,24 @@ const sites = [
 		alias: ['ddfvr'],
 		url: 'https://ddfnetworkvr.com',
 		description: 'VR Porn Videos shot Exclusively in 180 3D 4K Virtual Reality featuring the Hottest European & American VR Pornstar Babes',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
+		parameters: {
+			latest: 'https://ddfnetworkvr.com/home/tagfiltered/keywords/-/',
+		},
 	},
 	{
 		slug: 'eurogirlsongirls',
 		name: 'Euro Girls on Girls',
 		url: 'https://eurogirlsongirls.com',
 		description: 'Hot Lesbian Sex & Glamour Lesbian Porn Videos and Photos Starring Gorgeous European Pornstars in 4K and Full HD VR.',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
 	},
 	{
 		slug: '1byday',
 		name: '1By-Day',
 		url: 'https://1by-day.com',
 		description: 'Ultra Sexy Exclusive Solo Masturbation Videos in VR, 4K and full HD showcasing Glamour Babes & Intense Orgasms',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
 	},
 	{
 		slug: 'euroteenerotica',
@@ -1501,14 +1504,14 @@ const sites = [
 		alias: ['ete'],
 		url: 'https://euroteenerotica.com',
 		description: 'Teen Threesomes & Barely Legal Porn Videos in 4K, VR and FULL HD with Hot Nymphomaniac Teen Babes',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
 	},
 	{
 		slug: 'hotlegsandfeet',
 		name: 'Hot Legs and Feet',
 		url: 'https://hotlegsandfeet.com',
 		description: 'Foot Fetish & Sexy Legs Porn Videos with Hot and Sexy Euro Pornstars',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
 	},
 	{
 		slug: 'onlyblowjob',
@@ -1516,22 +1519,24 @@ const sites = [
 		alias: ['obj'],
 		url: 'https://onlyblowjob.com',
 		description: 'Fantasy Blowjobs & POV Cock Sucking Videos and Photos Produced in VR, 4K and full HD featuring Sexy European Pornstars',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
 	},
 	{
 		slug: 'fuckinhd',
 		name: 'Fuck in HD',
 		url: 'https://fuckinhd.com',
 		description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
-		parameters: { native: true },
 	},
 	{
 		slug: 'bustylover',
 		name: 'Busty Lover',
 		url: 'https://bustylover.com',
-		parent: 'ddfnetwork',
+		parent: 'pornworld',
-		parameters: { native: true },
+		parameters: {
+			latest: 'https://bustylover.com/videos/',
+			blockLayout: true,
+		},
 	},
 	// DIGITAL PLAYGROUND
 	{
@@ -7434,7 +7439,7 @@ exports.seed = knex => Promise.resolve()
 			slug: site.slug,
 			name: site.name,
 			type: site.type || 'channel',
-			alias: (site.alias || []).join(','),
+			alias: site.alias,
 			description: site.description,
 			url: site.url,
 			parameters: site.parameters,

src/scrapers/ddfnetwork.js

@@ -2,53 +2,60 @@
 
 const bhttp = require('bhttp');
 
-const { ed, ex, exa, get } = require('../utils/q');
+const qu = require('../utils/qu');
 const slugify = require('../utils/slugify');
 
-/* eslint-disable newline-per-chained-call */
+function scrapeAll(scenes, site, origin) {
-function scrapeAll(html, site, origin) {
+	return scenes.map(({ query }) => {
-	return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
 		const release = {};
 
-		release.title = q('a', 'title');
+		release.title = query.q('.card-title a, .videoContent h4 a', 'title');
-		release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`;
+		release.url = `${site?.url || origin || 'https://pornworld.com'}${query.q('a', 'href')}`;
-		[release.entryId] = release.url.split('/').slice(-1);
+		release.entryId = release.url.match(/\/(\d+)$/)[1];
 
-		release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
+		release.date = query.date('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
-		release.actors = qa('.card-subtitle a', true).filter(Boolean);
+		release.actors = query.all('.card-subtitle a, .featuring a', true).filter(Boolean);
 
-		const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
+		release.description = query.q('h4 + p', true);
+
+		const duration = parseInt(query.q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
 		if (duration) release.duration = duration;
+		else release.duration = query.dur('.time');
 
-		release.poster = q('img').dataset.src;
+		release.poster = query.img();
 
 		return release;
 	});
 }
 
-async function scrapeScene(html, url, _site) {
+async function scrapeScene({ query }, url, _site) {
-	const { qu } = ex(html);
 	const release = {};
 
-	[release.entryId] = url.split('/').slice(-1);
+	release.entryId = url.match(/\/(\d+)$/)[1];
 
-	release.title = qu.meta('itemprop=name');
+	release.title = query.meta('itemprop=name') || query.q('.video-title h1', true) || query.q('.about-text .story-title') || query.q('h3', true);
-	release.description = qu.q('.descr-box p', true);
+	release.description = query.q('.descr-box p', true) || query.q('.about-text p:not(.story-title)', true) || query.text('.description p');
-	release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
-        || qu.date('.title-border:nth-child(2) p', 'MM.DD.YYYY');
 
-	release.actors = qu.all('.pornstar-card > a', 'title');
+	release.date = query.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
-	release.tags = qu.all('.tags-tab .tags a', true);
+		|| query.date('.actors time', 'MMMM DD, YYYY')
+        || query.date('.title-border:nth-child(2) p', 'MM.DD.YYYY')
+		|| query.date('.length', 'MMMM DD, YYYY', /\w+ \d{2}, \d{4}/);
 
-	release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60;
+	if (query.exists('.pornstar-card > a')) release.actors = query.all('.pornstar-card > a', 'title');
-	release.likes = Number(qu.q('.icon-like-red + span', true));
+	else if (query.exists('.actors a')) release.actors = query.all('.actors a', true);
 
-	release.poster = qu.poster();
+	if (query.exists('.tags-tab')) release.tags = query.all('.tags-tab .tags a', true);
-	release.photos = qu.urls('.photo-slider-guest .card a');
+	else if (query.exists('.tags-box')) release.tags = query.all('.tags-box .tags li', true);
 
-	release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({
+	release.duration = parseInt(query.q('.icon-video-red + span', true), 10) * 60 || query.dur('.length') || null;
+	release.likes = Number(query.q('.icon-like-red + span', true)) || null;
+
+	release.poster = query.poster() || query.poster('dl8-video');
+	release.photos = query.urls('.photo-slider-guest .card a');
+
+	release.trailer = query.all('source[type="video/mp4"]').map(trailer => ({
 		src: trailer.src,
-		quality: Number(trailer.attributes.res.value),
+		quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null,
 	}));
 
 	return release;
@@ -59,9 +66,9 @@ async function fetchActorReleases(urls) {
 	const sources = urls.filter(url => !/ddfnetwork/.test(url));
 
 	const releases = await Promise.all(sources.map(async (url) => {
-		const { html } = await get(url);
+		const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)');
 
-		return scrapeAll(html, null, new URL(url).origin);
+		return res.ok ? scrapeAll(res.items, null, new URL(url).origin) : null;
 	}));
 
 	// DDF cross-releases scenes between sites, filter duplicates by entryId
@@ -71,11 +78,9 @@ async function fetchActorReleases(urls) {
 		.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
 }
 
-async function scrapeProfile(html, _url, actorName) {
+async function scrapeProfile({ query }, _url, actorName) {
-	const { qu } = ex(html);
+	const keys = query.all('.about-title', true).map(key => slugify(key, '_'));
-
+	const values = query.all('.about-info').map((el) => {
-	const keys = qu.all('.about-title', true).map(key => slugify(key, '_'));
-	const values = qu.all('.about-info').map((el) => {
 		if (el.children.length > 0) {
 			return Array.from(el.children, child => child.textContent.trim()).join(', ');
 		}
@@ -96,8 +101,8 @@ async function scrapeProfile(html, _url, actorName) {
 		name: actorName,
 	};
 
-	profile.description = qu.q('.description-box', true);
+	profile.description = query.q('.description-box', true);
-	profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY');
+	profile.birthdate = qu.extractDate(bio.birthday, 'MMMM DD, YYYY');
 
 	if (bio.nationality) profile.nationality = bio.nationality;
 
@@ -118,34 +123,37 @@ async function scrapeProfile(html, _url, actorName) {
 
 	if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
 
-	const avatarEl = qu.q('.pornstar-details .card-img-top');
+	const avatarEl = query.q('.pornstar-details .card-img-top');
 	if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
 
-	profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a'));
+	profile.releases = await fetchActorReleases(query.urls('.find-me-tab li a'));
 
 	return profile;
 }
 
-async function fetchLatest(site, page = 1) {
+async function fetchLatest(channel, page = 1) {
+	/* ddfnetwork.com redirects to pornworld.com
 	const url = site.parameters?.native
 		? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
 		: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
+	*/
 
-	const res = await bhttp.get(url);
+	const url = channel.parameters?.latest || `${channel.url}/videos/search/latest/ever/allsite/-/${page}`;
+	const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card), .allVideos .videoBlock');
 
-	if (res.statusCode === 200) {
+	if (res.ok) {
-		return scrapeAll(res.body.toString(), site);
+		return scrapeAll(res.items, channel);
 	}
 
-	return res.statusCode;
+	return res.status;
 }
 
 async function fetchScene(url, site) {
 	// DDF's main site moved to Porn World
 	// const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
-	const res = await bhttp.get(url);
+	const res = await qu.get(url, '.content, #content, .taspVideoPage');
 
-	return scrapeScene(res.body.toString(), url, site);
+	return res.ok ? scrapeScene(res.item, url, site) : res.status;
 }
 
 async function fetchProfile(actorName) {
@@ -155,7 +163,7 @@ async function fetchProfile(actorName) {
 			word: actorName,
 		},
 		{
-			decodeJSON: true,
+			decodeJSON: false,
 			headers: {
 				'x-requested-with': 'XMLHttpRequest',
 			},

src/scrapers/scrapers.js

@@ -83,7 +83,7 @@ module.exports = {
 		brazzers,
 		burningangel,
 		cherrypimps,
-		ddfnetwork,
+		pornworld: ddfnetwork,
 		digitalplayground,
 		dogfart,
 		dogfartnetwork: dogfart,
@@ -155,7 +155,7 @@ module.exports = {
 		brazzers,
 		burningangel,
 		cherrypimps,
-		ddfnetwork,
+		pornworld: ddfnetwork,
 		deeper: vixen,
 		deeplush: nubiles,
 		devilsfilm: famedigital,

src/store-releases.js

@@ -184,10 +184,10 @@ async function updateReleasesSearch(releaseIds) {
                 COALESCE(releases.title, '') || ' ' ||
                 entities.name || ' ' ||
                 entities.slug || ' ' ||
-                COALESCE(entities.alias, '') || ' ' ||
+                COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
                 COALESCE(parents.name, '') || ' ' ||
                 COALESCE(parents.slug, '') || ' ' ||
-                COALESCE(parents.alias, '') || ' ' ||
+                COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
                 COALESCE(releases.shoot_id, '') || ' ' ||
                 COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' ||
                 STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||

src/utils/qu.js

@@ -127,16 +127,16 @@ function date(context, selector, format, match, attr = 'textContent') {
 
 function image(context, selector = 'img', attr, origin, protocol = 'https') {
 	const imageEl = (attr && q(context, selector, attr))
-		|| q(context, selector, 'src')
+		|| q(context, selector, 'data-src')
-		|| q(context, selector, 'data-src');
+		|| q(context, selector, 'src');
 
 	return prefixUrl(imageEl, origin, protocol);
 }
 
 function images(context, selector = 'img', attr, origin, protocol = 'https') {
 	const attribute = attr
-		|| (q(context, selector, 'src') && 'src')
+		|| (q(context, selector, 'data-src') && 'data-src')
-		|| (q(context, selector, 'data-src') && 'data-src');
+		|| (q(context, selector, 'src') && 'src');
 
 	const imageEls = all(context, selector, attribute);