Separated actor expand buttons. Refactored Brazzers scraper. Fixed actor releases not included in shallow scrape. Added number query and data-src default to qu img. Updated README. Removed post-install migrate and seed.

2020-07-09 02:00:54 +02:00
parent 17d46e804e
commit 44a8ced30c
10 changed files with 289 additions and 326 deletions
--- a/src/scrapers/brazzers.js
+++ b/src/scrapers/brazzers.js
@@ -1,142 +1,111 @@
 'use strict';

 /* eslint-disable newline-per-chained-call */
-const bhttp = require('bhttp');
-const cheerio = require('cheerio');
-const { JSDOM } = require('jsdom');
-const moment = require('moment');
-
-const { get, ex } = require('../utils/q');
+const qu = require('../utils/qu');
 const slugify = require('../utils/slugify');
 const { heightToCm, lbsToKg } = require('../utils/convert');

-const hairMap = {
-	Blonde: 'blonde',
-	Brunette: 'brown',
-	'Black Hair': 'black',
-	Redhead: 'red',
-};
-
-function scrapeAll(html, site, upcoming) {
-	const $ = cheerio.load(html, { normalizeWhitespace: true });
-	const sceneElements = $('.release-card.scene').toArray();
-
-	return sceneElements.reduce((acc, element) => {
-		const isUpcoming = $(element).find('.icon-upcoming.active').length === 1;
+function scrapeAll(items, channel, upcoming) {
+	return items.reduce((acc, { query }) => {
+		const isUpcoming = query.exists('.icon-upcoming.active');

 		if ((upcoming && !isUpcoming) || (!upcoming && isUpcoming)) {
 			return acc;
 		}

-		const sceneLinkElement = $(element).find('a');
+		const release = {};
+		const pathname = query.url('a');

-		const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`;
-		const title = sceneLinkElement.attr('title');
-		const entryId = url.split('/').slice(-3, -2)[0];
+		release.url = `https://www.brazzers.com${pathname}`;
+		release.entryId = pathname.match(/(\/view\/id\/|\/episode\/)(\d+)/)[2];

-		const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate();
-		const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
+		release.title = query.q('a', 'title');
+		release.date = query.date('time', 'MMMM DD, YYYY');

-		const likes = Number($(element).find('.label-rating .like-amount').text());
-		const dislikes = Number($(element).find('.label-rating .dislike-amount').text());
+		release.actors = query.all('.model-names a', 'title');

-		const poster = `https:${$(element).find('.card-main-img').attr('data-src')}`;
-		const photos = $(element).find('.card-overlay .image-under').map((photoIndex, photoElement) => `https:${$(photoElement).attr('data-src')}`).toArray();
+		release.likes = query.number('.label-rating .like-amount');
+		release.dislikes = query.number('.label-rating .dislike-amount');

-		const channel = slugify($(element).find('.collection').attr('title'), '');
+		release.poster = query.img('.card-main-img');
+		release.photos = query.imgs('.card-overlay .image-under');

-		return acc.concat({
-			url,
-			entryId,
-			title,
-			actors,
-			date,
-			poster,
-			photos,
-			rating: {
-				likes,
-				dislikes,
-			},
-			channel,
-			site,
-		});
+		release.channel = slugify(query.q('.collection', 'title'), '');
+
+		return acc.concat(release);
 	}, []);
 }

-async function scrapeScene(html, url, _site) {
-	const $ = cheerio.load(html, { normalizeWhitespace: true });
+function getVideoData(html) {
+	try {
+		const videoScriptStart = html.indexOf('window.videoUiOptions');
+		const videoScript = html.slice(videoScriptStart, html.indexOf('};', videoScriptStart));
+		const videoString = videoScript.slice(videoScript.indexOf('{"stream_info"'), videoScript.lastIndexOf('},') + 1);
+
+		return JSON.parse(videoString);
+	} catch (error) {
+		return null;
+	}
+}
+
+async function scrapeScene({ query, html }, url, _site) {
 	const release = {};

-	const videoJson = $('script:contains("window.videoUiOptions")').html();
-	const videoString = videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('},') + 1);
-	const videoData = JSON.parse(videoString);
+	release.entryId = new URL(url).pathname.match(/(\/view\/id\/|\/episode\/)(\d+)/)[2];

-	[release.entryId] = url.split('/').slice(-3, -2);
-	release.title = $('.scene-title[itemprop="name"]').text();
+	release.title = query.q('.scene-title[itemprop="name"]', true);
+	release.description = query.text('#scene-description p[itemprop="description"]');

-	release.description = $('#scene-description p[itemprop="description"]')
-		.contents()
-		.first()
-		.text()
-		.trim();
+	release.date = query.date('.more-scene-info .scene-date', 'MMMM DD, YYYY');
+	release.duration = Number(query.q('#trailer-player-container', 'data-duration')) // more accurate
+		|| Number(query.q('.scene-length[itemprop="duration"]', 'content').slice(1, -1) * 60);

-	release.date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate();
-	release.duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60;
+	// actor cards have avatar, but truncated name
+	const actorImagesByActorId = query.imgs('.featured-model .card-image img').reduce((acc, img) => ({
+		...acc,
+		[img.match(/\/models\/(\d+)/)[1]]: [
+			img.replace('medium', 'large'),
+			img,
+		],
+	}), {});

-	const actorsFromCards = $('.featured-model .card-image a').map((actorIndex, actorElement) => {
-		const avatar = `https:${$(actorElement).find('img').attr('data-src')}`;
+	release.actors = query.all('.related-model a').map(actorEl => ({
+		name: query.q(actorEl, null, 'title'),
+		avatar: actorImagesByActorId[query.url(actorEl, null).match(/\/view\/id\/(\d+)/)?.[1]],
+	}));

-		return {
-			name: $(actorElement).attr('title'),
-			avatar: [avatar.replace('medium.jpg', 'large.jpg'), avatar],
-		};
-	}).toArray();
+	release.likes = query.number('.label-rating .like');
+	release.dislikes = query.number('.label-rating .dislike');

-	release.actors = actorsFromCards || $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
-
-	release.likes = Number($('.label-rating .like').text());
-	release.dislikes = Number($('.label-rating .dislike').text());
-
-	const siteElement = $('.niche-site-logo');
-	// const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
-	const siteName = siteElement.attr('title');
-	release.channel = slugify(siteName, '');
-
-	const tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-	const categories = $('.timeline a[href*="/categories"]').map((tagIndex, categoryElement) => $(categoryElement).attr('title')).toArray();
+	const tags = query.all('.tag-card-container a', true);
+	const categories = query.all('.timeline a[href*="/categories"]', 'title');

 	release.tags = tags.concat(categories);
-	release.photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray();
+	release.channel = slugify(query.q('.scene-site .label-text', true) || query.q('.niche-site-logo', 'title'), '');

-	const posterPath = videoData?.poster || $('meta[itemprop="thumbnailUrl"]').attr('content') || $('#trailer-player-container').attr('data-player-img');
-	if (posterPath) release.poster = `https:${posterPath}`;
+	const videoData = getVideoData(html);
+	const poster = videoData?.poster || query.meta('itemprop="thumbnailUrl"') || query.q('#trailer-player-container', 'data-player-img');
+
+	release.poster = qu.prefixUrl(poster);
+	release.photos = query.urls('.carousel-thumb a');

 	if (videoData) {
 		release.trailer = Object.entries(videoData.stream_info.http.paths).map(([quality, path]) => ({
-			src: `https:${path}`,
+			src: qu.prefixUrl(path),
 			quality: Number(quality.match(/\d{3,}/)[0]),
 		}));
 	}

-	console.log(release);
-
 	return release;
 }

-function scrapeActorSearch(html, url, actorName) {
-	const { document } = new JSDOM(html).window;
-	const actorLink = document.querySelector(`a[title="${actorName}" i]`);
-
-	return actorLink ? actorLink.href : null;
-}
-
-async function fetchActorReleases({ qu, html }, accReleases = []) {
-	const releases = scrapeAll(html);
-	const next = qu.url('.pagination .next a');
+async function fetchActorReleases({ query }, accReleases = []) {
+	const releases = scrapeAll(qu.initAll(query.all('.release-card.scene')));
+	const next = query.url('.pagination .next a');

 	if (next) {
 		const url = `https://www.brazzers.com${next}`;
-		const res = await get(url);
+		const res = await qu.get(url);

 		if (res.ok) {
 			return fetchActorReleases(res.item, accReleases.concat(releases));
@@ -146,12 +115,9 @@ async function fetchActorReleases({ qu, html }, accReleases = []) {
 	return accReleases.concat(releases);
 }

-async function scrapeProfile(html, url, actorName, include) {
-	const qProfile = ex(html);
-	const { q, qa } = qProfile;
-
-	const bioKeys = qa('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim());
-	const bioValues = qa('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim());
+async function scrapeProfile({ query }, url, actorName, include) {
+	const bioKeys = query.all('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim());
+	const bioValues = query.all('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim());

 	const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key]: bioValues[index] }), {});

@@ -159,17 +125,17 @@ async function scrapeProfile(html, url, actorName, include) {
 		name: actorName,
 	};

-	profile.description = q('.model-profile-specs p', true);
+	profile.description = query.q('.model-profile-specs p', true);

 	if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
 	if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
-	if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate();
+	if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = qu.extractDate(bio['Date of Birth'], 'MMMM DD, YYYY');
 	if (bio['Birth Location']) profile.birthPlace = bio['Birth Location'];
 	if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase();

 	if (bio.Height) profile.height = heightToCm(bio.Height);
 	if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]);
-	if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();
+	if (bio['Hair Color']) profile.hair = bio['Hair Color'].toLowerCase();

 	if (bio['Tits Type'] && bio['Tits Type'].match('Natural')) profile.naturalBoobs = true;
 	if (bio['Tits Type'] && bio['Tits Type'].match('Enhanced')) profile.naturalBoobs = false;
@@ -177,49 +143,60 @@ async function scrapeProfile(html, url, actorName, include) {
 	if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
 	if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;

-	const avatarEl = q('.big-pic-model-container img');
+	const avatarEl = query.q('.big-pic-model-container img');
 	if (avatarEl) profile.avatar = `https:${avatarEl.src}`;

 	if (include.releases) {
-		profile.releases = await fetchActorReleases(qProfile);
+		profile.releases = await fetchActorReleases({ query });
 	}

 	return profile;
 }

-async function fetchLatest(site, page = 1) {
-	const res = await bhttp.get(`${site.url}/page/${page}/`);
+async function fetchLatest(channel, page = 1) {
+	const res = await qu.getAll(`${channel.url}/page/${page}/`, '.release-card.scene');

-	return scrapeAll(res.body.toString(), site, false);
+	if (res.ok) {
+		return scrapeAll(res.items, channel, false);
+	}
+
+	return res.status;
 }

-async function fetchUpcoming(site) {
-	const res = await bhttp.get(`${site.url}/`);
+async function fetchUpcoming(channel) {
+	const res = await qu.getAll(`${channel.url}/page/1`, '.release-card.scene');

-	return scrapeAll(res.body.toString(), site, true);
+	if (res.ok) {
+		return scrapeAll(res.items, channel, true);
+	}
+
+	return res.status;
 }

 async function fetchScene(url, site) {
-	const res = await bhttp.get(url);
+	const res = await qu.get(url);

-	return scrapeScene(res.body.toString(), url, site);
+	if (res.ok) {
+		return scrapeScene(res.item, url, site);
+	}
+
+	return res.status;
 }

 async function fetchProfile(actorName, context, include) {
-	const searchUrl = 'https://brazzers.com/pornstars-search/';
-	const searchRes = await bhttp.get(searchUrl, {
-		headers: {
-			Cookie: `textSearch=${encodeURIComponent(actorName)};`,
-		},
+	const searchRes = await qu.get('https://brazzers.com/pornstars-search/', `a[title="${actorName}" i]`, {
+		Cookie: `textSearch=${encodeURIComponent(actorName)};`,
 	});

-	const actorLink = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
+	const actorLink = searchRes.ok && searchRes.item.qu.url(null);

 	if (actorLink) {
 		const url = `https://brazzers.com${actorLink}`;
-		const res = await bhttp.get(url);
+		const res = await qu.get(url);

-		return scrapeProfile(res.body.toString(), url, actorName, include);
+		if (res.ok) {
+			return scrapeProfile(res.item, url, actorName, include);
+		}
 	}

 	return null;