Refactored Hitzefrei to use unprint.

seeds/01_networks.js

@@ -426,7 +426,7 @@ const networks = [
 	{
 		slug: 'hitzefrei',
 		name: 'Hitzefrei',
-		url: 'http://www.hitzefrei.com',
+		url: 'http://tour.hitzefrei.com',
 	},
 	{
 		slug: 'hussiepass',

src/scrapers/hitzefrei.js

@@ -1,9 +1,8 @@
 'use strict';
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
-const http = require('../utils/http');
+
-const { lbsToKg, feetInchesToCm } = require('../utils/convert');
+const { convert } = require('../utils/convert');
-const slugify = require('../utils/slugify');
 
 function scrapeAll(scenes) {
 	return scenes.map(({ query }) => {
@@ -12,39 +11,50 @@ function scrapeAll(scenes) {
 		release.url = query.url('.content-title a');
 		release.entryId = new URL(release.url).pathname.match(/\/view\/(\d+)/)[1];
 
-		release.title = query.cnt('.content-title a');
+		release.title = query.content('.content-title a');
 		release.date = query.date('.content-date strong', 'DD/MM/YYYY');
-		release.duration = query.dur('.content-date');
+		release.duration = query.duration('.content-date');
 
-		release.actors = query.cnts('.content-models a');
+		release.actors = query.all('.content-models a').map((actorEl) => ({
+			name: unprint.query.content(actorEl),
+			url: unprint.query.url(actorEl, null),
+		}));
 
 		release.poster = query.img('.content-thumbnail img, .large-thumbnail img') || query.poster('.content-thumbnail video, .large-thumbnail video');
 		release.photos = query.imgs('.small-thumbnails img'); // actor page only
 
-		const teaser = query.video('.vid-hover source');
+		release.teaser = query.video('.vid-hover source');
-		release.teaser = { src: teaser };
-
-		release.channel = slugify(query.cnt('.content-site a'), '');
 
 		return release;
 	});
 }
 
+async function fetchLatest(channel, page = 1) {
+	const res = await unprint.get(`https://tour.hitzefrei.com/videos?site=${channel.parameters.siteId}&page=${page}`, { selectAll: '.hitem' });
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel);
+	}
+
+	return res.status;
+}
+
 function scrapeScene({ query }, url) {
 	const release = {};
 
 	release.entryId = new URL(url).pathname.match(/\/view\/(\d+)/)[1];
 
-	release.title = query.cnt('.content-title');
+	release.title = query.content('.content-title');
-	release.description = query.cnt('.content-description p');
+	release.description = query.content('.content-description p');
 
 	release.date = query.date('.content-metas span:nth-child(4)', 'DD/MM/YYYY');
-	release.duration = query.dur('.content-metas span:nth-child(2)');
+	release.duration = query.duration('.content-metas span:nth-child(2)');
 	release.likes = query.number('.content-metas span:nth-child(6)');
 
-	release.actors = query.all('.model-thumb img').map((el) => ({
+	release.actors = query.all('.model-thumb a').map((actorEl) => ({
-		name: query.q(el, null, 'alt'),
+		name: unprint.query.attribute(actorEl, 'img', 'alt'),
-		avatar: query.img(el, null, 'src'),
+		url: unprint.query.url(actorEl, null),
+		avatar: unprint.query.img(actorEl),
 	}));
 
 	release.poster = query.poster('.content-video video');
@@ -54,48 +64,50 @@ function scrapeScene({ query }, url) {
 		photo.replace('/full', '/thumbs'),
 	]);
 
-	const trailer = query.video('.content-video source');
+	release.trailer = query.video('.content-video source');
-	release.trailer = { src: trailer };
 
 	return release;
 }
 
+async function fetchScene(url) {
+	const res = await unprint.get(url, { select: '#content-details' });
+
+	if (res.ok) {
+		return scrapeScene(res.context, url);
+	}
+
+	return res.status;
+}
+
 async function fetchActorScenes({ query }, accReleases = []) {
-	const releases = scrapeAll(qu.initAll(query.all('.container-large-video-thumb')));
+	const releases = scrapeAll(unprint.initAll(query.all('.container-large-video-thumb')));
 	const nextPage = query.url('.pagination li:nth-last-child(2) a');
 
 	if (nextPage) {
-		const res = await qu.get(nextPage);
+		const res = await unprint.get(nextPage);
 
 		if (res.ok) {
-			return fetchActorScenes(res.item, accReleases.concat(releases));
+			return fetchActorScenes(res.context, accReleases.concat(releases));
 		}
 	}
 
 	return accReleases.concat(releases);
 }
 
-async function scrapeProfile({ query }, include) {
+async function scrapeProfile({ query }, _entity, include) {
 	const profile = {};
 
-	/*
-	const bio = query.all('.model-stats-info div div').reduce((acc, el) => ({
-		...acc,
-		[slugify(query.cnt(el, '.stat-label'), '_')]: query.cnt(el, '.stat-value'),
-	}), {});
-	*/
-
 	profile.dateOfBirth = query.date('.col-birtdate .stat-value, .col-birthdate .stat-value', 'YYYY-MM-DD'); // sic
-	profile.birthPlace = query.cnt('.col-birth .stat-value');
+	profile.birthPlace = query.content('.col-birth .stat-value');
 
-	[profile.bust, profile.waist, profile.hip] = query.cnt('.col-measurements .stat-value').split('-').map(Number);
+	profile.measurements = query.content('.col-measurements .stat-value');
-	profile.height = feetInchesToCm(query.cnt('.col-height .stat-value'));
+	profile.height = convert(query.content('.col-height .stat-value'), 'cm');
-	profile.weight = lbsToKg(query.number('.col-weight .stat-value'));
+	profile.weight = convert(query.number('.col-weight .stat-value'), 'lb', 'kg');
 
-	profile.eyes = query.cnt('.col-eyes .stat-value');
+	profile.eyes = query.content('.col-eyes .stat-value');
-	profile.hair = query.cnt('.col-hair .stat-value');
+	profile.hair = query.content('.col-hair .stat-value');
 
-	profile.description = query.cnt('.model-profile .model-profile');
+	profile.description = query.content('.model-profile .model-profile');
 	profile.avatar = query.img('.model-thumbnail img');
 
 	if (include.releases) {
@@ -105,28 +117,12 @@ async function scrapeProfile({ query }, include) {
 	return profile;
 }
 
-async function fetchLatest(channel, page = 1) {
+async function getActorUrl(baseActor) {
-	const res = await qu.getAll(`https://tour.hitzefrei.com/videos?site=${channel.parameters.siteId}&page=${page}`, '.hitem');
+	if (baseActor.url) {
-
+		return baseActor.url;
-	if (res.ok) {
-		return scrapeAll(res.items, channel);
 	}
 
-	return res.status;
+	const res = await unprint.post('https://tour.hitzefrei.com/search-preview', {
-}
-
-async function fetchScene(url) {
-	const res = await qu.get(url, '#content-details');
-
-	if (res.ok) {
-		return scrapeScene(res.item, url);
-	}
-
-	return res.status;
-}
-
-async function fetchProfile(baseActor, entity, include) {
-	const searchRes = await http.post('https://tour.hitzefrei.com/search-preview', {
 		q: baseActor.name,
 	}, {
 		headers: {
@@ -134,23 +130,31 @@ async function fetchProfile(baseActor, entity, include) {
 		},
 	});
 
-	if (searchRes.ok) {
+	if (res.ok) {
-		const actor = searchRes.body.find((result) => result.type === 'model' && result.title === baseActor.name);
+		const actor = res.data.find((result) => result.type === 'model' && result.title === baseActor.name);
 
 		if (actor) {
-			const actorRes = await qu.get(actor.url);
+			return actor.url;
-
-			if (actorRes.ok) {
-				return scrapeProfile(actorRes.item, include);
-			}
-
-			return actorRes.status;
 		}
-
-		return null;
 	}
 
-	return searchRes.status;
+	return null;
+}
+
+async function fetchProfile(baseActor, entity, include) {
+	const actorUrl = await getActorUrl(baseActor);
+
+	if (actorUrl) {
+		const res = await unprint.get(actorUrl);
+
+		if (res.ok) {
+			return scrapeProfile(res.context, entity, include);
+		}
+
+		return res.status;
+	}
+
+	return null;
 }
 
 module.exports = {

tests/profiles.js

@@ -183,6 +183,7 @@ const actors = [
 	{ entity: 'score', name: 'Vanessa Blue', fields: ['avatar', 'gender', 'placeOfResidence', 'ethnicity', 'height', 'weight', 'measurements', 'hairColor', 'dateOfBirth'] },
 	{ entity: 'pierrewoodman', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] },
 	{ entity: 'dorcelclub', name: 'Clea Gaultier', fields: ['avatar'] },
+	{ entity: 'hitzefrei', name: 'Jolee Love', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hair', 'description'] },
 ];
 
 const actorScrapers = scrapers.actors;