diff --git a/seeds/01_networks.js b/seeds/01_networks.js index f5ca8352..9c979568 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -426,7 +426,7 @@ const networks = [ { slug: 'hitzefrei', name: 'Hitzefrei', - url: 'http://www.hitzefrei.com', + url: 'http://tour.hitzefrei.com', }, { slug: 'hussiepass', diff --git a/src/scrapers/hitzefrei.js b/src/scrapers/hitzefrei.js index 503f9dc8..ce55cbad 100755 --- a/src/scrapers/hitzefrei.js +++ b/src/scrapers/hitzefrei.js @@ -1,9 +1,8 @@ 'use strict'; -const qu = require('../utils/qu'); -const http = require('../utils/http'); -const { lbsToKg, feetInchesToCm } = require('../utils/convert'); -const slugify = require('../utils/slugify'); +const unprint = require('unprint'); + +const { convert } = require('../utils/convert'); function scrapeAll(scenes) { return scenes.map(({ query }) => { @@ -12,39 +11,50 @@ function scrapeAll(scenes) { release.url = query.url('.content-title a'); release.entryId = new URL(release.url).pathname.match(/\/view\/(\d+)/)[1]; - release.title = query.cnt('.content-title a'); + release.title = query.content('.content-title a'); release.date = query.date('.content-date strong', 'DD/MM/YYYY'); - release.duration = query.dur('.content-date'); + release.duration = query.duration('.content-date'); - release.actors = query.cnts('.content-models a'); + release.actors = query.all('.content-models a').map((actorEl) => ({ + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null), + })); release.poster = query.img('.content-thumbnail img, .large-thumbnail img') || query.poster('.content-thumbnail video, .large-thumbnail video'); release.photos = query.imgs('.small-thumbnails img'); // actor page only - const teaser = query.video('.vid-hover source'); - release.teaser = { src: teaser }; - - release.channel = slugify(query.cnt('.content-site a'), ''); + release.teaser = query.video('.vid-hover source'); return release; }); } +async function fetchLatest(channel, page = 1) { + const res = await unprint.get(`https://tour.hitzefrei.com/videos?site=${channel.parameters.siteId}&page=${page}`, { selectAll: '.hitem' }); + + if (res.ok) { + return scrapeAll(res.context, channel); + } + + return res.status; +} + function scrapeScene({ query }, url) { const release = {}; release.entryId = new URL(url).pathname.match(/\/view\/(\d+)/)[1]; - release.title = query.cnt('.content-title'); - release.description = query.cnt('.content-description p'); + release.title = query.content('.content-title'); + release.description = query.content('.content-description p'); release.date = query.date('.content-metas span:nth-child(4)', 'DD/MM/YYYY'); - release.duration = query.dur('.content-metas span:nth-child(2)'); + release.duration = query.duration('.content-metas span:nth-child(2)'); release.likes = query.number('.content-metas span:nth-child(6)'); - release.actors = query.all('.model-thumb img').map((el) => ({ - name: query.q(el, null, 'alt'), - avatar: query.img(el, null, 'src'), + release.actors = query.all('.model-thumb a').map((actorEl) => ({ + name: unprint.query.attribute(actorEl, 'img', 'alt'), + url: unprint.query.url(actorEl, null), + avatar: unprint.query.img(actorEl), })); release.poster = query.poster('.content-video video'); @@ -54,48 +64,50 @@ function scrapeScene({ query }, url) { photo.replace('/full', '/thumbs'), ]); - const trailer = query.video('.content-video source'); - release.trailer = { src: trailer }; + release.trailer = query.video('.content-video source'); return release; } +async function fetchScene(url) { + const res = await unprint.get(url, { select: '#content-details' }); + + if (res.ok) { + return scrapeScene(res.context, url); + } + + return res.status; +} + async function fetchActorScenes({ query }, accReleases = []) { - const releases = scrapeAll(qu.initAll(query.all('.container-large-video-thumb'))); + const releases = scrapeAll(unprint.initAll(query.all('.container-large-video-thumb'))); const nextPage = query.url('.pagination li:nth-last-child(2) a'); if (nextPage) { - const res = await qu.get(nextPage); + const res = await unprint.get(nextPage); if (res.ok) { - return fetchActorScenes(res.item, accReleases.concat(releases)); + return fetchActorScenes(res.context, accReleases.concat(releases)); } } return accReleases.concat(releases); } -async function scrapeProfile({ query }, include) { +async function scrapeProfile({ query }, _entity, include) { const profile = {}; - /* - const bio = query.all('.model-stats-info div div').reduce((acc, el) => ({ - ...acc, - [slugify(query.cnt(el, '.stat-label'), '_')]: query.cnt(el, '.stat-value'), - }), {}); - */ - profile.dateOfBirth = query.date('.col-birtdate .stat-value, .col-birthdate .stat-value', 'YYYY-MM-DD'); // sic - profile.birthPlace = query.cnt('.col-birth .stat-value'); + profile.birthPlace = query.content('.col-birth .stat-value'); - [profile.bust, profile.waist, profile.hip] = query.cnt('.col-measurements .stat-value').split('-').map(Number); - profile.height = feetInchesToCm(query.cnt('.col-height .stat-value')); - profile.weight = lbsToKg(query.number('.col-weight .stat-value')); + profile.measurements = query.content('.col-measurements .stat-value'); + profile.height = convert(query.content('.col-height .stat-value'), 'cm'); + profile.weight = convert(query.number('.col-weight .stat-value'), 'lb', 'kg'); - profile.eyes = query.cnt('.col-eyes .stat-value'); - profile.hair = query.cnt('.col-hair .stat-value'); + profile.eyes = query.content('.col-eyes .stat-value'); + profile.hair = query.content('.col-hair .stat-value'); - profile.description = query.cnt('.model-profile .model-profile'); + profile.description = query.content('.model-profile .model-profile'); profile.avatar = query.img('.model-thumbnail img'); if (include.releases) { @@ -105,28 +117,12 @@ async function scrapeProfile({ query }, include) { return profile; } -async function fetchLatest(channel, page = 1) { - const res = await qu.getAll(`https://tour.hitzefrei.com/videos?site=${channel.parameters.siteId}&page=${page}`, '.hitem'); - - if (res.ok) { - return scrapeAll(res.items, channel); +async function getActorUrl(baseActor) { + if (baseActor.url) { + return baseActor.url; } - return res.status; -} - -async function fetchScene(url) { - const res = await qu.get(url, '#content-details'); - - if (res.ok) { - return scrapeScene(res.item, url); - } - - return res.status; -} - -async function fetchProfile(baseActor, entity, include) { - const searchRes = await http.post('https://tour.hitzefrei.com/search-preview', { + const res = await unprint.post('https://tour.hitzefrei.com/search-preview', { q: baseActor.name, }, { headers: { @@ -134,23 +130,31 @@ async function fetchProfile(baseActor, entity, include) { }, }); - if (searchRes.ok) { - const actor = searchRes.body.find((result) => result.type === 'model' && result.title === baseActor.name); + if (res.ok) { + const actor = res.data.find((result) => result.type === 'model' && result.title === baseActor.name); if (actor) { - const actorRes = await qu.get(actor.url); - - if (actorRes.ok) { - return scrapeProfile(actorRes.item, include); - } - - return actorRes.status; + return actor.url; } - - return null; } - return searchRes.status; + return null; +} + +async function fetchProfile(baseActor, entity, include) { + const actorUrl = await getActorUrl(baseActor); + + if (actorUrl) { + const res = await unprint.get(actorUrl); + + if (res.ok) { + return scrapeProfile(res.context, entity, include); + } + + return res.status; + } + + return null; } module.exports = { diff --git a/tests/profiles.js b/tests/profiles.js index 8a81eee6..161e6684 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -183,6 +183,7 @@ const actors = [ { entity: 'score', name: 'Vanessa Blue', fields: ['avatar', 'gender', 'placeOfResidence', 'ethnicity', 'height', 'weight', 'measurements', 'hairColor', 'dateOfBirth'] }, { entity: 'pierrewoodman', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] }, { entity: 'dorcelclub', name: 'Clea Gaultier', fields: ['avatar'] }, + { entity: 'hitzefrei', name: 'Jolee Love', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hair', 'description'] }, ]; const actorScrapers = scrapers.actors;