DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Refactored Hitzefrei to use unprint.

This commit is contained in:
DebaucheryLibrarian 2026-01-19 02:18:36 +01:00
parent ba2fd254ff
commit d380c64f7c
3 changed files with 75 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
seeds/01_networks.js
View File

@ -426,7 +426,7 @@ const networks = [
{
slug: 'hitzefrei',
name: 'Hitzefrei',
url: 'http://www.hitzefrei.com',
url: 'http://tour.hitzefrei.com',
},
{
slug: 'hussiepass',

142
src/scrapers/hitzefrei.js
View File

@ -1,9 +1,8 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const { lbsToKg, feetInchesToCm } = require('../utils/convert');
const slugify = require('../utils/slugify');
const unprint = require('unprint');
const { convert } = require('../utils/convert');
function scrapeAll(scenes) {
return scenes.map(({ query }) => {
@ -12,39 +11,50 @@ function scrapeAll(scenes) {
release.url = query.url('.content-title a');
release.entryId = new URL(release.url).pathname.match(/\/view\/(\d+)/)[1];
release.title = query.cnt('.content-title a');
release.title = query.content('.content-title a');
release.date = query.date('.content-date strong', 'DD/MM/YYYY');
release.duration = query.dur('.content-date');
release.duration = query.duration('.content-date');
release.actors = query.cnts('.content-models a');
release.actors = query.all('.content-models a').map((actorEl) => ({
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null),
}));
release.poster = query.img('.content-thumbnail img, .large-thumbnail img') || query.poster('.content-thumbnail video, .large-thumbnail video');
release.photos = query.imgs('.small-thumbnails img'); // actor page only
const teaser = query.video('.vid-hover source');
release.teaser = { src: teaser };
release.channel = slugify(query.cnt('.content-site a'), '');
release.teaser = query.video('.vid-hover source');
return release;
});
}
async function fetchLatest(channel, page = 1) {
const res = await unprint.get(`https://tour.hitzefrei.com/videos?site=${channel.parameters.siteId}&page=${page}`, { selectAll: '.hitem' });
if (res.ok) {
return scrapeAll(res.context, channel);
}
return res.status;
}
function scrapeScene({ query }, url) {
const release = {};
release.entryId = new URL(url).pathname.match(/\/view\/(\d+)/)[1];
release.title = query.cnt('.content-title');
release.description = query.cnt('.content-description p');
release.title = query.content('.content-title');
release.description = query.content('.content-description p');
release.date = query.date('.content-metas span:nth-child(4)', 'DD/MM/YYYY');
release.duration = query.dur('.content-metas span:nth-child(2)');
release.duration = query.duration('.content-metas span:nth-child(2)');
release.likes = query.number('.content-metas span:nth-child(6)');
release.actors = query.all('.model-thumb img').map((el) => ({
name: query.q(el, null, 'alt'),
avatar: query.img(el, null, 'src'),
release.actors = query.all('.model-thumb a').map((actorEl) => ({
name: unprint.query.attribute(actorEl, 'img', 'alt'),
url: unprint.query.url(actorEl, null),
avatar: unprint.query.img(actorEl),
}));
release.poster = query.poster('.content-video video');
@ -54,48 +64,50 @@ function scrapeScene({ query }, url) {
photo.replace('/full', '/thumbs'),
]);
const trailer = query.video('.content-video source');
release.trailer = { src: trailer };
release.trailer = query.video('.content-video source');
return release;
}
async function fetchScene(url) {
const res = await unprint.get(url, { select: '#content-details' });
if (res.ok) {
return scrapeScene(res.context, url);
}
return res.status;
}
async function fetchActorScenes({ query }, accReleases = []) {
const releases = scrapeAll(qu.initAll(query.all('.container-large-video-thumb')));
const releases = scrapeAll(unprint.initAll(query.all('.container-large-video-thumb')));
const nextPage = query.url('.pagination li:nth-last-child(2) a');
if (nextPage) {
const res = await qu.get(nextPage);
const res = await unprint.get(nextPage);
if (res.ok) {
return fetchActorScenes(res.item, accReleases.concat(releases));
return fetchActorScenes(res.context, accReleases.concat(releases));
}
}
return accReleases.concat(releases);
}
async function scrapeProfile({ query }, include) {
async function scrapeProfile({ query }, _entity, include) {
const profile = {};
/*
const bio = query.all('.model-stats-info div div').reduce((acc, el) => ({
...acc,
[slugify(query.cnt(el, '.stat-label'), '_')]: query.cnt(el, '.stat-value'),
}), {});
*/
profile.dateOfBirth = query.date('.col-birtdate .stat-value, .col-birthdate .stat-value', 'YYYY-MM-DD'); // sic
profile.birthPlace = query.cnt('.col-birth .stat-value');
profile.birthPlace = query.content('.col-birth .stat-value');
[profile.bust, profile.waist, profile.hip] = query.cnt('.col-measurements .stat-value').split('-').map(Number);
profile.height = feetInchesToCm(query.cnt('.col-height .stat-value'));
profile.weight = lbsToKg(query.number('.col-weight .stat-value'));
profile.measurements = query.content('.col-measurements .stat-value');
profile.height = convert(query.content('.col-height .stat-value'), 'cm');
profile.weight = convert(query.number('.col-weight .stat-value'), 'lb', 'kg');
profile.eyes = query.cnt('.col-eyes .stat-value');
profile.hair = query.cnt('.col-hair .stat-value');
profile.eyes = query.content('.col-eyes .stat-value');
profile.hair = query.content('.col-hair .stat-value');
profile.description = query.cnt('.model-profile .model-profile');
profile.description = query.content('.model-profile .model-profile');
profile.avatar = query.img('.model-thumbnail img');
if (include.releases) {
@ -105,28 +117,12 @@ async function scrapeProfile({ query }, include) {
return profile;
}
async function fetchLatest(channel, page = 1) {
const res = await qu.getAll(`https://tour.hitzefrei.com/videos?site=${channel.parameters.siteId}&page=${page}`, '.hitem');
if (res.ok) {
return scrapeAll(res.items, channel);
async function getActorUrl(baseActor) {
if (baseActor.url) {
return baseActor.url;
}
return res.status;
}
async function fetchScene(url) {
const res = await qu.get(url, '#content-details');
if (res.ok) {
return scrapeScene(res.item, url);
}
return res.status;
}
async function fetchProfile(baseActor, entity, include) {
const searchRes = await http.post('https://tour.hitzefrei.com/search-preview', {
const res = await unprint.post('https://tour.hitzefrei.com/search-preview', {
q: baseActor.name,
}, {
headers: {
@ -134,23 +130,31 @@ async function fetchProfile(baseActor, entity, include) {
},
});
if (searchRes.ok) {
const actor = searchRes.body.find((result) => result.type === 'model' && result.title === baseActor.name);
if (res.ok) {
const actor = res.data.find((result) => result.type === 'model' && result.title === baseActor.name);
if (actor) {
const actorRes = await qu.get(actor.url);
if (actorRes.ok) {
return scrapeProfile(actorRes.item, include);
}
return actorRes.status;
return actor.url;
}
return null;
}
return searchRes.status;
return null;
}
async function fetchProfile(baseActor, entity, include) {
const actorUrl = await getActorUrl(baseActor);
if (actorUrl) {
const res = await unprint.get(actorUrl);
if (res.ok) {
return scrapeProfile(res.context, entity, include);
}
return res.status;
}
return null;
}
module.exports = {

1
tests/profiles.js
View File

@ -183,6 +183,7 @@ const actors = [
{ entity: 'score', name: 'Vanessa Blue', fields: ['avatar', 'gender', 'placeOfResidence', 'ethnicity', 'height', 'weight', 'measurements', 'hairColor', 'dateOfBirth'] },
{ entity: 'pierrewoodman', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] },
{ entity: 'dorcelclub', name: 'Clea Gaultier', fields: ['avatar'] },
{ entity: 'hitzefrei', name: 'Jolee Love', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hair', 'description'] },
];
const actorScrapers = scrapers.actors;