Changed qu's HTML element detection. Passing base actor instead of actorName to profile scrapers.

This commit is contained in:
DebaucheryLibrarian
2020-07-21 01:16:26 +02:00
parent 0e4c0d8fff
commit 939eba8e61
9 changed files with 84 additions and 16 deletions

View File

@@ -69,10 +69,12 @@ async function scrapeScene({ query, html }, url, _site) {
],
}), {});
release.actors = query.all('.related-model a').map(actorEl => ({
name: query.q(actorEl, null, 'title'),
avatar: actorImagesByActorId[query.url(actorEl, null).match(/\/view\/id\/(\d+)/)?.[1]],
}));
release.actors = query.all('.related-model a').map((actorEl) => {
const name = query.q(actorEl, null, 'title');
const avatar = actorImagesByActorId[query.url(actorEl, null).match(/\/view\/id\/(\d+)/)?.[1]];
return { name, avatar };
});
release.likes = query.number('.label-rating .like');
release.dislikes = query.number('.label-rating .dislike');

View File

@@ -1,8 +1,8 @@
'use strict';
const qu = require('../utils/qu');
// TODO: profile scraping
const slugify = require('../utils/slugify');
const { feetInchesToCm } = require('../utils/convert');
function scrapeLatestBlog(scenes, channel) {
return scenes.map(({ query }) => {
@@ -106,6 +106,33 @@ function scrapeScene({ query, html }, url, channel) {
return release;
}
function scrapeProfile({ query }, entity) {
const profile = {};
const bio = query.cnts('.info p').reduce((acc, info) => {
const [key, value] = info.match(/(\w+):\s*(.*)/).slice(1);
return { ...acc, [slugify(key, '_')]: value };
}, {});
profile.age = Number(bio.age);
profile.height = feetInchesToCm(bio.height);
profile.eyes = bio.eyes || bio.eyecolor;
if (bio.figure || bio.measurements) {
const [bust, cup, waist, hip] = (bio.figure || bio.measurements)?.match(/(\d+)(\w+)-(\d+)-(\d+)/).slice(1);
profile.bust = Number(bust);
profile.cup = cup;
profile.waist = Number(waist);
profile.hip = Number(hip);
}
profile.avatar = query.img('img.main-preview', 'src', { origin: entity.url });
return profile;
}
async function fetchLatestBlog(channel, page) {
const url = `${channel.url}/free/updates/videos/${(page - 1) * 10}`;
const res = await qu.getAll(url, '.videos');
@@ -148,8 +175,32 @@ async function fetchScene(url, channel) {
return res.status;
}
async function fetchProfile(baseActor, entity) {
const modelsRes = await qu.getAll(`${entity.url}/free/girls.php?alpha=${baseActor.name.slice(0, 1)}`, '.model');
console.log(baseActor);
if (modelsRes.ok) {
const models = modelsRes.items.filter(({ query }) => query.cnt('strong') === baseActor.name);
return Promise.all(models.map(async (model) => {
const modelUrl = model.query.url('a', 'href', { origin: entity.url });
const modelRes = await qu.get(modelUrl);
if (modelRes.ok) {
return scrapeProfile(modelRes.item, entity);
}
return modelRes.status;
}));
}
return modelsRes.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchUpcoming,
fetchProfile,
};

View File

@@ -176,6 +176,7 @@ module.exports = {
evilangel,
eyeontheguy: hush,
fakehub,
exploitedcollegegirls: fcuk,
forbondage: porndoe,
freeones,
gangbangcreampie: aziani,