Added Top Web Models profile scraper.

This commit is contained in:
DebaucheryLibrarian 2021-01-16 04:10:43 +01:00
parent b9e4764516
commit e3ef0a0d69
5 changed files with 98 additions and 0 deletions

5
package-lock.json generated
View File

@ -3011,6 +3011,11 @@
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
"integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
}, },
"convert": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/convert/-/convert-1.6.2.tgz",
"integrity": "sha512-sPoB9KMlewWV7BRdwAEU2zQw85t5a/TTu7WU2qjcZ7t1NTp9l0HrGpAAxxce++qdoyLdF/ZpsA3y7MMPWBHjig=="
},
"convert-source-map": { "convert-source-map": {
"version": "1.7.0", "version": "1.7.0",
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz",

View File

@ -86,6 +86,7 @@
"cli-confirm": "^1.0.1", "cli-confirm": "^1.0.1",
"config": "^3.2.5", "config": "^3.2.5",
"connect-session-knex": "^2.0.0", "connect-session-knex": "^2.0.0",
"convert": "^1.6.2",
"csv-stringify": "^5.3.6", "csv-stringify": "^5.3.6",
"dayjs": "^1.8.21", "dayjs": "^1.8.21",
"dompurify": "^2.0.11", "dompurify": "^2.0.11",

View File

@ -267,6 +267,7 @@ const scrapers = {
sexyhub: mindgeek, sexyhub: mindgeek,
silverstonedvd: famedigital, silverstonedvd: famedigital,
silviasaint: famedigital, silviasaint: famedigital,
topwebmodels,
swallowed: mikeadriano, swallowed: mikeadriano,
teamskeet, teamskeet,
teencoreclub, teencoreclub,

View File

@ -3,6 +3,7 @@
const qu = require('../utils/qu'); const qu = require('../utils/qu');
const http = require('../utils/http'); const http = require('../utils/http');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
function scrapeSceneX(scene) { function scrapeSceneX(scene) {
const release = {}; const release = {};
@ -41,6 +42,43 @@ function scrapeAll(scenes) {
return scenes.map(scrapeSceneX); return scenes.map(scrapeSceneX);
} }
async function scrapeProfile(actor, options) {
const profile = {};
profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch
profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch
profile.gender = actor.attributes.gender || null;
profile.height = convert(actor.attributes.height.value, 'cm');
profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg');
const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || [];
profile.bust = Number(bust);
profile.cup = cup;
profile.waist = Number(waist);
profile.hip = Number(hip);
profile.ethnicity = actor.attributes.ethnicity.value;
profile.birthPlace = actor.attributes.born.value;
profile.eyes = actor.attributes.eyes.value;
profile.hairColor = actor.attributes.hair.value.split('/')[0];
profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`;
profile.avatar = actor.thumb;
if (options.includeActorScenes) {
const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } });
if (res.ok) {
profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items);
}
}
return profile;
}
async function fetchLatest(channel, page) { async function fetchLatest(channel, page) {
const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, { const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
headers: { headers: {
@ -71,7 +109,30 @@ async function fetchScene(url) {
return res.status; return res.status;
} }
async function fetchProfile(baseActor, entity, options) {
const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, {
headers: {
Referer: 'https://tour.topwebmodels.com',
'api-key': entity.parameters?.apiKey,
'x-Requested-With': 'XMLHttpRequest',
},
});
if (!searchRes.ok) {
return searchRes.status;
}
const actor = searchRes.body.models.items.find(model => slugify(model.name) === slugify(baseActor.name));
if (actor) {
return scrapeProfile(actor, options);
}
return null;
}
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchProfile,
}; };

View File

@ -1,5 +1,9 @@
'use strict'; 'use strict';
const { convert, convertMany } = require('convert');
const logger = require('../logger')(__filename);
function inchesToCm(inches) { function inchesToCm(inches) {
if (!inches) return null; if (!inches) return null;
@ -54,6 +58,31 @@ function kgToLbs(kgs) {
return Math.round(Number(kilos) / 0.453592); return Math.round(Number(kilos) / 0.453592);
} }
function convertManyApi(input, to) {
const curatedInput = input
.replace('\'', 'ft')
.replace('"', 'in');
return Math.round(convertMany(curatedInput).to(to)) || null;
}
function convertApi(input, fromOrTo, to) {
if (!input) {
return null;
}
try {
if (typeof input === 'string' && to === undefined) {
return convertManyApi(input, fromOrTo);
}
return Math.round(convert(input).from(fromOrTo).to(to)) || null;
} catch (error) {
logger.error(error);
return null;
}
}
module.exports = { module.exports = {
cmToFeetInches, cmToFeetInches,
cmToInches, cmToInches,
@ -62,4 +91,5 @@ module.exports = {
inchesToCm, inchesToCm,
lbsToKg, lbsToKg,
kgToLbs, kgToLbs,
convert: convertApi,
}; };