diff --git a/package-lock.json b/package-lock.json index 9843e23d..db3a306b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3011,6 +3011,11 @@ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, + "convert": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/convert/-/convert-1.6.2.tgz", + "integrity": "sha512-sPoB9KMlewWV7BRdwAEU2zQw85t5a/TTu7WU2qjcZ7t1NTp9l0HrGpAAxxce++qdoyLdF/ZpsA3y7MMPWBHjig==" + }, "convert-source-map": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz", diff --git a/package.json b/package.json index 936d3f34..1f7d4acf 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "cli-confirm": "^1.0.1", "config": "^3.2.5", "connect-session-knex": "^2.0.0", + "convert": "^1.6.2", "csv-stringify": "^5.3.6", "dayjs": "^1.8.21", "dompurify": "^2.0.11", diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index 786a2f1f..ee1c0558 100644 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -267,6 +267,7 @@ const scrapers = { sexyhub: mindgeek, silverstonedvd: famedigital, silviasaint: famedigital, + topwebmodels, swallowed: mikeadriano, teamskeet, teencoreclub, diff --git a/src/scrapers/topwebmodels.js b/src/scrapers/topwebmodels.js index 10382f3e..2f0ad889 100644 --- a/src/scrapers/topwebmodels.js +++ b/src/scrapers/topwebmodels.js @@ -3,6 +3,7 @@ const qu = require('../utils/qu'); const http = require('../utils/http'); const slugify = require('../utils/slugify'); +const { convert } = require('../utils/convert'); function scrapeSceneX(scene) { const release = {}; @@ -41,6 +42,43 @@ function scrapeAll(scenes) { return scenes.map(scrapeSceneX); } +async function scrapeProfile(actor, options) { + const profile = {}; + + profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch + profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch + profile.gender = actor.attributes.gender || null; + + profile.height = convert(actor.attributes.height.value, 'cm'); + profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg'); + + const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || []; + + profile.bust = Number(bust); + profile.cup = cup; + profile.waist = Number(waist); + profile.hip = Number(hip); + + profile.ethnicity = actor.attributes.ethnicity.value; + profile.birthPlace = actor.attributes.born.value; + + profile.eyes = actor.attributes.eyes.value; + profile.hairColor = actor.attributes.hair.value.split('/')[0]; + + profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`; + profile.avatar = actor.thumb; + + if (options.includeActorScenes) { + const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } }); + + if (res.ok) { + profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items); + } + } + + return profile; +} + async function fetchLatest(channel, page) { const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, { headers: { @@ -71,7 +109,30 @@ async function fetchScene(url) { return res.status; } +async function fetchProfile(baseActor, entity, options) { + const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, { + headers: { + Referer: 'https://tour.topwebmodels.com', + 'api-key': entity.parameters?.apiKey, + 'x-Requested-With': 'XMLHttpRequest', + }, + }); + + if (!searchRes.ok) { + return searchRes.status; + } + + const actor = searchRes.body.models.items.find(model => slugify(model.name) === slugify(baseActor.name)); + + if (actor) { + return scrapeProfile(actor, options); + } + + return null; +} + module.exports = { fetchLatest, fetchScene, + fetchProfile, }; diff --git a/src/utils/convert.js b/src/utils/convert.js index d4794116..26e6cb06 100644 --- a/src/utils/convert.js +++ b/src/utils/convert.js @@ -1,5 +1,9 @@ 'use strict'; +const { convert, convertMany } = require('convert'); + +const logger = require('../logger')(__filename); + function inchesToCm(inches) { if (!inches) return null; @@ -54,6 +58,31 @@ function kgToLbs(kgs) { return Math.round(Number(kilos) / 0.453592); } +function convertManyApi(input, to) { + const curatedInput = input + .replace('\'', 'ft') + .replace('"', 'in'); + + return Math.round(convertMany(curatedInput).to(to)) || null; +} + +function convertApi(input, fromOrTo, to) { + if (!input) { + return null; + } + + try { + if (typeof input === 'string' && to === undefined) { + return convertManyApi(input, fromOrTo); + } + + return Math.round(convert(input).from(fromOrTo).to(to)) || null; + } catch (error) { + logger.error(error); + return null; + } +} + module.exports = { cmToFeetInches, cmToInches, @@ -62,4 +91,5 @@ module.exports = { inchesToCm, lbsToKg, kgToLbs, + convert: convertApi, };