Added Top Web Models profile scraper.

package-lock.json

@@ -3011,6 +3011,11 @@
             "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
             "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
         },
+        "convert": {
+            "version": "1.6.2",
+            "resolved": "https://registry.npmjs.org/convert/-/convert-1.6.2.tgz",
+            "integrity": "sha512-sPoB9KMlewWV7BRdwAEU2zQw85t5a/TTu7WU2qjcZ7t1NTp9l0HrGpAAxxce++qdoyLdF/ZpsA3y7MMPWBHjig=="
+        },
         "convert-source-map": {
             "version": "1.7.0",
             "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz",

package.json

@@ -86,6 +86,7 @@
         "cli-confirm": "^1.0.1",
         "config": "^3.2.5",
         "connect-session-knex": "^2.0.0",
+        "convert": "^1.6.2",
         "csv-stringify": "^5.3.6",
         "dayjs": "^1.8.21",
         "dompurify": "^2.0.11",

src/scrapers/scrapers.js

@@ -267,6 +267,7 @@ const scrapers = {
 		sexyhub: mindgeek,
 		silverstonedvd: famedigital,
 		silviasaint: famedigital,
+		topwebmodels,
 		swallowed: mikeadriano,
 		teamskeet,
 		teencoreclub,

src/scrapers/topwebmodels.js

@@ -3,6 +3,7 @@
 const qu = require('../utils/qu');
 const http = require('../utils/http');
 const slugify = require('../utils/slugify');
+const { convert } = require('../utils/convert');
 
 function scrapeSceneX(scene) {
 	const release = {};
@@ -41,6 +42,43 @@ function scrapeAll(scenes) {
 	return scenes.map(scrapeSceneX);
 }
 
+async function scrapeProfile(actor, options) {
+	const profile = {};
+
+	profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch
+	profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch
+	profile.gender = actor.attributes.gender || null;
+
+	profile.height = convert(actor.attributes.height.value, 'cm');
+	profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg');
+
+	const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || [];
+
+	profile.bust = Number(bust);
+	profile.cup = cup;
+	profile.waist = Number(waist);
+	profile.hip = Number(hip);
+
+	profile.ethnicity = actor.attributes.ethnicity.value;
+	profile.birthPlace = actor.attributes.born.value;
+
+	profile.eyes = actor.attributes.eyes.value;
+	profile.hairColor = actor.attributes.hair.value.split('/')[0];
+
+	profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`;
+	profile.avatar = actor.thumb;
+
+	if (options.includeActorScenes) {
+		const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } });
+
+		if (res.ok) {
+			profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items);
+		}
+	}
+
+	return profile;
+}
+
 async function fetchLatest(channel, page) {
 	const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
 		headers: {
@@ -71,7 +109,30 @@ async function fetchScene(url) {
 	return res.status;
 }
 
+async function fetchProfile(baseActor, entity, options) {
+	const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, {
+		headers: {
+			Referer: 'https://tour.topwebmodels.com',
+			'api-key': entity.parameters?.apiKey,
+			'x-Requested-With': 'XMLHttpRequest',
+		},
+	});
+
+	if (!searchRes.ok) {
+		return searchRes.status;
+	}
+
+	const actor = searchRes.body.models.items.find(model => slugify(model.name) === slugify(baseActor.name));
+
+	if (actor) {
+		return scrapeProfile(actor, options);
+	}
+
+	return null;
+}
+
 module.exports = {
 	fetchLatest,
 	fetchScene,
+	fetchProfile,
 };

src/utils/convert.js

@@ -1,5 +1,9 @@
 'use strict';
 
+const { convert, convertMany } = require('convert');
+
+const logger = require('../logger')(__filename);
+
 function inchesToCm(inches) {
 	if (!inches) return null;
 
@@ -54,6 +58,31 @@ function kgToLbs(kgs) {
 	return Math.round(Number(kilos) / 0.453592);
 }
 
+function convertManyApi(input, to) {
+	const curatedInput = input
+		.replace('\'', 'ft')
+		.replace('"', 'in');
+
+	return Math.round(convertMany(curatedInput).to(to)) || null;
+}
+
+function convertApi(input, fromOrTo, to) {
+	if (!input) {
+		return null;
+	}
+
+	try {
+		if (typeof input === 'string' && to === undefined) {
+			return convertManyApi(input, fromOrTo);
+		}
+
+		return Math.round(convert(input).from(fromOrTo).to(to)) || null;
+	} catch (error) {
+		logger.error(error);
+		return null;
+	}
+}
+
 module.exports = {
 	cmToFeetInches,
 	cmToInches,
@@ -62,4 +91,5 @@ module.exports = {
 	inchesToCm,
 	lbsToKg,
 	kgToLbs,
+	convert: convertApi,
 };