Added Radical metadata layout scraper and the Got Filled and Inserted channels.

config/default.js

@@ -306,6 +306,10 @@ module.exports = {
 			interval: 1000,
 			concurrency: 1,
 		},
+		'www.twistys.com': {
+			interval: 1000,
+			concurrency: 1,
+		},
 		'westcoastproductions.com': {
 			interval: 100,
 			concurrency: 1,

seeds/02_sites.js

@@ -7554,6 +7554,26 @@ const sites = [
 		independent: true,
 		parent: 'radical',
 	},
+	{
+		name: 'Got Filled',
+		slug: 'gotfilled',
+		url: 'https://gotfilled.com',
+		independent: true,
+		parent: 'radical',
+		parameters: {
+			layout: 'metadata',
+		},
+	},
+	{
+		name: 'Inserted',
+		slug: 'inserted',
+		url: 'https://inserted.com',
+		independent: true,
+		parent: 'radical',
+		parameters: {
+			layout: 'metadata',
+		},
+	},
 	// REALITY KINGS
 	{
 		name: 'Look At Her Now',

src/scrapers/radical.js

@@ -0,0 +1,143 @@
+'use strict';
+
+const http = require('../utils/http');
+const qu = require('../utils/qu');
+const slugify = require('../utils/slugify');
+const { lbsToKg, feetInchesToCm } = require('../utils/convert');
+
+function scrapeSceneMetadata(data, channel) {
+	const release = {};
+
+	release.entryId = data.id;
+	release.url = `${channel.url}/tour/videos/${data.id}/${slugify(data.title, '-', { removePunctuation: true })}`;
+
+	release.title = data.title;
+	release.description = data.description;
+
+	release.date = new Date(data.release_date);
+	release.duration = qu.durationToSeconds(data.videos_duration);
+
+	release.actors = data.models.map((model) => ({
+		entryId: model.id,
+		name: model.name,
+		gender: model.gender,
+		avatar: model.thumb,
+		url: `${channel.url}/tour/models/${model.id}/${slugify(model.name, '-', { removePunctuation: true })}`,
+	}));
+
+	release.poster = data.trailer?.poster || [data.thumb?.replace('mobile.jpg', '.jpg'), data.thumb];
+	release.photos = [
+		data.extra_thumbs?.find((url) => /portrait1.jpg/.test(url)),
+		data.extra_thumbs?.find((url) => /scene.jpg/.test(url)),
+		data.extra_thumbs?.find((url) => /portrait2.jpg/.test(url)),
+	]; // ordered by chronology: portrait1.jpg and scene.jpg are usually pre-shoot poses, portrait2.jpg is the cumshot aftermath
+
+	release.trailer = data.trailer && {
+		src: data.trailer.src,
+		type: data.trailer.type,
+	};
+
+	release.teaser = data.special_thumbs;
+
+	release.tags = [].concat(data.tags?.map((tag) => tag.name));
+	release.qualities = data.downloads && Object.values(data.downloads)?.map((download) => download.meta_data.height);
+	release.stars = data.rating;
+
+	return release;
+}
+
+function scrapeAllMetadata(scenes, channel) {
+	return scenes.map((data) => scrapeSceneMetadata(data, channel));
+}
+
+function scrapeProfileMetadata(data, channel) {
+	const profile = {};
+
+	profile.entryId = data.id;
+	profile.url = `${channel.url}/tour/models/${data.id}/${slugify(data.name, '-', { removePunctuation: true })}`;
+
+	profile.description = data.attributes.bio?.value;
+	profile.dateOfBirth = qu.parseDate(data.attributes.birthdate?.value, 'YYYY-MM-DD');
+	profile.gender = data.gender;
+	profile.age = data.attributes.age?.value;
+	profile.birthPlace = data.attributes.born?.value;
+
+	profile.measurements = data.attributes.measurements?.value;
+	profile.height = feetInchesToCm(data.attributes.height?.value);
+	profile.weight = lbsToKg(data.attributes.weight?.value);
+
+	profile.eyes = data.attributes.eyes?.value;
+	profile.hairColor = data.attributes.hair?.value;
+
+	profile.avatar = data.thumb;
+	profile.date = new Date(data.publish_date);
+
+	return profile;
+}
+
+async function fetchLatestMetadata(channel, page = 1) {
+	const url = `${channel.url}/tour/videos?page=${page}`;
+	const res = await http.get(url, {
+		parse: true,
+		extract: {
+			runScripts: 'dangerously',
+		},
+	});
+
+	if (res.ok && res.window.__DATA__) {
+		return scrapeAllMetadata(res.window.__DATA__.videos.items, channel);
+	}
+
+	if (res.ok) {
+		return res.window.__DATA__?.error || null;
+	}
+
+	return res.status;
+}
+
+async function fetchSceneMetadata(url, channel) {
+	const res = await http.get(url, {
+		parse: true,
+		extract: {
+			runScripts: 'dangerously',
+		},
+	});
+
+	if (res.ok && res.window.__DATA__?.video) {
+		return scrapeSceneMetadata(res.window.__DATA__.video, channel);
+	}
+
+	if (res.ok) {
+		return res.window.__DATA__?.error || null;
+	}
+
+	return res.status;
+}
+
+async function fetchProfileMetadata(actor, channel) {
+	const res = await http.get(`${channel.url}/tour/search-preview/${actor.name}`, {
+		headers: {
+			'X-Requested-With': 'XMLHttpRequest',
+		},
+	});
+
+	if (res.ok) {
+		const model = res.body.models?.items.find((modelX) => slugify(modelX.name) === actor.slug);
+
+		if (model) {
+			return scrapeProfileMetadata(model, channel);
+		}
+
+		return null;
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	metadata: {
+		fetchLatest: fetchLatestMetadata,
+		fetchScene: fetchSceneMetadata,
+		fetchProfile: fetchProfileMetadata,
+	},
+};

src/scrapers/scrapers.js

@@ -52,6 +52,7 @@ const pierrewoodman = require('./pierrewoodman');
 const pinkyxxx = require('./pinkyxxx');
 const privateNetwork = require('./private'); // reserved keyword
 const purgatoryx = require('./purgatoryx'); // reserved keyword
+const radical = require('./radical');
 const score = require('./score');
 const spizoo = require('./spizoo');
 const teamskeet = require('./teamskeet');
@@ -138,6 +139,7 @@ const scrapers = {
 		pornpros: whalemember,
 		private: privateNetwork,
 		purgatoryx,
+		radical,
 		score,
 		sexyhub: mindgeek,
 		spizoo,
@@ -208,6 +210,7 @@ const scrapers = {
 		gaywire: bangbros,
 		girlfaction: fullpornnetwork,
 		gloryholesecrets: aziani,
+		gotfilled: radical,
 		hergape: fullpornnetwork,
 		hitzefrei,
 		homemadeanalwhores: fullpornnetwork,
@@ -216,6 +219,7 @@ const scrapers = {
 		hushpass: hush,
 		hussiepass: hush,
 		iconmale: mindgeek,
+		inserted: radical,
 		interracialpass: hush,
 		interracialpovs: hush,
 		inthecrack,