Added Top Web Models update and scene scraper.

\

@@ -0,0 +1,68 @@
+'use strict';
+
+const qu = require('../utils/qu');
+const http = require('../utils/http');
+const slugify = require('../utils/slugify');
+
+function scrapeAll(scenes) {
+	return scenes.map((scene) => {
+		const release = {};
+
+		release.entryId = scene.id;
+		release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title)}`;
+
+		release.title = scene.title;
+		release.description = scene.description;
+
+		release.duration = qu.durationToSeconds(scene.videos_duration);
+		release.date = new Date(scene.release_date);
+
+		release.actors = scene.models.map(actor => ({
+			name: actor.name,
+			gender: actor.gender || null,
+			avatar: actor.thumb,
+			url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
+		}));
+
+		release.stars = scene.rating;
+		release.tags = scene.tags.map(tag => tag.name);
+
+		release.poster = scene.thumb;
+
+		release.channel = slugify(scene.sites[0]?.name, '');
+
+		console.log(scene);
+		console.log(release);
+		return release;
+	});
+}
+
+async function fetchLatest(channel, page) {
+	console.log(channel);
+	const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
+		headers: {
+			Referer: 'https://tour.topwebmodels.com',
+			'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey,
+			'x-Requested-With': 'XMLHttpRequest',
+		},
+	});
+
+	console.log(res.body, res.request);
+
+	if (res.ok) {
+		return scrapeAll(res.body.videos.items);
+	}
+
+	return res.status;
+}
+
+async function fetchScene(url) {
+	const res = await http.get(url, { extract: { runScripts: 'dangerously', } });
+
+	console.log(res.);
+}
+
+module.exports = {
+	fetchLatest,
+	fetchScene,
+};

assets/components/expand/expand.vue

@@ -49,14 +49,14 @@ export default {
 
 .expand-dark {
 	.icon {
-		fill: var(--lighten);
+		fill: var(--darken-weak);
 	}
 
 	&:hover {
-		background: var(--lighten-hint);
+		background: var(--darken-hint);
 
 		.icon {
-			fill: var(--text-light);
+			fill: var(--darken);
 		}
 	}
 }

assets/components/releases/media.vue

@@ -115,7 +115,7 @@ function photos() {
 	const uniqueClipPosters = Array.from(new Set(clips.map(clip => clip.poster.id) || [])).map(posterId => clipPostersById[posterId]);
 	const photosWithClipPosters = (this.release.photos || []).concat(uniqueClipPosters);
 
-	if (this.release.trailer || this.release.teaser) {
+	if (this.release.trailer || (this.release.teaser && this.release.teaser.mime !== 'image/gif')) {
 		// poster will be on trailer video
 		return photosWithClipPosters;
 	}

assets/components/scroll/scroll.vue

@@ -1,5 +1,12 @@
 <template>
 	<div class="scroll">
+		<Expand
+			v-if="expanded"
+			:expanded="expanded"
+			class="expand-light"
+			@expand="(state) => $emit('expand', state)"
+		/>
+
 		<Expand
 			v-if="expanded"
 			:expanded="expanded"
@@ -8,13 +15,6 @@
 		/>
 
 		<div class="scrollable">
-			<Expand
-				v-if="expanded"
-				:expanded="expanded"
-				class="expand-light"
-				@expand="(state) => $emit('expand', state)"
-			/>
-
 			<div
 				v-show="enabled && !expanded"
 				class="scroll-button scroll-left noselect"
@@ -40,14 +40,14 @@
 		<Expand
 			v-if="expanded || (expandable && scrollable)"
 			:expanded="expanded"
-			class="expand-dark"
+			class="expand-light"
 			@expand="(state) => $emit('expand', state)"
 		/>
 
 		<Expand
 			v-if="expanded || (expandable && scrollable)"
 			:expanded="expanded"
-			class="expand-light"
+			class="expand-dark"
 			@expand="(state) => $emit('expand', state)"
 		/>
 	</div>
@@ -155,18 +155,6 @@ export default {
     }
 }
 
-.expand-light {
-	display: none;
-}
-
-.expand-dark {
-	display: none;
-}
-
-.expand-light {
-	display: block;
-}
-
 .scroll-button {
 	height: 100%;
 	display: flex;
@@ -222,6 +210,14 @@ export default {
 	padding: 1rem .5rem 1rem 2rem;
 }
 
+.scroll-light .expand-dark {
+	display: none;
+}
+
+.scroll-dark .expand-light {
+	display: none;
+}
+
 @media(max-width: $breakpoint) {
 	.scroll-button {
 		display: none;

assets/components/tags/tag.vue

@@ -21,7 +21,7 @@
 				v-if="hasMedia"
 				v-slot="scroll"
 				:expanded="expanded"
-				class="scroll-light"
+				class="scroll-dark"
 				@expand="(state) => expanded = state"
 			>
 				<Photos

seeds/01_networks.js

@@ -468,6 +468,9 @@ const networks = [
 		slug: 'topwebmodels',
 		name: 'Top Web Models',
 		url: 'https://tour.topwebmodels.com',
+		parameters: {
+			apiKey: '5b637cd8c4bc59cd13686f1c38dcb780',
+		},
 	},
 	{
 		slug: 'transbella',

seeds/04_media.js

@@ -795,6 +795,7 @@ const tagPhotos = [
 	['facial', 3, 'Paige Owens in "Oral Restraint" for Babes'],
 	['facial', 'poster', 'Jynx Maze'],
 	['facial', 2, 'Ashly Anderson for Hookup Hotshot'],
+	['facial', 4, 'Kendra Heart for Facials Forever'],
 	['facefucking', 6, 'Halle Hayes in "Towering Temptress" for 5K Porn'],
 	['facefucking', 1, 'Paige Owens in "Dark Meat 12" for Evil Angel'],
 	['facefucking', 0, 'Ashly Anderson in "Rough Love" for Hookup Hotshot'],

src/entities.js

@@ -32,6 +32,19 @@ function curateEntity(entity, includeParameters = false) {
 		parent: curateEntity(entity.parent, includeParameters),
 	} : {};
 
+	if (entity.tags) {
+		curatedEntity.tags = entity.tags.map(tag => ({
+			id: tag.id,
+			name: tag.name,
+			slug: tag.slug,
+			priority: tag.priority,
+		}));
+	}
+
+	if (includeParameters) {
+		curatedEntity.parameters = entity.parameters;
+	}
+
 	if (entity.children) {
 		curatedEntity.children = entity.children.map(child => curateEntity({
 			...child,
@@ -46,19 +59,6 @@ function curateEntity(entity, includeParameters = false) {
 		}, includeParameters));
 	}
 
-	if (entity.tags) {
-		curatedEntity.tags = entity.tags.map(tag => ({
-			id: tag.id,
-			name: tag.name,
-			slug: tag.slug,
-			priority: tag.priority,
-		}));
-	}
-
-	if (includeParameters) {
-		curatedEntity.parameters = entity.parameters;
-	}
-
 	return curatedEntity;
 }
 

src/media.js

@@ -400,7 +400,7 @@ async function storeFile(media, options) {
 			}
 		}
 
-		if (media.meta.type === 'image') {
+		if (media.meta.type === 'image' && media.meta.subtype !== 'gif') {
 			return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options);
 		}
 

src/scrapers/topwebmodels.js

@@ -4,50 +4,48 @@ const qu = require('../utils/qu');
 const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 
+function scrapeSceneX(scene) {
+	const release = {};
+
+	release.entryId = scene.id;
+	release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title, '-', { removePunctuation: true })}`;
+
+	release.title = scene.title;
+	release.description = scene.description;
+
+	release.duration = qu.durationToSeconds(scene.videos_duration);
+	release.date = new Date(scene.release_date);
+
+	release.actors = scene.models.map(actor => ({
+		name: actor.name,
+		gender: actor.gender || null,
+		avatar: actor.thumb,
+		url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
+	}));
+
+	release.stars = scene.rating;
+	release.tags = scene.tags.map(tag => tag.name);
+
+	release.poster = scene.thumb;
+
+	release.channel = slugify(scene.sites[0]?.name, '');
+
+	return release;
+}
+
 function scrapeAll(scenes) {
-	return scenes.map((scene) => {
-		const release = {};
-
-		release.entryId = scene.id;
-		release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title)}`;
-
-		release.title = scene.title;
-		release.description = scene.description;
-
-		release.duration = qu.durationToSeconds(scene.videos_duration);
-		release.date = new Date(scene.release_date);
-
-		release.actors = scene.models.map(actor => ({
-			name: actor.name,
-			gender: actor.gender || null,
-			avatar: actor.thumb,
-			url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
-		}));
-
-		release.stars = scene.rating;
-		release.tags = scene.tags.map(tag => tag.name);
-
-		release.poster = scene.thumb;
-
-		release.channel = slugify(scene.sites[0]?.name, '');
-
-		console.log(scene);
-		console.log(release);
-		return release;
-	});
+	return scenes.map(scrapeSceneX);
 }
 
 async function fetchLatest(channel, page) {
-	const session = http.session();
-
-	await http.get(channel.url);
-
 	const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
-		session,
+		headers: {
+			Referer: 'https://tour.topwebmodels.com',
+			'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey,
+			'x-Requested-With': 'XMLHttpRequest',
+		},
 	});
 
-	console.log(res);
-
 	if (res.ok) {
 		return scrapeAll(res.body.videos.items);
 	}
@@ -55,6 +53,21 @@ async function fetchLatest(channel, page) {
 	return res.status;
 }
 
+async function fetchScene(url) {
+	const res = await http.get(url, { extract: { runScripts: 'dangerously' } });
+
+	if (res.ok) {
+		return {
+			...scrapeSceneX(res.window.__DATA__.data.video),
+			...(/\.gif/.test(res.window.__DATA__.data.video.thumb) && { teaser: res.window.__DATA__.data.video.thumb }),
+			poster: res.window.__DATA__.data.file_poster,
+		};
+	}
+
+	return res.status;
+}
+
 module.exports = {
 	fetchLatest,
+	fetchScene,
 };