1.199.16

assets/components/actors/actor.vue

@@ -361,7 +361,10 @@
 					:available-actors="actor.actors"
 				/>
 
-				<Releases :releases="releases" />
+				<Releases
+					:releases="releases"
+					:done="done"
+				/>
 
 				<Pagination
 					:items-total="totalCount"
@@ -389,6 +392,8 @@ import Social from './social.vue';
 import StashButton from '../stashes/button.vue';
 
 async function fetchActor(scroll = true) {
+	this.done = false;
+
 	const { actor, releases, totalCount } = await this.$store.dispatch('fetchActorById', {
 		actorId: Number(this.$route.params.actorId),
 		limit: this.limit,
@@ -400,6 +405,7 @@ async function fetchActor(scroll = true) {
 	this.releases = releases;
 	this.totalCount = totalCount;
 	this.stashedBy = actor.stashes;
+	this.done = true;
 
 	if (this.$refs.filter && scroll) {
 		this.$refs.filter.$el.scrollIntoView();
@@ -463,6 +469,7 @@ export default {
 		return {
 			actor: null,
 			releases: null,
+			done: false,
 			totalCount: 0,
 			limit: 20,
 			pageTitle: null,

assets/components/auth/login.vue

@@ -35,11 +35,11 @@
 				class="button button-primary"
 			>Log in</button>
 
-			<router-link
+			<RouterLink
 				v-if="$store.state.auth.signup"
 				:to="{ name: 'signup', query: { ref: $route.query.ref } }"
 				class="link link-external signup"
-			>Sign up</router-link>
+			>Sign up</RouterLink>
 		</template>
 	</form>
 </template>

assets/components/auth/signup.vue

@@ -43,11 +43,11 @@
 				class="button button-primary"
 			>Sign up</button>
 
-			<router-link
+			<RouterLink
 				v-if="$store.state.auth.login"
 				:to="{ name: 'login', query: { ref: $route.query.ref } }"
 				class="link link-external login"
-			>Log in</router-link>
+			>Log in</RouterLink>
 		</template>
 	</form>
 </template>

assets/components/entities/entity.vue

@@ -111,7 +111,10 @@
 			/>
 
 			<div class="releases">
-				<Releases :releases="entity.releases" />
+				<Releases
+					:releases="entity.releases"
+					:done="done"
+				/>
 
 				<Pagination
 					:items-total="totalCount"
@@ -134,6 +137,7 @@ import Scroll from '../scroll/scroll.vue';
 import Campaign from '../campaigns/campaign.vue';
 
 async function fetchEntity(scroll = true) {
+	this.done = false;
 	this.entityUrl = null;
 
 	const { entity, totalCount } = await this.$store.dispatch('fetchEntityBySlugAndType', {
@@ -158,6 +162,7 @@ async function fetchEntity(scroll = true) {
 	}).toString();
 
 	this.entityUrl = campaign?.url || campaign?.affiliate?.url || `${entity.url}${campaign?.affiliate?.parameters ? `?${affiliateParams}` : ''}`;
+	this.done = true;
 
 	if (scroll && this.$refs.filter?.$el) {
 		this.$refs.filter.$el.scrollIntoView();
@@ -189,6 +194,7 @@ export default {
 			entity: null,
 			pageTitle: null,
 			totalCount: null,
+			done: false,
 			limit: Number(this.$route.query.limit) || 20,
 			expanded: false,
 			entityUrl: null,

assets/components/loading/ellipsis.vue

@@ -0,0 +1,90 @@
+<template>
+	<div class="load-container">
+		<div class="load-ellipsis">
+			<div />
+			<div />
+			<div />
+			<div />
+		</div>
+	</div>
+</template>
+
+<style lang="scss" scoped>
+.load-container {
+	display: inline-flex;
+	position: relative;
+	padding: 1rem;
+}
+
+.load-ellipsis {
+    display: inline-block;
+    position: relative;
+    width: 5rem;
+    height: .75rem;
+}
+
+.load-ellipsis div {
+    position: absolute;
+    top: 0;
+    width: .75rem;
+    height: .75rem;
+    border-radius: 50%;
+    background: var(--primary);
+}
+
+.load-ellipsis div:nth-child(1) {
+    left: .5rem;
+    animation: load-ellipsis1 0.5s infinite linear;
+}
+
+.load-ellipsis div:nth-child(2) {
+    left: .5rem;
+    animation: load-ellipsis2 0.5s infinite linear;
+}
+
+.load-ellipsis div:nth-child(3) {
+    left: 2rem;
+    animation: load-ellipsis3 0.5s infinite linear;
+}
+
+.load-ellipsis div:nth-child(4) {
+    left: 3.5rem;
+    animation: load-ellipsis4 0.5s infinite linear;
+}
+
+@keyframes load-ellipsis1 {
+	0% {
+		transform: scale(0);
+	}
+	100% {
+		transform: scale(0.5);
+	}
+}
+
+@keyframes load-ellipsis2 {
+	0% {
+		transform: translate(0, 0) scale(0.5);
+	}
+	100% {
+		transform: translate(1.5rem, 0) scale(1);
+	}
+}
+
+@keyframes load-ellipsis3 {
+	0% {
+		transform: translate(0, 0) scale(1);
+	}
+	100% {
+		transform: translate(1.5rem, 0) scale(0.5);
+	}
+}
+
+@keyframes load-ellipsis4 {
+    0% {
+		transform: scale(0.5);
+    }
+    100% {
+		transform: scale(0);
+    }
+}
+</style>

assets/components/releases/releases.vue

@@ -5,8 +5,10 @@
 			class="heading"
 		><span class="range">{{ range }}</span> releases for '{{ context }}'</h3>
 
+		<Ellipsis v-if="!done" />
+
 		<ul
-			v-if="releases.length > 0"
+			v-else-if="releases.length > 0"
 			:key="sfw"
 			class="nolist tiles"
 		>
@@ -25,7 +27,7 @@
 		</ul>
 
 		<span
-			v-if="releases.length === 0 && range !== 'all'"
+			v-else-if="releases.length === 0 && range !== 'all'"
 			class="empty"
 		>No {{ range }} releases</span>
 
@@ -37,6 +39,7 @@
 </template>
 
 <script>
+import Ellipsis from '../loading/ellipsis.vue';
 import SceneTile from './scene-tile.vue';
 
 function range() {
@@ -49,6 +52,7 @@ function sfw() {
 
 export default {
 	components: {
+		Ellipsis,
 		SceneTile,
 	},
 	props: {
@@ -60,6 +64,10 @@ export default {
 			type: String,
 			default: null,
 		},
+		done: {
+			type: Boolean,
+			default: true,
+		},
 		referer: {
 			type: String,
 			default: null,

assets/components/tags/tag.vue

@@ -64,7 +64,10 @@
 				:fetch-releases="fetchReleases"
 			/>
 
-			<Releases :releases="releases" />
+			<Releases
+				:releases="releases"
+				:done="done"
+			/>
 
 			<Pagination
 				:items-total="totalCount"
@@ -93,6 +96,8 @@ import Campaign from '../campaigns/campaign.vue';
 const converter = new Converter();
 
 async function fetchReleases(scroll = true) {
+	this.done = false;
+
 	const { tag, releases, totalCount } = await this.$store.dispatch('fetchTagBySlug', {
 		tagSlug: this.$route.params.tagSlug,
 		pageNumber: Number(this.$route.params.pageNumber),
@@ -107,6 +112,8 @@ async function fetchReleases(scroll = true) {
 	this.hasMedia = this.tag.poster || this.tag.photos.length > 0;
 	this.description = this.tag.description && converter.makeHtml(escapeHtml(this.tag.description));
 
+	this.done = true;
+
 	if (this.hasMedia) {
 		this.showBannerCampaign = true;
 	}
@@ -146,6 +153,7 @@ export default {
 			tag: null,
 			description: null,
 			releases: null,
+			done: false,
 			totalCount: 0,
 			limit: 20,
 			pageTitle: null,

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.199.15",
+    "version": "1.199.16",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.199.15",
+            "version": "1.199.16",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.199.15",
+    "version": "1.199.16",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

seeds/04_media.js

@@ -665,12 +665,13 @@ const tagMedia = [
 	['blonde', 'shawna_lenee_sunrisekings', 'Shawna Lenee', 'sunrisekings'],
 	['blonde', 2, 'Isabelle Deltore', 'herlimit'],
 	['blowbang', 'ana_foxxx_hardx', 'Ana Foxxx in "Facialized Vol. 4"', 'hardx'],
+	['blowbang', 'lisey_sweet_legalporno', 'Lisey Sweet in GIO816', 'legalporno'],
 	['blowbang', 'angela_white_julesjordan', 'Angela White in "Her Biggest Gangbang Ever"', 'julesjordan'],
 	['blowbang', 'monika_fox_legalporno', 'Monika Fox in GL479', 'legalporno'],
 	['blowbang', 0, 'Lacy Lennon in "Lacy Lennon\'s First Blowbang"', 'hardx'],
 	['blowbang', 'zaawaadi_roccosiffredi_1', 'Zaawaadi in "My Name Is Zaawaadi"', 'roccosiffredi'],
-	['blowbang', 'gina_gerson_assholefever', 'Gina Gerson in "Oppa Gangbang Style"', 'assholefever'],
 	['blowbang', 1, 'Nicole Black in GIO1680', 'legalporno'],
+	['blowbang', 'gina_gerson_assholefever', 'Gina Gerson in "Oppa Gangbang Style"', 'assholefever'],
 	['blowjob', 'clanddi_jinkcego_ddfbusty_1', 'Clanddi Jinkcego', 'ddfbusty'],
 	['blowjob', 'juelz_ventura_babygotboobs', 'Juelz Ventura in "A Deep DP For Dessert"', 'babygotboobs'],
 	['blowjob', 4, 'Chloe Cherry in "Chloe\'s Big Anal"', 'darkx'],
@@ -1020,6 +1021,7 @@ const tagMedia = [
 	['trainbang', 1, 'Ria Sunn', 'private'],
 	['trainbang', 0, 'Nicole Black in GIO971', 'legalporno'],
 	['tap', 4, 'Francys Belle in GIO1103', 'legalporno'],
+	['tap', 'lisey_sweet_legalporno', 'Lisey Sweet in GIO816', 'legalporno'],
 	['tap', 3, 'Julia Red in GIO1007', 'legalporno'],
 	['tap', 1, 'Natasha Teen in SZ2098', 'legalporno'],
 	['tap', 2, 'Kira Thorn in GIO1018', 'legalporno'],

src/app.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const util = require('util');
-const log = require('why-is-node-running');
+// const log = require('why-is-node-running');
 const Inspector = require('inspector-api');
 const fs = require('fs').promises;
 const dayjs = require('dayjs');
@@ -24,8 +24,10 @@ const getFileEntries = require('./utils/file-entries');
 const inspector = new Inspector();
 
 function logActive() {
+	console.log('log active!');
+
 	setTimeout(() => {
-		log();
+		// log();
 		logActive();
 	}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
 }

src/deep.js

@@ -5,7 +5,6 @@ const { mergeAdvanced: merge } = require('object-merge-advanced');
 
 const argv = require('./argv');
 const include = require('./utils/argv-include')(argv);
-const { resolveScraper, resolveLayoutScraper } = require('./scrapers/resolve');
 const { fetchReleaseEntities, urlToSiteSlug } = require('./entities');
 const logger = require('./logger')(__filename);
 const qu = require('./utils/qu');
@@ -96,10 +95,9 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
 		};
 	}
 
-	const scraper = resolveScraper(entity);
+	const layoutScraper = entity.scraper;
-	const layoutScraper = resolveLayoutScraper(entity, scraper);
 
-	if (!layoutScraper) {
+	if (!entity.scraper) {
 		logger.warn(`Could not find scraper for ${baseRelease.url}`);
 		return baseRelease;
 	}
@@ -114,13 +112,16 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
 
 		const options = {
 			...include,
+			beforeFetchScene: entity.preData,
 			parameters: getRecursiveParameters(entity),
 		};
 
-		const scrapedRelease = type === 'scene'
+		const rawScrapedRelease = type === 'scene'
 			? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options, null)
 			: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, options, null);
 
+		const scrapedRelease = rawScrapedRelease?.scene || rawScrapedRelease;
+
 		if (!scrapedRelease || typeof scrapedRelease !== 'object' || Array.isArray(scrapedRelease)) {
 			// scraper is unable to fetch the releases and returned a HTTP code or null
 			throw new Error(`Scraper returned '${scrapedRelease}' when fetching latest from '${entity.name}' (${entity.parent?.name})`);
@@ -170,9 +171,21 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
 }
 
 async function scrapeReleases(baseReleases, entitiesBySlug, type) {
+	const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesBySlug).map(async ([slug, entity]) => {
+		if (entity.scraper?.beforeFetchScene) {
+			const preData = await entity.scraper.beforeFetchScene(entity);
+
+			return [slug, { ...entity, preData }];
+		}
+
+		return null;
+	}));
+
+	const entitiesWithBeforeDataBySlug = Object.fromEntries(entitiesWithBeforeDataEntries.filter(Boolean));
+
 	return Promise.map(
 		baseReleases,
-		async baseRelease => scrapeRelease(baseRelease, entitiesBySlug, type),
+		async baseRelease => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
 		{ concurrency: 10 },
 	);
 }

src/entities.js

@@ -8,6 +8,7 @@ const argv = require('./argv');
 const knex = require('./knex');
 const { deleteScenes, deleteMovies } = require('./releases');
 const { flushOrphanedMedia } = require('./media');
+const { resolveScraper, resolveLayoutScraper } = require('./scrapers/resolve');
 
 function getRecursiveParent(entity) {
 	if (!entity) {
@@ -71,6 +72,9 @@ function curateEntity(entity, includeParameters = false) {
 		}, includeParameters));
 	}
 
+	const scraper = resolveScraper(curatedEntity);
+	curatedEntity.scraper = resolveLayoutScraper(entity, scraper);
+
 	return curatedEntity;
 }
 
@@ -187,7 +191,6 @@ async function fetchEntitiesBySlug(entitySlugs, sort = 'asc') {
 				array['parent'] as parent_path
 			FROM entities
 			WHERE slug = ANY(:entitySlugs)
-			OR substring(url from 'https%://%#"[a-z0-9-]+#".(com|net)%' for '#') = ANY(:entitySlugs)
 
 			UNION ALL
 

src/scrapers/mindgeek.js

@@ -171,6 +171,7 @@ async function getSession(site, parameters) {
 		session,
 		interval: parameters?.interval,
 		concurrency: parameters?.concurrency,
+		parse: false,
 	});
 
 	if (res.statusCode === 200) {
@@ -293,7 +294,7 @@ async function fetchScene(url, site, baseScene, options) {
 	}
 
 	const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
-	const { session, instanceToken } = await getSession(site, options.parameters);
+	const { session, instanceToken } = options.beforeFetchScene || await getSession(site, options.parameters);
 
 	const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
 		session,
@@ -305,7 +306,9 @@ async function fetchScene(url, site, baseScene, options) {
 	});
 
 	if (res.status === 200 && res.body.result) {
-		return scrapeScene(res.body.result, url, site);
+		return {
+			scene: scrapeScene(res.body.result, url, site),
+		};
 	}
 
 	return null;
@@ -360,6 +363,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
 }
 
 module.exports = {
+	beforeFetchScene: getSession,
 	scrapeLatestX,
 	fetchLatest,
 	fetchUpcoming,

src/utils/http.js

@@ -23,6 +23,7 @@ Promise.config({
 const defaultOptions = {
 	timeout: argv.requestTimeout,
 	encodeJSON: true,
+	parse: false,
 	headers: {
 		'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
 	},
@@ -114,14 +115,14 @@ async function finalizeResult(res, options) {
 
 	if (Buffer.isBuffer(res.body)) {
 		const html = res.body.toString();
-		const window = new JSDOM(html, { virtualConsole, ...options.extract }).window;
+		const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null;
 
 		return {
 			...res,
 			body: html,
 			html,
 			status: res.statusCode,
-			document: window.document,
+			document: window?.document || null,
 			window,
 			ok: res.statusCode >= 200 && res.statusCode <= 299,
 		};

src/utils/qu.js

@@ -521,7 +521,11 @@ function extractAll(htmlValue, selector, options) {
 async function request(method = 'get', urlValue, body, selector, headers, options, queryAll = false) {
 	const res = await (method === 'post'
 		? http.post(urlValue, body, { ...options, headers })
-		: http[method](urlValue, { ...options, headers }));
+		: http[method](urlValue, {
+			...options,
+			headers,
+			parse: true,
+		}));
 
 	if (res.ok) {
 		const item = queryAll