1.239.0

config/default.js

@@ -189,12 +189,8 @@ module.exports = {
 			'hotcrazymess',
 			'thatsitcomshow',
 		],
-		[
-			// Adult DVD Empire
-			'elegantangel',
-			'westcoastproductions',
-		],
 		'21sextury',
+		'adultempire',
 		'julesjordan',
 		'dorcelclub',
 		'bang',

migrations/20240815013526_entities_options.js

@@ -0,0 +1,65 @@
+const config = require('config');
+
+exports.up = async (knex) => {
+	await knex.schema.alterTable('entities', (table) => {
+		// internal options, as opposed to parameters for scraper options
+		table.json('options');
+	});
+
+	await knex.schema.alterTable('releases', (table) => {
+		table.dropForeign('entity_id');
+
+		table.foreign('entity_id')
+			.references('id')
+			.inTable('entities')
+			.onDelete('cascade');
+	});
+
+	await knex.schema.alterTable('releases_caps', (table) => {
+		table.unique(['release_id', 'media_id']);
+	});
+
+	await knex.schema.createTable('movies_tags', (table) => {
+		table.integer('tag_id')
+			.references('id')
+			.inTable('tags');
+
+		table.integer('movie_id')
+			.notNullable()
+			.references('id')
+			.inTable('movies')
+			.onDelete('cascade');
+
+		table.text('original_tag');
+
+		table.text('source')
+			.defaultTo('scraper');
+
+		table.unique(['tag_id', 'movie_id']);
+	});
+
+	await knex.raw('GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor;', {
+		visitor: knex.raw(config.database.query.user),
+	});
+};
+
+exports.down = async (knex) => {
+	await knex.schema.alterTable('entities', (table) => {
+		table.dropColumn('options');
+	});
+
+	await knex.schema.alterTable('releases', (table) => {
+		table.dropForeign('entity_id');
+
+		table.foreign('entity_id')
+			.references('id')
+			.inTable('entities')
+			.onDelete('no action');
+	});
+
+	await knex.schema.alterTable('releases_caps', (table) => {
+		table.dropUnique(['release_id', 'media_id']);
+	});
+
+	await knex.schema.dropTable('movies_tags');
+};

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.238.8",
+    "version": "1.239.0",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.238.8",
+            "version": "1.239.0",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",
@@ -47,7 +47,7 @@
                 "express-session": "^1.17.3",
                 "face-api.js": "^0.22.2",
                 "file-type": "^18.7.0",
-                "fluent-ffmpeg": "^2.1.2",
+                "fluent-ffmpeg": "^2.1.3",
                 "fs-extra": "^11.1.1",
                 "graphile-build": "^4.14.0",
                 "graphile-utils": "^4.14.0",
@@ -88,7 +88,7 @@
                 "tunnel": "0.0.6",
                 "ua-parser-js": "^1.0.37",
                 "undici": "^5.28.1",
-                "unprint": "^0.11.5",
+                "unprint": "^0.11.8",
                 "url-pattern": "^1.0.3",
                 "v-tooltip": "^2.1.3",
                 "video.js": "^8.6.1",
@@ -9851,17 +9851,22 @@
             "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ=="
         },
         "node_modules/fluent-ffmpeg": {
-            "version": "2.1.2",
+            "version": "2.1.3",
-            "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
+            "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.3.tgz",
-            "integrity": "sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q==",
+            "integrity": "sha512-Be3narBNt2s6bsaqP6Jzq91heDgOEaDCJAXcE3qcma/EJBSy5FB4cvO31XBInuAuKBx8Kptf8dkhjK0IOru39Q==",
             "dependencies": {
-                "async": ">=0.2.9",
+                "async": "^0.2.9",
                 "which": "^1.1.1"
             },
             "engines": {
-                "node": ">=0.8.0"
+                "node": ">=18"
             }
         },
+        "node_modules/fluent-ffmpeg/node_modules/async": {
+            "version": "0.2.10",
+            "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
+            "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ=="
+        },
         "node_modules/fluent-ffmpeg/node_modules/which": {
             "version": "1.3.1",
             "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
@@ -18293,9 +18298,9 @@
             }
         },
         "node_modules/unprint": {
-            "version": "0.11.5",
+            "version": "0.11.8",
-            "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.11.5.tgz",
+            "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.11.8.tgz",
-            "integrity": "sha512-tLhiFGeSU40GN12625+9oqmNGDFSToMPME60pB+DSGT9wd9fJM0L/lyZMQeNFmWMSThwa/id/FHAOnN7cE1aOw==",
+            "integrity": "sha512-UCtfdbbHSNS/F0hlFwMa+ZmUqkVdp7V3SZVJjcMNnb0GUKm/7VWjhdvzHe+dIejhRdJykHfXWkI/BCbKwl51Vg==",
             "dependencies": {
                 "axios": "^0.27.2",
                 "bottleneck": "^2.19.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.238.8",
+    "version": "1.239.0",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {
@@ -106,7 +106,7 @@
         "express-session": "^1.17.3",
         "face-api.js": "^0.22.2",
         "file-type": "^18.7.0",
-        "fluent-ffmpeg": "^2.1.2",
+        "fluent-ffmpeg": "^2.1.3",
         "fs-extra": "^11.1.1",
         "graphile-build": "^4.14.0",
         "graphile-utils": "^4.14.0",
@@ -147,7 +147,7 @@
         "tunnel": "0.0.6",
         "ua-parser-js": "^1.0.37",
         "undici": "^5.28.1",
-        "unprint": "^0.11.5",
+        "unprint": "^0.11.8",
         "url-pattern": "^1.0.3",
         "v-tooltip": "^2.1.3",
         "video.js": "^8.6.1",

seeds/00_tags.js

@@ -1251,6 +1251,10 @@ const tags = [
 		name: 'voodoo',
 		slug: 'voodoo',
 	},
+	{
+		name: 'bikini',
+		slug: 'bikini',
+	},
 ];
 
 const aliases = [
@@ -2545,6 +2549,30 @@ const aliases = [
 		name: 'parasites',
 		for: 'parasite',
 	},
+	{
+		name: 'threesome - fmm',
+		for: 'mfm',
+	},
+	{
+		name: '4k ultra hd',
+		for: '4k',
+	},
+	{
+		name: 'sex toy play',
+		for: 'toys',
+	},
+	{
+		name: 'cumshots',
+		for: 'cumshot',
+	},
+	{
+		name: 'bikini babes',
+		for: 'bikini',
+	},
+	{
+		name: 'threesomes',
+		for: 'threesome',
+	},
 ];
 
 const priorities = [ // higher index is higher priority

seeds/01_networks.js

@@ -104,6 +104,12 @@ const networks = [
 		},
 		parent: '21sextury',
 	},
+	{
+		slug: 'adultempire',
+		name: 'Adult Empire',
+		url: 'https://www.adultempire.com',
+		type: 'info',
+	},
 	{
 		slug: 'adulttime',
 		name: 'Adult Time',

seeds/02_sites.js

@@ -3270,6 +3270,15 @@ const sites = [
 		slug: 'elegantangel',
 		name: 'Elegant Angel',
 		url: 'https://www.elegantangel.com',
+		options: {
+			spawn: [
+				{
+					parameters: {
+						latest: 'https://www.elegantangel.com/watch-exclusive-elegant-angel-scenes.html',
+					},
+				},
+			],
+		},
 	},
 	// EVIL ANGEL
 	{
@@ -13478,7 +13487,6 @@ const sites = [
 		tags: ['black-cock'],
 		parameters: {
 			studio: false,
-			layout: 'grid',
 		},
 	},
 	// WHALE MEMBER
@@ -13713,27 +13721,36 @@ exports.seed = (knex) => Promise.resolve()
 	.then(async () => {
 		await Promise.all(sites.map(async (channel) => {
 			if (channel.rename) {
-				return knex('entities')
+				await knex('entities')
 					.where({
 						type: channel.type || 'channel',
 						slug: channel.rename,
 					})
 					.update('slug', channel.slug);
+
+				return;
 			}
 
-			return null;
+			if (channel.delete) {
+				await knex('entities')
+					.where({
+						type: channel.type || 'channel',
+						slug: channel.slug,
+					})
+					.delete();
+			}
 		}).filter(Boolean));
 
 		const networks = await knex('entities')
 			.where('type', 'network')
 			.orWhereNull('parent_id');
 
-		const networksMap = networks.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
+		const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
 
 		const tags = await knex('tags').select('*').whereNull('alias_for');
 		const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
 
-		const sitesWithNetworks = sites.map((site) => ({
+		const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({
 			slug: site.slug,
 			name: site.name,
 			type: site.type || 'channel',
@@ -13741,6 +13758,7 @@ exports.seed = (knex) => Promise.resolve()
 			description: site.description,
 			url: site.url,
 			parameters: site.parameters,
+			options: site.options,
 			parent_id: networksMap[site.parent],
 			priority: site.priority || 0,
 			independent: !!site.independent,

src/actors.js

@@ -410,7 +410,7 @@ async function curateProfile(profile, actor) {
 		curatedProfile.ethnicity = ethnicities[profile.ethnicity?.trim().toLowerCase()] || null;
 		curatedProfile.hairType = profile.hairType?.trim() || null;
 		curatedProfile.hairColor = hairColors[(profile.hairColor || profile.hair)?.toLowerCase().replace('hair', '').trim()] || null;
-		curatedProfile.eyes = eyeColors[profile.eyes?.trim().toLowerCase()] || null;
+		curatedProfile.eyes = eyeColors[profile.eyes?.replace(/eyes?/i).trim().toLowerCase()] || null;
 
 		curatedProfile.tattoos = profile.tattoos?.trim() || null;
 		curatedProfile.piercings = profile.piercings?.trim() || null;
@@ -878,7 +878,7 @@ async function scrapeActors(argNames) {
 	const entitySlugs = sources.flat();
 
 	const [entitiesBySlug, existingActorEntries] = await Promise.all([
-		fetchEntitiesBySlug(entitySlugs, 'desc'),
+		fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'] }),
 		knex('actors')
 			.select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
 			.whereIn('actors.slug', baseActors.map((baseActor) => baseActor.slug))

src/deep.js

@@ -84,7 +84,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
 	}
 
 	if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
-		if (scraper.useUnprint || scraper.scrapeScene?.unprint || scraper.scrapeMovie?.unprint) {
+		if (scraper.useUnprint || (type === 'scene' && scraper.scrapeScene?.unprint) || (type === 'movie' && scraper.scrapeMovie?.unprint)) {
 			return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
 		}
 

src/entities.js

@@ -55,7 +55,8 @@ function curateEntity(entity, includeParameters = false) {
 	}
 
 	if (includeParameters) {
-		curatedEntity.parameters = entity.parameters;
+		curatedEntity.options = entity.options; // global internal options
+		curatedEntity.parameters = entity.parameters; // scraper-specific parameters
 	}
 
 	if (entity.children) {
@@ -66,10 +67,25 @@ function curateEntity(entity, includeParameters = false) {
 	}
 
 	if (entity.included_children) {
-		curatedEntity.includedChildren = entity.included_children.map((child) => curateEntity({
+		curatedEntity.includedChildren = entity.included_children.flatMap((child) => {
+			const curatedChild = curateEntity({
 				...child,
 				parent: curatedEntity.id ? curatedEntity : null,
-		}, includeParameters));
+			}, includeParameters);
+
+			// allow entities to 'spawn' virtual copies of themselves, this is useful for sites that use two separate update pages (i.e. Elegant Angel)
+			if (child.options?.spawn) {
+				return [
+					curatedChild,
+					...child.options.spawn.map((spawnEntity) => ({
+						...curatedChild,
+						...spawnEntity,
+					})),
+				];
+			}
+
+			return curatedChild;
+		});
 	}
 
 	const scraper = resolveScraper(curatedEntity);
@@ -199,7 +215,7 @@ async function fetchIncludedEntities() {
 	return curatedNetworks;
 }
 
-async function fetchEntitiesBySlug(entitySlugs, prefer = 'channel') {
+async function fetchEntitiesBySlug(entitySlugs, options = { prefer: 'channel' }) {
 	const entities = await knex.raw(`
 		WITH RECURSIVE entity_tree as (
 			SELECT to_jsonb(entities) as entity,
@@ -208,7 +224,7 @@ async function fetchEntitiesBySlug(entitySlugs, prefer = 'channel') {
 			FROM entities
 			WHERE (slug = ANY(:entitySlugs)
 			OR url ILIKE ANY(:entityHosts))
-			AND type IN ('channel', 'network')
+			AND type = ANY(:entityTypes)
 
 			UNION ALL
 
@@ -236,7 +252,8 @@ async function fetchEntitiesBySlug(entitySlugs, prefer = 'channel') {
 	`, {
 		entitySlugs: entitySlugs.filter((slug) => !slug.includes('.')),
 		entityHosts: entitySlugs.filter((slug) => slug.includes('.')).map((hostname) => `%${hostname}`),
-		sort: knex.raw(prefer === 'channel' ? 'asc' : 'desc'),
+		entityTypes: options.types || ['channel', 'network'],
+		sort: knex.raw(options.prefer === 'channel' ? 'asc' : 'desc'),
 	});
 
 	// channel entity will overwrite network entity
@@ -263,7 +280,7 @@ async function fetchReleaseEntities(baseReleases) {
 			.filter(Boolean),
 	));
 
-	return fetchEntitiesBySlug(entitySlugs, argv.prefer || 'network');
+	return fetchEntitiesBySlug(entitySlugs, { prefer: argv.prefer || 'network' });
 }
 
 async function fetchEntity(entityId, type) {

src/media.js

@@ -648,7 +648,12 @@ streamQueue.define('fetchStreamSource', async ({ source, tempFileTarget, hashStr
 		.format('mp4')
 		.outputOptions(['-movflags frag_keyframe+empty_moov'])
 		.on('start', (cmd) => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
-		.on('error', (error) => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
+		.on('error', (error) => {
+			logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`);
+
+			hashStream.end();
+			tempFileTarget.end();
+		})
 		.pipe();
 
 	// await pipeline(video, hashStream, tempFileTarget);

src/scrapers/adultempire.js

@@ -1,97 +1,61 @@
 'use strict';
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
+
 const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 const { feetInchesToCm, lbsToKg } = require('../utils/convert');
 
-async function getPhotos(entryId, channel) {
+function scrapeAll(scenes, channel, _options) {
-	const res = await http.get(`${channel.url}/Membership/GetScreenshots?sceneID=scene_${entryId}`);
-
-	if (res.ok) {
-		return res.body.split(/[\s,]+/).filter(Boolean);
-	}
-
-	return [];
-}
-
-function scrapeAllTour(scenes, channel) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.url = query.url('.scene-update-details, .feature-update-details', 'href', { origin: channel.url });
+		release.url = query.url('a.scene-title, a.scene-img', { origin: channel.url });
-		release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
+		release.entryId = query.attribute('article[data-scene-id]', 'data-scene-id') || new URL(release.url).pathname.match(/^\/(\d+)/)?.[1];
 
-		release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, '');
+		release.title = query.content('.scene-title')?.trim();
+		release.duration = query.duration('.scene-length');
 
-		release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY');
+		release.actors = query.content('.scene-performer-names')?.split(/[,&]/).map((actor) => actor.trim());
-		release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map((actor) => actor.trim());
 
-		const poster = query.img('.scene-img-wrapper img');
+		release.poster = query.sourceSet('.screenshot', 'data-srcset');
-		release.poster = [
-			poster.replace(/\/res\/\d+/, '/res/1920'),
-			poster.replace(/\/res\/\d+/, '/res/1600'),
-			poster,
-		];
 
-		release.trailer = { src: query.video('.scene-img-wrapper source') };
+		const sceneId = query.attribute('article[data-scene-id]', 'data-scene-id');
+		const masterId = query.attribute('article[data-master-id]', 'data-master-id');
+
+		if (sceneId && masterId) {
+			release.teaser = `https://video.adultempire.com/hls/previewscene/${masterId}/${sceneId}/index-f1-v1.m3u8`;
+		}
 
 		return release;
 	});
 }
 
-async function scrapeAllGrid(scenes, channel, options) {
+const photoRegex = /(\/\w\/\d+\/)\d+/;
-	return Promise.all(scenes.map(async ({ query, el }) => {
-		const release = {};
-		const uri = query.url('.grid-item-title') || query.url('a.animated-screen');
 
-		release.entryId = el.id.match(/\d+/)?.[0] || uri.match(/^(\d+)\//)?.[1];
+async function scrapeRelease({ query, html, element }, { url, entity, baseRelease, parameters }) {
-
-		release.title = query.cnt('.grid-item-title');
-		release.url = qu.prefixUrl(uri, channel.url);
-
-		release.poster = query.img('.screenshot');
-
-		if (options.includePhotos) {
-			release.photos = await getPhotos(release.entryId, channel);
-		}
-
-		return release;
-	}));
-}
-
-function scrapeMovieScenes(scenes) {
-	return scenes.map(({ query }) => {
-		const release = {};
-
-		release.title = query.cnt('.scene-title a');
-		release.url = query.url('.scene-title a', 'href', { origin: 'https://www.elegantangel.com' });
-		release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
-
-		release.duration = query.number('.scene-length') * 60;
-		release.actors = query.cnts('.scene-cast-list a');
-
-		release.poster = query.img('a img');
-
-		return release;
-	});
-}
-
-async function scrapeRelease({ query, html }, url, channel, baseRelease, options) {
 	const release = {};
 	const type = query.exists('.scene-list-header') ? 'movie' : 'scene';
 
 	release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1];
 
-	release.title = query.cnt('.scene-page .description, .video-page .description');
+	const title = query.content('.scene-page .description, .video-page .description');
+
+	if (/^scene \d+$/i.test(title)) {
+		release.sceneIndex = unprint.extractNumber(title);
+	} else {
+		release.title = title;
+	}
+
 	release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w{3} \d{2}, \d{4}/);
+	release.duration = query.duration('.release-date:last-child');
 
 	release.actors = query.all('.video-performer').map((el) => {
-		const avatar = qu.query.img(el, 'img', 'data-bgsrc');
+		const avatar = unprint.query.img(el, 'img', 'data-bgsrc');
 
 		return {
-			name: qu.query.cnt(el, 'span'),
+			name: unprint.query.content(el, 'span').trim(),
-			url: qu.query.url(el, 'a', 'href', { origin: channel.url }),
+			url: unprint.query.url(el, 'a', { origin: entity.url }),
 			avatar: [
 				avatar.replace(/\/actor\/\d+/, '/actor/1600'),
 				avatar,
@@ -99,8 +63,8 @@ async function scrapeRelease({ query, html }, url, channel, baseRelease, options
 		};
 	});
 
-	release.tags = query.cnts('.tags a, .categories a');
+	release.tags = query.contents('.tags a, .categories a');
-	release.studio = options?.parameters.studio === false ? null : slugify(query.cnt('.studio span:last-child'), '');
+	release.studio = parameters?.studio === false ? null : slugify(query.content('.studio span:last-child, .studio a'), '');
 
 	if (type === 'scene') {
 		release.director = query.text('.director');
@@ -109,87 +73,44 @@ async function scrapeRelease({ query, html }, url, channel, baseRelease, options
 	}
 
 	if (type === 'movie') {
-		release.director = query.cnt('.director a');
+		release.director = query.content('.director a');
-		release.covers = query.imgs('.carousel-item > img');
+		release.covers = [query.sourceSet('.carousel-item .boxcover-image', 'data-srcset')];
 
-		release.scenes = scrapeMovieScenes(qu.initAll(query.all('#scenes .grid-item')), channel);
+		release.scenes = scrapeAll(unprint.initAll(element, '#scenes .grid-item'), entity);
 	}
 
 	if (query.exists('.video-title .movie-title')) {
 		release.movie = {
-			title: query.cnt('#viewLargeBoxcover .modal-title a'),
+			title: query.content('#viewLargeBoxcover .modal-title a'),
-			url: query.url('#viewLargeBoxcover .modal-title a', 'href', { origin: channel.url }),
+			url: query.url('#viewLargeBoxcover .modal-title a', 'href', { origin: entity.url }),
 			entryId: query.url('#viewLargeBoxcover .modal-title a')?.match(/(\d+)\//)[1],
 			covers: query.imgs('#viewLargeBoxcover #viewLargeBoxcoverCarousel .carousel-item > img'),
 		};
 	}
 
-	release.photos = query.imgs('#dv_frames a > img').map((photo) => [
+	release.caps = query.imgs('#dv_frames a > img', { attribute: 'data-src' }).map((photo) => [
-		photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`),
+		photo.replace(photoRegex, (match, path) => `${path}1920`),
-		photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`),
+		photo.replace(photoRegex, (match, path) => `${path}1280`),
 		photo,
 	]);
 
 	const trailerId = html.match(/item: (\d+),/)?.[1];
 
 	if (trailerId) {
-		const trailerUrl = `https://www.adultempire.com/videoEmbed/${trailerId}?type=preview`;
+		release.trailer = `https://trailer.adultempire.com/hls/trailer/${trailerId}/master.m3u8`;
-		const trailerRes = await qu.get(trailerUrl);
-
-		if (trailerRes.ok) {
-			const stream = trailerRes.item.query.video();
-
-			release.trailer = { stream };
 	}
+
+	if (query.exists('.user-actions .btn-4k')) {
+		release.qualities = [2160];
 	}
 
 	return release;
 }
 
-function scrapeMovies(movies, channel) {
+async function scrapeProfile({ query }) {
-	return movies.map(({ query }) => {
-		const release = {};
-
-		release.url = query.url('.boxcover', 'href', { origin: channel.url });
-		release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
-
-		release.title = query.cnt('span');
-
-		const cover = query.img('picture img');
-
-		release.covers = [
-			// filename is ignored, back-cover has suffix after media ID
-			cover.replace('_sq.jpg', '/front.jpg').replace(/\/product\/\d+/, '/product/500'),
-			cover.replace('_sq.jpg', 'b/back.jpg').replace(/\/product\/\d+/, '/product/500'),
-		];
-
-		return release;
-	});
-}
-
-function scrapeActorScenes(scenes, channel) {
-	return scenes.map(({ query }) => {
-		const release = {};
-
-		release.url = query.url('a', 'href', { origin: channel.url });
-		release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
-
-		release.title = query.cnt('.grid-item-title');
-
-		const poster = query.img('a img');
-		release.poster = [
-			poster.replace(/\/\d+\//, '/1600/'),
-			poster,
-		];
-
-		return release;
-	});
-}
-
-async function scrapeProfile({ query }, url, channel, include) {
 	const profile = {};
 
-	const bio = query.cnts('.performer-page-header li').reduce((acc, info) => {
+	const bio = query.contents('#profileModal .well li').reduce((acc, info) => {
 		const [key, value] = info.split(':');
 
 		return {
@@ -198,11 +119,14 @@ async function scrapeProfile({ query }, url, channel, include) {
 		};
 	}, {});
 
-	const measurements = bio.meas?.match(/(\d+)(\w+)-(\d+)-(\d+)/);
+	const bioText = query.content('#profileModal .well');
 
-	if (measurements) {
+	profile.description = query.content('#profileModal .modal-body')
-		[profile.bust, profile.cup, profile.waist, profile.hip] = measurements.slice(1);
+		.slice(bioText.length)
-	}
+		.replace(/Biography Text ©Adult DVD Empire/i, '')
+		.trim();
+
+	profile.measurements = bio.measurements?.replace(/["\s]+/g, '');
 
 	profile.hair = bio.hair;
 	profile.eyes = bio.eyes;
@@ -211,79 +135,41 @@ async function scrapeProfile({ query }, url, channel, include) {
 	profile.height = feetInchesToCm(bio.height);
 	profile.weight = lbsToKg(bio.weight);
 
-	profile.avatar = query.img('picture img');
+	const avatar = query.img('picture img, .performer-image-container img');
 
-	if (include) {
+	if (avatar) {
-		const actorId = new URL(url).pathname.match(/\/(\d+)/)[1];
+		profile.avatar = [
-		const res = await qu.getAll(`${channel.url}/www.elegantangel.com/streaming-video-by-scene.html?cast=${actorId}`, '.grid-item', null, {
+			avatar
-			rejectUnauthorized: false,
+				.replace('_bust', '_body')
-		});
+				.replace(/\/actor\/\d+\//i, '/actor/1000/'),
-
+			avatar,
-		if (res.ok) {
+		];
-			profile.releases = scrapeActorScenes(res.items, channel);
-		}
 	}
 
 	return profile;
 }
 
-async function fetchLatestTour(channel, page = 1) {
+async function fetchLatest(channel, page, options) {
-	const url = `${channel.url}/tour?page=${page}`;
+	// const res = await qu.getAll(`${channel.url}/watch-newest-clips-and-scenes.html?page=${page}&hybridview=member`, '.item-grid-scene .grid-item');
-	const res = await qu.getAll(url, '.scene-update', null, {
+	const res = await unprint.get(options.parameters?.latest
-		// invalid certificate
+		? `${options.parameters.latest}?page=${page}&view=grid`
-		rejectUnauthorized: false,
+		: `${channel.url}/watch-newest-clips-and-scenes.html?page=${page}&view=grid`, { selectAll: '.item-grid-scene .grid-item' });
-	});
 
 	if (res.ok) {
-		return scrapeAllTour(res.items, channel);
+		return scrapeAll(res.context, channel, options);
 	}
 
 	return res.status;
 }
 
-async function fetchLatestGrid(channel, page, options) {
+async function fetchProfilePage(actorUrl) {
-	const res = await qu.getAll(`${channel.url}/watch-newest-clips-and-scenes.html?page=${page}&hybridview=member`, '.item-grid-scene .grid-item');
+	const res = await unprint.get(actorUrl, {
-
+		select: '#content',
-	if (res.ok) {
-		return scrapeAllGrid(res.items, channel, options);
-	}
-
-	return res.status;
-}
-
-async function fetchMovie(url, channel, baseRelease, options) {
-	const res = await qu.get(url, null, null, {
-		// invalid certificate
 		rejectUnauthorized: false,
 	});
 
 	if (res.ok) {
-		return scrapeRelease(res.item, url, channel, baseRelease, options);
+		return scrapeProfile(res.context);
-	}
-
-	return res.status;
-}
-
-async function fetchMovies(channel, page = 1) {
-	const res = await qu.getAll(`https://www.elegantangel.com/streaming-elegant-angel-dvds-on-video.html?page=${page}`, '.grid-item', null, {
-		// invalid certificate
-		rejectUnauthorized: false,
-	});
-
-	if (res.ok) {
-		return scrapeMovies(res.items, channel);
-	}
-
-	return res.status;
-}
-
-async function fetchProfilePage(actorUrl, channel, include) {
-	const res = await qu.get(actorUrl, '.performer-page', null, {
-		rejectUnauthorized: false,
-	});
-
-	if (res.ok) {
-		return scrapeProfile(res.item, actorUrl, channel, include);
 	}
 
 	return res.status;
@@ -298,13 +184,15 @@ async function fetchProfile(baseActor, channel, include) {
 		}
 	}
 
-	const searchRes = await http.get(`${channel.url}/search/SearchAutoComplete_Agg_ByMedia?rows=9&name_startsWith=${slugify(baseActor.name, '+')}`);
+	const searchRes = await http.get(`https://www.adultempire.com/search/SearchAutoComplete_Agg_EmpireDTRank?search_type=Pornstars&rows=9&name_startsWith=${slugify(baseActor.name, '+')}`);
 
-	if (searchRes.ok) {
+	if (searchRes.ok && searchRes.body.Results) {
 		const actorResult = searchRes.body.Results.find((result) => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description));
 
 		if (actorResult) {
-			return fetchProfilePage(`${channel.url}${actorResult.BasicResponseGroup.id}`, channel, include);
+			const url = `https://www.adultempire.com/${actorResult.BasicResponseGroup.id}`;
+
+			return fetchProfilePage(url);
 		}
 
 		return null;
@@ -314,16 +202,15 @@ async function fetchProfile(baseActor, channel, include) {
 }
 
 module.exports = {
-	fetchLatest: fetchLatestTour,
+	fetchLatest,
-	fetchMovies,
+	// fetchMovies,
-	fetchMovie,
-	fetchProfile,
-	scrapeScene: scrapeRelease,
-	scrapeMovie: scrapeRelease,
-	grid: {
-		fetchLatest: fetchLatestGrid,
-		scrapeScene: scrapeRelease,
-		fetchMovie,
 	fetchProfile,
+	scrapeScene: {
+		scraper: scrapeRelease,
+		unprint: true,
+	},
+	scrapeMovie: {
+		scraper: scrapeRelease,
+		unprint: true,
 	},
 };

src/scrapers/resolve.js

@@ -3,6 +3,10 @@
 const scrapers = require('./scrapers');
 
 function resolveScraper(entity) {
+	if (entity.parameters?.useScraper && scrapers.releases[entity.parameters.useScraper]) {
+		return scrapers.releases[entity.parameters.useScraper];
+	}
+
 	if (scrapers.releases[entity.slug]) {
 		return scrapers.releases[entity.slug];
 	}

src/scrapers/scrapers.js

@@ -177,6 +177,7 @@ const scrapers = {
 	actors: {
 		'18vr': badoink,
 		'21sextury': gamma,
+		adultempire,
 		allanal: mikeadriano,
 		amateureuro: porndoe,
 		americanpornstar,
@@ -217,7 +218,6 @@ const scrapers = {
 		dorcelclub: dorcel,
 		doubleviewcasting: firstanalquest,
 		dtfsluts: fullpornnetwork,
-		elegantangel: adultempire,
 		evilangel: gamma,
 		exploitedcollegegirls: elevatedx,
 		eyeontheguy: hush,
@@ -323,7 +323,6 @@ const scrapers = {
 		vixen,
 		vrcosplayx: badoink,
 		wankzvr,
-		westcoastproductions: adultempire,
 		wicked: gamma,
 		wildoncam: cherrypimps,
 		xempire: gamma,

src/store-releases.js

@@ -288,7 +288,11 @@ async function associateMovieScenes(movies, movieScenes) {
 		},
 	}), {});
 
-	const associations = movieScenes.map((scene) => {
+	const associations = movieScenes
+		.toSorted((sceneA, sceneB) => {
+			return (sceneA.sceneIndex || 1) - (sceneB.sceneIndex || 1);
+		})
+		.map((scene) => {
 			if (!scene.movie) {
 				return null;
 			}
@@ -304,7 +308,8 @@ async function associateMovieScenes(movies, movieScenes) {
 			}
 
 			return null;
-	}).filter(Boolean);
+		})
+		.filter(Boolean);
 
 	await bulkInsert('movies_scenes', associations, false);
 }
@@ -354,6 +359,7 @@ async function storeMovies(movies, useBatchId) {
 
 	await updateMovieSearch(moviesWithId.map((movie) => movie.id));
 	await associateReleaseMedia(moviesWithId, 'movie');
+	await associateReleaseTags(moviesWithId, 'movie');
 
 	return moviesWithId;
 }

src/updates.js

@@ -298,6 +298,8 @@ async function scrapeNetworkParallel(networkEntity) {
 async function fetchUpdates() {
 	const includedNetworks = await fetchIncludedEntities();
 
+	// console.log(includedNetworks[0]);
+
 	const scrapedNetworks = await Promise.map(
 		includedNetworks,
 		async (networkEntity) => (networkEntity.parameters?.sequential