Added m3u8 stream support to media module. Added Elegant Angel. Added regex parameter to qu's number method. Various tags.

assets/components/tags/tags.vue

@@ -52,6 +52,7 @@ async function mounted() {
 			'ebony',
 			'latina',
 			'caucasian',
+			'indian',
 			'natural-boobs',
 			'fake-boobs',
 			'blonde',

package-lock.json

@@ -4614,6 +4614,15 @@
             "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==",
             "dev": true
         },
+        "fluent-ffmpeg": {
+            "version": "2.1.2",
+            "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
+            "integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=",
+            "requires": {
+                "async": ">=0.2.9",
+                "which": "^1.1.1"
+            }
+        },
         "flush-write-stream": {
             "version": "1.1.1",
             "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz",
@@ -5721,28 +5730,12 @@
                 "minimalistic-assert": "^1.0.1"
             }
         },
-        "hashish": {
-            "version": "0.0.4",
-            "resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz",
-            "integrity": "sha1-bWC8b/r3Ebav1g5CbQd5iAFOZVQ=",
-            "requires": {
-                "traverse": ">=0.2.4"
-            }
-        },
         "he": {
             "version": "1.2.0",
             "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
             "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
             "dev": true
         },
-        "hh-mm-ss": {
-            "version": "1.2.0",
-            "resolved": "https://registry.npmjs.org/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz",
-            "integrity": "sha512-f4I9Hz1dLpX/3mrEs7yq30+FiuO3tt5NWAqAGeBTaoeoBfB8vhcQ3BphuDc5DjZb/K809agqrAaFlP0jhEU/8w==",
-            "requires": {
-                "zero-fill": "^2.2.3"
-            }
-        },
         "hmac-drbg": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
@@ -5845,11 +5838,6 @@
                 "debug": "^3.1.0"
             }
         },
-        "human-signals": {
-            "version": "1.1.1",
-            "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
-            "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw=="
-        },
         "iconv-lite": {
             "version": "0.5.1",
             "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.1.tgz",
@@ -7168,11 +7156,6 @@
                 }
             }
         },
-        "merge-stream": {
-            "version": "2.0.0",
-            "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
-            "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
-        },
         "mersenne-twister": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz",
@@ -10456,14 +10439,6 @@
                 "mixme": "^0.3.1"
             }
         },
-        "streamify": {
-            "version": "0.2.9",
-            "resolved": "https://registry.npmjs.org/streamify/-/streamify-0.2.9.tgz",
-            "integrity": "sha512-8pUxeLEef9UO1FxtTt5iikAiyzGI4SZRnGuJ3sz8axZ5Xk+/7ezEV5kuJQsMEFxw7AKYw3xp0Ow+20mmSaJbQQ==",
-            "requires": {
-                "hashish": "~0.0.4"
-            }
-        },
         "strict-uri-encode": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
@@ -10542,11 +10517,6 @@
             "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
             "dev": true
         },
-        "strip-final-newline": {
-            "version": "2.0.0",
-            "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
-            "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="
-        },
         "strip-indent": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz",
@@ -10966,11 +10936,6 @@
                 "punycode": "^2.1.1"
             }
         },
-        "traverse": {
-            "version": "0.6.6",
-            "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz",
-            "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc="
-        },
         "trim-newlines": {
             "version": "1.0.0",
             "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
@@ -12110,127 +12075,6 @@
                 "camelcase": "^5.0.0",
                 "decamelize": "^1.2.0"
             }
-        },
-        "youtube-dl": {
-            "version": "3.0.2",
-            "resolved": "https://registry.npmjs.org/youtube-dl/-/youtube-dl-3.0.2.tgz",
-            "integrity": "sha512-LFFfpsYbRLpqKsnb4gzbnyN7fm190tJw3gJVSvfoEfnb/xYIPNT6i9G3jdzPDp/U5cwB3OSq63nUa7rUwxXAGA==",
-            "requires": {
-                "debug": "~4.1.1",
-                "execa": "~3.2.0",
-                "hh-mm-ss": "~1.2.0",
-                "mkdirp": "~0.5.1",
-                "request": "~2.88.0",
-                "streamify": "~0.2.9",
-                "universalify": "~0.1.2"
-            },
-            "dependencies": {
-                "cross-spawn": {
-                    "version": "7.0.3",
-                    "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-                    "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
-                    "requires": {
-                        "path-key": "^3.1.0",
-                        "shebang-command": "^2.0.0",
-                        "which": "^2.0.1"
-                    }
-                },
-                "debug": {
-                    "version": "4.1.1",
-                    "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-                    "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
-                    "requires": {
-                        "ms": "^2.1.1"
-                    }
-                },
-                "execa": {
-                    "version": "3.2.0",
-                    "resolved": "https://registry.npmjs.org/execa/-/execa-3.2.0.tgz",
-                    "integrity": "sha512-kJJfVbI/lZE1PZYDI5VPxp8zXPO9rtxOkhpZ0jMKha56AI9y2gGVC6bkukStQf0ka5Rh15BA5m7cCCH4jmHqkw==",
-                    "requires": {
-                        "cross-spawn": "^7.0.0",
-                        "get-stream": "^5.0.0",
-                        "human-signals": "^1.1.1",
-                        "is-stream": "^2.0.0",
-                        "merge-stream": "^2.0.0",
-                        "npm-run-path": "^4.0.0",
-                        "onetime": "^5.1.0",
-                        "p-finally": "^2.0.0",
-                        "signal-exit": "^3.0.2",
-                        "strip-final-newline": "^2.0.0"
-                    }
-                },
-                "get-stream": {
-                    "version": "5.1.0",
-                    "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
-                    "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
-                    "requires": {
-                        "pump": "^3.0.0"
-                    }
-                },
-                "is-stream": {
-                    "version": "2.0.0",
-                    "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
-                    "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw=="
-                },
-                "mimic-fn": {
-                    "version": "2.1.0",
-                    "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
-                    "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="
-                },
-                "npm-run-path": {
-                    "version": "4.0.1",
-                    "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
-                    "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
-                    "requires": {
-                        "path-key": "^3.0.0"
-                    }
-                },
-                "onetime": {
-                    "version": "5.1.0",
-                    "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz",
-                    "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==",
-                    "requires": {
-                        "mimic-fn": "^2.1.0"
-                    }
-                },
-                "p-finally": {
-                    "version": "2.0.1",
-                    "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz",
-                    "integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw=="
-                },
-                "path-key": {
-                    "version": "3.1.1",
-                    "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
-                    "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="
-                },
-                "shebang-command": {
-                    "version": "2.0.0",
-                    "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
-                    "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
-                    "requires": {
-                        "shebang-regex": "^3.0.0"
-                    }
-                },
-                "shebang-regex": {
-                    "version": "3.0.0",
-                    "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
-                    "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
-                },
-                "which": {
-                    "version": "2.0.2",
-                    "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
-                    "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
-                    "requires": {
-                        "isexe": "^2.0.0"
-                    }
-                }
-            }
-        },
-        "zero-fill": {
-            "version": "2.2.3",
-            "resolved": "https://registry.npmjs.org/zero-fill/-/zero-fill-2.2.3.tgz",
-            "integrity": "sha1-o97wa6XjmuZEhQu0yirUEStIVek="
         }
     }
 }

package.json

@@ -91,6 +91,7 @@
         "express-react-views": "^0.11.0",
         "face-api.js": "^0.21.0",
         "file-type": "^14.1.4",
+        "fluent-ffmpeg": "^2.1.2",
         "fs-extra": "^7.0.1",
         "graphile-utils": "^4.5.6",
         "iconv-lite": "^0.5.1",
@@ -124,7 +125,6 @@
         "vuex": "^3.1.2",
         "winston": "^3.2.1",
         "winston-daily-rotate-file": "^4.4.2",
-        "yargs": "^13.3.0",
+        "yargs": "^13.3.0"
-        "youtube-dl": "^3.0.2"
     }
 }

seeds/00_tags.js

@@ -514,6 +514,10 @@ const tags = [
 		name: 'humiliation',
 		slug: 'humiliation',
 	},
+	{
+		name: 'indian',
+		slug: 'indian',
+	},
 	{
 		name: 'innie pussy',
 		slug: 'innie-pussy',

seeds/04_media.js

@@ -594,7 +594,7 @@ const tagPosters = [
 	['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
 	['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'],
 	['behind-the-scenes', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'],
-	['blonde', 0, 'Anikka Albrite for Digital Playground'],
+	['blonde', 0, 'Anikka Albrite in "Black Owned 4" for Jules Jordan'],
 	['blowbang', 0, 'Lacy Lennon in "Lacy Lennon\'s First Blowbang" for HardX'],
 	['blowjob', 0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
 	['brunette', 0, 'Nicole Black in GIO971 for LegalPorno'],
@@ -615,11 +615,12 @@ const tagPosters = [
 	['ebony', 1, 'Ana Foxxx in "DP Me 4" for HardX'],
 	['facefucking', 2, 'Jynx Maze for Throated'],
 	['facial', 0, 'Brooklyn Gray in "All About Ass 4" for Evil Angel'],
-	['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
+	['fake-boobs', 2, 'Gia Milana in "Hot Anal Latina" for HardX'],
 	['family', 0, 'Teanna Trump in "A Family Appear: Part One" for Brazzers'],
 	['femdom', 0, 'Alina Li in "Asian Domination… She Holds Jules Jordan\'s Cock Hostage!" for Jules Jordan'],
 	['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9" for Jules Jordan'],
 	['gaping', 1, 'Vina Sky in "Vina Sky Does Anal" for HardX'],
+	['indian', 0, 'Resha in "Casting Resha" for Watch 4 Beauty'],
 	['interracial', 0, 'Jaye Summers and Prince Yahshua in "Platinum Pussy 3" for Jules Jordan'],
 	['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'],
 	['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'],
@@ -699,6 +700,8 @@ const tagPhotos = [
 	['facial', 'poster', 'Jynx Maze'],
 	['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'],
 	['facefucking', 1, 'Carrie for Young Throats'],
+	['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
+	['fake-boobs', 3, 'Ashly Anderson for Passion HD'],
 	// ['fake-boobs', 0, 'Marsha May in "Once You Go Black 7" for Jules Jordan'],
 	['gangbang', 'poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'],
 	['gangbang', 0, '"4 On 1 Gangbangs" for Doghouse Digital'],

src/actors.js

@@ -134,7 +134,7 @@ function toBaseActors(actorsOrNames, release) {
 	});
 }
 
-function curateActor(actor, withDetails = false) {
+function curateActor(actor, withDetails = false, isProfile = false) {
 	if (!actor) {
 		return null;
 	}
@@ -174,7 +174,7 @@ function curateActor(actor, withDetails = false) {
 			hasPiercings: actor.has_piercings,
 			tattoos: actor.tattoos,
 			piercings: actor.piercings,
-			description: actor.description,
+			...(isProfile && { description: actor.description }),
 			placeOfBirth: actor.birth_country && {
 				country: {
 					alpha2: actor.birth_country.alpha2,
@@ -201,6 +201,7 @@ function curateActor(actor, withDetails = false) {
 				size: actor.avatar.size,
 				source: actor.avatar.source,
 			},
+			...(actor.profiles && { profiles: actor.profiles?.map(profile => curateActor(profile, true, true)) }),
 		}),
 	};
 
@@ -504,12 +505,14 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
 			return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
 				try {
 					const scraper = scrapers[scraperSlug];
+					const entity = entitiesBySlug[scraperSlug] || null;
+
 					const context = {
-						...entitiesBySlug[scraperSlug],
+						...entity,
 						// legacy
-						site: entitiesBySlug[scraperSlug] || null,
+						site: entity,
-						network: entitiesBySlug[scraperSlug] || null,
+						network: entity?.parent,
-						entity: entitiesBySlug[scraperSlug] || null,
+						entity,
 						scraper: scraperSlug,
 					};
 
@@ -547,7 +550,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
 					return await curateProfile({
 						...actor,
 						...profile,
-						...context,
+						entity,
 						update: existingProfile?.id || false,
 					});
 				} catch (error) {
@@ -715,7 +718,8 @@ async function fetchActor(actorId) {
 			row_to_json(actor_alias) as alias,
 			row_to_json(birth_country) as birth_country,
 			row_to_json(residence_country) as residence_country,
-			row_to_json(media) as avatar
+			row_to_json(media) as avatar,
+			json_agg(actors_profiles) as profiles
 		`))
 		.modify((queryBuilder) => {
 			if (Number.isNaN(Number(actorId))) {
@@ -726,10 +730,12 @@ async function fetchActor(actorId) {
 			queryBuilder.where('actors.id', actorId);
 		})
 		.leftJoin('actors as actor_alias', 'actor_alias.id', 'actors.alias_for')
+		.leftJoin('actors_profiles', 'actors.id', 'actors_profiles.actor_id')
 		.leftJoin('entities', 'entities.id', 'actors.entity_id')
 		.leftJoin('countries as birth_country', 'birth_country.alpha2', 'actors.birth_country_alpha2')
 		.leftJoin('countries as residence_country', 'residence_country.alpha2', 'actors.residence_country_alpha2')
 		.leftJoin('media', 'media.id', 'actors.avatar_media_id')
+		.groupBy('actors.id', 'entities.id', 'actor_alias.id', 'birth_country.alpha2', 'residence_country.alpha2', 'media.id')
 		.first();
 
 	return curateActor(actor, true);

src/media.js

@@ -10,7 +10,7 @@ const stream = require('stream');
 const nanoid = require('nanoid/non-secure');
 const mime = require('mime');
 // const fileType = require('file-type');
-const youtubeDl = require('youtube-dl');
+const ffmpeg = require('fluent-ffmpeg');
 const sharp = require('sharp');
 const blake2 = require('blake2');
 
@@ -418,22 +418,21 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
 	};
 }
 
-async function fetchStreamSource(source, tempFileTarget, hashStream) {
+async function fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream) {
-	const video = youtubeDl(source.stream);
+	const meta = { mimetype: 'video/mp4' };
 
-	video.on('info', (info) => {
+	const video = ffmpeg(source.stream)
-		console.log(info);
+		.format('mp4')
-		logger.verbose(`Starting fetching stream from ${source.stream}`);
+		.outputOptions(['-movflags frag_keyframe+empty_moov'])
-	});
+		.on('start', cmd => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
-
+		.on('error', error => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
-	video.on('end', (info) => {
+		.pipe();
-		console.log(info);
-		logger.verbose(`Finished fetching stream from ${source.stream}`);
-	});
 
 	await pipeline(video, hashStream, tempFileTarget);
 
-	return { mimetype: null };
+	logger.verbose(`Finished fetching stream from ${source.stream}`);
+
+	return meta;
 }
 
 async function fetchSource(source, baseMedia) {
@@ -457,7 +456,7 @@ async function fetchSource(source, baseMedia) {
 			});
 
 			const { mimetype } = source.stream
-				? await fetchStreamSource(source, tempFileTarget, hashStream)
+				? await fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream)
 				: await fetchHttpSource(source, tempFileTarget, hashStream);
 
 			hasher.end();

src/releases.js

@@ -7,8 +7,6 @@ function curateRelease(release, withMedia = false) {
 		return null;
 	}
 
-	const network = release.site_network || release.network;
-
 	return {
 		id: release.id,
 		entryId: release.entry_id,
@@ -18,22 +16,22 @@ function curateRelease(release, withMedia = false) {
 		date: release.date,
 		description: release.description,
 		duration: release.duration,
-		site: release.site && {
+		entity: release.entity && {
-			id: release.site.id,
+			id: release.entity.id,
-			name: release.site.name,
+			name: release.entity.name,
-			slug: release.site.slug,
+			slug: release.entity.slug,
-		},
+			parent: release.entity.parent && {
-		network: network && {
+				id: release.entity.parent.id,
-			id: network.id,
+				name: release.entity.parent.name,
-			name: network.name,
+				slug: release.entity.parent.slug,
-			slug: network.slug,
+			},
 		},
 		actors: (release.actors || []).map(actor => ({
 			id: actor.id,
 			name: actor.name,
 			slug: actor.slug,
 			gender: actor.gender,
-			networkId: actor.network_id,
+			entityId: actor.entity_id,
 			aliasFor: actor.alias_for,
 		})),
 		tags: (release.tags || []).map(tag => ({
@@ -67,23 +65,21 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') {
 	queryBuilder
 		.select(knex.raw(`
 			releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
-			row_to_json(sites) as site,
+			row_to_json(entities) as entity,
-			row_to_json(networks) as network,
+			row_to_json(parents) as parent,
-			row_to_json(site_networks) as site_network,
 			COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
 			COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
 		`))
-		.where('type', type)
+		.where('releases.type', type)
-		.leftJoin('sites', 'sites.id', 'releases.site_id')
+		.leftJoin('entities', 'entities.id', 'releases.entity_id')
-		.leftJoin('networks', 'networks.id', 'releases.network_id')
+		.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
-		.leftJoin('networks as site_networks', 'site_networks.id', 'sites.network_id')
 		.leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id')
 		.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
 		.leftJoin('releases_tags', 'releases_tags.release_id', 'releases.id')
 		.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
 		.groupBy(knex.raw(`
 			releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
-			sites.id, networks.id, site_networks.id
+			entities.id, parents.id
 		`));
 
 	if (withMedia) {

src/scrapers/elegantangel.js

@@ -0,0 +1,136 @@
+'use strict';
+
+const qu = require('../utils/q');
+const slugify = require('../utils/slugify');
+
+function scrapeAll(scenes, channel) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.url = query.url('.scene-update-details, .feature-update-details', 'href', { origin: channel.url });
+		release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
+
+		release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, '');
+
+		release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY');
+		release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map(actor => actor.trim());
+
+		const poster = query.img('.scene-img-wrapper img');
+		release.poster = [
+			poster.replace(/\/res\/\d+/, '/res/1920'),
+			poster.replace(/\/res\/\d+/, '/res/1600'),
+			poster,
+		];
+
+		release.trailer = { src: query.video('.scene-img-wrapper source') };
+
+		return release;
+	});
+}
+
+async function scrapeScene({ query, html }, url) {
+	const release = {};
+
+	release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1];
+
+	release.title = query.cnt('.scene-page .description');
+	release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w{3} \d{2}, \d{4}/);
+	release.duration = query.number('.release-date:last-child') * 60;
+
+	release.actors = query.all('.video-performer').map((el) => {
+		const avatar = qu.query.img(el, 'img', 'data-bgsrc');
+
+		return {
+			name: qu.query.cnt(el, 'span'),
+			avatar: [
+				avatar.replace(/\/actor\/\d+/, '/actor/1600'),
+				avatar,
+			],
+		};
+	});
+
+	release.tags = query.cnts('.tags a');
+	release.poster = query.url('link[rel="image_src"]') || query.meta('property="og:image"');
+
+	release.photos = query.imgs('#dv_frames a > img').map(photo => [
+		photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`),
+		photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`),
+		photo,
+	]);
+
+	const trailerId = html.match(/item: (\d+),/)?.[1];
+
+	if (trailerId) {
+		const trailerUrl = `https://www.adultempire.com/videoEmbed/${trailerId}?type=preview`;
+		const trailerRes = await qu.get(trailerUrl);
+
+		if (trailerRes.ok) {
+			const stream = trailerRes.item.query.video();
+
+			release.trailer = { stream };
+		}
+	}
+
+	// console.log(release);
+	return release;
+}
+
+function scrapeProfile({ query, el }, actorName, entity, include) {
+	const profile = {};
+
+	profile.description = query.cnt('.bio-text');
+	profile.birthPlace = query.cnt('.birth-place span');
+
+	profile.avatar = query.img('.actor-photo img');
+
+	if (include.releases) {
+		return scrapeAll(qu.initAll(el, '.scene'));
+	}
+
+	console.log(profile);
+	return profile;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const url = `${channel.url}/tour?page=${page}`;
+	const res = await qu.getAll(url, '.scene-update', null, {
+		// invalid certificate
+		rejectUnauthorized: false,
+	});
+
+	if (res.ok) {
+		return scrapeAll(res.items, channel);
+	}
+
+	return res.status;
+}
+
+async function fetchScene(url, channel) {
+	const res = await qu.get(url, null, null, {
+		// invalid certificate
+		rejectUnauthorized: false,
+	});
+
+	if (res.ok) {
+		return scrapeScene(res.item, url, channel);
+	}
+
+	return res.status;
+}
+
+async function fetchProfile(actorName, entity, include) {
+	const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		return scrapeProfile(res.item, actorName, entity, include);
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchScene,
+	fetchProfile,
+};

src/scrapers/template.js

@@ -3,13 +3,12 @@
 const qu = require('../utils/q');
 const slugify = require('../utils/slugify');
 
-function scrapeAll(scenes, site) {
+function scrapeAll(scenes) {
 	return scenes.map(({ query }) => {
 		const release = {};
-		const pathname = query.url('.title a');
 
-		release.entryId = pathname.match(/\/scene\/(\d+)/)[1];
+		release.url = query.url('.title a');
-		release.url = `${site.url}${pathname}`;
+		release.entryId = new URL(release.url).pathname.match(/\/scene\/(\d+)/)[1];
 
 		release.title = query.cnt('.title a');
 		release.description = query.cnt('.description');
@@ -17,7 +16,8 @@ function scrapeAll(scenes, site) {
 		release.date = query.date('.date', 'MMM DD, YYYY');
 		release.actors = query.cnts('.models a.model');
 
-		release.poster = query.q('img.poster');
+		release.poster = query.img('img.poster');
+		release.teaser = { src: query.video('.teaser video') };
 
 		release.stars = query.number('.rating');
 		release.likes = query.number('.likes');
@@ -27,16 +27,16 @@ function scrapeAll(scenes, site) {
 	});
 }
 
-function scrapeScene({ query }) {
+function scrapeScene({ query }, url) {
 	const release = {};
 
+	release.entryId = new URL(url).pathname.match(/\/scene\/(\d+)/)[1];
+
 	release.title = query.cnt('h3.title');
 	release.description = query.cnt('p.description');
 
 	[release.poster, ...release.photos] = query.imgs('.preview-thumb');
-
+	release.trailer = { src: query.video('.trailer video') };
-	const trailer = query.video('.trailer video');
-	release.trailer = { src: trailer };
 
 	console.log(release);
 	return release;
@@ -73,7 +73,7 @@ async function fetchScene(url, channel) {
 	const res = await qu.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, channel);
+		return scrapeScene(res.item, url, channel);
 	}
 
 	return res.status;

src/utils/qu.js

@@ -140,10 +140,18 @@ function styles(context, selector, styleAttr) {
 	return elStyles;
 }
 
-function number(context, selector, attr = true) {
+function number(context, selector, match = /\d+/, attr = 'textContent') {
 	const value = q(context, selector, attr);
 
-	return value ? Number(value) : null;
+	if (value && match) {
+		return Number(value.match(match)?.[0]);
+	}
+
+	if (value) {
+		return Number(value);
+	}
+
+	return null;
 }
 
 function meta(context, selector, attrArg = 'content', applyTrim = true) {
@@ -280,6 +288,8 @@ const quFuncs = {
 	date,
 	dur: duration,
 	duration,
+	element: q,
+	el: q,
 	exists,
 	image,
 	images,