Added S3 support for media files. Fixed MindGeek scraper for new poster data structure.

assets/components/actors/actor.vue

@@ -42,13 +42,13 @@
 			>
 				<a
 					v-if="actor.avatar"
-					:href="`/media/${actor.avatar.path}`"
+					:href="getPath(actor.avatar)"
 					target="_blank"
 					rel="noopener noreferrer"
 					class="avatar-link"
 				>
 					<img
-						:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
+						:src="getPath(actor.avatar, 'thumbnail')"
 						:title="actor.avatar.credit && `© ${actor.avatar.credit}`"
 						class="avatar"
 					>

assets/components/actors/photos.vue

@@ -8,14 +8,14 @@
 	>
 		<a
 			v-if="actor.avatar"
-			:href="`/media/${actor.avatar.path}`"
+			:href="getPath(actor.avatar)"
 			target="_blank"
 			rel="noopener noreferrer"
 			class="avatar-link photo-link"
 		>
 			<img
-				:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
+				:src="getPath(actor.avatar, 'thumbnail')"
-				:style="{ 'background-image': sfw ? `/img/${actor.avatar.sfw.lazy}` : `/media/${actor.avatar.lazy}` }"
+				:style="{ 'background-image': getBgPath(actor.avatar, 'lazy') }"
 				:title="actor.avatar.credit && `© ${actor.avatar.credit}`"
 				loading="lazy"
 				class="avatar photo"
@@ -26,14 +26,14 @@
 		<a
 			v-for="photo in photos"
 			:key="`photo-${photo.id}`"
-			:href="`/media/${photo.path}`"
+			:href="getPath(photo)"
 			target="_blank"
 			rel="noopener noreferrer"
 			class="photo-link"
 		>
 			<img
-				:src="sfw ? `/img/${photo.sfw.thumbnail}` : `/media/${photo.thumbnail}`"
+				:src="getPath(photo, 'thumbnail')"
-				:style="{ 'background-image': sfw ? `/img/${photo.sfw.lazy}` : `/media/${photo.lazy}` }"
+				:style="{ 'background-image': getBgPath(photo, 'lazy') }"
 				:title="`© ${photo.credit || photo.entity.name}`"
 				loading="lazy"
 				class="photo"

assets/components/actors/tile.vue

@@ -45,8 +45,8 @@
 			<div class="avatar-container">
 				<img
 					v-if="actor.avatar"
-					:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
+					:src="getPath(actor.avatar, 'thumbnail')"
-					:style="{ 'background-image': sfw ? `url(/img/${actor.avatar.sfw.lazy})`: `url(/img/${actor.avatar.lazy})` }"
+					:style="{ 'background-image': getBgPath(actor.avatar, 'lazy') }"
 					loading="lazy"
 					class="avatar"
 				>

assets/components/album/album.vue

@@ -21,12 +21,12 @@
 					class="item-container"
 				>
 					<a
-						:href="`${path}/${item.path}`"
+						:href="getPath(item, null, { local })"
 						class="item-link"
 						target="_blank"
 					>
 						<img
-							:src="`${path}/${item.thumbnail}`"
+							:src="getPath(item, 'thumbnail', { local })"
 							:title="item.title"
 							loading="lazy"
 							class="item image"
@@ -63,9 +63,9 @@ export default {
 			type: String,
 			default: null,
 		},
-		path: {
+		local: {
-			type: String,
+			type: Boolean,
-			default: '/media',
+			default: false,
 		},
 		portrait: {
 			type: Boolean,

assets/components/releases/banner.vue

@@ -31,7 +31,7 @@
 
 				<img
 					v-else-if="release.teaser && /^image\//.test(release.teaser.mime)"
-					:src="sfw ? `/img/${release.teaser.sfw.thumbnail}` : `/media/${release.teaser.path}`"
+					:src="getPath(release.teaser, 'thumbnail', { original: true })"
 					:alt="release.title"
 					loading="lazy"
 					class="item trailer"
@@ -40,7 +40,7 @@
 				<a
 					v-if="release.poster"
 					v-tooltip="'View poster'"
-					:href="`/media/${release.poster.path}`"
+					:href="`${config.media.mediaPath}/${release.poster.path}`"
 					:class="{ playing }"
 					target="_blank"
 					rel="noopener noreferrer"
@@ -59,13 +59,13 @@
 				<a
 					v-for="cover in release.covers"
 					:key="`cover-${cover.id}`"
-					:href="`/media/${cover.path}`"
+					:href="`${config.media.mediaPath}/${cover.path}`"
 					target="_blank"
 					rel="noopener noreferrer"
 				>
 					<img
-						:src="`/media/${cover.thumbnail}`"
+						:src="getPath(cover, 'thumbnail')"
-						:style="{ 'background-image': sfw ? `url(/media/${cover.sfw.lazy})` : `url(/media/${cover.lazy})` }"
+						:style="{ 'background-image': getBgPath(cover, 'lazy') }"
 						class="item cover"
 						loading="lazy"
 						@load="$emit('load', $event)"
@@ -79,15 +79,15 @@
 				class="item-container"
 			>
 				<a
-					:href="`/media/${photo.path}`"
+					:href="getPath(photo)"
 					:class="{ sfw }"
 					class="item-link"
 					target="_blank"
 					rel="noopener noreferrer"
 				>
 					<img
-						:src="sfw ? `/img/${photo.sfw.thumbnail}` : `/media/${photo.thumbnail}`"
+						:src="getPath(photo, 'thumbnail')"
-						:style="{ 'background-image': sfw ? `url(/img/${photo.sfw.lazy})` : `url(/media/${photo.lazy})` }"
+						:style="{ 'background-image': getPath(photo, 'lazy') }"
 						:alt="`Photo ${photo.index + 1}`"
 						loading="lazy"
 						class="item"
@@ -115,15 +115,15 @@ function sfw() {
 
 function poster() {
 	if (this.release.poster) {
-		return this.sfw ? `/img/${this.release.poster.sfw.thumbnail}` : `/media/${this.release.poster.thumbnail}`;
+		return this.getPath(this.release.poster, 'thumbnail');
 	}
 
 	if (this.release.covers?.length > 0) {
-		return this.sfw ? `/img/${this.release.covers[0].sfw.path}` : `/media/${this.release.covers[0].path}`;
+		return this.getPath(this.release.covers[0], 'thumbnail');
 	}
 
 	if (this.photos?.length > 0) {
-		return this.sfw ? `/img/${this.photos[0].sfw.thumbnail}` : `/media/${this.photos[0].thumbnail}`;
+		return this.getPath(this.release.photos[0], 'thumbnail');
 	}
 
 	return null;

assets/components/releases/movie-tile.vue

@@ -9,8 +9,8 @@
 			>
 				<img
 					v-if="movie.covers[0]"
-					:src="sfw ? `/img/${movie.covers[0].sfw.thumbnail}` : `/media/${movie.covers[0].thumbnail}`"
+					:src="getPath(movie.covers[0], 'thumbnail')"
-					:style="{ 'background-image': sfw ? `/img/${movie.covers[0].sfw.lazy}` : `/media/${movie.covers[0].lazy }` }"
+					:style="{ 'background-image': getBgPath(movie.covers[0], 'lazy') }"
 					loading="lazy"
 				>
 			</router-link>

assets/components/releases/release.vue

@@ -29,6 +29,7 @@
 			v-if="showAlbum"
 			:items="[release.poster, ...release.photos]"
 			:title="release.title"
+			:path="config.media.mediaPath"
 			@close="$router.go(-1)"
 		/>
 

assets/components/releases/scene-tile.vue

@@ -19,8 +19,8 @@
 				>
 					<img
 						v-if="release.poster"
-						:src="sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`"
+						:src="getPath(release.poster, 'thumbnail')"
-						:style="{ 'background-image': sfw ? `/img/${release.poster.sfw.lazy}` : `/media/${release.poster.lazy}` }"
+						:style="{ 'background-image': getBgPath(release.poster, 'lazy') }"
 						:alt="release.title"
 						class="thumbnail"
 						loading="lazy"
@@ -28,8 +28,8 @@
 
 					<img
 						v-else-if="release.photos && release.photos.length > 0"
-						:src="sfw ? `/img/${release.photos[0].sfw.thumbnail}` : `/media/${release.photos[0].thumbnail}`"
+						:src="getPath(release.photos[0], 'thumbnail')"
-						:style="{ 'background-image': sfw ? `/img/${release.photos[0].sfw.lazy}` : `/media/${release.photos[0].lazy}` }"
+						:style="{ 'background-image': getBgPath(release.photos[0], 'lazy') } "
 						:alt="release.title"
 						class="thumbnail"
 						loading="lazy"
@@ -130,10 +130,6 @@
 <script>
 import Details from './tile-details.vue';
 
-function sfw() {
-	return this.$store.state.ui.sfw;
-}
-
 export default {
 	components: {
 		Details,
@@ -144,9 +140,6 @@ export default {
 			default: null,
 		},
 	},
-	computed: {
-		sfw,
-	},
 };
 </script>
 

assets/components/tags/tag.vue

@@ -39,7 +39,7 @@
 				v-if="showAlbum"
 				:items="[tag.poster, ...tag.photos]"
 				:title="tag.name"
-				path="/img"
+				:local="true"
 				class="portrait"
 				@close="$router.go(-1)"
 			/>

assets/js/actors/actions.js

@@ -89,6 +89,7 @@ function initActorActions(store, router) {
 						hash
 						comment
 						credit
+						isS3
 						sfw: sfwMedia {
 							id
 							thumbnail
@@ -118,6 +119,7 @@ function initActorActions(store, router) {
 							thumbnail
 							lazy
 							hash
+							isS3
 							comment
 							credit
 							entropy
@@ -320,6 +322,7 @@ function initActorActions(store, router) {
 							lazy
 							comment
 							credit
+							isS3
 							sfw: sfwMedia {
 								id
 								thumbnail

assets/js/config/default.js

@@ -2,6 +2,11 @@ export default {
 	api: {
 		url: `${window.location.origin}/api`,
 	},
+	media: {
+		assetPath: '/img',
+		mediaPath: '/media',
+		s3Path: 'https://s3.eu-central-1.wasabisys.com/traxxx',
+	},
 	showDisclaimer: false,
 	disclaimer: 'This site is in early development, and content may occasionally disappear. Please stay tuned, you will be able to use traxxx to its full potential in the near future!',
 	selectableTags: [

assets/js/fragments.js

@@ -100,6 +100,7 @@ const releasePosterFragment = `
             path
             thumbnail
             lazy
+			isS3
             comment
             sfw: sfwMedia {
                 id
@@ -120,6 +121,7 @@ const releaseCoversFragment = `
             path
             thumbnail
             lazy
+			isS3
             comment
             sfw: sfwMedia {
                 id
@@ -140,6 +142,7 @@ const releasePhotosFragment = `
             path
             thumbnail
             lazy
+			isS3
             comment
             sfw: sfwMedia {
                 id
@@ -160,6 +163,7 @@ const releaseTrailerFragment = `
             path
             thumbnail
             mime
+			isS3
 			isVr
         }
     }
@@ -173,6 +177,7 @@ const releaseTeaserFragment = `
             path
             thumbnail
             mime
+			isS3
         }
     }
 `;

assets/js/main.js

@@ -24,6 +24,17 @@ async function init() {
 	const app = createApp(Container);
 	const events = mitt();
 
+	function getPath(media, type, options) {
+		const path = (store.state.ui.sfw && media.assetPath)
+			|| (media.isS3 && config.media.s3Path)
+			|| (options?.local && config.media.assetPath)
+			|| config.media.mediaPath;
+
+		const filename = type && !options?.original ? media[type] : media.path;
+
+		return `${path}/${filename}`;
+	}
+
 	initUiObservers(store, router);
 
 	if (window.env.sfw) {
@@ -64,6 +75,8 @@ async function init() {
 			formatDuration,
 			isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB),
 			isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB),
+			getPath,
+			getBgPath: (media, type) => `url(${getPath(media, type)})`,
 		},
 		beforeCreate() {
 			this.uid = uid;

assets/js/ui/state.js

@@ -3,10 +3,12 @@ const storedBatch = localStorage.getItem('batch');
 const storedSfw = localStorage.getItem('sfw');
 const storedTheme = localStorage.getItem('theme');
 
+const deviceTheme = window.matchMedia?.('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
+
 export default {
 	tagFilter: storedTagFilter ? storedTagFilter.split(',') : [],
 	range: 'latest',
 	batch: storedBatch || 'all',
 	sfw: storedSfw === 'true' || false,
-	theme: storedTheme || 'light',
+	theme: storedTheme || deviceTheme,
 };

migrations/20190325001339_releases.js

@@ -28,6 +28,9 @@ exports.up = knex => Promise.resolve()
 		table.integer('index');
 		table.text('mime');
 
+		table.boolean('is_s3')
+			.defaultTo(false);
+
 		table.text('hash');
 
 		table.bigInteger('size', 12);
@@ -1020,17 +1023,17 @@ exports.up = knex => Promise.resolve()
 			CREATE FUNCTION search_entities(search text) RETURNS SETOF entities AS $$
 				SELECT * FROM entities
 				WHERE
-				name ILIKE ('%' || search || '%') OR
+				name ILIKE ('%' || TRIM(search) || '%') OR
-				slug ILIKE ('%' || search || '%') OR
+				slug ILIKE ('%' || TRIM(search) || '%') OR
-				array_to_string(alias, '') ILIKE ('%' || search || '%') OR
+				array_to_string(alias, '') ILIKE ('%' || TRIM(search) || '%') OR
-				replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || search || '%') OR
+				replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || TRIM(search) || '%') OR
 				url ILIKE ('%' || search || '%')
 			$$ LANGUAGE SQL STABLE;
 
 			CREATE FUNCTION search_actors(search text, min_length numeric DEFAULT 2) RETURNS SETOF actors AS $$
 				SELECT * FROM actors
 				WHERE length(search) >= min_length
-				AND name ILIKE ('%' || search || '%')
+				AND name ILIKE ('%' || TRIM(search) || '%')
 			$$ LANGUAGE SQL STABLE;
 
 			CREATE FUNCTION actors_tags(actor actors, selectable_tags text[]) RETURNS SETOF tags AS $$

src/media.js

@@ -14,6 +14,7 @@ const ffmpeg = require('fluent-ffmpeg');
 const sharp = require('sharp');
 const blake2 = require('blake2');
 const taskQueue = require('promise-task-queue');
+const AWS = require('aws-sdk');
 
 const logger = require('./logger')(__filename);
 const argv = require('./argv');
@@ -25,6 +26,17 @@ const { get } = require('./utils/qu');
 const pipeline = util.promisify(stream.pipeline);
 const streamQueue = taskQueue();
 
+const endpoint = new AWS.Endpoint('s3.wasabisys.com');
+
+const s3 = new AWS.S3({
+	// region: 'eu-central-1',
+	endpoint,
+	credentials: {
+		accessKeyId: config.s3.accessKey,
+		secretAccessKey: config.s3.secretKey,
+	},
+});
+
 function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 	// limit media sets, use extras as fallbacks
 	if (medias.length <= limit) {
@@ -303,6 +315,58 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
 	throw new Error(`Could not extract source from ${baseSource.url}: ${res.status}`);
 }
 
+async function storeS3Object(filepath, media) {
+	const fullFilepath = path.join(config.media.path, filepath);
+	const file = fs.createReadStream(fullFilepath);
+
+	const status = await s3.upload({
+		Bucket: config.s3.bucket,
+		Body: file,
+		Key: filepath,
+		ContentType: media.meta.mimetype,
+	}).promise();
+
+	await fsPromises.unlink(fullFilepath);
+
+	return status;
+}
+
+async function writeImage(image, media, info, filepath, isProcessed) {
+	if (isProcessed && info.pages) {
+		// convert animated image to WebP and write to permanent location
+		await image
+			.webp()
+			.toFile(path.join(config.media.path, filepath));
+	}
+
+	if (isProcessed) {
+		// convert to JPEG and write to permanent location
+		await image
+			.jpeg()
+			.toFile(path.join(config.media.path, filepath));
+	}
+}
+
+async function writeThumbnail(image, thumbpath) {
+	return image
+		.resize({
+			height: config.media.thumbnailSize,
+			withoutEnlargement: true,
+		})
+		.jpeg({ quality: config.media.thumbnailQuality })
+		.toFile(path.join(config.media.path, thumbpath));
+}
+
+async function writeLazy(image, lazypath) {
+	return image
+		.resize({
+			height: config.media.lazySize,
+			withoutEnlargement: true,
+		})
+		.jpeg({ quality: config.media.lazyQuality })
+		.toFile(path.join(config.media.path, lazypath));
+}
+
 async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options) {
 	logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`);
 
@@ -343,46 +407,28 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
 			});
 		}
 
-		if (isProcessed) {
-			if (info.pages) {
-				// convert animated image to WebP and write to permanent location
-				await image
-					.webp()
-					.toFile(path.join(config.media.path, filepath));
-			} else {
-				// convert to JPEG and write to permanent location
-				await image
-					.jpeg()
-					.toFile(path.join(config.media.path, filepath));
-			}
-		}
-
-		// generate thumbnail and lazy
 		await Promise.all([
-			image
+			writeImage(image, media, info, filepath, isProcessed),
-				.resize({
+			writeThumbnail(image, thumbpath),
-					height: config.media.thumbnailSize,
+			writeLazy(image, lazypath),
-					withoutEnlargement: true,
-				})
-				.jpeg({ quality: config.media.thumbnailQuality })
-				.toFile(path.join(config.media.path, thumbpath)),
-			image
-				.resize({
-					height: config.media.lazySize,
-					withoutEnlargement: true,
-				})
-				.jpeg({ quality: config.media.lazyQuality })
-				.toFile(path.join(config.media.path, lazypath)),
 		]);
 
 		if (isProcessed) {
-			// remove temp file
+			// file already stored, remove temporary file
 			await fsPromises.unlink(media.file.path);
 		} else {
-			// move temp file to permanent location
+			// image not processed, simply move temporary file to final location
 			await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
 		}
 
+		if (config.s3.enabled) {
+			await Promise.all([
+				storeS3Object(filepath, media),
+				storeS3Object(thumbpath, media),
+				storeS3Object(lazypath, media),
+			]);
+		}
+
 		logger.silly(`Stored thumbnail, lazy and permanent media file for ${media.id} from ${media.src} at ${filepath}`);
 
 		return {
@@ -521,7 +567,6 @@ async function fetchSource(source, baseMedia) {
 
 		try {
 			const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
-
 			const tempFileTarget = fs.createWriteStream(tempFilePath);
 			const hashStream = new stream.PassThrough();
 			let size = 0;
@@ -648,6 +693,7 @@ function curateMediaEntry(media, index) {
 		path: media.file.path,
 		thumbnail: media.file.thumbnail,
 		lazy: media.file.lazy,
+		is_s3: config.s3.enabled,
 		index,
 		mime: media.meta.mimetype,
 		hash: media.meta.hash,

src/scrapers/mindgeek.js

@@ -13,7 +13,9 @@ const { cookieToData } = require('../utils/cookies');
 
 function getThumbs(scene) {
 	if (scene.images.poster) {
-		return scene.images.poster.map(image => image.xl.url);
+		return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
+			.filter(img => typeof img === 'object') // remove alternateText property
+			.map(image => image.xl.url);
 	}
 
 	if (scene.images.card_main_rect) {