Merge branch 'master' into experimental

assets/components/actors/actor.vue

@@ -42,13 +42,13 @@
 			>
 				<a
 					v-if="actor.avatar"
-					:href="`/media/${actor.avatar.path}`"
+					:href="getPath(actor.avatar)"
 					target="_blank"
 					rel="noopener noreferrer"
 					class="avatar-link"
 				>
 					<img
-						:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
+						:src="getPath(actor.avatar, 'thumbnail')"
 						:title="actor.avatar.credit && `© ${actor.avatar.credit}`"
 						class="avatar"
 					>

assets/components/actors/photos.vue

@@ -8,14 +8,14 @@
 	>
 		<a
 			v-if="actor.avatar"
-			:href="`/media/${actor.avatar.path}`"
+			:href="getPath(actor.avatar)"
 			target="_blank"
 			rel="noopener noreferrer"
 			class="avatar-link photo-link"
 		>
 			<img
-				:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
+				:src="getPath(actor.avatar, 'thumbnail')"
-				:style="{ 'background-image': sfw ? `/img/${actor.avatar.sfw.lazy}` : `/media/${actor.avatar.lazy}` }"
+				:style="{ 'background-image': getBgPath(actor.avatar, 'lazy') }"
 				:title="actor.avatar.credit && `© ${actor.avatar.credit}`"
 				loading="lazy"
 				class="avatar photo"
@@ -26,14 +26,14 @@
 		<a
 			v-for="photo in photos"
 			:key="`photo-${photo.id}`"
-			:href="`/media/${photo.path}`"
+			:href="getPath(photo)"
 			target="_blank"
 			rel="noopener noreferrer"
 			class="photo-link"
 		>
 			<img
-				:src="sfw ? `/img/${photo.sfw.thumbnail}` : `/media/${photo.thumbnail}`"
+				:src="getPath(photo, 'thumbnail')"
-				:style="{ 'background-image': sfw ? `/img/${photo.sfw.lazy}` : `/media/${photo.lazy}` }"
+				:style="{ 'background-image': getBgPath(photo, 'lazy') }"
 				:title="`© ${photo.credit || photo.entity.name}`"
 				loading="lazy"
 				class="photo"

assets/components/actors/tile.vue

@@ -45,8 +45,8 @@
 			<div class="avatar-container">
 				<img
 					v-if="actor.avatar"
-					:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
+					:src="getPath(actor.avatar, 'thumbnail')"
-					:style="{ 'background-image': sfw ? `url(/img/${actor.avatar.sfw.lazy})`: `url(/img/${actor.avatar.lazy})` }"
+					:style="{ 'background-image': getBgPath(actor.avatar, 'lazy') }"
 					loading="lazy"
 					class="avatar"
 				>

assets/components/album/album.vue

@@ -21,12 +21,12 @@
 					class="item-container"
 				>
 					<a
-						:href="`${path}/${item.path}`"
+						:href="getPath(item, null, { local })"
 						class="item-link"
 						target="_blank"
 					>
 						<img
-							:src="`${path}/${item.thumbnail}`"
+							:src="getPath(item, 'thumbnail', { local })"
 							:title="item.title"
 							loading="lazy"
 							class="item image"
@@ -63,9 +63,9 @@ export default {
 			type: String,
 			default: null,
 		},
-		path: {
+		local: {
-			type: String,
+			type: Boolean,
-			default: '/media',
+			default: false,
 		},
 		portrait: {
 			type: Boolean,

assets/components/releases/banner.vue

@@ -31,7 +31,7 @@
 
 				<img
 					v-else-if="release.teaser && /^image\//.test(release.teaser.mime)"
-					:src="sfw ? `/img/${release.teaser.sfw.thumbnail}` : `/media/${release.teaser.path}`"
+					:src="getPath(release.teaser, 'thumbnail', { original: true })"
 					:alt="release.title"
 					loading="lazy"
 					class="item trailer"
@@ -40,7 +40,7 @@
 				<a
 					v-if="release.poster"
 					v-tooltip="'View poster'"
-					:href="`/media/${release.poster.path}`"
+					:href="`${config.media.mediaPath}/${release.poster.path}`"
 					:class="{ playing }"
 					target="_blank"
 					rel="noopener noreferrer"
@@ -59,13 +59,13 @@
 				<a
 					v-for="cover in release.covers"
 					:key="`cover-${cover.id}`"
-					:href="`/media/${cover.path}`"
+					:href="getPath(cover)"
 					target="_blank"
 					rel="noopener noreferrer"
 				>
 					<img
-						:src="`/media/${cover.thumbnail}`"
+						:src="getPath(cover, 'thumbnail')"
-						:style="{ 'background-image': sfw ? `url(/media/${cover.sfw.lazy})` : `url(/media/${cover.lazy})` }"
+						:style="{ 'background-image': getBgPath(cover, 'lazy') }"
 						class="item cover"
 						loading="lazy"
 						@load="$emit('load', $event)"
@@ -79,15 +79,15 @@
 				class="item-container"
 			>
 				<a
-					:href="`/media/${photo.path}`"
+					:href="getPath(photo)"
 					:class="{ sfw }"
 					class="item-link"
 					target="_blank"
 					rel="noopener noreferrer"
 				>
 					<img
-						:src="sfw ? `/img/${photo.sfw.thumbnail}` : `/media/${photo.thumbnail}`"
+						:src="getPath(photo, 'thumbnail')"
-						:style="{ 'background-image': sfw ? `url(/img/${photo.sfw.lazy})` : `url(/media/${photo.lazy})` }"
+						:style="{ 'background-image': getPath(photo, 'lazy') }"
 						:alt="`Photo ${photo.index + 1}`"
 						loading="lazy"
 						class="item"
@@ -115,15 +115,15 @@ function sfw() {
 
 function poster() {
 	if (this.release.poster) {
-		return this.sfw ? `/img/${this.release.poster.sfw.thumbnail}` : `/media/${this.release.poster.thumbnail}`;
+		return this.getPath(this.release.poster, 'thumbnail');
 	}
 
 	if (this.release.covers?.length > 0) {
-		return this.sfw ? `/img/${this.release.covers[0].sfw.path}` : `/media/${this.release.covers[0].path}`;
+		return this.getPath(this.release.covers[0], 'thumbnail');
 	}
 
 	if (this.photos?.length > 0) {
-		return this.sfw ? `/img/${this.photos[0].sfw.thumbnail}` : `/media/${this.photos[0].thumbnail}`;
+		return this.getPath(this.release.photos[0], 'thumbnail');
 	}
 
 	return null;

assets/components/releases/movie-tile.vue

@@ -9,8 +9,8 @@
 			>
 				<img
 					v-if="movie.covers[0]"
-					:src="sfw ? `/img/${movie.covers[0].sfw.thumbnail}` : `/media/${movie.covers[0].thumbnail}`"
+					:src="getPath(movie.covers[0], 'thumbnail')"
-					:style="{ 'background-image': sfw ? `/img/${movie.covers[0].sfw.lazy}` : `/media/${movie.covers[0].lazy }` }"
+					:style="{ 'background-image': getBgPath(movie.covers[0], 'lazy') }"
 					loading="lazy"
 				>
 			</router-link>

assets/components/releases/release.vue

@@ -29,6 +29,7 @@
 			v-if="showAlbum"
 			:items="[release.poster, ...release.photos]"
 			:title="release.title"
+			:path="config.media.mediaPath"
 			@close="$router.go(-1)"
 		/>
 
@@ -83,7 +84,7 @@
 						<span class="movie-title">{{ movie.title }}</span>
 						<img
 							v-if="movie.covers.length > 0"
-							:src="`/media/${movie.covers[0].thumbnail}`"
+							:src="getPath(movie.covers[0], 'thumbnail')"
 							class="movie-cover"
 						>
 					</router-link>

assets/components/releases/scene-tile.vue

@@ -19,8 +19,8 @@
 				>
 					<img
 						v-if="release.poster"
-						:src="sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`"
+						:src="getPath(release.poster, 'thumbnail')"
-						:style="{ 'background-image': sfw ? `/img/${release.poster.sfw.lazy}` : `/media/${release.poster.lazy}` }"
+						:style="{ 'background-image': getBgPath(release.poster, 'lazy') }"
 						:alt="release.title"
 						class="thumbnail"
 						loading="lazy"
@@ -28,8 +28,8 @@
 
 					<img
 						v-else-if="release.photos && release.photos.length > 0"
-						:src="sfw ? `/img/${release.photos[0].sfw.thumbnail}` : `/media/${release.photos[0].thumbnail}`"
+						:src="getPath(release.photos[0], 'thumbnail')"
-						:style="{ 'background-image': sfw ? `/img/${release.photos[0].sfw.lazy}` : `/media/${release.photos[0].lazy}` }"
+						:style="{ 'background-image': getBgPath(release.photos[0], 'lazy') } "
 						:alt="release.title"
 						class="thumbnail"
 						loading="lazy"
@@ -130,10 +130,6 @@
 <script>
 import Details from './tile-details.vue';
 
-function sfw() {
-	return this.$store.state.ui.sfw;
-}
-
 export default {
 	components: {
 		Details,
@@ -144,9 +140,6 @@ export default {
 			default: null,
 		},
 	},
-	computed: {
-		sfw,
-	},
 };
 </script>
 

assets/components/tags/tag.vue

@@ -39,7 +39,7 @@
 				v-if="showAlbum"
 				:items="[tag.poster, ...tag.photos]"
 				:title="tag.name"
-				path="/img"
+				:local="true"
 				class="portrait"
 				@close="$router.go(-1)"
 			/>

assets/components/video/player.vue

@@ -7,7 +7,7 @@
 		@pause="$emit('pause')"
 	>
 		<source
-			:src="`/media/${video.path}`"
+			:src="getPath(video)"
 			type="video/mp4"
 		>
 	</video>

assets/css/_theme.scss

@@ -44,7 +44,7 @@ $breakpoint4: 1500px;
     --text-contrast: #fff;
 
     --background: var(--background-light);
-    --background-dim: #fafafa;
+    --background-dim: #f5f5f5;
     --background-soft: #fdfdfd;
 
     --profile: #222;

assets/js/actors/actions.js

@@ -89,6 +89,7 @@ function initActorActions(store, router) {
 						hash
 						comment
 						credit
+						isS3
 						sfw: sfwMedia {
 							id
 							thumbnail
@@ -118,6 +119,7 @@ function initActorActions(store, router) {
 							thumbnail
 							lazy
 							hash
+							isS3
 							comment
 							credit
 							entropy
@@ -320,6 +322,7 @@ function initActorActions(store, router) {
 							lazy
 							comment
 							credit
+							isS3
 							sfw: sfwMedia {
 								id
 								thumbnail

assets/js/config/default.js

@@ -2,6 +2,11 @@ export default {
 	api: {
 		url: `${window.location.origin}/api`,
 	},
+	media: {
+		assetPath: '/img',
+		mediaPath: '/media',
+		s3Path: 'https://s3.wasabisys.com',
+	},
 	showDisclaimer: false,
 	disclaimer: 'This site is in early development, and content may occasionally disappear. Please stay tuned, you will be able to use traxxx to its full potential in the near future!',
 	selectableTags: [

assets/js/fragments.js

@@ -100,6 +100,7 @@ const releasePosterFragment = `
             path
             thumbnail
             lazy
+			isS3
             comment
             sfw: sfwMedia {
                 id
@@ -120,6 +121,7 @@ const releaseCoversFragment = `
             path
             thumbnail
             lazy
+			isS3
             comment
             sfw: sfwMedia {
                 id
@@ -140,6 +142,7 @@ const releasePhotosFragment = `
             path
             thumbnail
             lazy
+			isS3
             comment
             sfw: sfwMedia {
                 id
@@ -160,6 +163,7 @@ const releaseTrailerFragment = `
             path
             thumbnail
             mime
+			isS3
 			isVr
         }
     }
@@ -173,6 +177,7 @@ const releaseTeaserFragment = `
             path
             thumbnail
             mime
+			isS3
         }
     }
 `;
@@ -310,6 +315,7 @@ const releaseFragment = `
 					path
 					thumbnail
 					lazy
+					isS3
 					comment
 					sfw: sfwMedia {
 						id

assets/js/main.js

@@ -24,6 +24,17 @@ async function init() {
 	const app = createApp(Container);
 	const events = mitt();
 
+	function getPath(media, type, options) {
+		const path = (store.state.ui.sfw && media.assetPath)
+			|| (media.isS3 && config.media.s3Path)
+			|| (options?.local && config.media.assetPath)
+			|| config.media.mediaPath;
+
+		const filename = type && !options?.original ? media[type] : media.path;
+
+		return `${path}/${filename}`;
+	}
+
 	initUiObservers(store, router);
 
 	if (window.env.sfw) {
@@ -64,6 +75,8 @@ async function init() {
 			formatDuration,
 			isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB),
 			isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB),
+			getPath,
+			getBgPath: (media, type) => `url(${getPath(media, type)})`,
 		},
 		beforeCreate() {
 			this.uid = uid;

assets/js/releases/actions.js

@@ -103,6 +103,7 @@ function initReleasesActions(store, router) {
 								path
 								thumbnail
 								lazy
+								isS3
 								sfw: sfwMedia {
 									id
 									path
@@ -157,6 +158,16 @@ function initReleasesActions(store, router) {
 							path
 							thumbnail
 							lazy
+							isS3
+						}
+					}
+					posters: moviesPosterByMovieId {
+						media {
+							id
+							path
+							thumbnail
+							lazy
+							isS3
 						}
 					}
 					covers: moviesCovers {
@@ -165,12 +176,14 @@ function initReleasesActions(store, router) {
 							path
 							thumbnail
 							lazy
+							isS3
 						}
 					}
 					trailer: moviesTrailerByMovieId {
 						media {
 							id
 							path
+							isS3
 						}
 					}
 					scenes: moviesScenes {
@@ -189,6 +202,7 @@ function initReleasesActions(store, router) {
 						path
 						thumbnail
 						lazy
+						isS3
 						comment
 						sfw: sfwMedia {
 							id

assets/js/ui/state.js

@@ -3,10 +3,12 @@ const storedBatch = localStorage.getItem('batch');
 const storedSfw = localStorage.getItem('sfw');
 const storedTheme = localStorage.getItem('theme');
 
+const deviceTheme = window.matchMedia?.('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
+
 export default {
 	tagFilter: storedTagFilter ? storedTagFilter.split(',') : [],
 	range: 'latest',
 	batch: storedBatch || 'all',
 	sfw: storedSfw === 'true' || false,
-	theme: storedTheme || 'light',
+	theme: storedTheme || deviceTheme,
 };

config/default.js

@@ -19,6 +19,12 @@ module.exports = {
 			},
 		},
 	},
+	s3: {
+		enabled: false,
+		bucket: 'traxxx',
+		accessKey: 'ABCDEFGHIJ1234567890',
+		secretKey: 'abcdefghijklmnopqrstuvwxyz1234567890ABCD',
+	},
 	exclude: {
 		channels: [
 			// 21sextreme, no longer updated

migrations/20190325001339_releases.js

@@ -28,6 +28,9 @@ exports.up = knex => Promise.resolve()
 		table.integer('index');
 		table.text('mime');
 
+		table.boolean('is_s3')
+			.defaultTo(false);
+
 		table.text('hash');
 
 		table.bigInteger('size', 12);
@@ -898,7 +901,7 @@ exports.up = knex => Promise.resolve()
 			.references('id')
 			.inTable('media');
 
-		table.unique(['movie_id', 'media_id']);
+		table.unique('movie_id');
 	}))
 	.then(() => knex.schema.createTable('clips', (table) => {
 		table.increments('id', 16);
@@ -1020,17 +1023,17 @@ exports.up = knex => Promise.resolve()
 			CREATE FUNCTION search_entities(search text) RETURNS SETOF entities AS $$
 				SELECT * FROM entities
 				WHERE
-				name ILIKE ('%' || search || '%') OR
+				name ILIKE ('%' || TRIM(search) || '%') OR
-				slug ILIKE ('%' || search || '%') OR
+				slug ILIKE ('%' || TRIM(search) || '%') OR
-				array_to_string(alias, '') ILIKE ('%' || search || '%') OR
+				array_to_string(alias, '') ILIKE ('%' || TRIM(search) || '%') OR
-				replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || search || '%') OR
+				replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || TRIM(search) || '%') OR
 				url ILIKE ('%' || search || '%')
 			$$ LANGUAGE SQL STABLE;
 
 			CREATE FUNCTION search_actors(search text, min_length numeric DEFAULT 2) RETURNS SETOF actors AS $$
 				SELECT * FROM actors
 				WHERE length(search) >= min_length
-				AND name ILIKE ('%' || search || '%')
+				AND name ILIKE ('%' || TRIM(search) || '%')
 			$$ LANGUAGE SQL STABLE;
 
 			CREATE FUNCTION actors_tags(actor actors, selectable_tags text[]) RETURNS SETOF tags AS $$

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.172.3",
+    "version": "1.173.2",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {
@@ -49,20 +49,20 @@
         "babel-loader": "^8.0.6",
         "babel-preset-airbnb": "^3.3.2",
         "css-loader": "^5.0.1",
-        "eslint": "^5.16.0",
+        "eslint": "^7.20.0",
         "eslint-config-airbnb": "^17.1.1",
         "eslint-config-airbnb-base": "^13.2.0",
-        "eslint-loader": "^2.2.1",
         "eslint-plugin-import": "^2.20.1",
         "eslint-plugin-jsx-a11y": "^6.2.3",
         "eslint-plugin-react": "^7.18.3",
         "eslint-plugin-vue": "^6.2.1",
         "eslint-watch": "^4.0.2",
+        "eslint-webpack-plugin": "^2.5.2",
         "mini-css-extract-plugin": "^1.3.3",
-        "node-sass": "^4.13.1",
+        "node-sass": "^5.0.0",
         "postcss-loader": "^3.0.0",
         "raw-loader": "^4.0.2",
-        "sass-loader": "^7.3.1",
+        "sass-loader": "^11.0.1",
         "style-loader": "^0.23.1",
         "vue-loader": "^16.1.2",
         "webpack": "^5.11.0",
@@ -71,9 +71,9 @@
     "dependencies": {
         "@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
         "@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
-        "@tensorflow/tfjs-node": "^1.5.2",
         "@thependulum/bhttp": "^1.2.6",
         "acorn": "^8.0.4",
+        "aws-sdk": "^2.847.0",
         "babel-polyfill": "^6.26.0",
         "bhttp": "^1.2.6",
         "blake2": "^4.0.0",
@@ -121,7 +121,7 @@
         "prop-types": "^15.7.2",
         "react": "^16.13.0",
         "react-dom": "^16.13.0",
-        "sharp": "^0.26.3",
+        "sharp": "^0.27.2",
         "showdown": "^1.9.1",
         "source-map-support": "^0.5.16",
         "template-format": "^1.2.5",

src/media.js

@@ -14,6 +14,7 @@ const ffmpeg = require('fluent-ffmpeg');
 const sharp = require('sharp');
 const blake2 = require('blake2');
 const taskQueue = require('promise-task-queue');
+const AWS = require('aws-sdk');
 
 const logger = require('./logger')(__filename);
 const argv = require('./argv');
@@ -25,6 +26,17 @@ const { get } = require('./utils/qu');
 const pipeline = util.promisify(stream.pipeline);
 const streamQueue = taskQueue();
 
+const endpoint = new AWS.Endpoint('s3.eu-central-1.wasabisys.com');
+
+const s3 = new AWS.S3({
+	// region: 'eu-central-1',
+	endpoint,
+	credentials: {
+		accessKeyId: config.s3.accessKey,
+		secretAccessKey: config.s3.secretKey,
+	},
+});
+
 function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 	// limit media sets, use extras as fallbacks
 	if (medias.length <= limit) {
@@ -303,14 +315,68 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
 	throw new Error(`Could not extract source from ${baseSource.url}: ${res.status}`);
 }
 
+async function storeS3Object(filepath, media) {
+	const fullFilepath = path.join(config.media.path, filepath);
+	const file = fs.createReadStream(fullFilepath);
+
+	const status = await s3.upload({
+		Bucket: config.s3.bucket,
+		Body: file,
+		Key: filepath,
+		ContentType: media.meta.mimetype,
+	}).promise();
+
+	await fsPromises.unlink(fullFilepath);
+
+	logger.silly(`Uploaded '${media.id}' from ${media.src} to S3 bucket '${status.Bucket}' at ${status.Location}`);
+
+	return status;
+}
+
+async function writeImage(image, media, info, filepath, isProcessed) {
+	if (isProcessed && info.pages) {
+		// convert animated image to WebP and write to permanent location
+		await image
+			.webp()
+			.toFile(path.join(config.media.path, filepath));
+	}
+
+	if (isProcessed) {
+		// convert to JPEG and write to permanent location
+		await image
+			.jpeg()
+			.toFile(path.join(config.media.path, filepath));
+	}
+}
+
+async function writeThumbnail(image, thumbpath) {
+	return image
+		.resize({
+			height: config.media.thumbnailSize,
+			withoutEnlargement: true,
+		})
+		.jpeg({ quality: config.media.thumbnailQuality })
+		.toFile(path.join(config.media.path, thumbpath));
+}
+
+async function writeLazy(image, lazypath) {
+	return image
+		.resize({
+			height: config.media.lazySize,
+			withoutEnlargement: true,
+		})
+		.jpeg({ quality: config.media.lazyQuality })
+		.toFile(path.join(config.media.path, lazypath));
+}
+
 async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options) {
 	logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`);
 
 	try {
-		const thumbdir = path.join(media.role, 'thumbs', hashDir, hashSubDir);
+		const thumbdir = config.s3.enabled ? path.join(media.role, 'thumbs') : path.join(media.role, 'thumbs', hashDir, hashSubDir);
 		const thumbpath = path.join(thumbdir, filename);
 
-		const lazydir = path.join(media.role, 'lazy', hashDir, hashSubDir);
+		const lazydir = config.s3.enabled ? path.join(media.role, 'lazy') : path.join(media.role, 'lazy', hashDir, hashSubDir);
 		const lazypath = path.join(lazydir, filename);
 
 		await Promise.all([
@@ -343,46 +409,28 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
 			});
 		}
 
-		if (isProcessed) {
-			if (info.pages) {
-				// convert animated image to WebP and write to permanent location
-				await image
-					.webp()
-					.toFile(path.join(config.media.path, filepath));
-			} else {
-				// convert to JPEG and write to permanent location
-				await image
-					.jpeg()
-					.toFile(path.join(config.media.path, filepath));
-			}
-		}
-
-		// generate thumbnail and lazy
 		await Promise.all([
-			image
+			writeImage(image, media, info, filepath, isProcessed),
-				.resize({
+			writeThumbnail(image, thumbpath),
-					height: config.media.thumbnailSize,
+			writeLazy(image, lazypath),
-					withoutEnlargement: true,
-				})
-				.jpeg({ quality: config.media.thumbnailQuality })
-				.toFile(path.join(config.media.path, thumbpath)),
-			image
-				.resize({
-					height: config.media.lazySize,
-					withoutEnlargement: true,
-				})
-				.jpeg({ quality: config.media.lazyQuality })
-				.toFile(path.join(config.media.path, lazypath)),
 		]);
 
 		if (isProcessed) {
-			// remove temp file
+			// file already stored, remove temporary file
 			await fsPromises.unlink(media.file.path);
 		} else {
-			// move temp file to permanent location
+			// image not processed, simply move temporary file to final location
 			await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
 		}
 
+		if (config.s3.enabled) {
+			await Promise.all([
+				storeS3Object(filepath, media),
+				storeS3Object(thumbpath, media),
+				storeS3Object(lazypath, media),
+			]);
+		}
+
 		logger.silly(`Stored thumbnail, lazy and permanent media file for ${media.id} from ${media.src} at ${filepath}`);
 
 		return {
@@ -413,13 +461,13 @@ async function storeFile(media, options) {
 	try {
 		const hashDir = media.meta.hash.slice(0, 2);
 		const hashSubDir = media.meta.hash.slice(2, 4);
-		const hashFilename = media.meta.hash.slice(4);
+		const hashFilename = config.s3.enabled ? media.meta.hash : media.meta.hash.slice(4);
 
 		const filename = media.quality
 			? `${hashFilename}_${media.quality}.${media.meta.extension}`
 			: `${hashFilename}.${media.meta.extension}`;
 
-		const filedir = path.join(media.role, hashDir, hashSubDir);
+		const filedir = config.s3.enabled ? media.role : path.join(media.role, hashDir, hashSubDir);
 		const filepath = path.join(filedir, filename);
 
 		if (argv.force) {
@@ -447,6 +495,11 @@ async function storeFile(media, options) {
 		// move temp file to permanent location
 		await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
 
+		if (config.s3.enabled) {
+			// upload the file to S3 storage, will remove original
+			await storeS3Object(filepath, media);
+		}
+
 		logger.silly(`Stored permanent media file for ${media.id} from ${media.src} at ${filepath}`);
 
 		return {
@@ -521,7 +574,6 @@ async function fetchSource(source, baseMedia) {
 
 		try {
 			const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
-
 			const tempFileTarget = fs.createWriteStream(tempFilePath);
 			const hashStream = new stream.PassThrough();
 			let size = 0;
@@ -648,6 +700,7 @@ function curateMediaEntry(media, index) {
 		path: media.file.path,
 		thumbnail: media.file.thumbnail,
 		lazy: media.file.lazy,
+		is_s3: config.s3.enabled,
 		index,
 		mime: media.meta.mimetype,
 		hash: media.meta.hash,
@@ -816,6 +869,29 @@ async function associateAvatars(profiles) {
 	return profilesWithAvatarIds;
 }
 
+async function deleteS3Objects(media) {
+	const objects = media
+		.map(item => [
+			{ Key: item.path },
+			{ Key: item.thumbnail },
+			{ Key: item.lazy },
+		])
+		.flat()
+		.filter(item => item.Key);
+
+	const status = await s3.deleteObjects({
+		Bucket: config.s3.bucket,
+		Delete: {
+			Objects: objects,
+			Quiet: false,
+		},
+	}).promise();
+
+	logger.info(`Removed ${status.Deleted.length} media files from S3 bucket '${config.s3.bucket}', ${status.Errors.length} errors`);
+
+	return status;
+}
+
 async function flushOrphanedMedia() {
 	const orphanedMedia = await knex('media')
 		.where('is_sfw', false)
@@ -843,10 +919,10 @@ async function flushOrphanedMedia() {
 				)
 				.whereRaw('associations.media_id = media.id'),
 		)
-		.returning(['media.path', 'media.thumbnail', 'media.lazy'])
+		.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
 		.delete();
 
-	await Promise.all(orphanedMedia.map(media => Promise.all([
+	await Promise.all(orphanedMedia.filter(media => !media.is_s3).map(media => Promise.all([
 		media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),
 		media.thumbnail && fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }),
 		media.lazy && fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }),
@@ -854,6 +930,10 @@ async function flushOrphanedMedia() {
 
 	logger.info(`Removed ${orphanedMedia.length} media files from database and storage`);
 
+	if (config.s3.enabled) {
+		await deleteS3Objects(orphanedMedia.filter(media => media.is_s3));
+	}
+
 	await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });
 
 	logger.info('Cleared temporary media directory');

src/scrapers/gamma.js

@@ -200,8 +200,6 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
 			];
 		}
 
-		console.log(release);
-
 		return release;
 	});
 }

src/scrapers/mindgeek.js

@@ -13,7 +13,9 @@ const { cookieToData } = require('../utils/cookies');
 
 function getThumbs(scene) {
 	if (scene.images.poster) {
-		return scene.images.poster.map(image => image.xl.url);
+		return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
+			.filter(img => typeof img === 'object') // remove alternateText property
+			.map(image => image.xl.url);
 	}
 
 	if (scene.images.card_main_rect) {

src/utils/s3.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const config = require('config');
+const AWS = require('aws-sdk');
+const fs = require('fs');
+const nanoid = require('nanoid');
+
+async function init() {
+	const filepath = './public/img/sfw/animals/j0iiByCxGfA.jpeg';
+	const endpoint = new AWS.Endpoint('s3.wasabisys.com');
+
+	const s3 = new AWS.S3({
+		// region: 'eu-central-1',
+		endpoint,
+		credentials: {
+			accessKeyId: config.s3.accessKey,
+			secretAccessKey: config.s3.secretKey,
+		},
+	});
+
+	try {
+		const data = await s3.listBuckets().promise();
+		const file = fs.createReadStream(filepath);
+		const key = `img/${nanoid()}.jpg`;
+
+		const status = await s3.upload({
+			Bucket: config.s3.bucket,
+			Body: file,
+			Key: key,
+			ContentType: 'image/jpeg',
+		}).promise();
+
+		console.log(data);
+		console.log(status);
+	} catch (error) {
+		console.log(error);
+	}
+}
+
+init();

src/utils/timeout.js

@@ -1,14 +1,14 @@
 'use strict';
 
-const bhttp = require('bhttp');
+const http = require('./http');
 
 const sleep = 5000;
 const timeout = 1000;
 
 async function init() {
 	try {
-		const res = await bhttp.get(`https://httpstat.us/200?sleep=${sleep}`, {
+		const res = await http.get(`https://httpstat.us/200?sleep=${sleep}`, {
-			responseTimeout: timeout,
+			timeout,
 		});
 
 		console.log(res.statusCode);
@@ -17,15 +17,4 @@ async function init() {
 	}
 }
 
-/*
-/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:159
-  err.response = response;
-               ^
-
-    TypeError: Cannot assign to read only property 'response' of object '[object Object]'
-    at addErrorData (/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:159:16)
-    at Timeout.timeoutHandler [as _onTimeout] (/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:525:27)
-*/
-
-
 init();

webpack.config.js

@@ -3,6 +3,7 @@ const path = require('path');
 const { VueLoaderPlugin } = require('vue-loader');
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const autoprefixer = require('autoprefixer');
+const ESLintPlugin = require('eslint-webpack-plugin');
 
 module.exports = {
 	entry: './assets/js/main.js',
@@ -35,7 +36,6 @@ module.exports = {
 							],
 						},
 					},
-					'eslint-loader',
 				],
 			},
 			{
@@ -56,7 +56,12 @@ module.exports = {
 							sourceMap: true,
 						},
 					},
-					'sass-loader?sourceMap',
+					{
+						loader: 'sass-loader',
+						options: {
+							sourceMap: true,
+						},
+					},
 				],
 			},
 			{
@@ -67,6 +72,7 @@ module.exports = {
 	},
 	plugins: [
 		new VueLoaderPlugin(),
+		new ESLintPlugin(),
 		new MiniCssExtractPlugin({
 			filename: '../css/style.css',
 		}),