diff --git a/assets/components/actors/actor.vue b/assets/components/actors/actor.vue index 0eeea1a4..c858ea09 100644 --- a/assets/components/actors/actor.vue +++ b/assets/components/actors/actor.vue @@ -42,13 +42,13 @@ > diff --git a/assets/components/actors/photos.vue b/assets/components/actors/photos.vue index f2b39efa..aece2042 100644 --- a/assets/components/actors/photos.vue +++ b/assets/components/actors/photos.vue @@ -8,14 +8,14 @@ > diff --git a/assets/components/album/album.vue b/assets/components/album/album.vue index b10c0cf1..9f267fe0 100644 --- a/assets/components/album/album.vue +++ b/assets/components/album/album.vue @@ -21,12 +21,12 @@ class="item-container" > 0) { - return this.sfw ? `/img/${this.release.covers[0].sfw.path}` : `/media/${this.release.covers[0].path}`; + return this.getPath(this.release.covers[0], 'thumbnail'); } if (this.photos?.length > 0) { - return this.sfw ? `/img/${this.photos[0].sfw.thumbnail}` : `/media/${this.photos[0].thumbnail}`; + return this.getPath(this.release.photos[0], 'thumbnail'); } return null; diff --git a/assets/components/releases/movie-tile.vue b/assets/components/releases/movie-tile.vue index b17eb3e2..9d2b07ad 100644 --- a/assets/components/releases/movie-tile.vue +++ b/assets/components/releases/movie-tile.vue @@ -9,8 +9,8 @@ > diff --git a/assets/components/releases/release.vue b/assets/components/releases/release.vue index 245a5416..07379d0e 100644 --- a/assets/components/releases/release.vue +++ b/assets/components/releases/release.vue @@ -29,6 +29,7 @@ v-if="showAlbum" :items="[release.poster, ...release.photos]" :title="release.title" + :path="config.media.mediaPath" @close="$router.go(-1)" /> diff --git a/assets/components/releases/scene-tile.vue b/assets/components/releases/scene-tile.vue index dd28d0a2..5fd595a4 100644 --- a/assets/components/releases/scene-tile.vue +++ b/assets/components/releases/scene-tile.vue @@ -19,8 +19,8 @@ > import Details from './tile-details.vue'; -function sfw() { - return this.$store.state.ui.sfw; -} - export default { components: { Details, @@ -144,9 +140,6 @@ export default { default: null, }, }, - computed: { - sfw, - }, }; diff --git a/assets/components/tags/tag.vue b/assets/components/tags/tag.vue index 6956ac71..6f6af1ba 100644 --- a/assets/components/tags/tag.vue +++ b/assets/components/tags/tag.vue @@ -39,7 +39,7 @@ v-if="showAlbum" :items="[tag.poster, ...tag.photos]" :title="tag.name" - path="/img" + :local="true" class="portrait" @close="$router.go(-1)" /> diff --git a/assets/js/actors/actions.js b/assets/js/actors/actions.js index 23523e7e..de4b00ea 100644 --- a/assets/js/actors/actions.js +++ b/assets/js/actors/actions.js @@ -89,6 +89,7 @@ function initActorActions(store, router) { hash comment credit + isS3 sfw: sfwMedia { id thumbnail @@ -118,6 +119,7 @@ function initActorActions(store, router) { thumbnail lazy hash + isS3 comment credit entropy @@ -320,6 +322,7 @@ function initActorActions(store, router) { lazy comment credit + isS3 sfw: sfwMedia { id thumbnail diff --git a/assets/js/config/default.js b/assets/js/config/default.js index 4baac88d..671eb4d7 100644 --- a/assets/js/config/default.js +++ b/assets/js/config/default.js @@ -2,6 +2,11 @@ export default { api: { url: `${window.location.origin}/api`, }, + media: { + assetPath: '/img', + mediaPath: '/media', + s3Path: 'https://s3.eu-central-1.wasabisys.com/traxxx', + }, showDisclaimer: false, disclaimer: 'This site is in early development, and content may occasionally disappear. Please stay tuned, you will be able to use traxxx to its full potential in the near future!', selectableTags: [ diff --git a/assets/js/fragments.js b/assets/js/fragments.js index bc95747a..7845898f 100644 --- a/assets/js/fragments.js +++ b/assets/js/fragments.js @@ -100,6 +100,7 @@ const releasePosterFragment = ` path thumbnail lazy + isS3 comment sfw: sfwMedia { id @@ -120,6 +121,7 @@ const releaseCoversFragment = ` path thumbnail lazy + isS3 comment sfw: sfwMedia { id @@ -140,6 +142,7 @@ const releasePhotosFragment = ` path thumbnail lazy + isS3 comment sfw: sfwMedia { id @@ -160,6 +163,7 @@ const releaseTrailerFragment = ` path thumbnail mime + isS3 isVr } } @@ -173,6 +177,7 @@ const releaseTeaserFragment = ` path thumbnail mime + isS3 } } `; diff --git a/assets/js/main.js b/assets/js/main.js index 9bb12c55..2b57093e 100644 --- a/assets/js/main.js +++ b/assets/js/main.js @@ -24,6 +24,17 @@ async function init() { const app = createApp(Container); const events = mitt(); + function getPath(media, type, options) { + const path = (store.state.ui.sfw && media.assetPath) + || (media.isS3 && config.media.s3Path) + || (options?.local && config.media.assetPath) + || config.media.mediaPath; + + const filename = type && !options?.original ? media[type] : media.path; + + return `${path}/${filename}`; + } + initUiObservers(store, router); if (window.env.sfw) { @@ -64,6 +75,8 @@ async function init() { formatDuration, isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB), isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB), + getPath, + getBgPath: (media, type) => `url(${getPath(media, type)})`, }, beforeCreate() { this.uid = uid; diff --git a/assets/js/ui/state.js b/assets/js/ui/state.js index 302692eb..e6818dfe 100644 --- a/assets/js/ui/state.js +++ b/assets/js/ui/state.js @@ -3,10 +3,12 @@ const storedBatch = localStorage.getItem('batch'); const storedSfw = localStorage.getItem('sfw'); const storedTheme = localStorage.getItem('theme'); +const deviceTheme = window.matchMedia?.('(prefers-color-scheme: dark)').matches ? 'dark' : 'light'; + export default { tagFilter: storedTagFilter ? storedTagFilter.split(',') : [], range: 'latest', batch: storedBatch || 'all', sfw: storedSfw === 'true' || false, - theme: storedTheme || 'light', + theme: storedTheme || deviceTheme, }; diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 951d0128..c1b9d90d 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -28,6 +28,9 @@ exports.up = knex => Promise.resolve() table.integer('index'); table.text('mime'); + table.boolean('is_s3') + .defaultTo(false); + table.text('hash'); table.bigInteger('size', 12); @@ -1020,17 +1023,17 @@ exports.up = knex => Promise.resolve() CREATE FUNCTION search_entities(search text) RETURNS SETOF entities AS $$ SELECT * FROM entities WHERE - name ILIKE ('%' || search || '%') OR - slug ILIKE ('%' || search || '%') OR - array_to_string(alias, '') ILIKE ('%' || search || '%') OR - replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || search || '%') OR + name ILIKE ('%' || TRIM(search) || '%') OR + slug ILIKE ('%' || TRIM(search) || '%') OR + array_to_string(alias, '') ILIKE ('%' || TRIM(search) || '%') OR + replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || TRIM(search) || '%') OR url ILIKE ('%' || search || '%') $$ LANGUAGE SQL STABLE; CREATE FUNCTION search_actors(search text, min_length numeric DEFAULT 2) RETURNS SETOF actors AS $$ SELECT * FROM actors WHERE length(search) >= min_length - AND name ILIKE ('%' || search || '%') + AND name ILIKE ('%' || TRIM(search) || '%') $$ LANGUAGE SQL STABLE; CREATE FUNCTION actors_tags(actor actors, selectable_tags text[]) RETURNS SETOF tags AS $$ diff --git a/src/media.js b/src/media.js index e3646f88..de92bc85 100644 --- a/src/media.js +++ b/src/media.js @@ -14,6 +14,7 @@ const ffmpeg = require('fluent-ffmpeg'); const sharp = require('sharp'); const blake2 = require('blake2'); const taskQueue = require('promise-task-queue'); +const AWS = require('aws-sdk'); const logger = require('./logger')(__filename); const argv = require('./argv'); @@ -25,6 +26,17 @@ const { get } = require('./utils/qu'); const pipeline = util.promisify(stream.pipeline); const streamQueue = taskQueue(); +const endpoint = new AWS.Endpoint('s3.wasabisys.com'); + +const s3 = new AWS.S3({ + // region: 'eu-central-1', + endpoint, + credentials: { + accessKeyId: config.s3.accessKey, + secretAccessKey: config.s3.secretKey, + }, +}); + function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) { // limit media sets, use extras as fallbacks if (medias.length <= limit) { @@ -303,6 +315,58 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) { throw new Error(`Could not extract source from ${baseSource.url}: ${res.status}`); } +async function storeS3Object(filepath, media) { + const fullFilepath = path.join(config.media.path, filepath); + const file = fs.createReadStream(fullFilepath); + + const status = await s3.upload({ + Bucket: config.s3.bucket, + Body: file, + Key: filepath, + ContentType: media.meta.mimetype, + }).promise(); + + await fsPromises.unlink(fullFilepath); + + return status; +} + +async function writeImage(image, media, info, filepath, isProcessed) { + if (isProcessed && info.pages) { + // convert animated image to WebP and write to permanent location + await image + .webp() + .toFile(path.join(config.media.path, filepath)); + } + + if (isProcessed) { + // convert to JPEG and write to permanent location + await image + .jpeg() + .toFile(path.join(config.media.path, filepath)); + } +} + +async function writeThumbnail(image, thumbpath) { + return image + .resize({ + height: config.media.thumbnailSize, + withoutEnlargement: true, + }) + .jpeg({ quality: config.media.thumbnailQuality }) + .toFile(path.join(config.media.path, thumbpath)); +} + +async function writeLazy(image, lazypath) { + return image + .resize({ + height: config.media.lazySize, + withoutEnlargement: true, + }) + .jpeg({ quality: config.media.lazyQuality }) + .toFile(path.join(config.media.path, lazypath)); +} + async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options) { logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`); @@ -343,46 +407,28 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil }); } - if (isProcessed) { - if (info.pages) { - // convert animated image to WebP and write to permanent location - await image - .webp() - .toFile(path.join(config.media.path, filepath)); - } else { - // convert to JPEG and write to permanent location - await image - .jpeg() - .toFile(path.join(config.media.path, filepath)); - } - } - - // generate thumbnail and lazy await Promise.all([ - image - .resize({ - height: config.media.thumbnailSize, - withoutEnlargement: true, - }) - .jpeg({ quality: config.media.thumbnailQuality }) - .toFile(path.join(config.media.path, thumbpath)), - image - .resize({ - height: config.media.lazySize, - withoutEnlargement: true, - }) - .jpeg({ quality: config.media.lazyQuality }) - .toFile(path.join(config.media.path, lazypath)), + writeImage(image, media, info, filepath, isProcessed), + writeThumbnail(image, thumbpath), + writeLazy(image, lazypath), ]); if (isProcessed) { - // remove temp file + // file already stored, remove temporary file await fsPromises.unlink(media.file.path); } else { - // move temp file to permanent location + // image not processed, simply move temporary file to final location await fsPromises.rename(media.file.path, path.join(config.media.path, filepath)); } + if (config.s3.enabled) { + await Promise.all([ + storeS3Object(filepath, media), + storeS3Object(thumbpath, media), + storeS3Object(lazypath, media), + ]); + } + logger.silly(`Stored thumbnail, lazy and permanent media file for ${media.id} from ${media.src} at ${filepath}`); return { @@ -521,7 +567,6 @@ async function fetchSource(source, baseMedia) { try { const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`); - const tempFileTarget = fs.createWriteStream(tempFilePath); const hashStream = new stream.PassThrough(); let size = 0; @@ -648,6 +693,7 @@ function curateMediaEntry(media, index) { path: media.file.path, thumbnail: media.file.thumbnail, lazy: media.file.lazy, + is_s3: config.s3.enabled, index, mime: media.meta.mimetype, hash: media.meta.hash, diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index f92faa5e..6e98522a 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -13,7 +13,9 @@ const { cookieToData } = require('../utils/cookies'); function getThumbs(scene) { if (scene.images.poster) { - return scene.images.poster.map(image => image.xl.url); + return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array + .filter(img => typeof img === 'object') // remove alternateText property + .map(image => image.xl.url); } if (scene.images.card_main_rect) {