Added S3 support for media files. Fixed MindGeek scraper for new poster data structure.

This commit is contained in:
DebaucheryLibrarian 2021-02-22 02:33:39 +01:00
parent 9a65d8c0eb
commit 37e39dc1ec
17 changed files with 152 additions and 79 deletions

View File

@ -42,13 +42,13 @@
>
<a
v-if="actor.avatar"
:href="`/media/${actor.avatar.path}`"
:href="getPath(actor.avatar)"
target="_blank"
rel="noopener noreferrer"
class="avatar-link"
>
<img
:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
:src="getPath(actor.avatar, 'thumbnail')"
:title="actor.avatar.credit && `© ${actor.avatar.credit}`"
class="avatar"
>

View File

@ -8,14 +8,14 @@
>
<a
v-if="actor.avatar"
:href="`/media/${actor.avatar.path}`"
:href="getPath(actor.avatar)"
target="_blank"
rel="noopener noreferrer"
class="avatar-link photo-link"
>
<img
:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
:style="{ 'background-image': sfw ? `/img/${actor.avatar.sfw.lazy}` : `/media/${actor.avatar.lazy}` }"
:src="getPath(actor.avatar, 'thumbnail')"
:style="{ 'background-image': getBgPath(actor.avatar, 'lazy') }"
:title="actor.avatar.credit && `© ${actor.avatar.credit}`"
loading="lazy"
class="avatar photo"
@ -26,14 +26,14 @@
<a
v-for="photo in photos"
:key="`photo-${photo.id}`"
:href="`/media/${photo.path}`"
:href="getPath(photo)"
target="_blank"
rel="noopener noreferrer"
class="photo-link"
>
<img
:src="sfw ? `/img/${photo.sfw.thumbnail}` : `/media/${photo.thumbnail}`"
:style="{ 'background-image': sfw ? `/img/${photo.sfw.lazy}` : `/media/${photo.lazy}` }"
:src="getPath(photo, 'thumbnail')"
:style="{ 'background-image': getBgPath(photo, 'lazy') }"
:title="`© ${photo.credit || photo.entity.name}`"
loading="lazy"
class="photo"

View File

@ -45,8 +45,8 @@
<div class="avatar-container">
<img
v-if="actor.avatar"
:src="sfw ? `/img/${actor.avatar.sfw.thumbnail}` : `/media/${actor.avatar.thumbnail}`"
:style="{ 'background-image': sfw ? `url(/img/${actor.avatar.sfw.lazy})`: `url(/img/${actor.avatar.lazy})` }"
:src="getPath(actor.avatar, 'thumbnail')"
:style="{ 'background-image': getBgPath(actor.avatar, 'lazy') }"
loading="lazy"
class="avatar"
>

View File

@ -21,12 +21,12 @@
class="item-container"
>
<a
:href="`${path}/${item.path}`"
:href="getPath(item, null, { local })"
class="item-link"
target="_blank"
>
<img
:src="`${path}/${item.thumbnail}`"
:src="getPath(item, 'thumbnail', { local })"
:title="item.title"
loading="lazy"
class="item image"
@ -63,9 +63,9 @@ export default {
type: String,
default: null,
},
path: {
type: String,
default: '/media',
local: {
type: Boolean,
default: false,
},
portrait: {
type: Boolean,

View File

@ -31,7 +31,7 @@
<img
v-else-if="release.teaser && /^image\//.test(release.teaser.mime)"
:src="sfw ? `/img/${release.teaser.sfw.thumbnail}` : `/media/${release.teaser.path}`"
:src="getPath(release.teaser, 'thumbnail', { original: true })"
:alt="release.title"
loading="lazy"
class="item trailer"
@ -40,7 +40,7 @@
<a
v-if="release.poster"
v-tooltip="'View poster'"
:href="`/media/${release.poster.path}`"
:href="`${config.media.mediaPath}/${release.poster.path}`"
:class="{ playing }"
target="_blank"
rel="noopener noreferrer"
@ -59,13 +59,13 @@
<a
v-for="cover in release.covers"
:key="`cover-${cover.id}`"
:href="`/media/${cover.path}`"
:href="`${config.media.mediaPath}/${cover.path}`"
target="_blank"
rel="noopener noreferrer"
>
<img
:src="`/media/${cover.thumbnail}`"
:style="{ 'background-image': sfw ? `url(/media/${cover.sfw.lazy})` : `url(/media/${cover.lazy})` }"
:src="getPath(cover, 'thumbnail')"
:style="{ 'background-image': getBgPath(cover, 'lazy') }"
class="item cover"
loading="lazy"
@load="$emit('load', $event)"
@ -79,15 +79,15 @@
class="item-container"
>
<a
:href="`/media/${photo.path}`"
:href="getPath(photo)"
:class="{ sfw }"
class="item-link"
target="_blank"
rel="noopener noreferrer"
>
<img
:src="sfw ? `/img/${photo.sfw.thumbnail}` : `/media/${photo.thumbnail}`"
:style="{ 'background-image': sfw ? `url(/img/${photo.sfw.lazy})` : `url(/media/${photo.lazy})` }"
:src="getPath(photo, 'thumbnail')"
:style="{ 'background-image': getPath(photo, 'lazy') }"
:alt="`Photo ${photo.index + 1}`"
loading="lazy"
class="item"
@ -115,15 +115,15 @@ function sfw() {
function poster() {
if (this.release.poster) {
return this.sfw ? `/img/${this.release.poster.sfw.thumbnail}` : `/media/${this.release.poster.thumbnail}`;
return this.getPath(this.release.poster, 'thumbnail');
}
if (this.release.covers?.length > 0) {
return this.sfw ? `/img/${this.release.covers[0].sfw.path}` : `/media/${this.release.covers[0].path}`;
return this.getPath(this.release.covers[0], 'thumbnail');
}
if (this.photos?.length > 0) {
return this.sfw ? `/img/${this.photos[0].sfw.thumbnail}` : `/media/${this.photos[0].thumbnail}`;
return this.getPath(this.release.photos[0], 'thumbnail');
}
return null;

View File

@ -9,8 +9,8 @@
>
<img
v-if="movie.covers[0]"
:src="sfw ? `/img/${movie.covers[0].sfw.thumbnail}` : `/media/${movie.covers[0].thumbnail}`"
:style="{ 'background-image': sfw ? `/img/${movie.covers[0].sfw.lazy}` : `/media/${movie.covers[0].lazy }` }"
:src="getPath(movie.covers[0], 'thumbnail')"
:style="{ 'background-image': getBgPath(movie.covers[0], 'lazy') }"
loading="lazy"
>
</router-link>

View File

@ -29,6 +29,7 @@
v-if="showAlbum"
:items="[release.poster, ...release.photos]"
:title="release.title"
:path="config.media.mediaPath"
@close="$router.go(-1)"
/>

View File

@ -19,8 +19,8 @@
>
<img
v-if="release.poster"
:src="sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`"
:style="{ 'background-image': sfw ? `/img/${release.poster.sfw.lazy}` : `/media/${release.poster.lazy}` }"
:src="getPath(release.poster, 'thumbnail')"
:style="{ 'background-image': getBgPath(release.poster, 'lazy') }"
:alt="release.title"
class="thumbnail"
loading="lazy"
@ -28,8 +28,8 @@
<img
v-else-if="release.photos && release.photos.length > 0"
:src="sfw ? `/img/${release.photos[0].sfw.thumbnail}` : `/media/${release.photos[0].thumbnail}`"
:style="{ 'background-image': sfw ? `/img/${release.photos[0].sfw.lazy}` : `/media/${release.photos[0].lazy}` }"
:src="getPath(release.photos[0], 'thumbnail')"
:style="{ 'background-image': getBgPath(release.photos[0], 'lazy') } "
:alt="release.title"
class="thumbnail"
loading="lazy"
@ -130,10 +130,6 @@
<script>
import Details from './tile-details.vue';
function sfw() {
return this.$store.state.ui.sfw;
}
export default {
components: {
Details,
@ -144,9 +140,6 @@ export default {
default: null,
},
},
computed: {
sfw,
},
};
</script>

View File

@ -39,7 +39,7 @@
v-if="showAlbum"
:items="[tag.poster, ...tag.photos]"
:title="tag.name"
path="/img"
:local="true"
class="portrait"
@close="$router.go(-1)"
/>

View File

@ -89,6 +89,7 @@ function initActorActions(store, router) {
hash
comment
credit
isS3
sfw: sfwMedia {
id
thumbnail
@ -118,6 +119,7 @@ function initActorActions(store, router) {
thumbnail
lazy
hash
isS3
comment
credit
entropy
@ -320,6 +322,7 @@ function initActorActions(store, router) {
lazy
comment
credit
isS3
sfw: sfwMedia {
id
thumbnail

View File

@ -2,6 +2,11 @@ export default {
api: {
url: `${window.location.origin}/api`,
},
media: {
assetPath: '/img',
mediaPath: '/media',
s3Path: 'https://s3.eu-central-1.wasabisys.com/traxxx',
},
showDisclaimer: false,
disclaimer: 'This site is in early development, and content may occasionally disappear. Please stay tuned, you will be able to use traxxx to its full potential in the near future!',
selectableTags: [

View File

@ -100,6 +100,7 @@ const releasePosterFragment = `
path
thumbnail
lazy
isS3
comment
sfw: sfwMedia {
id
@ -120,6 +121,7 @@ const releaseCoversFragment = `
path
thumbnail
lazy
isS3
comment
sfw: sfwMedia {
id
@ -140,6 +142,7 @@ const releasePhotosFragment = `
path
thumbnail
lazy
isS3
comment
sfw: sfwMedia {
id
@ -160,6 +163,7 @@ const releaseTrailerFragment = `
path
thumbnail
mime
isS3
isVr
}
}
@ -173,6 +177,7 @@ const releaseTeaserFragment = `
path
thumbnail
mime
isS3
}
}
`;

View File

@ -24,6 +24,17 @@ async function init() {
const app = createApp(Container);
const events = mitt();
function getPath(media, type, options) {
const path = (store.state.ui.sfw && media.assetPath)
|| (media.isS3 && config.media.s3Path)
|| (options?.local && config.media.assetPath)
|| config.media.mediaPath;
const filename = type && !options?.original ? media[type] : media.path;
return `${path}/${filename}`;
}
initUiObservers(store, router);
if (window.env.sfw) {
@ -64,6 +75,8 @@ async function init() {
formatDuration,
isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB),
isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB),
getPath,
getBgPath: (media, type) => `url(${getPath(media, type)})`,
},
beforeCreate() {
this.uid = uid;

View File

@ -3,10 +3,12 @@ const storedBatch = localStorage.getItem('batch');
const storedSfw = localStorage.getItem('sfw');
const storedTheme = localStorage.getItem('theme');
const deviceTheme = window.matchMedia?.('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
export default {
tagFilter: storedTagFilter ? storedTagFilter.split(',') : [],
range: 'latest',
batch: storedBatch || 'all',
sfw: storedSfw === 'true' || false,
theme: storedTheme || 'light',
theme: storedTheme || deviceTheme,
};

View File

@ -28,6 +28,9 @@ exports.up = knex => Promise.resolve()
table.integer('index');
table.text('mime');
table.boolean('is_s3')
.defaultTo(false);
table.text('hash');
table.bigInteger('size', 12);
@ -1020,17 +1023,17 @@ exports.up = knex => Promise.resolve()
CREATE FUNCTION search_entities(search text) RETURNS SETOF entities AS $$
SELECT * FROM entities
WHERE
name ILIKE ('%' || search || '%') OR
slug ILIKE ('%' || search || '%') OR
array_to_string(alias, '') ILIKE ('%' || search || '%') OR
replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || search || '%') OR
name ILIKE ('%' || TRIM(search) || '%') OR
slug ILIKE ('%' || TRIM(search) || '%') OR
array_to_string(alias, '') ILIKE ('%' || TRIM(search) || '%') OR
replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || TRIM(search) || '%') OR
url ILIKE ('%' || search || '%')
$$ LANGUAGE SQL STABLE;
CREATE FUNCTION search_actors(search text, min_length numeric DEFAULT 2) RETURNS SETOF actors AS $$
SELECT * FROM actors
WHERE length(search) >= min_length
AND name ILIKE ('%' || search || '%')
AND name ILIKE ('%' || TRIM(search) || '%')
$$ LANGUAGE SQL STABLE;
CREATE FUNCTION actors_tags(actor actors, selectable_tags text[]) RETURNS SETOF tags AS $$

View File

@ -14,6 +14,7 @@ const ffmpeg = require('fluent-ffmpeg');
const sharp = require('sharp');
const blake2 = require('blake2');
const taskQueue = require('promise-task-queue');
const AWS = require('aws-sdk');
const logger = require('./logger')(__filename);
const argv = require('./argv');
@ -25,6 +26,17 @@ const { get } = require('./utils/qu');
const pipeline = util.promisify(stream.pipeline);
const streamQueue = taskQueue();
const endpoint = new AWS.Endpoint('s3.wasabisys.com');
const s3 = new AWS.S3({
// region: 'eu-central-1',
endpoint,
credentials: {
accessKeyId: config.s3.accessKey,
secretAccessKey: config.s3.secretKey,
},
});
function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
// limit media sets, use extras as fallbacks
if (medias.length <= limit) {
@ -303,6 +315,58 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
throw new Error(`Could not extract source from ${baseSource.url}: ${res.status}`);
}
async function storeS3Object(filepath, media) {
const fullFilepath = path.join(config.media.path, filepath);
const file = fs.createReadStream(fullFilepath);
const status = await s3.upload({
Bucket: config.s3.bucket,
Body: file,
Key: filepath,
ContentType: media.meta.mimetype,
}).promise();
await fsPromises.unlink(fullFilepath);
return status;
}
async function writeImage(image, media, info, filepath, isProcessed) {
if (isProcessed && info.pages) {
// convert animated image to WebP and write to permanent location
await image
.webp()
.toFile(path.join(config.media.path, filepath));
}
if (isProcessed) {
// convert to JPEG and write to permanent location
await image
.jpeg()
.toFile(path.join(config.media.path, filepath));
}
}
async function writeThumbnail(image, thumbpath) {
return image
.resize({
height: config.media.thumbnailSize,
withoutEnlargement: true,
})
.jpeg({ quality: config.media.thumbnailQuality })
.toFile(path.join(config.media.path, thumbpath));
}
async function writeLazy(image, lazypath) {
return image
.resize({
height: config.media.lazySize,
withoutEnlargement: true,
})
.jpeg({ quality: config.media.lazyQuality })
.toFile(path.join(config.media.path, lazypath));
}
async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options) {
logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`);
@ -343,46 +407,28 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
});
}
if (isProcessed) {
if (info.pages) {
// convert animated image to WebP and write to permanent location
await image
.webp()
.toFile(path.join(config.media.path, filepath));
} else {
// convert to JPEG and write to permanent location
await image
.jpeg()
.toFile(path.join(config.media.path, filepath));
}
}
// generate thumbnail and lazy
await Promise.all([
image
.resize({
height: config.media.thumbnailSize,
withoutEnlargement: true,
})
.jpeg({ quality: config.media.thumbnailQuality })
.toFile(path.join(config.media.path, thumbpath)),
image
.resize({
height: config.media.lazySize,
withoutEnlargement: true,
})
.jpeg({ quality: config.media.lazyQuality })
.toFile(path.join(config.media.path, lazypath)),
writeImage(image, media, info, filepath, isProcessed),
writeThumbnail(image, thumbpath),
writeLazy(image, lazypath),
]);
if (isProcessed) {
// remove temp file
// file already stored, remove temporary file
await fsPromises.unlink(media.file.path);
} else {
// move temp file to permanent location
// image not processed, simply move temporary file to final location
await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
}
if (config.s3.enabled) {
await Promise.all([
storeS3Object(filepath, media),
storeS3Object(thumbpath, media),
storeS3Object(lazypath, media),
]);
}
logger.silly(`Stored thumbnail, lazy and permanent media file for ${media.id} from ${media.src} at ${filepath}`);
return {
@ -521,7 +567,6 @@ async function fetchSource(source, baseMedia) {
try {
const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
const tempFileTarget = fs.createWriteStream(tempFilePath);
const hashStream = new stream.PassThrough();
let size = 0;
@ -648,6 +693,7 @@ function curateMediaEntry(media, index) {
path: media.file.path,
thumbnail: media.file.thumbnail,
lazy: media.file.lazy,
is_s3: config.s3.enabled,
index,
mime: media.meta.mimetype,
hash: media.meta.hash,

View File

@ -13,7 +13,9 @@ const { cookieToData } = require('../utils/cookies');
function getThumbs(scene) {
if (scene.images.poster) {
return scene.images.poster.map(image => image.xl.url);
return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
.filter(img => typeof img === 'object') // remove alternateText property
.map(image => image.xl.url);
}
if (scene.images.card_main_rect) {