DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added series.

This commit is contained in:
DebaucheryLibrarian 2022-03-26 17:40:20 +01:00
parent 661b8b716b
commit fd8170f223
13 changed files with 377 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
assets/components/container/container.vue
View File

@ -62,7 +62,7 @@ async function setConsent(consent, includeQueer) {
}
if (includeQueer) {
this.$store.dispatch('setTagFilter', this.$store.state.ui.tagFilter.filter(tag => !['gay', 'bisexual', 'transsexual'].includes(tag)));
this.$store.dispatch('setTagFilter', this.$store.state.ui.tagFilter.filter((tag) => !['gay', 'bisexual', 'transsexual'].includes(tag)));
return;
}

2
assets/components/icon/icon.vue
View File

@ -28,7 +28,7 @@ export default {
};
},
beforeMount() {
this.svg = require(`../../img/icons/${this.icon}.svg`).default;
this.svg = require(`../../img/icons/${this.icon}.svg`).default; // eslint-disable-line global-require, import/no-dynamic-require
},
};
</script>

15
assets/components/releases/release.vue
View File

@ -79,22 +79,23 @@
/>
<div
v-if="release.movies && release.movies.length > 0"
v-if="release.movies?.length > 0 || release.series?.length > 0"
class="row"
>
<span class="row-label">Part of</span>
<div class="movies">
<router-link
v-for="movie in release.movies"
v-for="movie in [...release.movies, ...release.series]"
:key="`movie-${movie.id}`"
:to="{ name: 'movie', params: { releaseId: movie.id, releaseSlug: movie.slug } }"
:to="{ name: movie.type || 'movie', params: { releaseId: movie.id, releaseSlug: movie.slug } }"
class="movie"
>
<span class="movie-title">{{ movie.title }}</span>
<img
v-if="movie.covers.length > 0"
:src="getPath(movie.covers[0], 'thumbnail')"
v-if="movie.covers.length > 0 || movie.poster"
:src="getPath(movie.covers[0] || movie.poster, 'thumbnail')"
class="movie-cover"
>
</router-link>
@ -243,6 +244,10 @@ async function fetchRelease(scroll = true) {
this.release = await this.$store.dispatch('fetchMovieById', this.$route.params.releaseId);
}
if (this.$route.name === 'serie') {
this.release = await this.$store.dispatch('fetchSerieById', this.$route.params.releaseId);
}
if (scroll && this.$refs.content) {
this.$refs.content.scrollTop = 0;
}

8
assets/js/curate.js
View File

@ -65,16 +65,18 @@ function curateActor(actor, release) {
return curatedActor;
}
function curateRelease(release) {
function curateRelease(release, type = 'scene') {
const curatedRelease = {
...release,
type: release.type || type,
actors: [],
poster: release.poster && release.poster.media,
tags: release.tags ? release.tags.map((tag) => tag.tag || tag) : [],
};
if (release.scenes) curatedRelease.scenes = release.scenes.filter(Boolean).map(({ scene }) => curateRelease(scene));
if (release.movies) curatedRelease.movies = release.movies.filter(Boolean).map(({ movie }) => curateRelease(movie));
if (release.scenes) curatedRelease.scenes = release.scenes.filter(Boolean).map(({ scene }) => curateRelease(scene, 'scene'));
if (release.movies) curatedRelease.movies = release.movies.filter(Boolean).map(({ movie }) => curateRelease(movie, 'movie'));
if (release.series) curatedRelease.series = release.series.filter(Boolean).map(({ serie }) => curateRelease(serie, 'serie'));
if (release.chapters) curatedRelease.chapters = release.chapters.filter(Boolean).map((chapter) => curateRelease(chapter));
if (release.photos) curatedRelease.photos = release.photos.filter(Boolean).map((photo) => photo.media || photo);
if (release.covers) curatedRelease.covers = release.covers.filter(Boolean).map(({ media }) => media);

32
assets/js/fragments.js
View File

@ -442,6 +442,25 @@ const releasesFragment = `
}
`;
const mediaFragment = `
media {
id
index
path
thumbnail
lazy
isS3
comment
sfw: sfwMedia {
id
thumbnail
lazy
path
comment
}
}
`;
const releaseFragment = `
release(id: $releaseId) {
id
@ -536,6 +555,19 @@ const releaseFragment = `
}
}
}
series: seriesScenesBySceneId {
serie {
id
title
slug
covers: seriesCoversBySerieId(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) {
${mediaFragment}
}
poster: seriesPosterBySerieId {
${mediaFragment}
}
}
}
isFavorited
isStashed(includeFavorites: false)
stashes: stashesScenesBySceneId(

29
assets/js/releases/actions.js
View File

@ -141,16 +141,16 @@ function initReleasesActions(store, router) {
};
}
async function fetchMovieById({ _commit }, movieId) {
async function fetchCollectionById({ _commit }, movieId, type = 'movie') {
// const release = await get(`/releases/${releaseId}`);
const { movie } = await graphql(`
const result = await graphql(`
query Movie(
$movieId: Int!
$hasAuth: Boolean!
$userId: Int
) {
movie(id: $movieId) {
${type}(id: $movieId) {
id
title
description
@ -182,7 +182,7 @@ function initReleasesActions(store, router) {
isS3
}
}
poster: moviesPoster {
poster: ${type === 'series' ? 'seriesPosterBySerieId' : 'moviesPoster'} {
media {
id
path
@ -195,7 +195,7 @@ function initReleasesActions(store, router) {
isS3
}
}
covers: moviesCovers(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) {
covers: ${type === 'series' ? 'seriesCoversBySerieId' : 'moviesCovers'}(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) {
media {
id
path
@ -208,14 +208,14 @@ function initReleasesActions(store, router) {
isS3
}
}
trailer: moviesTrailer {
trailer: ${type === 'series' ? 'seriesTrailerBySerieId' : 'moviesTrailer'} {
media {
id
path
isS3
}
}
scenes: moviesScenes {
scenes: ${type === 'series' ? 'seriesScenesBySerieId' : 'moviesScenes'} {
scene {
${releaseFields}
}
@ -259,7 +259,7 @@ function initReleasesActions(store, router) {
hasLogo
}
}
stashes: stashesMovies(
stashes: ${type === 'series' ? 'stashesSeriesBySerieId' : 'stashesMovies'}(
filter: {
stash: {
userId: {
@ -283,12 +283,20 @@ function initReleasesActions(store, router) {
userId: store.state.auth.user?.id,
});
if (!movie) {
if (!result[type]) {
router.replace('/not-found');
return null;
}
return curateRelease(movie);
return curateRelease(result[type]);
}
async function fetchMovieById(context, movieId) {
return fetchCollectionById(context, movieId, 'movie');
}
async function fetchSerieById(context, serieId) {
return fetchCollectionById(context, serieId, 'series');
}
return {
@ -296,6 +304,7 @@ function initReleasesActions(store, router) {
fetchReleaseById,
fetchMovies,
fetchMovieById,
fetchSerieById,
searchMovies,
};
}

5
assets/js/router.js
View File

@ -71,6 +71,11 @@ const routes = [
component: Release,
name: 'movie',
},
{
path: '/serie/:releaseId/:releaseSlug?',
component: Release,
name: 'serie',
},
{
path: '/actor/:actorId/:actorSlug',
name: 'actor',

2
migrations/20220227215315_entity_filters.js
View File

@ -19,5 +19,5 @@ exports.up = async (knex) => knex.raw(`
`);
exports.down = async (knex) => knex.raw(`
DROP FUNCTION IF EXISTS entities_tags;
DROP FUNCTION IF EXISTS entities_scene_tags;
`);

127
migrations/20220304233846_series.js
View File

@ -1,3 +1,5 @@
const config = require('config');
exports.up = async (knex) => Promise.resolve()
.then(() => knex.schema.createTable('series', (table) => {
table.increments('id', 16);
@ -89,10 +91,127 @@ exports.up = async (knex) => Promise.resolve()
.onDelete('cascade');
table.unique('serie_id');
}))
.then(() => knex.schema.createTable('series_covers', (table) => {
table.integer('serie_id', 16)
.notNullable()
.references('id')
.inTable('series')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media');
table.unique(['serie_id', 'media_id']);
}))
.then(() => knex.schema.createTable('series_search', (table) => {
table.integer('serie_id', 16)
.references('id')
.inTable('series')
.onDelete('cascade');
}))
.then(() => knex.schema.createTable('stashes_series', (table) => {
table.integer('stash_id')
.notNullable()
.references('id')
.inTable('stashes')
.onDelete('cascade');
table.integer('serie_id')
.notNullable()
.references('id')
.inTable('series')
.onDelete('cascade');
table.unique(['stash_id', 'serie_id']);
table.string('comment');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
}))
.then(() => knex.raw(`
ALTER TABLE series_search ADD COLUMN document tsvector;
CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
CREATE INDEX series_search_index ON series_search USING GIN (document);
CREATE FUNCTION series_actors(serie series) RETURNS SETOF actors AS $$
SELECT actors.*
FROM series_scenes
LEFT JOIN
releases ON releases.id = series_scenes.scene_id
LEFT JOIN
releases_actors ON releases_actors.release_id = releases.id
LEFT JOIN
actors ON actors.id = releases_actors.actor_id
WHERE series_scenes.serie_id = serie.id
AND actors.id IS NOT NULL
GROUP BY actors.id
ORDER BY actors.name, actors.gender
$$ LANGUAGE SQL STABLE;
CREATE FUNCTION series_tags(serie series) RETURNS SETOF tags AS $$
SELECT tags.*
FROM series_scenes
LEFT JOIN
releases ON releases.id = series_scenes.scene_id
LEFT JOIN
releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN
tags ON tags.id = releases_tags.tag_id
WHERE series_scenes.serie_id = serie.id
AND tags.id IS NOT NULL
GROUP BY tags.id
ORDER BY tags.priority DESC
$$ LANGUAGE SQL STABLE;
CREATE FUNCTION series_photos(serie series) RETURNS SETOF media AS $$
SELECT media.*
FROM series_scenes
LEFT JOIN
releases ON releases.id = series_scenes.scene_id
INNER JOIN
releases_photos ON releases_photos.release_id = releases.id
LEFT JOIN
media ON media.id = releases_photos.media_id
WHERE series_scenes.serie_id = serie.id
GROUP BY media.id
ORDER BY media.index ASC
$$ LANGUAGE SQL STABLE;
COMMENT ON FUNCTION search_movies IS E'@sortable';
GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO :visitor;
ALTER TABLE stashes_series ENABLE ROW LEVEL SECURITY;
CREATE POLICY stashes_policy ON stashes_series
USING (EXISTS (
SELECT *
FROM stashes
WHERE stashes.id = stashes_series.stash_id
AND (stashes.user_id = current_user_id() OR stashes.public)
));
`, {
visitor: knex.raw(config.database.query.user),
}));
exports.down = async (knex) => Promise.resolve()
.then(() => knex.schema.dropTable('series_covers'))
.then(() => knex.schema.dropTable('series_trailers'))
.then(() => knex.schema.dropTable('series_posters'))
.then(() => knex.schema.dropTable('series'));
.then(() => knex.raw(`
DROP FUNCTION IF EXISTS series_actors;
DROP FUNCTION IF EXISTS series_tags;
DROP FUNCTION IF EXISTS series_photos;
DROP TABLE IF EXISTS stashes_series CASCADE;
DROP TABLE IF EXISTS series_scenes CASCADE;
DROP TABLE IF EXISTS series_trailers CASCADE;
DROP TABLE IF EXISTS series_posters CASCADE;
DROP TABLE IF EXISTS series_covers CASCADE;
DROP TABLE IF EXISTS series_search CASCADE;
DROP TABLE IF EXISTS series CASCADE;
`));

2
src/.eslintrc
View File

@ -14,6 +14,6 @@
"prefer-destructuring": "off",
"template-curly-spacing": "off",
"object-curly-newline": "off",
"max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}],
"max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}]
}
}

32
src/scrapers/mindgeek.js
View File

@ -33,18 +33,26 @@ function getThumbs(scene) {
return [];
}
function getCovers(images) {
return [
[
images.cover[0].md?.url,
images.cover[0].sm?.url,
images.cover[0].xs?.url,
function getCovers(images, target = 'cover') {
if (!images[target]) {
return [];
}
const covers = [
images[target][0].md?.url,
images[target][0].sm?.url,
images[target][0].xs?.url,
// bigger but usually upscaled
images.cover[0].xx?.url,
images.cover[0].xl?.url,
images.cover[0].lg?.url,
],
images[target][0].xx?.url,
images[target][0].xl?.url,
images[target][0].lg?.url,
];
if (target === 'poster') {
return covers;
}
return [covers];
}
function getVideos(data) {
@ -153,12 +161,12 @@ function scrapeRelease(data, url, channel, networkName) {
if (data.parent?.type === 'movie' || data.parent?.type === 'serie') {
release[data.parent.type] = {
entryId: data.parent.id,
url: `${getBasePath(channel, '/movie')}/${data.parent.id}/${slugify(data.parent.title, '-', { removePunctuation: true })}`,
url: `${getBasePath(channel, data.parent.type === 'movie' ? '/movie' : '/series')}/${data.parent.id}/${slugify(data.parent.title, '-', { removePunctuation: true })}`,
title: data.parent.title,
description: data.parent.description,
date: new Date(data.parent.dateReleased),
channel: slugify(data.parent.collections?.name || data.parent.brand),
covers: getCovers(data.parent.images),
poster: getCovers(data.parent.images, 'poster'),
shallow: true,
};
}

229
src/store-releases.js
View File

@ -146,7 +146,7 @@ function attachReleaseIds(releases, storedReleases, batchId) {
const releasesWithId = releases.map((release) => {
if (!release.entity) {
logger.error(`No entitity available for ${release.url}`);
logger.error(`No entity available for ${release.url}`);
return null;
}
@ -315,6 +315,142 @@ async function storeChapters(releases) {
await associateReleaseMedia(chaptersWithId, 'chapter');
}
async function associateMovieScenes(movies, movieScenes) {
const moviesByEntityIdAndEntryId = movies.reduce((acc, movie) => ({
...acc,
[movie.entity.id]: {
...acc[movie.entity.id],
[movie.entryId]: movie,
},
}), {});
const associations = movieScenes.map((scene) => {
if (!scene.movie) {
return null;
}
const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]
|| moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId];
if (sceneMovie?.id) {
return {
movie_id: sceneMovie.id,
scene_id: scene.id,
};
}
return null;
}).filter(Boolean);
await bulkInsert('movies_scenes', associations, false);
}
async function associateSerieScenes(series, serieScenes) {
const seriesByEntityIdAndEntryId = series.reduce((acc, serie) => ({
...acc,
[serie.entity.id]: {
...acc[serie.entity.id],
[serie.entryId]: serie,
},
}), {});
const associations = serieScenes.map((scene) => {
if (!scene.serie) {
return null;
}
const sceneSerie = seriesByEntityIdAndEntryId[scene.entity.id]?.[scene.serie.entryId]
|| seriesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.serie.entryId];
if (sceneSerie?.id) {
return {
serie_id: sceneSerie.id,
scene_id: scene.id,
};
}
return null;
}).filter(Boolean);
await bulkInsert('series_scenes', associations, false);
}
async function updateMovieSearch(movieIds, target = 'movie') {
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
const documents = await knex.raw(`
SELECT
${target}s.id AS ${target}_id,
TO_TSVECTOR(
'english',
COALESCE(${target}s.title, '') || ' ' ||
entities.name || ' ' ||
entities.slug || ' ' ||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ')
) as document
FROM ${target}s
LEFT JOIN entities ON ${target}s.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN actors ON actors.id = releases_actors.actor_id
LEFT JOIN tags ON tags.id = releases_tags.tag_id
${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
`, movieIds && [movieIds]);
if (documents.rows?.length > 0) {
await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
}
}
async function storeMovies(movies, useBatchId) {
if (!movies || movies.length === 0) {
return [];
}
const { uniqueReleases } = await filterDuplicateReleases(movies);
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
const moviesWithId = attachReleaseIds(movies, storedMovies);
await updateMovieSearch(moviesWithId.map((movie) => movie.id));
await associateReleaseMedia(moviesWithId, 'movie');
return moviesWithId;
}
async function storeSeries(series, useBatchId) {
if (!series || series.length === 0) {
return [];
}
const { uniqueReleases } = await filterDuplicateReleases(series);
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
const curatedSerieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'serie')));
const storedSeries = await bulkInsert('series', curatedSerieEntries, ['entity_id', 'entry_id'], true);
const seriesWithId = attachReleaseIds(series, storedSeries);
await updateMovieSearch(seriesWithId.map((serie) => serie.id), 'serie');
await associateReleaseMedia(seriesWithId, 'serie');
return seriesWithId;
}
async function storeScenes(releases, useBatchId) {
if (!releases || releases.length === 0) {
return [];
@ -355,12 +491,14 @@ async function storeScenes(releases, useBatchId) {
scenes: JSON.stringify(duplicateReleasesWithId),
});
const [actors] = await Promise.all([
const [actors, storedSeries] = await Promise.all([
associateActors(releasesWithId, batchId),
storeSeries(releasesWithId.map((release) => release.serie && { ...release.serie, entity: release.entity }).filter(Boolean), batchId),
associateReleaseTags(releasesWithId),
storeChapters(releasesWithId),
]);
await associateSerieScenes(storedSeries, releasesWithId);
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
await updateSceneSearch(releasesWithId.map((release) => release.id));
@ -378,93 +516,6 @@ async function storeScenes(releases, useBatchId) {
return releasesWithId;
}
async function associateMovieScenes(movies, movieScenes) {
const moviesByEntityIdAndEntryId = movies.reduce((acc, movie) => ({
...acc,
[movie.entity.id]: {
...acc[movie.entity.id],
[movie.entryId]: movie,
},
}), {});
const associations = movieScenes.map((scene) => {
if (!scene.movie) {
return null;
}
const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]
|| moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId];
if (sceneMovie?.id) {
return {
movie_id: sceneMovie.id,
scene_id: scene.id,
};
}
return null;
}).filter(Boolean);
await bulkInsert('movies_scenes', associations, false);
}
async function updateMovieSearch(movieIds) {
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } movies`);
const documents = await knex.raw(`
SELECT
movies.id AS movie_id,
TO_TSVECTOR(
'english',
COALESCE(movies.title, '') || ' ' ||
entities.name || ' ' ||
entities.slug || ' ' ||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(TO_CHAR(movies.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ')
) as document
FROM movies
LEFT JOIN entities ON movies.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
LEFT JOIN releases ON releases.id = movies_scenes.scene_id
LEFT JOIN releases_actors ON releases_actors.release_id = movies_scenes.scene_id
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN actors ON actors.id = releases_actors.actor_id
LEFT JOIN tags ON tags.id = releases_tags.tag_id
${movieIds ? 'WHERE movies.id = ANY(?)' : ''}
GROUP BY movies.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
`, movieIds && [movieIds]);
if (documents.rows?.length > 0) {
await bulkInsert('movies_search', documents.rows, ['movie_id']);
}
}
async function storeMovies(movies, useBatchId) {
if (!movies || movies.length === 0) {
return [];
}
const { uniqueReleases } = await filterDuplicateReleases(movies);
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
const moviesWithId = attachReleaseIds(movies, storedMovies);
await updateMovieSearch(moviesWithId.map((movie) => movie.id));
await associateReleaseMedia(moviesWithId, 'movie');
return moviesWithId;
}
module.exports = {
associateMovieScenes,
storeScenes,

18
src/tools/knex-update.js Normal file
View File

@ -0,0 +1,18 @@
'use strict';
const knex = require('../knex');
async function update() {
const query = knex('bans')
.update('type', {
type: 'mute',
username_original: 'charles',
})
.where('id', 2754);
console.log(query.toSQL());
await query;
}
update();