Added series.
This commit is contained in:
DebaucheryLibrarian
@@ -14,6 +14,6 @@
|
||||
"prefer-destructuring": "off",
|
||||
"template-curly-spacing": "off",
|
||||
"object-curly-newline": "off",
|
||||
"max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}],
|
||||
"max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,18 +33,26 @@ function getThumbs(scene) {
|
||||
return [];
|
||||
}
|
||||
|
||||
function getCovers(images) {
|
||||
return [
|
||||
[
|
||||
images.cover[0].md?.url,
|
||||
images.cover[0].sm?.url,
|
||||
images.cover[0].xs?.url,
|
||||
// bigger but usually upscaled
|
||||
images.cover[0].xx?.url,
|
||||
images.cover[0].xl?.url,
|
||||
images.cover[0].lg?.url,
|
||||
],
|
||||
function getCovers(images, target = 'cover') {
|
||||
if (!images[target]) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const covers = [
|
||||
images[target][0].md?.url,
|
||||
images[target][0].sm?.url,
|
||||
images[target][0].xs?.url,
|
||||
// bigger but usually upscaled
|
||||
images[target][0].xx?.url,
|
||||
images[target][0].xl?.url,
|
||||
images[target][0].lg?.url,
|
||||
];
|
||||
|
||||
if (target === 'poster') {
|
||||
return covers;
|
||||
}
|
||||
|
||||
return [covers];
|
||||
}
|
||||
|
||||
function getVideos(data) {
|
||||
@@ -153,12 +161,12 @@ function scrapeRelease(data, url, channel, networkName) {
|
||||
if (data.parent?.type === 'movie' || data.parent?.type === 'serie') {
|
||||
release[data.parent.type] = {
|
||||
entryId: data.parent.id,
|
||||
url: `${getBasePath(channel, '/movie')}/${data.parent.id}/${slugify(data.parent.title, '-', { removePunctuation: true })}`,
|
||||
url: `${getBasePath(channel, data.parent.type === 'movie' ? '/movie' : '/series')}/${data.parent.id}/${slugify(data.parent.title, '-', { removePunctuation: true })}`,
|
||||
title: data.parent.title,
|
||||
description: data.parent.description,
|
||||
date: new Date(data.parent.dateReleased),
|
||||
channel: slugify(data.parent.collections?.name || data.parent.brand),
|
||||
covers: getCovers(data.parent.images),
|
||||
poster: getCovers(data.parent.images, 'poster'),
|
||||
shallow: true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -146,7 +146,7 @@ function attachReleaseIds(releases, storedReleases, batchId) {
|
||||
|
||||
const releasesWithId = releases.map((release) => {
|
||||
if (!release.entity) {
|
||||
logger.error(`No entitity available for ${release.url}`);
|
||||
logger.error(`No entity available for ${release.url}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -315,6 +315,142 @@ async function storeChapters(releases) {
|
||||
await associateReleaseMedia(chaptersWithId, 'chapter');
|
||||
}
|
||||
|
||||
async function associateMovieScenes(movies, movieScenes) {
|
||||
const moviesByEntityIdAndEntryId = movies.reduce((acc, movie) => ({
|
||||
...acc,
|
||||
[movie.entity.id]: {
|
||||
...acc[movie.entity.id],
|
||||
[movie.entryId]: movie,
|
||||
},
|
||||
}), {});
|
||||
|
||||
const associations = movieScenes.map((scene) => {
|
||||
if (!scene.movie) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]
|
||||
|| moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId];
|
||||
|
||||
if (sceneMovie?.id) {
|
||||
return {
|
||||
movie_id: sceneMovie.id,
|
||||
scene_id: scene.id,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
|
||||
await bulkInsert('movies_scenes', associations, false);
|
||||
}
|
||||
|
||||
async function associateSerieScenes(series, serieScenes) {
|
||||
const seriesByEntityIdAndEntryId = series.reduce((acc, serie) => ({
|
||||
...acc,
|
||||
[serie.entity.id]: {
|
||||
...acc[serie.entity.id],
|
||||
[serie.entryId]: serie,
|
||||
},
|
||||
}), {});
|
||||
|
||||
const associations = serieScenes.map((scene) => {
|
||||
if (!scene.serie) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sceneSerie = seriesByEntityIdAndEntryId[scene.entity.id]?.[scene.serie.entryId]
|
||||
|| seriesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.serie.entryId];
|
||||
|
||||
if (sceneSerie?.id) {
|
||||
return {
|
||||
serie_id: sceneSerie.id,
|
||||
scene_id: scene.id,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
|
||||
await bulkInsert('series_scenes', associations, false);
|
||||
}
|
||||
|
||||
async function updateMovieSearch(movieIds, target = 'movie') {
|
||||
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
|
||||
|
||||
const documents = await knex.raw(`
|
||||
SELECT
|
||||
${target}s.id AS ${target}_id,
|
||||
TO_TSVECTOR(
|
||||
'english',
|
||||
COALESCE(${target}s.title, '') || ' ' ||
|
||||
entities.name || ' ' ||
|
||||
entities.slug || ' ' ||
|
||||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
||||
COALESCE(parents.name, '') || ' ' ||
|
||||
COALESCE(parents.slug, '') || ' ' ||
|
||||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
||||
COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
|
||||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(tags.name, ''), ' ')
|
||||
) as document
|
||||
FROM ${target}s
|
||||
LEFT JOIN entities ON ${target}s.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
|
||||
LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
|
||||
LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
|
||||
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
|
||||
LEFT JOIN actors ON actors.id = releases_actors.actor_id
|
||||
LEFT JOIN tags ON tags.id = releases_tags.tag_id
|
||||
${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
|
||||
GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
|
||||
`, movieIds && [movieIds]);
|
||||
|
||||
if (documents.rows?.length > 0) {
|
||||
await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
|
||||
}
|
||||
}
|
||||
|
||||
async function storeMovies(movies, useBatchId) {
|
||||
if (!movies || movies.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const { uniqueReleases } = await filterDuplicateReleases(movies);
|
||||
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
|
||||
|
||||
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
|
||||
|
||||
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
|
||||
const moviesWithId = attachReleaseIds(movies, storedMovies);
|
||||
|
||||
await updateMovieSearch(moviesWithId.map((movie) => movie.id));
|
||||
await associateReleaseMedia(moviesWithId, 'movie');
|
||||
|
||||
return moviesWithId;
|
||||
}
|
||||
|
||||
async function storeSeries(series, useBatchId) {
|
||||
if (!series || series.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const { uniqueReleases } = await filterDuplicateReleases(series);
|
||||
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
|
||||
|
||||
const curatedSerieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'serie')));
|
||||
|
||||
const storedSeries = await bulkInsert('series', curatedSerieEntries, ['entity_id', 'entry_id'], true);
|
||||
const seriesWithId = attachReleaseIds(series, storedSeries);
|
||||
|
||||
await updateMovieSearch(seriesWithId.map((serie) => serie.id), 'serie');
|
||||
await associateReleaseMedia(seriesWithId, 'serie');
|
||||
|
||||
return seriesWithId;
|
||||
}
|
||||
|
||||
async function storeScenes(releases, useBatchId) {
|
||||
if (!releases || releases.length === 0) {
|
||||
return [];
|
||||
@@ -355,12 +491,14 @@ async function storeScenes(releases, useBatchId) {
|
||||
scenes: JSON.stringify(duplicateReleasesWithId),
|
||||
});
|
||||
|
||||
const [actors] = await Promise.all([
|
||||
const [actors, storedSeries] = await Promise.all([
|
||||
associateActors(releasesWithId, batchId),
|
||||
storeSeries(releasesWithId.map((release) => release.serie && { ...release.serie, entity: release.entity }).filter(Boolean), batchId),
|
||||
associateReleaseTags(releasesWithId),
|
||||
storeChapters(releasesWithId),
|
||||
]);
|
||||
|
||||
await associateSerieScenes(storedSeries, releasesWithId);
|
||||
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
|
||||
await updateSceneSearch(releasesWithId.map((release) => release.id));
|
||||
|
||||
@@ -378,93 +516,6 @@ async function storeScenes(releases, useBatchId) {
|
||||
return releasesWithId;
|
||||
}
|
||||
|
||||
async function associateMovieScenes(movies, movieScenes) {
|
||||
const moviesByEntityIdAndEntryId = movies.reduce((acc, movie) => ({
|
||||
...acc,
|
||||
[movie.entity.id]: {
|
||||
...acc[movie.entity.id],
|
||||
[movie.entryId]: movie,
|
||||
},
|
||||
}), {});
|
||||
|
||||
const associations = movieScenes.map((scene) => {
|
||||
if (!scene.movie) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]
|
||||
|| moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId];
|
||||
|
||||
if (sceneMovie?.id) {
|
||||
return {
|
||||
movie_id: sceneMovie.id,
|
||||
scene_id: scene.id,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
|
||||
await bulkInsert('movies_scenes', associations, false);
|
||||
}
|
||||
|
||||
async function updateMovieSearch(movieIds) {
|
||||
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } movies`);
|
||||
|
||||
const documents = await knex.raw(`
|
||||
SELECT
|
||||
movies.id AS movie_id,
|
||||
TO_TSVECTOR(
|
||||
'english',
|
||||
COALESCE(movies.title, '') || ' ' ||
|
||||
entities.name || ' ' ||
|
||||
entities.slug || ' ' ||
|
||||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
||||
COALESCE(parents.name, '') || ' ' ||
|
||||
COALESCE(parents.slug, '') || ' ' ||
|
||||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
||||
COALESCE(TO_CHAR(movies.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
|
||||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(tags.name, ''), ' ')
|
||||
) as document
|
||||
FROM movies
|
||||
LEFT JOIN entities ON movies.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
|
||||
LEFT JOIN releases ON releases.id = movies_scenes.scene_id
|
||||
LEFT JOIN releases_actors ON releases_actors.release_id = movies_scenes.scene_id
|
||||
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
|
||||
LEFT JOIN actors ON actors.id = releases_actors.actor_id
|
||||
LEFT JOIN tags ON tags.id = releases_tags.tag_id
|
||||
${movieIds ? 'WHERE movies.id = ANY(?)' : ''}
|
||||
GROUP BY movies.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
|
||||
`, movieIds && [movieIds]);
|
||||
|
||||
if (documents.rows?.length > 0) {
|
||||
await bulkInsert('movies_search', documents.rows, ['movie_id']);
|
||||
}
|
||||
}
|
||||
|
||||
async function storeMovies(movies, useBatchId) {
|
||||
if (!movies || movies.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const { uniqueReleases } = await filterDuplicateReleases(movies);
|
||||
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
|
||||
|
||||
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
|
||||
|
||||
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
|
||||
const moviesWithId = attachReleaseIds(movies, storedMovies);
|
||||
|
||||
await updateMovieSearch(moviesWithId.map((movie) => movie.id));
|
||||
await associateReleaseMedia(moviesWithId, 'movie');
|
||||
|
||||
return moviesWithId;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
associateMovieScenes,
|
||||
storeScenes,
|
||||
|
||||
18
src/tools/knex-update.js
Normal file
18
src/tools/knex-update.js
Normal file
@@ -0,0 +1,18 @@
|
||||
'use strict';
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
async function update() {
|
||||
const query = knex('bans')
|
||||
.update('type', {
|
||||
type: 'mute',
|
||||
username_original: 'charles',
|
||||
})
|
||||
.where('id', 2754);
|
||||
|
||||
console.log(query.toSQL());
|
||||
|
||||
await query;
|
||||
}
|
||||
|
||||
update();
|
||||
Reference in New Issue
Block a user