DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Using paginated full text search for movies, combined actor search and fetch to allow combining search with filters.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-08-22 22:25:20 +02:00
parent e0905ab8fc
commit 6a8c9d89cb
21 changed files with 163 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/app.js
View File

@@ -10,7 +10,7 @@ const http = require('./utils/http');
const knex = require('./knex');
const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep');
const { storeScenes, storeMovies, updateReleasesSearch, associateMovieScenes } = require('./store-releases');
const { storeScenes, storeMovies, updateSceneSearch, updateMovieSearch, associateMovieScenes } = require('./store-releases');
const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
const { flushEntities } = require('./entities');
const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases');
@@ -35,7 +35,10 @@ async function init() {
}
if (argv.updateSearch) {
await updateReleasesSearch();
await Promise.all([
updateSceneSearch(),
updateMovieSearch(),
]);
}
if (argv.interpolateProfiles) {

44
src/store-releases.js
View File

@@ -220,7 +220,7 @@ async function filterDuplicateReleases(releases) {
};
}
async function updateReleasesSearch(releaseIds) {
async function updateSceneSearch(releaseIds) {
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
const documents = await knex.raw(`
@@ -355,7 +355,7 @@ async function storeScenes(releases) {
]);
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
await updateReleasesSearch(releasesWithId.map(release => release.id));
await updateSceneSearch(releasesWithId.map(release => release.id));
// media is more error-prone, associate separately
await associateReleaseMedia(releasesWithId);
@@ -400,6 +400,43 @@ async function associateMovieScenes(movies, movieScenes) {
await bulkInsert('movies_scenes', associations, false);
}
async function updateMovieSearch(movieIds) {
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } movies`);
const documents = await knex.raw(`
SELECT
movies.id AS movie_id,
TO_TSVECTOR(
'english',
COALESCE(movies.title, '') || ' ' ||
entities.name || ' ' ||
entities.slug || ' ' ||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(TO_CHAR(movies.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ')
) as document
FROM movies
LEFT JOIN entities ON movies.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
LEFT JOIN releases ON releases.id = movies_scenes.scene_id
LEFT JOIN releases_actors ON releases_actors.release_id = movies_scenes.scene_id
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN actors ON actors.id = releases_actors.actor_id
LEFT JOIN tags ON tags.id = releases_tags.tag_id
GROUP BY movies.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
`, movieIds && [movieIds]);
if (documents.rows?.length > 0) {
await bulkInsert('movies_search', documents.rows, ['movie_id']);
}
}
async function storeMovies(movies) {
if (!movies || movies.length === 0) {
return [];
@@ -422,5 +459,6 @@ module.exports = {
associateMovieScenes,
storeScenes,
storeMovies,
updateReleasesSearch,
updateSceneSearch,
updateMovieSearch,
};