DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updating Manticore scenes database on scene store.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2024-01-25 01:15:42 +01:00
parent ae2d3591ee
commit 86ffcc3316
31 changed files with 237 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/actors.js
View File

@@ -1074,6 +1074,7 @@ async function associatePeople(releases, batchId, type = 'actor') {
}
await bulkInsert(`releases_${type}`, validReleaseActorAssociations, false);
await knex.schema.refreshMaterializedView('actors_meta');
logger.verbose(`Associated ${releaseActorAssociations.length} actors to ${releases.length} scenes`);

24
src/stashes.js
View File

@@ -1,8 +1,13 @@
'use strict';
const config = require('config');
const knex = require('./knex');
const { HttpError } = require('./errors');
const slugify = require('./utils/slugify');
const logger = require('./logger')(__filename);
let lastActorsViewRefresh = 0;
function curateStash(stash) {
if (!stash) {
@@ -119,6 +124,21 @@ async function removeStash(stashId, sessionUser) {
}
}
async function refreshActorsView() {
if (new Date() - lastActorsViewRefresh > config.stashes.viewRefreshCooldown * 60000) {
// don't refresh actors view more than once an hour
lastActorsViewRefresh = new Date();
logger.debug('Refreshing actors view');
return knex.schema.refreshMaterializedView('actors_meta');
}
logger.silly('Skipping actors view refresh');
return false;
}
async function stashActor(actorId, stashId, sessionUser) {
const stash = await fetchStash(stashId, sessionUser);
@@ -128,6 +148,8 @@ async function stashActor(actorId, stashId, sessionUser) {
actor_id: actorId,
});
refreshActorsView();
return fetchStashes('actor', actorId, sessionUser);
}
@@ -166,6 +188,8 @@ async function unstashActor(actorId, stashId, sessionUser) {
.where('stashes.user_id', sessionUser.id))
.delete();
refreshActorsView();
return fetchStashes('actor', actorId, sessionUser);
}

13
src/tools/manticore-actors.js
View File

@@ -20,9 +20,10 @@ const update = args.update;
async function fetchActors() {
// manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC...
const actors = await knex.raw(`
SELECT actors.*, date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob
FROM actors
GROUP BY actors.id;
SELECT
actors_meta.*,
date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob
FROM actors_meta;
`);
return actors.rows;
@@ -44,7 +45,9 @@ async function init() {
cup string,
natural_boobs int,
penis_length int,
penis_girth int
penis_girth int,
stashed int,
scenes int
)`);
const actors = await fetchActors();
@@ -66,6 +69,8 @@ async function init() {
natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown
penis_length: actor.penis_length || undefined,
penis_girth: actor.penis_girth || undefined,
stashed: actor.stashed || 0,
scenes: actor.scenes || 0,
},
},
}));

43
src/tools/manticore.js
View File

@@ -20,11 +20,12 @@ const update = args.update;
async function fetchScenes() {
const scenes = await knex.raw(`
SELECT
releases.id AS id,
releases.title,
releases.created_at,
releases.date,
releases.entry_id,
scenes_meta.id AS id,
scenes_meta.title,
scenes_meta.created_at,
scenes_meta.date,
scenes_meta.entry_id,
scenes_meta.stashed,
entities.id as channel_id,
entities.slug as channel_slug,
entities.name as channel_name,
@@ -33,17 +34,31 @@ async function fetchScenes() {
parents.name as network_name,
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
FROM releases
LEFT JOIN entities ON releases.entity_id = entities.id
FROM scenes_meta
LEFT JOIN entities ON scenes_meta.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = scenes_meta.id
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = scenes_meta.id
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = scenes_meta.id
LEFT JOIN actors ON local_actors.actor_id = actors.id
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
GROUP BY releases.id, entities.id, entities.name, entities.slug, entities.alias, parents.id, parents.name, parents.slug, parents.alias;
GROUP BY
scenes_meta.id,
scenes_meta.title,
scenes_meta.created_at,
scenes_meta.date,
scenes_meta.entry_id,
scenes_meta.stashed,
entities.id,
entities.name,
entities.slug,
entities.alias,
parents.id,
parents.name,
parents.slug,
parents.alias;
`);
return scenes.rows;
@@ -68,13 +83,14 @@ async function init() {
tags text,
date timestamp,
created_at timestamp,
effective_date timestamp
effective_date timestamp,
stashed int
)`);
const scenes = await fetchScenes();
const docs = scenes.map((scene) => ({
insert: {
replace: {
index: 'scenes',
id: scene.id,
doc: {
@@ -93,6 +109,7 @@ async function init() {
actors: scene.actors.map((actor) => actor.f2).join(),
tag_ids: scene.tags.map((tag) => tag.f1),
tags: scene.tags.map((tag) => tag.f2).join(),
stashed: scene.stashed || 0,
},
},
}));

98
src/update-search.js
View File

@@ -1,10 +1,99 @@
'use strict';
const manticore = require('manticoresearch');
const knex = require('./knex');
const logger = require('./logger')(__filename);
const bulkInsert = require('./utils/bulk-insert');
async function updateSceneSearch(releaseIds) {
const mantiClient = new manticore.ApiClient();
const indexApi = new manticore.IndexApi(mantiClient);
async function updateManticoreSearch(releaseIds) {
const scenes = await knex.raw(`
SELECT
scenes_meta.id AS id,
scenes_meta.title,
scenes_meta.created_at,
scenes_meta.date,
scenes_meta.entry_id,
scenes_meta.stashed,
entities.id as channel_id,
entities.slug as channel_slug,
entities.name as channel_name,
parents.id as network_id,
parents.slug as network_slug,
parents.name as network_name,
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
FROM scenes_meta
LEFT JOIN entities ON scenes_meta.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = scenes_meta.id
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = scenes_meta.id
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = scenes_meta.id
LEFT JOIN actors ON local_actors.actor_id = actors.id
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
${releaseIds ? 'WHERE scenes_meta.id = ANY(?)' : ''}
GROUP BY
scenes_meta.id,
scenes_meta.title,
scenes_meta.created_at,
scenes_meta.date,
scenes_meta.entry_id,
scenes_meta.stashed,
entities.id,
entities.name,
entities.slug,
entities.alias,
parents.id,
parents.name,
parents.slug,
parents.alias;
`, releaseIds && [releaseIds]);
console.log(releaseIds);
console.log(scenes);
const docs = scenes.rows.map((scene) => ({
replace: {
index: 'scenes',
id: scene.id,
doc: {
title: scene.title || undefined,
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
created_at: Math.round(scene.created_at.getTime() / 1000),
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
entry_id: scene.entry_id,
channel_id: scene.channel_id,
channel_slug: scene.channel_slug,
channel_name: scene.channel_name,
network_id: scene.network_id || undefined,
network_slug: scene.network_slug || undefined,
network_name: scene.network_name || undefined,
actor_ids: scene.actors.map((actor) => actor.f1),
actors: scene.actors.map((actor) => actor.f2).join(),
tag_ids: scene.tags.map((tag) => tag.f1),
tags: scene.tags.map((tag) => tag.f2).join(),
stashed: scene.stashed || 0,
},
},
}));
console.log('docs', docs);
if (docs.length === 0) {
return;
}
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
console.log('data', data);
}
async function updateSqlSearch(releaseIds) {
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
const documents = await knex.raw(`
@@ -48,6 +137,13 @@ async function updateSceneSearch(releaseIds) {
await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
}
async function updateSceneSearch(releaseIds) {
await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;');
await updateSqlSearch(releaseIds);
await updateManticoreSearch(releaseIds);
}
async function updateMovieSearch(movieIds, target = 'movie') {
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);