DebaucheryLibrarian/traxxx-web
1
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Added movies stash page.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2024-03-24 23:36:25 +01:00
parent c06b6d5568
commit 72af9add7d
11 changed files with 739 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/actors.js
View File

@@ -442,7 +442,7 @@ async function queryManticoreSql(filters, options, _reqUser) {
]);
} else if (options.order?.[0] === 'results') {
builder.orderBy([
{ column: 'actors._score', order: options.order[1] },
{ column: '_score', order: options.order[1] },
{ column: 'actors.slug', order: 'asc' },
]);
} else if (options.order?.[0] === 'stashed' && filters.stashId) {

229
src/movies.js
View File

@@ -1,7 +1,7 @@
import config from 'config';
import knex from './knex.js';
import { searchApi } from './manticore.js';
import { knexQuery as knex, knexManticore } from './knex.js';
import { utilsApi } from './manticore.js';
import { HttpError } from './errors.js';
import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js';
import { fetchTagsById } from './tags.js';
@@ -166,6 +166,7 @@ function curateOptions(options) {
};
}
/*
function buildQuery(filters = {}) {
const query = {
bool: {
@@ -257,16 +258,6 @@ function buildQuery(filters = {}) {
});
}
/* tag filter
must_not: [
{
in: {
'any(tag_ids)': [101, 180, 32],
},
},
],
*/
return { query, sort };
}
@@ -304,20 +295,9 @@ function buildAggregates(options) {
return aggregates;
}
function countAggregations(buckets) {
if (!buckets) {
return null;
}
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
}
export async function fetchMovies(filters, rawOptions) {
const options = curateOptions(rawOptions);
async function queryManticoreJson(filters, options) {
const { query, sort } = buildQuery(filters);
console.log('filters', filters);
console.log('options', options);
console.log('query', query.bool.must);
console.time('manticore');
@@ -345,23 +325,202 @@ export async function fetchMovies(filters, rawOptions) {
},
});
console.timeEnd('manticore');
const movies = result.hits.hits.map((hit) => ({
id: hit._id,
...hit._source,
_score: hit._score,
}));
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
return {
movies,
total: result.hits.total,
aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])),
};
}
*/
console.time('fetch aggregations');
async function queryManticoreSql(filters, options) {
const aggSize = config.database.manticore.maxAggregateSize;
const sqlQuery = knexManticore.raw(`
:query:
OPTION
field_weights=(
title_filtered=7,
actors=10,
tags=9,
meta=6,
channel_name=2,
channel_slug=3,
network_name=1,
network_slug=1
),
max_matches=:maxMatches:,
max_query_time=:maxQueryTime:
:actorsFacet:
:tagsFacet:
:channelsFacet:;
show meta;
`, {
query: knexManticore(filters.stashId ? 'movies_stashed' : 'movies')
.modify((builder) => {
if (filters.stashId) {
builder.select(knex.raw(`
movies.id as id,
movies.title as title,
movies.actor_ids as actor_ids,
movies.entity_ids as entity_ids,
movies.tag_ids as tag_ids,
movies.channel_id as channel_id,
movies.network_id as network_id,
movies.effective_date as effective_date,
movies.stashed as stashed,
movies.created_at,
created_at as stashed_at,
weight() as _score
`));
builder
.innerJoin('movies', 'movies.id', 'movies_stashed.movie_id')
.where('stash_id', filters.stashId);
} else {
builder.select(knex.raw('*, weight() as _score'));
}
if (filters.query) {
builder.whereRaw('match(\'@!title :query:\', movies)', { query: filters.query });
}
filters.tagIds?.forEach((tagId) => {
builder.where('any(tag_ids)', tagId);
});
filters.actorIds?.forEach((actorId) => {
builder.where('any(actor_ids)', actorId);
});
if (filters.entityId) {
builder.whereRaw('any(entity_ids) = ?', filters.entityId);
}
if (typeof filters.isShowcased === 'boolean') {
builder.where('is_showcased', filters.isShowcased);
}
if (!filters.scope || filters.scope === 'latest') {
builder
.where('effective_date', '<=', Math.round(Date.now() / 1000))
.orderBy('movies.effective_date', 'desc'); // can't seem to use alias if it matches column-name? behavior not fully understand, but this works
} else if (filters.scope === 'upcoming') {
builder
.where('effective_date', '>', Math.round(Date.now() / 1000))
.orderBy('movies.effective_date', 'asc');
} else if (filters.scope === 'new') {
builder.orderBy([
{ column: 'movies.created_at', order: 'desc' },
{ column: 'movies.effective_date', order: 'asc' },
]);
} else if (filters.scope === 'likes') {
builder.orderBy([
{ column: 'movies.stashed', order: 'desc' },
{ column: 'movies.effective_date', order: 'desc' },
]);
} else if (filters.scope === 'results') {
builder.orderBy([
{ column: '_score', order: 'desc' },
{ column: 'movies.effective_date', order: 'desc' },
]);
} else if (filters.scope === 'stashed' && filters.stashId) {
builder.orderBy([
{ column: 'stashed_at', order: 'desc' },
{ column: 'movies.effective_date', order: 'desc' },
]);
} else {
builder.orderBy('movies.effective_date', 'desc');
}
})
.limit(options.limit)
.offset((options.page - 1) * options.limit)
.toString(),
// option threads=1 fixes actors, but drastically slows down performance, wait for fix
actorsFacet: options.aggregateActors ? knex.raw('facet movies.actor_ids order by count(*) desc limit ?', [aggSize]) : null,
tagsFacet: options.aggregateTags ? knex.raw('facet movies.tag_ids order by count(*) desc limit ?', [aggSize]) : null,
channelsFacet: options.aggregateChannels ? knex.raw('facet movies.channel_id order by count(*) desc limit ?', [aggSize]) : null,
maxMatches: config.database.manticore.maxMatches,
maxQueryTime: config.database.manticore.maxQueryTime,
}).toString();
// manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around
const curatedSqlQuery = filters.stashId
? sqlQuery
: sqlQuery.replace(/movies\./g, '');
if (process.env.NODE_ENV === 'development') {
console.log(curatedSqlQuery);
}
const results = await utilsApi.sql(curatedSqlQuery);
// console.log(results[0]);
const actorIds = results
.find((result) => (result.columns[0].actor_ids || result.columns[0]['movies.actor_ids']) && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.actor_ids || row['movies.actor_ids'], doc_count: row['count(*)'] }))
|| [];
const tagIds = results
.find((result) => (result.columns[0].tag_ids || result.columns[0]['movies.tag_ids']) && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.tag_ids || row['movies.tag_ids'], doc_count: row['count(*)'] }))
|| [];
const channelIds = results
.find((result) => (result.columns[0].channel_id || result.columns[0]['movies.channel_id']) && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.channel_id || row['movies.channel_id'], doc_count: row['count(*)'] }))
|| [];
const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0;
return {
movies: results[0].data,
total,
aggregations: {
actorIds,
tagIds,
channelIds,
},
};
}
function countAggregations(buckets) {
if (!buckets) {
return null;
}
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
}
export async function fetchMovies(filters, rawOptions) {
const options = curateOptions(rawOptions);
console.log(options);
console.log(filters);
const result = await queryManticoreSql(filters, options);
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds);
const [aggActors, aggTags, aggChannels] = await Promise.all([
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
]);
console.timeEnd('fetch aggregations');
console.log(result.aggregations);
console.log(aggActors);
const movieIds = result.hits.hits.map((hit) => Number(hit._id));
const movieIds = result.movies.map((movie) => Number(movie.id));
const movies = await fetchMoviesById(movieIds);
return {
@@ -369,7 +528,7 @@ export async function fetchMovies(filters, rawOptions) {
aggActors,
aggTags,
aggChannels,
total: result.hits.total,
total: result.total,
limit: options.limit,
};
}

11
src/scenes.js
View File

@@ -190,8 +190,6 @@ export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) {
const sceneStashes = stashes.filter((stash) => stash.scene_id === sceneId);
const sceneActorStashes = sceneActors.map((actor) => actorStashes.find((stash) => stash.actor_id === actor.id)).filter(Boolean);
console.log(sceneActors);
return curateScene(scene, {
channel: sceneChannel,
actors: sceneActors,
@@ -411,6 +409,7 @@ async function queryManticoreSql(filters, options, _reqUser) {
scenes.id as id,
scenes.title as title,
scenes.actor_ids as actor_ids,
scenes.entity_ids as entity_ids,
scenes.tag_ids as tag_ids,
scenes.channel_id as channel_id,
scenes.network_id as network_id,
@@ -441,11 +440,15 @@ async function queryManticoreSql(filters, options, _reqUser) {
});
if (filters.entityId) {
builder.whereRaw('any(entity_ids) = ?', filters.entityId);
/* manticore does not support OR if both left and right table are queried https://github.com/manticoresoftware/manticoresearch/issues/1978#issuecomment-2010470068
builder.where((whereBuilder) => {
whereBuilder
.where('scenes.channel_id', filters.entityId)
.orWhere('scenes.network_id', filters.entityId);
});
*/
}
if (typeof filters.isShowcased === 'boolean') {
@@ -472,7 +475,7 @@ async function queryManticoreSql(filters, options, _reqUser) {
]);
} else if (filters.scope === 'results') {
builder.orderBy([
{ column: 'scenes._score', order: 'desc' },
{ column: '_score', order: 'desc' },
{ column: 'scenes.effective_date', order: 'desc' },
]);
} else if (filters.scope === 'stashed' && filters.stashId) {
@@ -523,7 +526,7 @@ async function queryManticoreSql(filters, options, _reqUser) {
?.data.map((row) => ({ key: row.channel_id || row['scenes.channel_id'], doc_count: row['count(*)'] }))
|| [];
const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found').Value);
const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0;
return {
scenes: results[0].data,

33
src/tools/sync-stashes.js
View File

@@ -1,31 +1,29 @@
import { indexApi } from '../manticore.js';
import { indexApi, utilsApi } from '../manticore.js';
import { knexOwner as knex } from '../knex.js';
import chunk from '../utils/chunk.js';
async function syncActorStashes() {
const stashes = await knex('stashes_actors')
async function syncStashes(domain = 'scene') {
await utilsApi.sql(`truncate table ${domain}s_stashed`);
const stashes = await knex(`stashes_${domain}s`)
.select(
'stashes_actors.id as stashed_id',
'stashes_actors.actor_id',
`stashes_${domain}s.id as stashed_id`,
`stashes_${domain}s.${domain}_id`,
'stashes.id as stash_id',
'stashes.user_id as user_id',
'stashes_actors.created_at as created_at',
`stashes_${domain}s.created_at as created_at`,
)
.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id');
if (stashes.length > 0) {
console.log(stashes);
}
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
await chain;
const stashDocs = stashChunk.map((stash) => ({
replace: {
index: 'actors_stashed',
index: `${domain}s_stashed`,
id: stash.stashed_id,
doc: {
actor_id: stash.actor_id,
[`${domain}_id`]: stash[`${domain}_id`],
stash_id: stash.stash_id,
user_id: stash.user_id,
created_at: Math.round(stash.created_at.getTime() / 1000),
@@ -33,16 +31,17 @@ async function syncActorStashes() {
},
}));
console.log(stashDocs);
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} actor stashes`);
console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
}, Promise.resolve());
}
async function init() {
await syncActorStashes();
await syncStashes('scene');
await syncStashes('actor');
await syncStashes('movie');
console.log('Done!');
knex.destroy();

1
src/web/movies.js
View File

@@ -12,6 +12,7 @@ export async function curateMoviesQuery(query) {
tagIds: await getIdsBySlug([query.tagSlug, ...(query.tags?.split(',') || [])], 'tags'),
entityId: query.e ? await getIdsBySlug([query.e], 'entities').then(([id]) => id) : query.entityId,
requireCover: query.cover,
stashId: Number(query.stashId) || null,
};
}