traxxx-web/src/scenes.js

598 lines
18 KiB
JavaScript

import config from 'config';
import util from 'util'; /* eslint-disable-line no-unused-vars */
import { knexQuery as knex, knexOwner, knexManticore } from './knex.js';
import { utilsApi } from './manticore.js';
import { HttpError } from './errors.js';
import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js';
import { fetchTagsById } from './tags.js';
import { fetchEntitiesById } from './entities.js';
import { curateStash } from './stashes.js';
import escape from '../utils/escape-manticore.js';
import promiseProps from '../utils/promise-props.js';
function curateMedia(media) {
if (!media) {
return null;
}
return {
id: media.id,
path: media.path,
thumbnail: media.thumbnail,
lazy: media.lazy,
hash: media.hash,
isS3: media.is_s3,
width: media.width,
height: media.height,
};
}
function curateScene(rawScene, assets) {
if (!rawScene) {
return null;
}
return {
id: rawScene.id,
title: rawScene.title,
slug: rawScene.slug,
url: rawScene.url,
date: rawScene.date,
createdAt: rawScene.created_at,
effectiveDate: rawScene.effective_date,
description: rawScene.description,
duration: rawScene.duration,
shootId: rawScene.shoot_id,
channel: {
id: assets.channel.id,
slug: assets.channel.slug,
name: assets.channel.name,
type: assets.channel.type,
isIndependent: assets.channel.independent,
hasLogo: assets.channel.has_logo,
},
network: assets.channel.network_id ? {
id: assets.channel.network_id,
slug: assets.channel.network_slug,
name: assets.channel.network_name,
type: assets.channel.network_type,
hasLogo: assets.channel.has_logo,
} : null,
actors: sortActorsByGender(assets.actors.map((actor) => curateActor(actor, {
sceneDate: rawScene.effective_date,
stashes: assets.actorStashes.filter((actorStash) => actorStash.actor_id === actor.id),
})), { title: rawScene.title }),
directors: assets.directors.map((director) => ({
id: director.id,
slug: director.slug,
name: director.name,
})),
tags: assets.tags.map((tag) => ({
id: tag.id,
slug: tag.slug,
name: tag.name,
})),
poster: curateMedia(assets.poster),
trailer: curateMedia(assets.trailer),
teaser: curateMedia(assets.teaser),
photos: assets.photos.map((photo) => curateMedia(photo)),
stashes: assets.stashes?.map((stash) => curateStash(stash)) || [],
createdBatchId: rawScene.created_batch_id,
updatedBatchId: rawScene.updated_batch_id,
isNew: assets.lastBatchId === rawScene.created_batch_id,
};
}
export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) {
const {
scenes,
channels,
actors,
directors,
tags,
posters,
photos,
trailers,
teasers,
stashes,
lastBatch: { id: lastBatchId },
} = await promiseProps({
scenes: knex('releases').whereIn('releases.id', sceneIds),
channels: knex('releases')
.select('channels.*', 'networks.id as network_id', 'networks.slug as network_slug', 'networks.name as network_name', 'networks.type as network_type')
.whereIn('releases.id', sceneIds)
.leftJoin('entities as channels', 'channels.id', 'releases.entity_id')
.leftJoin('entities as networks', 'networks.id', 'channels.parent_id')
.groupBy('channels.id', 'networks.id'),
actors: knex('releases_actors')
.select(
'actors.*',
'actors_meta.*',
'releases_actors.release_id',
)
.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
.leftJoin('actors_meta', 'actors_meta.actor_id', 'actors.id')
.whereIn('release_id', sceneIds),
directors: knex('releases_directors')
.whereIn('release_id', sceneIds)
.leftJoin('actors as directors', 'directors.id', 'releases_directors.director_id'),
tags: knex('releases_tags')
.select('id', 'slug', 'name', 'release_id')
.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
.whereNotNull('tags.id')
.whereIn('release_id', sceneIds)
.orderBy('priority', 'desc'),
posters: knex('releases_posters')
.whereIn('release_id', sceneIds)
.leftJoin('media', 'media.id', 'releases_posters.media_id'),
photos: knex.transaction(async (trx) => {
if (reqUser) {
await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
}
return trx('releases_photos')
.whereIn('release_id', sceneIds)
.leftJoin('media', 'media.id', 'releases_photos.media_id');
}),
trailers: knex.transaction(async (trx) => {
if (reqUser) {
await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
}
return trx('releases_trailers')
.whereIn('release_id', sceneIds)
.leftJoin('media', 'media.id', 'releases_trailers.media_id');
}),
teasers: knex.transaction(async (trx) => {
if (reqUser) {
await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
}
return trx('releases_teasers')
.whereIn('release_id', sceneIds)
.leftJoin('media', 'media.id', 'releases_teasers.media_id');
}),
lastBatch: knex('batches')
.select('id')
.where('showcased', true)
.orderBy('created_at', 'desc')
.first(),
stashes: reqUser
? knexOwner('stashes_scenes')
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
.where('stashes.user_id', reqUser.id)
.whereIn('stashes_scenes.scene_id', sceneIds)
: [],
});
const actorStashes = reqUser && context.actorStashes
? await knexOwner('stashes_actors')
.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id')
.where('stashes.user_id', reqUser.id)
.whereIn('stashes_actors.actor_id', actors.map((actor) => actor.id))
: [];
return sceneIds.map((sceneId) => {
const scene = scenes.find((sceneEntry) => sceneEntry.id === sceneId);
if (!scene) {
return null;
}
const sceneChannel = channels.find((entity) => entity.id === scene.entity_id);
const sceneActors = actors.filter((actor) => actor.release_id === sceneId);
const sceneDirectors = directors.filter((director) => director.release_id === sceneId);
const sceneTags = tags.filter((tag) => tag.release_id === sceneId);
const scenePoster = posters.find((poster) => poster.release_id === sceneId);
const scenePhotos = photos.filter((photo) => photo.release_id === sceneId);
const sceneTrailers = trailers.find((trailer) => trailer.release_id === sceneId);
const sceneTeasers = teasers.find((teaser) => teaser.release_id === sceneId);
const sceneStashes = stashes.filter((stash) => stash.scene_id === sceneId);
const sceneActorStashes = sceneActors.map((actor) => actorStashes.find((stash) => stash.actor_id === actor.id)).filter(Boolean);
return curateScene(scene, {
channel: sceneChannel,
actors: sceneActors,
directors: sceneDirectors,
tags: sceneTags,
poster: scenePoster,
photos: scenePhotos,
trailer: sceneTrailers,
teaser: sceneTeasers,
stashes: sceneStashes,
actorStashes: sceneActorStashes,
lastBatchId,
});
}).filter(Boolean);
}
const sqlImplied = ['scenes_stashed'];
function curateOptions(options) {
if (options?.limit > 100) {
throw new HttpError('Limit must be <= 100', 400);
}
return {
limit: options?.limit || 30,
page: Number(options?.page) || 1,
aggregate: options.aggregate ?? true,
aggregateActors: (options.aggregate ?? true) && (options.aggregateActors ?? true),
aggregateTags: (options.aggregate ?? true) && (options.aggregateTags ?? true),
aggregateChannels: (options.aggregate ?? true) && (options.aggregateChannels ?? true),
index: options.index || 'scenes',
useSql: options.useSql || (typeof options.useSql === 'undefined' && sqlImplied.includes(options.index)) || false,
};
}
/*
function buildQuery(filters = {}, options) {
const query = {
bool: {
must: [],
},
};
let sort = [{ effective_date: 'desc' }];
if (!filters.scope || filters.scope === 'latest') {
query.bool.must.push({
range: {
effective_date: {
lte: Math.round(Date.now() / 1000),
},
},
});
}
if (filters.scope === 'upcoming') {
query.bool.must.push({
range: {
effective_date: {
gt: Math.round(Date.now() / 1000),
},
},
});
sort = [{ effective_date: 'asc' }];
}
if (filters.scope === 'new') {
sort = [{ created_at: 'desc' }, { effective_date: 'asc' }];
}
if (filters.scope === 'likes') {
sort = [{ stashed: 'desc' }, { effective_date: 'desc' }];
}
if (filters.scope === 'results') {
sort = [{ _score: 'desc' }, { effective_date: 'desc' }];
}
if (filters.query) {
query.bool.must.push({ match: { '!title': filters.query } }); // title_filtered is matched instead of title
}
if (filters.tagIds) {
filters.tagIds.forEach((tagId) => {
query.bool.must.push({ equals: { 'any(tag_ids)': tagId } });
});
}
if (filters.entityId) {
query.bool.must.push({
bool: {
should: [
{ equals: { channel_id: filters.entityId } },
{ equals: { network_id: filters.entityId } },
],
},
});
}
if (filters.actorIds) {
filters.actorIds.forEach((actorId) => {
query.bool.must.push({ equals: { 'any(actor_ids)': actorId } });
});
}
if (filters.stashId && options.index === 'scenes_stashed') {
query.bool.must.push({ equals: { stash_id: filters.stashId } });
}
return { query, sort };
}
function buildAggregates(options) {
const aggregates = {};
if (options.aggregateActors) {
aggregates.actorIds = {
terms: {
field: 'actor_ids',
size: config.database.manticore.maxAggregateSize,
},
sort: [{ 'count(*)': { order: 'desc' } }],
};
}
if (options.aggregateTags) {
aggregates.tagIds = {
terms: {
field: 'tag_ids',
size: config.database.manticore.maxAggregateSize,
},
sort: [{ 'count(*)': { order: 'desc' } }],
};
}
if (options.aggregateChannels) {
aggregates.channelIds = {
terms: {
field: 'channel_id',
size: config.database.manticore.maxAggregateSize,
},
sort: [{ 'count(*)': { order: 'desc' } }],
};
}
return aggregates;
}
async function queryManticoreJson(filters, options, _reqUser) {
const { query, sort } = buildQuery(filters, options);
const result = await searchApi.search({
index: options.index,
query,
limit: options.limit,
offset: (options.page - 1) * options.limit,
sort,
aggs: buildAggregates(options),
options: {
max_matches: config.database.manticore.maxMatches,
max_query_time: config.database.manticore.maxQueryTime,
field_weights: {
title_filtered: 7,
actors: 10,
tags: 9,
meta: 6,
channel_name: 2,
channel_slug: 3,
network_name: 1,
network_slug: 1,
},
},
});
const scenes = result.hits.hits.map((hit) => ({
id: hit._id,
...hit._source,
_score: hit._score,
}));
return {
scenes,
total: result.hits.total,
aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])),
};
}
*/
async function queryManticoreSql(filters, options, _reqUser) {
const aggSize = config.database.manticore.maxAggregateSize;
const sqlQuery = knexManticore.raw(`
:query:
OPTION
field_weights=(
title_filtered=7,
actors=10,
tags=9,
meta=6,
channel_name=2,
channel_slug=3,
network_name=1,
network_slug=1
),
max_matches=:maxMatches:,
max_query_time=:maxQueryTime:
:actorsFacet:
:tagsFacet:
:channelsFacet:;
show meta;
`, {
query: knexManticore(filters.stashId ? 'scenes_stashed' : 'scenes')
.modify((builder) => {
if (filters.stashId) {
builder.select(knex.raw(`
scenes.id as id,
scenes.title as title,
scenes.actor_ids as actor_ids,
scenes.entity_ids as entity_ids,
scenes.tag_ids as tag_ids,
scenes.channel_id as channel_id,
scenes.network_id as network_id,
scenes.effective_date as effective_date,
scenes.stashed as stashed,
scenes.created_at,
created_at as stashed_at,
weight() as _score
`));
builder
.innerJoin('scenes', 'scenes.id', 'scenes_stashed.scene_id')
.where('stash_id', filters.stashId);
} else {
builder.select(knex.raw('*, weight() as _score'));
}
if (filters.query) {
builder.whereRaw('match(\'@!title :query:\', scenes)', { query: escape(filters.query) });
}
filters.tagIds?.forEach((tagId) => {
builder.where('any(tag_ids)', tagId);
});
filters.actorIds?.forEach((actorId) => {
builder.where('any(actor_ids)', actorId);
});
if (filters.entityId) {
builder.whereRaw('any(entity_ids) = ?', filters.entityId);
/* manticore does not currently support OR if both left and right table are queried https://github.com/manticoresoftware/manticoresearch/issues/1978#issuecomment-2010470068
builder.where((whereBuilder) => {
whereBuilder
.where('scenes.channel_id', filters.entityId)
.orWhere('scenes.network_id', filters.entityId);
});
*/
}
if (filters.movieId) {
builder.whereRaw('any(movie_ids) = ?', filters.movieId);
}
if (typeof filters.isShowcased === 'boolean') {
builder.where('is_showcased', filters.isShowcased);
}
if (!filters.scope || filters.scope === 'latest') {
builder
.where('effective_date', '<=', Math.round(Date.now() / 1000))
.orderBy('scenes.effective_date', 'desc'); // can't seem to use alias if it matches column-name? behavior not fully understand, but this works
} else if (filters.scope === 'upcoming') {
builder
.where('effective_date', '>', Math.round(Date.now() / 1000))
.orderBy('scenes.effective_date', 'asc');
} else if (filters.scope === 'new') {
builder.orderBy([
{ column: 'scenes.created_at', order: 'desc' },
{ column: 'scenes.effective_date', order: 'asc' },
]);
} else if (filters.scope === 'likes') {
builder.orderBy([
{ column: 'scenes.stashed', order: 'desc' },
{ column: 'scenes.effective_date', order: 'desc' },
]);
} else if (filters.scope === 'results') {
builder.orderBy([
{ column: '_score', order: 'desc' },
{ column: 'scenes.effective_date', order: 'desc' },
]);
} else if (filters.scope === 'stashed' && filters.stashId) {
builder.orderBy([
{ column: 'stashed_at', order: 'desc' },
{ column: 'scenes.effective_date', order: 'desc' },
]);
} else {
builder.orderBy('scenes.effective_date', 'desc');
}
})
.limit(options.limit)
.offset((options.page - 1) * options.limit),
// option threads=1 fixes actors, but drastically slows down performance, wait for fix
actorsFacet: options.aggregateActors ? knex.raw('facet scenes.actor_ids order by count(*) desc limit ?', [aggSize]) : null,
tagsFacet: options.aggregateTags ? knex.raw('facet scenes.tag_ids order by count(*) desc limit ?', [aggSize]) : null,
channelsFacet: options.aggregateChannels ? knex.raw('facet scenes.channel_id order by count(*) desc limit ?', [aggSize]) : null,
maxMatches: config.database.manticore.maxMatches,
maxQueryTime: config.database.manticore.maxQueryTime,
}).toString();
// manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around
const curatedSqlQuery = filters.stashId
? sqlQuery
: sqlQuery.replace(/scenes\./g, '');
if (process.env.NODE_ENV === 'development') {
console.log(curatedSqlQuery);
}
const results = await utilsApi.sql(curatedSqlQuery);
// console.log(results[0]);
const actorIds = results
.find((result) => (result.columns[0].actor_ids || result.columns[0]['scenes.actor_ids']) && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.actor_ids || row['scenes.actor_ids'], doc_count: row['count(*)'] }))
|| [];
const tagIds = results
.find((result) => (result.columns[0].tag_ids || result.columns[0]['scenes.tag_ids']) && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.tag_ids || row['scenes.tag_ids'], doc_count: row['count(*)'] }))
|| [];
const channelIds = results
.find((result) => (result.columns[0].channel_id || result.columns[0]['scenes.channel_id']) && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.channel_id || row['scenes.channel_id'], doc_count: row['count(*)'] }))
|| [];
const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0;
return {
scenes: results[0].data,
total,
aggregations: {
actorIds,
tagIds,
channelIds,
},
};
}
function countAggregations(buckets) {
if (!buckets) {
return null;
}
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
}
export async function fetchScenes(filters, rawOptions, reqUser) {
const options = curateOptions(rawOptions);
console.log('filters', filters);
console.log('options', options);
/*
const result = config.database.manticore.forceSql || filters.stashId
? await queryManticoreSql(filters, options, reqUser)
: await queryManticoreJson(filters, options, reqUser);
*/
console.time('manticore sql');
const result = await queryManticoreSql(filters, options, reqUser);
console.timeEnd('manticore sql');
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds);
console.time('fetch aggregations');
const [aggActors, aggTags, aggChannels] = await Promise.all([
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: actorCounts }) : [],
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: [knex.raw('lower(name)'), 'asc'], append: tagCounts }) : [],
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: channelCounts }) : [],
]);
console.timeEnd('fetch aggregations');
console.time('fetch full');
const sceneIds = result.scenes.map((scene) => Number(scene.id));
const scenes = await fetchScenesById(sceneIds, { reqUser });
console.timeEnd('fetch full');
return {
scenes,
aggActors,
aggTags,
aggChannels,
total: result.total,
limit: options.limit,
};
}