526 lines
15 KiB
JavaScript
526 lines
15 KiB
JavaScript
import config from 'config';
|
|
import util from 'util'; /* eslint-disable-line no-unused-vars */
|
|
|
|
import { knexOwner as knex, knexManticore } from './knex.js';
|
|
import { searchApi, utilsApi } from './manticore.js';
|
|
import { HttpError } from './errors.js';
|
|
import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js';
|
|
import { fetchTagsById } from './tags.js';
|
|
import { fetchEntitiesById } from './entities.js';
|
|
import { curateStash } from './stashes.js';
|
|
|
|
function curateMedia(media) {
|
|
if (!media) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
id: media.id,
|
|
path: media.path,
|
|
thumbnail: media.thumbnail,
|
|
lazy: media.lazy,
|
|
isS3: media.is_s3,
|
|
width: media.width,
|
|
height: media.height,
|
|
};
|
|
}
|
|
|
|
function curateScene(rawScene, assets) {
|
|
if (!rawScene) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
id: rawScene.id,
|
|
title: rawScene.title,
|
|
slug: rawScene.slug,
|
|
url: rawScene.url,
|
|
date: rawScene.date,
|
|
createdAt: rawScene.created_at,
|
|
effectiveDate: rawScene.effective_date,
|
|
description: rawScene.description,
|
|
duration: rawScene.duration,
|
|
channel: {
|
|
id: assets.channel.id,
|
|
slug: assets.channel.slug,
|
|
name: assets.channel.name,
|
|
type: assets.channel.type,
|
|
isIndependent: assets.channel.independent,
|
|
hasLogo: assets.channel.has_logo,
|
|
},
|
|
network: assets.channel.network_id ? {
|
|
id: assets.channel.network_id,
|
|
slug: assets.channel.network_slug,
|
|
name: assets.channel.network_name,
|
|
type: assets.channel.network_type,
|
|
hasLogo: assets.channel.has_logo,
|
|
} : null,
|
|
actors: sortActorsByGender(assets.actors.map((actor) => curateActor(actor, { sceneDate: rawScene.effective_date }))),
|
|
directors: assets.directors.map((director) => ({
|
|
id: director.id,
|
|
slug: director.slug,
|
|
name: director.name,
|
|
})),
|
|
tags: assets.tags.map((tag) => ({
|
|
id: tag.id,
|
|
slug: tag.slug,
|
|
name: tag.name,
|
|
})),
|
|
poster: curateMedia(assets.poster),
|
|
photos: assets.photos.map((photo) => curateMedia(photo)),
|
|
stashes: assets.stashes?.map((stash) => curateStash(stash)) || [],
|
|
createdBatchId: rawScene.created_batch_id,
|
|
updatedBatchId: rawScene.updated_batch_id,
|
|
};
|
|
}
|
|
|
|
export async function fetchScenesById(sceneIds, reqUser) {
|
|
const [scenes, channels, actors, directors, tags, posters, photos, stashes] = await Promise.all([
|
|
knex('releases').whereIn('releases.id', sceneIds),
|
|
knex('releases')
|
|
.select('channels.*', 'networks.id as network_id', 'networks.slug as network_slug', 'networks.name as network_name', 'networks.type as network_type')
|
|
.whereIn('releases.id', sceneIds)
|
|
.leftJoin('entities as channels', 'channels.id', 'releases.entity_id')
|
|
.leftJoin('entities as networks', 'networks.id', 'channels.parent_id')
|
|
.groupBy('channels.id', 'networks.id'),
|
|
knex('releases_actors')
|
|
.select(
|
|
'actors.*',
|
|
'actors_meta.*',
|
|
'releases_actors.release_id',
|
|
/* why would we need this for scenes?
|
|
'birth_countries.alpha2 as birth_country_alpha2',
|
|
knex.raw('COALESCE(birth_countries.alias, birth_countries.name) as birth_country_name'),
|
|
'residence_countries.alpha2 as residence_country_alpha2',
|
|
knex.raw('COALESCE(residence_countries.alias, residence_countries.name) as residence_country_name'),
|
|
*/
|
|
)
|
|
.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
|
|
.leftJoin('actors_meta', 'actors_meta.actor_id', 'actors.id')
|
|
.whereIn('release_id', sceneIds),
|
|
/*
|
|
.leftJoin('countries as birth_countries', 'birth_countries.alpha2', 'actors_meta.birth_country_alpha2')
|
|
.leftJoin('countries as residence_countries', 'residence_countries.alpha2', 'actors_meta.residence_country_alpha2'),
|
|
*/
|
|
knex('releases_directors')
|
|
.whereIn('release_id', sceneIds)
|
|
.leftJoin('actors as directors', 'directors.id', 'releases_directors.director_id'),
|
|
knex('releases_tags')
|
|
.select('id', 'slug', 'name', 'release_id')
|
|
.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
|
|
.whereNotNull('tags.id')
|
|
.whereIn('release_id', sceneIds)
|
|
.orderBy('priority', 'desc'),
|
|
knex('releases_posters')
|
|
.whereIn('release_id', sceneIds)
|
|
.leftJoin('media', 'media.id', 'releases_posters.media_id'),
|
|
knex('releases_photos')
|
|
.whereIn('release_id', sceneIds)
|
|
.leftJoin('media', 'media.id', 'releases_photos.media_id'),
|
|
reqUser
|
|
? knex('stashes_scenes')
|
|
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
|
|
.where('stashes.user_id', reqUser.id)
|
|
.whereIn('stashes_scenes.scene_id', sceneIds)
|
|
: [],
|
|
]);
|
|
|
|
return sceneIds.map((sceneId) => {
|
|
const scene = scenes.find((sceneEntry) => sceneEntry.id === sceneId);
|
|
|
|
if (!scene) {
|
|
return null;
|
|
}
|
|
|
|
const sceneChannel = channels.find((entity) => entity.id === scene.entity_id);
|
|
const sceneActors = actors.filter((actor) => actor.release_id === sceneId);
|
|
const sceneDirectors = directors.filter((director) => director.release_id === sceneId);
|
|
const sceneTags = tags.filter((tag) => tag.release_id === sceneId);
|
|
const scenePoster = posters.find((poster) => poster.release_id === sceneId);
|
|
const scenePhotos = photos.filter((photo) => photo.release_id === sceneId);
|
|
const sceneStashes = stashes.filter((stash) => stash.scene_id === sceneId);
|
|
|
|
return curateScene(scene, {
|
|
channel: sceneChannel,
|
|
actors: sceneActors,
|
|
directors: sceneDirectors,
|
|
tags: sceneTags,
|
|
poster: scenePoster,
|
|
photos: scenePhotos,
|
|
stashes: sceneStashes,
|
|
});
|
|
}).filter(Boolean);
|
|
}
|
|
|
|
const sqlImplied = ['scenes_stashed'];
|
|
|
|
function curateOptions(options) {
|
|
if (options?.limit > 100) {
|
|
throw new HttpError('Limit must be <= 100', 400);
|
|
}
|
|
|
|
return {
|
|
limit: options?.limit || 30,
|
|
page: Number(options?.page) || 1,
|
|
aggregate: options.aggregate ?? true,
|
|
aggregateActors: (options.aggregate ?? true) && (options.aggregateActors ?? true),
|
|
aggregateTags: (options.aggregate ?? true) && (options.aggregateTags ?? true),
|
|
aggregateChannels: (options.aggregate ?? true) && (options.aggregateChannels ?? true),
|
|
index: options.index || 'scenes',
|
|
useSql: options.useSql || (typeof options.useSql === 'undefined' && sqlImplied.includes(options.index)) || false,
|
|
};
|
|
}
|
|
|
|
function buildQuery(filters = {}, options) {
|
|
const query = {
|
|
bool: {
|
|
must: [],
|
|
},
|
|
};
|
|
|
|
let sort = [{ effective_date: 'desc' }];
|
|
|
|
if (!filters.scope || filters.scope === 'latest') {
|
|
query.bool.must.push({
|
|
range: {
|
|
effective_date: {
|
|
lte: Math.round(Date.now() / 1000),
|
|
},
|
|
},
|
|
});
|
|
}
|
|
|
|
if (filters.scope === 'upcoming') {
|
|
query.bool.must.push({
|
|
range: {
|
|
effective_date: {
|
|
gt: Math.round(Date.now() / 1000),
|
|
},
|
|
},
|
|
});
|
|
|
|
sort = [{ effective_date: 'asc' }];
|
|
}
|
|
|
|
if (filters.scope === 'new') {
|
|
sort = [{ created_at: 'desc' }, { effective_date: 'asc' }];
|
|
}
|
|
|
|
if (filters.scope === 'likes') {
|
|
sort = [{ stashed: 'desc' }, { effective_date: 'desc' }];
|
|
}
|
|
|
|
if (filters.scope === 'results') {
|
|
sort = [{ _score: 'desc' }, { effective_date: 'desc' }];
|
|
}
|
|
|
|
if (filters.query) {
|
|
/*
|
|
query.bool.must.push({
|
|
bool: {
|
|
should: [
|
|
{ match: { title_filtered: filters.query } },
|
|
{ match: { actors: filters.query } },
|
|
{ match: { tags: filters.query } },
|
|
{ match: { channel_name: filters.query } },
|
|
{ match: { network_name: filters.query } },
|
|
{ match: { channel_slug: filters.query } },
|
|
{ match: { network_slug: filters.query } },
|
|
{ match: { meta: filters.query } }, // date
|
|
],
|
|
},
|
|
});
|
|
*/
|
|
|
|
query.bool.must.push({ match: { '!title': filters.query } }); // title_filtered is matched instead of title
|
|
}
|
|
|
|
if (filters.tagIds) {
|
|
filters.tagIds.forEach((tagId) => {
|
|
query.bool.must.push({ equals: { 'any(tag_ids)': tagId } });
|
|
});
|
|
}
|
|
|
|
if (filters.entityId) {
|
|
query.bool.must.push({
|
|
bool: {
|
|
should: [
|
|
{ equals: { channel_id: filters.entityId } },
|
|
{ equals: { network_id: filters.entityId } },
|
|
],
|
|
},
|
|
});
|
|
}
|
|
|
|
if (filters.actorIds) {
|
|
filters.actorIds.forEach((actorId) => {
|
|
query.bool.must.push({ equals: { 'any(actor_ids)': actorId } });
|
|
});
|
|
}
|
|
|
|
if (filters.stashId && options.index === 'scenes_stashed') {
|
|
query.bool.must.push({ equals: { stash_id: filters.stashId } });
|
|
}
|
|
|
|
/* tag filter
|
|
must_not: [
|
|
{
|
|
in: {
|
|
'any(tag_ids)': [101, 180, 32],
|
|
},
|
|
},
|
|
],
|
|
*/
|
|
|
|
return { query, sort };
|
|
}
|
|
|
|
function buildAggregates(options) {
|
|
const aggregates = {};
|
|
|
|
if (options.aggregateActors) {
|
|
aggregates.actorIds = {
|
|
terms: {
|
|
field: 'actor_ids',
|
|
size: config.database.manticore.maxAggregateSize,
|
|
},
|
|
sort: [{ 'count(*)': { order: 'desc' } }],
|
|
};
|
|
}
|
|
|
|
if (options.aggregateTags) {
|
|
aggregates.tagIds = {
|
|
terms: {
|
|
field: 'tag_ids',
|
|
size: config.database.manticore.maxAggregateSize,
|
|
},
|
|
sort: [{ 'count(*)': { order: 'desc' } }],
|
|
};
|
|
}
|
|
|
|
if (options.aggregateChannels) {
|
|
aggregates.channelIds = {
|
|
terms: {
|
|
field: 'channel_id',
|
|
size: config.database.manticore.maxAggregateSize,
|
|
},
|
|
sort: [{ 'count(*)': { order: 'desc' } }],
|
|
};
|
|
}
|
|
|
|
return aggregates;
|
|
}
|
|
|
|
function countAggregations(buckets) {
|
|
if (!buckets) {
|
|
return null;
|
|
}
|
|
|
|
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
|
|
}
|
|
|
|
async function queryManticoreJson(filters, options, _reqUser) {
|
|
const { query, sort } = buildQuery(filters, options);
|
|
|
|
const result = await searchApi.search({
|
|
index: options.index,
|
|
query,
|
|
limit: options.limit,
|
|
offset: (options.page - 1) * options.limit,
|
|
sort,
|
|
aggs: buildAggregates(options),
|
|
options: {
|
|
max_matches: config.database.manticore.maxMatches,
|
|
max_query_time: config.database.manticore.maxQueryTime,
|
|
field_weights: {
|
|
title_filtered: 7,
|
|
actors: 10,
|
|
tags: 9,
|
|
meta: 6,
|
|
channel_name: 2,
|
|
channel_slug: 3,
|
|
network_name: 1,
|
|
network_slug: 1,
|
|
},
|
|
},
|
|
});
|
|
|
|
const scenes = result.hits.hits.map((hit) => ({
|
|
id: hit._id,
|
|
...hit._source,
|
|
_score: hit._score,
|
|
}));
|
|
|
|
return {
|
|
scenes,
|
|
total: result.hits.total,
|
|
aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])),
|
|
};
|
|
}
|
|
|
|
async function queryManticoreSql(filters, options, _reqUser) {
|
|
const aggSize = 10 || config.database.manticore.maxAggregateSize;
|
|
|
|
const sqlQuery = knexManticore.raw(`
|
|
:query:
|
|
OPTION field_weights=(
|
|
title_filtered=7,
|
|
actors=10,
|
|
tags=9,
|
|
meta=6,
|
|
channel_name=2,
|
|
channel_slug=3,
|
|
network_name=1,
|
|
network_slug=1
|
|
),
|
|
max_matches=:maxMatches:,
|
|
max_query_time=:maxQueryTime:
|
|
:actorsFacet:
|
|
:tagsFacet:
|
|
:channelsFacet:
|
|
`, {
|
|
query: knexManticore('scenes')
|
|
.select(knex.raw('*, weight() as _score'))
|
|
.modify((builder) => {
|
|
if (filters.stashId) {
|
|
builder
|
|
.innerJoin('scenes_stashed', 'scenes.id', 'scenes_stashed.scene_id')
|
|
.where('scenes_stashed.stash_id', filters.stashId);
|
|
}
|
|
|
|
if (filters.query) {
|
|
builder.whereRaw('match(\'@!title :query:\', scenes)', { query: filters.query });
|
|
}
|
|
|
|
if (filters.tagIds?.length > 0) {
|
|
builder.whereIn('any(tag_ids)', filters.tagIds);
|
|
}
|
|
|
|
if (filters.entityId) {
|
|
builder.where((whereBuilder) => {
|
|
whereBuilder
|
|
.where('channel_id', filters.entityId)
|
|
.orWhere('network_id', filters.entityId);
|
|
});
|
|
}
|
|
|
|
if (filters.actorIds?.length > 0) {
|
|
builder.whereIn('any(actor_ids)', filters.actorIds);
|
|
}
|
|
|
|
if (!filters.scope || filters.scope === 'latest') {
|
|
builder
|
|
.where('effective_date', '<=', Math.round(Date.now() / 1000))
|
|
.orderBy('effective_date', 'desc');
|
|
} else if (filters.scope === 'upcoming') {
|
|
builder
|
|
.where('effective_date', '>', Math.round(Date.now() / 1000))
|
|
.orderBy('effective_date', 'asc');
|
|
} else if (filters.scope === 'new') {
|
|
builder.orderBy([
|
|
{ column: 'created_at', order: 'desc' },
|
|
{ column: 'effective_date', order: 'asc' },
|
|
]);
|
|
} else if (filters.scope === 'likes') {
|
|
builder.orderBy([
|
|
{ column: 'stashed', order: 'desc' },
|
|
{ column: 'effective_date', order: 'desc' },
|
|
]);
|
|
} else if (filters.scope === 'results') {
|
|
builder.orderBy([
|
|
{ column: '_score', order: 'desc' },
|
|
{ column: 'effective_date', order: 'desc' },
|
|
]);
|
|
} else {
|
|
builder.orderBy('effective_date', 'desc');
|
|
}
|
|
})
|
|
.limit(options.limit)
|
|
.toString(),
|
|
// option threads=1 fixes actors, but drastically slows down performance, wait for fix
|
|
actorsFacet: options.aggregateActors ? knex.raw('facet actor_ids order by count(*) desc limit ?', [aggSize]) : null,
|
|
tagsFacet: options.aggregateTags ? knex.raw('facet tag_ids order by count(*) desc limit ?', [aggSize]) : null,
|
|
channelsFacet: options.aggregateChannels ? knex.raw('facet channel_id order by count(*) desc limit ?', [aggSize]) : null,
|
|
maxMatches: config.database.manticore.maxMatches,
|
|
maxQueryTime: config.database.manticore.maxQueryTime,
|
|
}).toString();
|
|
|
|
console.log(sqlQuery);
|
|
|
|
const results = await utilsApi.sql(sqlQuery);
|
|
|
|
const actorIds = results
|
|
.find((result) => result.columns[0].actor_ids && result.columns[1]['count(*)'])
|
|
?.data.map((row) => ({ key: row.actor_ids, doc_count: row['count(*)'] }))
|
|
|| [];
|
|
|
|
const tagIds = results
|
|
.find((result) => result.columns[0].tag_ids && result.columns[1]['count(*)'])
|
|
?.data.map((row) => ({ key: row.tag_ids, doc_count: row['count(*)'] }))
|
|
|| [];
|
|
|
|
const channelIds = results
|
|
.find((result) => result.columns[0].channel_id && result.columns[1]['count(*)'])
|
|
?.data.map((row) => ({ key: row.channel_id, doc_count: row['count(*)'] }))
|
|
|| [];
|
|
|
|
return {
|
|
scenes: results[0].data,
|
|
total: results[0].total,
|
|
aggregations: {
|
|
actorIds,
|
|
tagIds,
|
|
channelIds,
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function fetchScenes(filters, rawOptions, reqUser) {
|
|
const options = curateOptions(rawOptions);
|
|
|
|
console.log('filters', filters);
|
|
console.log('options', options);
|
|
|
|
/*
|
|
const result = config.database.manticore.forceSql || filters.stashId
|
|
? await queryManticoreSql(filters, options, reqUser)
|
|
: await queryManticoreJson(filters, options, reqUser);
|
|
*/
|
|
|
|
console.time('manticore sql');
|
|
const result = await queryManticoreSql(filters, options, reqUser);
|
|
console.timeEnd('manticore sql');
|
|
|
|
console.time('manticore json');
|
|
await queryManticoreJson(filters, options, reqUser);
|
|
console.timeEnd('manticore json');
|
|
|
|
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
|
|
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
|
|
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds);
|
|
|
|
console.time('fetch aggregations');
|
|
|
|
const [aggActors, aggTags, aggChannels] = await Promise.all([
|
|
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
|
|
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
|
|
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
|
|
]);
|
|
|
|
console.timeEnd('fetch aggregations');
|
|
|
|
console.time('fetch full');
|
|
const sceneIds = result.scenes.map((scene) => Number(scene.id));
|
|
const scenes = await fetchScenesById(sceneIds, reqUser);
|
|
console.timeEnd('fetch full');
|
|
|
|
return {
|
|
scenes,
|
|
aggActors,
|
|
aggTags,
|
|
aggChannels,
|
|
total: result.total,
|
|
limit: options.limit,
|
|
};
|
|
}
|