DebaucheryLibrarian/traxxx-web
1
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Added filterable stash pages.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2024-03-15 00:08:24 +01:00
parent 7f00e31fc4
commit a1b45cb721
39 changed files with 649218 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
src/scenes.js
View File

@@ -1,7 +1,8 @@
import config from 'config';
import util from 'util'; /* eslint-disable-line no-unused-vars */
import { knexOwner as knex } from './knex.js';
import { searchApi } from './manticore.js';
import { knexOwner as knex, knexManticore } from './knex.js';
import { searchApi, utilsApi } from './manticore.js';
import { HttpError } from './errors.js';
import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js';
import { fetchTagsById } from './tags.js';
@@ -151,6 +152,8 @@ export async function fetchScenesById(sceneIds, reqUser) {
}).filter(Boolean);
}
const sqlImplied = ['scenes_stashed'];
function curateOptions(options) {
if (options?.limit > 100) {
throw new HttpError('Limit must be <= 100', 400);
@@ -163,10 +166,12 @@ function curateOptions(options) {
aggregateActors: (options.aggregate ?? true) && (options.aggregateActors ?? true),
aggregateTags: (options.aggregate ?? true) && (options.aggregateTags ?? true),
aggregateChannels: (options.aggregate ?? true) && (options.aggregateChannels ?? true),
index: options.index || 'scenes',
useSql: options.useSql || (typeof options.useSql === 'undefined' && sqlImplied.includes(options.index)) || false,
};
}
function buildQuery(filters = {}) {
function buildQuery(filters = {}, options) {
const query = {
bool: {
must: [],
@@ -210,6 +215,7 @@ function buildQuery(filters = {}) {
}
if (filters.query) {
/*
query.bool.must.push({
bool: {
should: [
@@ -224,6 +230,9 @@ function buildQuery(filters = {}) {
],
},
});
*/
query.bool.must.push({ match: { '!title': filters.query } }); // title_filtered is matched instead of title
}
if (filters.tagIds) {
@@ -249,6 +258,10 @@ function buildQuery(filters = {}) {
});
}
if (filters.stashId && options.index === 'scenes_stashed') {
query.bool.must.push({ equals: { stash_id: filters.stashId } });
}
/* tag filter
must_not: [
{
@@ -281,6 +294,7 @@ function buildAggregates(options) {
field: 'tag_ids',
size: config.database.manticore.maxAggregateSize,
},
sort: [{ 'count(*)': { order: 'desc' } }],
};
}
@@ -290,6 +304,7 @@ function buildAggregates(options) {
field: 'channel_id',
size: config.database.manticore.maxAggregateSize,
},
sort: [{ 'count(*)': { order: 'desc' } }],
};
}
@@ -304,20 +319,11 @@ function countAggregations(buckets) {
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
}
export async function fetchScenes(filters, rawOptions, reqUser) {
const options = curateOptions(rawOptions);
const { query, sort } = buildQuery(filters);
console.log('filters', filters);
console.log('options', options);
console.log('query', query.bool.must);
console.log('request user', reqUser);
console.time('manticore');
async function queryManticoreJson(filters, options, _reqUser) {
const { query, sort } = buildQuery(filters, options);
const result = await searchApi.search({
index: 'scenes',
index: options.index,
query,
limit: options.limit,
offset: (options.page - 1) * options.limit,
@@ -339,31 +345,181 @@ export async function fetchScenes(filters, rawOptions, reqUser) {
},
});
console.timeEnd('manticore');
const scenes = result.hits.hits.map((hit) => ({
id: hit._id,
...hit._source,
_score: hit._score,
}));
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
return {
scenes,
total: result.hits.total,
aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])),
};
}
async function queryManticoreSql(filters, options, _reqUser) {
const aggSize = 10 || config.database.manticore.maxAggregateSize;
const sqlQuery = knexManticore.raw(`
:query:
OPTION field_weights=(
title_filtered=7,
actors=10,
tags=9,
meta=6,
channel_name=2,
channel_slug=3,
network_name=1,
network_slug=1
),
max_matches=:maxMatches:,
max_query_time=:maxQueryTime:
:actorsFacet:
:tagsFacet:
:channelsFacet:
`, {
query: knexManticore('scenes')
.select(knex.raw('*, weight() as _score'))
.modify((builder) => {
if (filters.stashId) {
builder
.innerJoin('scenes_stashed', 'scenes.id', 'scenes_stashed.scene_id')
.where('scenes_stashed.stash_id', filters.stashId);
}
if (filters.query) {
builder.whereRaw('match(\'@!title :query:\', scenes)', { query: filters.query });
}
if (filters.tagIds?.length > 0) {
builder.whereIn('any(tag_ids)', filters.tagIds);
}
if (filters.entityId) {
builder.where((whereBuilder) => {
whereBuilder
.where('channel_id', filters.entityId)
.orWhere('network_id', filters.entityId);
});
}
if (filters.actorIds?.length > 0) {
builder.whereIn('any(actor_ids)', filters.actorIds);
}
if (!filters.scope || filters.scope === 'latest') {
builder
.where('effective_date', '<=', Math.round(Date.now() / 1000))
.orderBy('effective_date', 'desc');
} else if (filters.scope === 'upcoming') {
builder
.where('effective_date', '>', Math.round(Date.now() / 1000))
.orderBy('effective_date', 'asc');
} else if (filters.scope === 'new') {
builder.orderBy([
{ column: 'created_at', order: 'desc' },
{ column: 'effective_date', order: 'asc' },
]);
} else if (filters.scope === 'likes') {
builder.orderBy([
{ column: 'stashed', order: 'desc' },
{ column: 'effective_date', order: 'desc' },
]);
} else if (filters.scope === 'results') {
builder.orderBy([
{ column: '_score', order: 'desc' },
{ column: 'effective_date', order: 'desc' },
]);
} else {
builder.orderBy('effective_date', 'desc');
}
})
.limit(options.limit)
.toString(),
// option threads=1 fixes actors, but drastically slows down performance, wait for fix
actorsFacet: options.aggregateActors ? knex.raw('facet actor_ids order by count(*) desc limit ?', [aggSize]) : null,
tagsFacet: options.aggregateTags ? knex.raw('facet tag_ids order by count(*) desc limit ?', [aggSize]) : null,
channelsFacet: options.aggregateChannels ? knex.raw('facet channel_id order by count(*) desc limit ?', [aggSize]) : null,
maxMatches: config.database.manticore.maxMatches,
maxQueryTime: config.database.manticore.maxQueryTime,
}).toString();
console.log(sqlQuery);
const results = await utilsApi.sql(sqlQuery);
const actorIds = results
.find((result) => result.columns[0].actor_ids && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.actor_ids, doc_count: row['count(*)'] }))
|| [];
const tagIds = results
.find((result) => result.columns[0].tag_ids && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.tag_ids, doc_count: row['count(*)'] }))
|| [];
const channelIds = results
.find((result) => result.columns[0].channel_id && result.columns[1]['count(*)'])
?.data.map((row) => ({ key: row.channel_id, doc_count: row['count(*)'] }))
|| [];
return {
scenes: results[0].data,
total: results[0].total,
aggregations: {
actorIds,
tagIds,
channelIds,
},
};
}
export async function fetchScenes(filters, rawOptions, reqUser) {
const options = curateOptions(rawOptions);
console.log('filters', filters);
console.log('options', options);
/*
const result = config.database.manticore.forceSql || filters.stashId
? await queryManticoreSql(filters, options, reqUser)
: await queryManticoreJson(filters, options, reqUser);
*/
console.time('manticore sql');
const result = await queryManticoreSql(filters, options, reqUser);
console.timeEnd('manticore sql');
console.time('manticore json');
await queryManticoreJson(filters, options, reqUser);
console.timeEnd('manticore json');
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds);
console.time('fetch aggregations');
const [aggActors, aggTags, aggChannels] = await Promise.all([
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
]);
console.timeEnd('fetch aggregations');
const sceneIds = result.hits.hits.map((hit) => Number(hit._id));
console.time('fetch full');
const sceneIds = result.scenes.map((scene) => Number(scene.id));
const scenes = await fetchScenesById(sceneIds, reqUser);
console.timeEnd('fetch full');
return {
scenes,
aggActors,
aggTags,
aggChannels,
total: result.hits.total,
total: result.total,
limit: options.limit,
};
}