Added movies stash page.
This commit is contained in:
229
src/movies.js
229
src/movies.js
@@ -1,7 +1,7 @@
|
||||
import config from 'config';
|
||||
|
||||
import knex from './knex.js';
|
||||
import { searchApi } from './manticore.js';
|
||||
import { knexQuery as knex, knexManticore } from './knex.js';
|
||||
import { utilsApi } from './manticore.js';
|
||||
import { HttpError } from './errors.js';
|
||||
import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js';
|
||||
import { fetchTagsById } from './tags.js';
|
||||
@@ -166,6 +166,7 @@ function curateOptions(options) {
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
function buildQuery(filters = {}) {
|
||||
const query = {
|
||||
bool: {
|
||||
@@ -257,16 +258,6 @@ function buildQuery(filters = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
/* tag filter
|
||||
must_not: [
|
||||
{
|
||||
in: {
|
||||
'any(tag_ids)': [101, 180, 32],
|
||||
},
|
||||
},
|
||||
],
|
||||
*/
|
||||
|
||||
return { query, sort };
|
||||
}
|
||||
|
||||
@@ -304,20 +295,9 @@ function buildAggregates(options) {
|
||||
return aggregates;
|
||||
}
|
||||
|
||||
function countAggregations(buckets) {
|
||||
if (!buckets) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
|
||||
}
|
||||
|
||||
export async function fetchMovies(filters, rawOptions) {
|
||||
const options = curateOptions(rawOptions);
|
||||
async function queryManticoreJson(filters, options) {
|
||||
const { query, sort } = buildQuery(filters);
|
||||
|
||||
console.log('filters', filters);
|
||||
console.log('options', options);
|
||||
console.log('query', query.bool.must);
|
||||
|
||||
console.time('manticore');
|
||||
@@ -345,23 +325,202 @@ export async function fetchMovies(filters, rawOptions) {
|
||||
},
|
||||
});
|
||||
|
||||
console.timeEnd('manticore');
|
||||
const movies = result.hits.hits.map((hit) => ({
|
||||
id: hit._id,
|
||||
...hit._source,
|
||||
_score: hit._score,
|
||||
}));
|
||||
|
||||
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
|
||||
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
|
||||
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
|
||||
return {
|
||||
movies,
|
||||
total: result.hits.total,
|
||||
aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])),
|
||||
};
|
||||
}
|
||||
*/
|
||||
|
||||
console.time('fetch aggregations');
|
||||
async function queryManticoreSql(filters, options) {
|
||||
const aggSize = config.database.manticore.maxAggregateSize;
|
||||
|
||||
const sqlQuery = knexManticore.raw(`
|
||||
:query:
|
||||
OPTION
|
||||
field_weights=(
|
||||
title_filtered=7,
|
||||
actors=10,
|
||||
tags=9,
|
||||
meta=6,
|
||||
channel_name=2,
|
||||
channel_slug=3,
|
||||
network_name=1,
|
||||
network_slug=1
|
||||
),
|
||||
max_matches=:maxMatches:,
|
||||
max_query_time=:maxQueryTime:
|
||||
:actorsFacet:
|
||||
:tagsFacet:
|
||||
:channelsFacet:;
|
||||
show meta;
|
||||
`, {
|
||||
query: knexManticore(filters.stashId ? 'movies_stashed' : 'movies')
|
||||
.modify((builder) => {
|
||||
if (filters.stashId) {
|
||||
builder.select(knex.raw(`
|
||||
movies.id as id,
|
||||
movies.title as title,
|
||||
movies.actor_ids as actor_ids,
|
||||
movies.entity_ids as entity_ids,
|
||||
movies.tag_ids as tag_ids,
|
||||
movies.channel_id as channel_id,
|
||||
movies.network_id as network_id,
|
||||
movies.effective_date as effective_date,
|
||||
movies.stashed as stashed,
|
||||
movies.created_at,
|
||||
created_at as stashed_at,
|
||||
weight() as _score
|
||||
`));
|
||||
|
||||
builder
|
||||
.innerJoin('movies', 'movies.id', 'movies_stashed.movie_id')
|
||||
.where('stash_id', filters.stashId);
|
||||
} else {
|
||||
builder.select(knex.raw('*, weight() as _score'));
|
||||
}
|
||||
|
||||
if (filters.query) {
|
||||
builder.whereRaw('match(\'@!title :query:\', movies)', { query: filters.query });
|
||||
}
|
||||
|
||||
filters.tagIds?.forEach((tagId) => {
|
||||
builder.where('any(tag_ids)', tagId);
|
||||
});
|
||||
|
||||
filters.actorIds?.forEach((actorId) => {
|
||||
builder.where('any(actor_ids)', actorId);
|
||||
});
|
||||
|
||||
if (filters.entityId) {
|
||||
builder.whereRaw('any(entity_ids) = ?', filters.entityId);
|
||||
}
|
||||
|
||||
if (typeof filters.isShowcased === 'boolean') {
|
||||
builder.where('is_showcased', filters.isShowcased);
|
||||
}
|
||||
|
||||
if (!filters.scope || filters.scope === 'latest') {
|
||||
builder
|
||||
.where('effective_date', '<=', Math.round(Date.now() / 1000))
|
||||
.orderBy('movies.effective_date', 'desc'); // can't seem to use alias if it matches column-name? behavior not fully understand, but this works
|
||||
} else if (filters.scope === 'upcoming') {
|
||||
builder
|
||||
.where('effective_date', '>', Math.round(Date.now() / 1000))
|
||||
.orderBy('movies.effective_date', 'asc');
|
||||
} else if (filters.scope === 'new') {
|
||||
builder.orderBy([
|
||||
{ column: 'movies.created_at', order: 'desc' },
|
||||
{ column: 'movies.effective_date', order: 'asc' },
|
||||
]);
|
||||
} else if (filters.scope === 'likes') {
|
||||
builder.orderBy([
|
||||
{ column: 'movies.stashed', order: 'desc' },
|
||||
{ column: 'movies.effective_date', order: 'desc' },
|
||||
]);
|
||||
} else if (filters.scope === 'results') {
|
||||
builder.orderBy([
|
||||
{ column: '_score', order: 'desc' },
|
||||
{ column: 'movies.effective_date', order: 'desc' },
|
||||
]);
|
||||
} else if (filters.scope === 'stashed' && filters.stashId) {
|
||||
builder.orderBy([
|
||||
{ column: 'stashed_at', order: 'desc' },
|
||||
{ column: 'movies.effective_date', order: 'desc' },
|
||||
]);
|
||||
} else {
|
||||
builder.orderBy('movies.effective_date', 'desc');
|
||||
}
|
||||
})
|
||||
.limit(options.limit)
|
||||
.offset((options.page - 1) * options.limit)
|
||||
.toString(),
|
||||
// option threads=1 fixes actors, but drastically slows down performance, wait for fix
|
||||
actorsFacet: options.aggregateActors ? knex.raw('facet movies.actor_ids order by count(*) desc limit ?', [aggSize]) : null,
|
||||
tagsFacet: options.aggregateTags ? knex.raw('facet movies.tag_ids order by count(*) desc limit ?', [aggSize]) : null,
|
||||
channelsFacet: options.aggregateChannels ? knex.raw('facet movies.channel_id order by count(*) desc limit ?', [aggSize]) : null,
|
||||
maxMatches: config.database.manticore.maxMatches,
|
||||
maxQueryTime: config.database.manticore.maxQueryTime,
|
||||
}).toString();
|
||||
|
||||
// manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around
|
||||
const curatedSqlQuery = filters.stashId
|
||||
? sqlQuery
|
||||
: sqlQuery.replace(/movies\./g, '');
|
||||
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
console.log(curatedSqlQuery);
|
||||
}
|
||||
|
||||
const results = await utilsApi.sql(curatedSqlQuery);
|
||||
|
||||
// console.log(results[0]);
|
||||
|
||||
const actorIds = results
|
||||
.find((result) => (result.columns[0].actor_ids || result.columns[0]['movies.actor_ids']) && result.columns[1]['count(*)'])
|
||||
?.data.map((row) => ({ key: row.actor_ids || row['movies.actor_ids'], doc_count: row['count(*)'] }))
|
||||
|| [];
|
||||
|
||||
const tagIds = results
|
||||
.find((result) => (result.columns[0].tag_ids || result.columns[0]['movies.tag_ids']) && result.columns[1]['count(*)'])
|
||||
?.data.map((row) => ({ key: row.tag_ids || row['movies.tag_ids'], doc_count: row['count(*)'] }))
|
||||
|| [];
|
||||
|
||||
const channelIds = results
|
||||
.find((result) => (result.columns[0].channel_id || result.columns[0]['movies.channel_id']) && result.columns[1]['count(*)'])
|
||||
?.data.map((row) => ({ key: row.channel_id || row['movies.channel_id'], doc_count: row['count(*)'] }))
|
||||
|| [];
|
||||
|
||||
const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0;
|
||||
|
||||
return {
|
||||
movies: results[0].data,
|
||||
total,
|
||||
aggregations: {
|
||||
actorIds,
|
||||
tagIds,
|
||||
channelIds,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function countAggregations(buckets) {
|
||||
if (!buckets) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
|
||||
}
|
||||
|
||||
export async function fetchMovies(filters, rawOptions) {
|
||||
const options = curateOptions(rawOptions);
|
||||
|
||||
console.log(options);
|
||||
console.log(filters);
|
||||
|
||||
const result = await queryManticoreSql(filters, options);
|
||||
|
||||
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
|
||||
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
|
||||
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds);
|
||||
|
||||
const [aggActors, aggTags, aggChannels] = await Promise.all([
|
||||
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
|
||||
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
|
||||
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
|
||||
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
|
||||
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
|
||||
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
|
||||
]);
|
||||
|
||||
console.timeEnd('fetch aggregations');
|
||||
console.log(result.aggregations);
|
||||
console.log(aggActors);
|
||||
|
||||
const movieIds = result.hits.hits.map((hit) => Number(hit._id));
|
||||
const movieIds = result.movies.map((movie) => Number(movie.id));
|
||||
const movies = await fetchMoviesById(movieIds);
|
||||
|
||||
return {
|
||||
@@ -369,7 +528,7 @@ export async function fetchMovies(filters, rawOptions) {
|
||||
aggActors,
|
||||
aggTags,
|
||||
aggChannels,
|
||||
total: result.hits.total,
|
||||
total: result.total,
|
||||
limit: options.limit,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user