import config from 'config'; import { differenceInYears } from 'date-fns'; import { unit } from 'mathjs'; import { knexOwner as knex, knexManticore } from './knex.js'; import { utilsApi } from './manticore.js'; import { HttpError } from './errors.js'; import { fetchCountriesByAlpha2 } from './countries.js'; import { curateStash } from './stashes.js'; import escape from '../utils/escape-manticore.js'; import slugify from '../utils/slugify.js'; export function curateActor(actor, context = {}) { return { id: actor.id, slug: actor.slug, name: actor.name, gender: actor.gender, age: actor.age, dateOfBirth: actor.date_of_birth, ageFromBirth: actor.date_of_birth && differenceInYears(Date.now(), actor.date_of_birth), ageThen: context.sceneDate && actor.date_of_birth && differenceInYears(context.sceneDate, actor.date_of_birth), bust: actor.bust, cup: actor.cup, waist: actor.waist, hip: actor.hip, naturalBoobs: actor.naturalBoobs, height: actor.height && { metric: actor.height, imperial: unit(actor.height, 'cm').splitUnit(['ft', 'in']).map((value) => Math.round(value.toNumber())), }, weight: actor.weight && { metric: actor.weight, imperial: Math.round(unit(actor.weight, 'kg').toNumeric('lbs')), }, eyes: actor.eyes, hairColor: actor.hairColor, hasTattoos: actor.has_tattoos, tattoos: actor.tattoos, hasPiercings: actor.has_piercings, piercings: actor.piercings, origin: actor.birth_country_alpha2 && { country: actor.birth_country_alpha2 && { alpha2: actor.birth_country_alpha2, name: actor.birth_country_name, }, }, residence: actor.residence_country_alpha2 && { country: actor.residence_country_alpha2 && { alpha2: actor.residence_country_alpha2, name: actor.residence_country_name, }, }, avatar: actor.avatar && { id: actor.avatar.id, path: actor.avatar.path, thumbnail: actor.avatar.thumbnail, lazy: actor.avatar.lazy, isS3: actor.avatar.is_s3, }, createdAt: actor.created_at, updatedAt: actor.updated_at, likes: actor.stashed, stashes: context.stashes?.map((stash) => curateStash(stash)) || [], ...context.append?.[actor.id], }; } export function sortActorsByGender(actors, context = {}) { if (!actors) { return actors; } const alphaActors = actors.sort((actorA, actorB) => actorA.name.localeCompare(actorB.name, 'en')); const genderActors = ['transsexual', 'female', undefined, null, 'male'].flatMap((gender) => alphaActors.filter((actor) => actor.gender === gender)); const titleSlug = slugify(context.title); const titleActors = titleSlug ? genderActors.sort((actorA, actorB) => { const actorASlug = actorA.slug.split('-')[0]; const actorBSlug = actorB.slug.split('-')[0]; if (titleSlug.includes(actorASlug) && !titleSlug.includes(actorBSlug)) { return -1; } if (titleSlug.includes(actorBSlug) && !titleSlug.includes(actorASlug)) { return 1; } return 0; }) : alphaActors; return titleActors; } export async function fetchActorsById(actorIds, options = {}, reqUser) { const [actors, stashes] = await Promise.all([ knex('actors') .select( 'actors.*', 'actors_meta.*', 'birth_countries.alpha2 as birth_country_alpha2', knex.raw('COALESCE(birth_countries.alias, birth_countries.name) as birth_country_name'), 'residence_countries.alpha2 as residence_country_alpha2', knex.raw('COALESCE(residence_countries.alias, residence_countries.name) as residence_country_name'), ) .leftJoin('actors_meta', 'actors_meta.actor_id', 'actors.id') .leftJoin('countries as birth_countries', 'birth_countries.alpha2', 'actors.birth_country_alpha2') .leftJoin('countries as residence_countries', 'residence_countries.alpha2', 'actors.residence_country_alpha2') .whereIn('actors.id', actorIds) .modify((builder) => { if (options.order) { builder.orderBy(...options.order); } }), reqUser ? knex('stashes_actors') .leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id') .where('stashes.user_id', reqUser.id) .whereIn('stashes_actors.actor_id', actorIds) : [], ]); if (options.order) { return actors.map((actorEntry) => curateActor(actorEntry, { stashes: stashes.filter((stash) => stash.actor_id === actorEntry.id), append: options.append, })); } const curatedActors = actorIds.map((actorId) => { const actor = actors.find((actorEntry) => actorEntry.id === actorId); if (!actor) { console.warn(`Can't match actor ${actorId}`); return null; } return curateActor(actor, { stashes: stashes.filter((stash) => stash.actor_id === actor.id), append: options.append, }); }).filter(Boolean); return curatedActors; } function curateOptions(options) { if (options?.limit > 120) { throw new HttpError('Limit must be <= 120', 400); } return { page: options?.page || 1, limit: options?.limit || 30, requireAvatar: options?.requireAvatar || false, order: [escape(options.order?.[0]) || 'name', escape(options.order?.[1]) || 'asc'], }; } /* const sortMap = { likes: 'stashed', scenes: 'scenes', relevance: '_score', }; function getSort(order) { if (order[0] === 'name') { return [{ slug: order[1], }]; } return [ { [sortMap[order[0]]]: order[1], }, { slug: 'asc', // sort by name where primary order is equal }, ]; } function buildQuery(filters) { const query = { bool: { must: [], }, }; const expressions = { age: 'if(date_of_birth, floor((now() - date_of_birth) / 31556952), 0)', }; if (filters.query) { query.bool.must.push({ match: { name: filters.query, }, }); } ['gender', 'country'].forEach((attribute) => { if (filters[attribute]) { query.bool.must.push({ equals: { [attribute]: filters[attribute], }, }); } }); ['age', 'height', 'weight'].forEach((attribute) => { if (filters[attribute]) { query.bool.must.push({ range: { [attribute]: { gte: filters[attribute][0], lte: filters[attribute][1], }, }, }); } }); if (filters.dateOfBirth && filters.dobType === 'dateOfBirth') { query.bool.must.push({ equals: { date_of_birth: Math.floor(filters.dateOfBirth.getTime() / 1000), }, }); } if (filters.dateOfBirth && filters.dobType === 'birthday') { expressions.month_of_birth = 'month(date_of_birth)'; expressions.day_of_birth = 'day(date_of_birth)'; const month = filters.dateOfBirth.getMonth() + 1; const day = filters.dateOfBirth.getDate(); query.bool.must.push({ bool: { must: [ { equals: { month_of_birth: month, }, }, { equals: { day_of_birth: day, }, }, ], }, }); } if (filters.cup) { expressions.cup_in_range = `regex(cup, '^[${filters.cup[0]}-${filters.cup[1]}]')`; query.bool.must.push({ equals: { cup_in_range: 1, }, }); } if (typeof filters.naturalBoobs === 'boolean') { query.bool.must.push({ equals: { natural_boobs: filters.naturalBoobs ? 2 : 1, // manticore boolean does not support null, so 0 = null, 1 = false (enhanced), 2 = true (natural) }, }); } if (filters.requireAvatar) { query.bool.must.push({ equals: { has_avatar: 1, }, }); } return { query, expressions }; } async function queryManticoreJson(filters, options) { const { query, expressions } = buildQuery(filters); const result = await searchApi.search({ index: 'actors', query, expressions, limit: options.limit, offset: (options.page - 1) * options.limit, sort: getSort(options.order, filters), aggs: { countries: { terms: { field: 'country', size: 300, }, sort: [{ country: { order: 'asc' } }], }, }, options: { max_matches: config.database.manticore.maxMatches, max_query_time: config.database.manticore.maxQueryTime, }, }); const actors = result.hits.hits.map((hit) => ({ id: hit._id, ...hit._source, _score: hit._score, })); return { actors, total: result.hits.total, aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])), }; } */ async function queryManticoreSql(filters, options, _reqUser) { const aggSize = config.database.manticore.maxAggregateSize; const sqlQuery = knexManticore.raw(` :query: OPTION max_matches=:maxMatches:, max_query_time=:maxQueryTime: :countriesFacet:; show meta; `, { query: knexManticore(filters.stashId ? 'actors_stashed' : 'actors') .modify((builder) => { if (filters.stashId) { builder.select(knex.raw(` actors.id as id, actors.slug, actors.gender as gender, actors.country as country, actors.height as height, actors.mass as mass, actors.cup as cup, actors.natural_boobs as natural_boobs, actors.date_of_birth as date_of_birth, actors.has_avatar as has_avatar, actors.scenes as scenes, actors.stashed as stashed, created_at as stashed_at, if(actors.date_of_birth, floor((now() - actors.date_of_birth) / 31556952), 0) as age, weight() as _score `)); builder .innerJoin('actors', 'actors.id', 'actors_stashed.actor_id') .where('stash_id', filters.stashId); } else { builder.select(knex.raw('*, weight() as _score')); } if (filters.query) { builder.whereRaw('match(\'@name :query:\', actors)', { query: escape(filters.query) }); } // attribute filters ['country'].forEach((attribute) => { if (filters[attribute]) { builder.where(attribute, filters[attribute]); } }); if (filters.gender === 'other') { builder.whereNull('gender'); } else if (filters.gender) { builder.where('gender', filters.gender); } if (filters.age) { builder.select('if(date_of_birth, floor((now() - date_of_birth) / 31556952), 0) as age'); } // range filters ['age', 'height'].forEach((attribute) => { if (filters[attribute]) { builder .where(attribute, '>=', filters[attribute][0]) .where(attribute, '<=', filters[attribute][1]); } }); if (filters.weight) { // weight is a reserved keyword in manticore builder .where('mass', '>=', filters.weight[0]) .where('mass', '<=', filters.weight[1]); } if (filters.dateOfBirth && filters.dobType === 'dateOfBirth') { builder.where('date_of_birth', Math.floor(filters.dateOfBirth.getTime() / 1000)); } if (filters.dateOfBirth && filters.dobType === 'birthday') { const month = filters.dateOfBirth.getMonth() + 1; const day = filters.dateOfBirth.getDate(); builder.select('month(date_of_birth) as month_of_birth, day(date_of_birth) as day_of_birth'); builder .where('month_of_birth', month) .where('day_of_birth', day); } if (filters.cup) { builder.select(`regex(actors.cup, '^[${filters.cup[0]}-${filters.cup[1]}]') as cup_in_range`); builder.where('cup_in_range', 1); } if (typeof filters.naturalBoobs === 'boolean') { builder.where('natural_boobs', filters.naturalBoobs ? 2 : 1); // manticore boolean does not support null, so 0 = null, 1 = false (enhanced), 2 = true (natural) } if (filters.requireAvatar) { builder.where('has_avatar', 1); } if (options.order?.[0] === 'name') { builder.orderBy('actors.slug', options.order[1]); } else if (options.order?.[0] === 'likes') { builder.orderBy([ { column: 'actors.stashed', order: options.order[1] }, { column: 'actors.slug', order: 'asc' }, ]); } else if (options.order?.[0] === 'scenes') { builder.orderBy([ { column: 'actors.scenes', order: options.order[1] }, { column: 'actors.slug', order: 'asc' }, ]); } else if (options.order?.[0] === 'results') { builder.orderBy([ { column: '_score', order: options.order[1] }, { column: 'actors.slug', order: 'asc' }, ]); } else if (options.order?.[0] === 'stashed' && filters.stashId) { builder.orderBy([ { column: 'stashed_at', order: options.order[1] }, { column: 'actors.slug', order: 'asc' }, ]); } else if (options.order) { builder.orderBy([ { column: `actors.${options.order[0]}`, order: options.order[1] }, { column: 'actors.slug', order: 'asc' }, ]); } else { builder.orderBy('actors.slug', 'asc'); } }) .limit(options.limit) .offset((options.page - 1) * options.limit) .toString(), // option threads=1 fixes actors, but drastically slows down performance, wait for fix countriesFacet: options.aggregateActors ? knex.raw('facet actors.country order by count(*) desc limit 300', [aggSize]) : null, maxMatches: config.database.manticore.maxMatches, maxQueryTime: config.database.manticore.maxQueryTime, }).toString(); // manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around const curatedSqlQuery = filters.stashId ? sqlQuery : sqlQuery.replace(/actors\./g, ''); if (process.env.NODE_ENV === 'development') { console.log(curatedSqlQuery); } const results = await utilsApi.sql(curatedSqlQuery); // console.log(results[0]); const countries = results .find((result) => (result.columns[0].actor_ids || result.columns[0]['scenes.country']) && result.columns[1]['count(*)']) ?.data.map((row) => ({ key: row.actor_ids || row['scenes.country'], doc_count: row['count(*)'] })) || []; const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found').Value); return { actors: results[0].data, total, aggregations: { countries, }, }; } export async function fetchActors(filters, rawOptions, reqUser) { const options = curateOptions(rawOptions); console.log('filters', filters); console.log('options', options); const result = await queryManticoreSql(filters, options, reqUser); // console.log('result', result); const actorIds = result.actors.map((actor) => Number(actor.id)); const [actors, countries] = await Promise.all([ fetchActorsById(actorIds, {}, reqUser), fetchCountriesByAlpha2(result.aggregations.countries.map((bucket) => bucket.key)), ]); return { actors, countries, total: result.total, limit: options.limit, }; }