Added unidentified actors and actor count filters.

This commit is contained in:
2026-11-11 22:23:50 +01:00
parent 10ac366e87
commit 76ccd3729f
24 changed files with 575 additions and 70 deletions
+62 -23
View File
@@ -24,7 +24,7 @@ import { curateMedia } from './media.js';
import redis from './redis.js';
import { curateRevision } from './revisions.js';
import { curateStash } from './stashes.js';
import { syncActors, syncScenes, syncStashes } from './sync.js';
import { syncActors, syncManticoreScenes, syncScenes, syncStashes } from './sync.js';
const logger = initLogger();
const mj = new MerkleJson();
@@ -235,15 +235,17 @@ export function sortActorsByGender(actors, context = {}) {
return actors;
}
const alphaActors = actors.sort((actorA, actorB) => actorA.name.localeCompare(actorB.name, 'en'));
const genderActors = ['transsexual', 'female', undefined, null, 'male'].flatMap((gender) => alphaActors.filter((actor) => actor.gender === gender));
const alphaActors = actors.toSorted((actorA, actorB) => actorA.name?.localeCompare(actorB.name, 'en'));
const titleSlug = slugify(context.title);
const titleActors = titleSlug
? genderActors.sort((actorA, actorB) => {
const actorASlug = actorA.slug.split('-')[0];
const actorBSlug = actorB.slug.split('-')[0];
? alphaActors.sort((actorA, actorB) => {
const actorASlug = actorA.slug?.split('-')[0];
const actorBSlug = actorB.slug?.split('-')[0];
if (!actorASlug || !actorBSlug) {
return 0;
}
if (titleSlug.includes(actorASlug) && !titleSlug.includes(actorBSlug)) {
return -1;
@@ -257,7 +259,10 @@ export function sortActorsByGender(actors, context = {}) {
})
: alphaActors;
return titleActors;
const genderActors = ['transsexual', 'female', undefined, null, 'male'].flatMap((gender) => titleActors.filter((actor) => actor.gender === gender));
const identifiedActors = genderActors.toSorted((actorA, actorB) => !!actorA.isUnknown - !!actorB.isUnknown);
return identifiedActors;
}
export async function fetchActorsById(actorIds, options = {}, reqUser) {
@@ -465,22 +470,21 @@ async function queryManticoreSql(filters, options, _reqUser) {
builder.select('if(date_of_birth, floor((now() - date_of_birth) / 31556952), 0) as age');
}
// range filters
['age', 'height'].forEach((attribute) => {
if (filters[attribute]) {
builder
.where(attribute, '>=', filters[attribute][0])
.where(attribute, '<=', filters[attribute][1]);
// range filters, either end may be open, weight is a reserved keyword in manticore
Object.entries({ age: 'age', height: 'height', weight: 'mass' }).forEach(([attribute, column]) => {
if (typeof filters[attribute]?.min === 'number') {
builder.where(column, '>=', filters[attribute].min);
}
if (typeof filters[attribute]?.max === 'number') {
builder.where(column, '<=', filters[attribute].max);
if (typeof filters[attribute].min !== 'number') {
builder.where(column, '>', 0); // unknown values are stored as 0, an open lower bound should not match them
}
}
});
if (filters.weight) {
// weight is a reserved keyword in manticore
builder
.where('mass', '>=', filters.weight[0])
.where('mass', '<=', filters.weight[1]);
}
if (filters.dateOfBirth && filters.dobType === 'dateOfBirth') {
builder.where('date_of_birth', Math.floor(filters.dateOfBirth.getTime() / 1000));
}
@@ -497,7 +501,8 @@ async function queryManticoreSql(filters, options, _reqUser) {
}
if (filters.cup) {
builder.select(`regex(actors.cup, '^[${filters.cup[0]}-${filters.cup[1]}]') as cup_in_range`);
// cup range is validated to single letters by the query parser
builder.select(`regex(actors.cup, '^[${filters.cup.min || 'A'}-${filters.cup.max || 'Z'}]') as cup_in_range`);
builder.where('cup_in_range', 1);
}
@@ -1195,6 +1200,36 @@ async function fetchMainProfile(actorId, wasCreated = false) {
return fetchMainProfile(actorId, true);
}
async function fetchActorGenders(actorIds) {
const actors = await knex('actors')
.select('id', 'gender')
.whereIn('id', actorIds);
return Object.fromEntries(actors.map((actor) => [actor.id, actor.gender]));
}
// scene search documents count actors per gender, so scenes need a resync when an actor's gender changes
// syncs the search documents directly, the materialized views don't depend on actor genders
async function syncGenderChangedScenes(actorIds, gendersBefore) {
const gendersAfter = await fetchActorGenders(actorIds);
const changedActorIds = actorIds.filter((actorId) => gendersAfter[actorId] !== gendersBefore[actorId]);
if (changedActorIds.length === 0) {
return;
}
const sceneIds = await knex('releases_actors')
.distinct('release_id')
.whereIn('actor_id', changedActorIds)
.pluck('release_id');
logger.info(`Gender changed for ${changedActorIds.length} actors, resyncing ${sceneIds.length} scenes`);
if (sceneIds.length > 0) { // without IDs, all scenes are synced
await syncManticoreScenes(sceneIds);
}
}
async function applyActorRevision(revisionIds, reqUser) {
const revisions = await knex('actors_revisions')
.whereIn('id', revisionIds)
@@ -1276,6 +1311,7 @@ async function applyActorRevision(revisionIds, reqUser) {
}, Promise.resolve());
const actorIds = Array.from(new Set(revisions.map((revision) => revision.actor_id)));
const gendersBefore = await fetchActorGenders(actorIds);
await interpolateProfiles(actorIds, {
knex,
@@ -1285,7 +1321,10 @@ async function applyActorRevision(revisionIds, reqUser) {
omit,
}, { refreshView: false });
await syncActors(actorIds);
await Promise.all([
syncActors(actorIds),
syncGenderChangedScenes(actorIds, gendersBefore),
]);
}
export async function reviewActorRevision(revisionId, isApproved, { feedback }, reqUser) {
+114 -14
View File
@@ -79,9 +79,12 @@ function curateScene(rawScene, assets, reqUser, context) {
entityId: assets.channel.affiliate.entity_id,
}
: null,
actors: sortActorsByGender(assets.actors.map((actor) => curateActor(actor, {
sceneDate: rawScene.effective_date,
stashes: assets.actorStashes.filter((actorStash) => actorStash.actor_id === actor.id),
actors: sortActorsByGender(assets.actors.map((actor) => ({
...curateActor(actor, {
sceneDate: rawScene.effective_date,
stashes: assets.actorStashes.filter((actorStash) => actorStash.actor_id === actor.id),
}),
isUnknown: !!actor.isUnknown,
})), { title: rawScene.title }),
directors: assets.directors.map((director) => ({
id: director.id,
@@ -228,6 +231,8 @@ export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) {
'countries.name as birth_country_name',
'countries.alias as birth_country_alias',
'releases_actors.release_id',
'releases_actors.id as release_actor_id',
'releases_actors.gender as release_actor_gender',
)
.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
.leftJoin('actors as aliases', 'aliases.id', 'releases_actors.alias_id')
@@ -236,7 +241,7 @@ export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) {
.leftJoin('media_biometrics', 'media_biometrics.media_id', 'actors.avatar_media_id')
.leftJoin('countries', 'countries.alpha2', 'actors.birth_country_alpha2')
.whereIn('release_id', sceneIds)
.groupBy('actors.id', 'aliases.id', 'releases_actors.release_id', 'avatars.id', 'media_biometrics.id', 'countries.name', 'countries.alias', 'sfw_media.id'),
.groupBy('releases_actors.id', 'actors.id', 'aliases.id', 'releases_actors.release_id', 'avatars.id', 'media_biometrics.id', 'countries.name', 'countries.alias', 'sfw_media.id'),
directors: knex('releases_directors')
.whereIn('release_id', sceneIds)
.leftJoin('actors as directors', 'directors.id', 'releases_directors.director_id'),
@@ -385,7 +390,15 @@ export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) {
const sceneChannel = channels.find((entity) => entity.id === scene.entity_id);
const sceneStudio = studios.find((entity) => entity.id === scene.studio_id);
const sceneActors = actors.filter((actor) => actor.release_id === sceneId);
const sceneActors = actors.filter((actor) => actor.release_id === sceneId).map((actor) => (actor.id
? actor
: {
...actor,
// unidentified actor, needs a unique ID for editing, encodes gender for storage
id: `${actor.release_actor_gender || 'other'}_${actor.release_actor_id}`,
gender: actor.release_actor_gender,
isUnknown: true,
}));
const sceneDirectors = directors.filter((director) => director.release_id === sceneId);
const sceneTags = tags.filter((tag) => tag.release_id === sceneId);
const sceneChapters = chapters.filter((chapter) => chapter.release_id === sceneId);
@@ -439,6 +452,7 @@ function curateOptions(options) {
aggregateActors: (options?.aggregate ?? true) && (options?.aggregateActors ?? true),
aggregateTags: (options?.aggregate ?? true) && (options?.aggregateTags ?? true),
aggregateChannels: (options?.aggregate ?? true) && (options?.aggregateChannels ?? true),
aggregateActorCounts: (options?.aggregate ?? true) && (options?.aggregateActorCounts ?? true),
dedupe: !!options?.dedupe,
index: options?.index || 'scenes',
useSql: options?.useSql || (typeof options?.useSql === 'undefined' && sqlImplied.includes(options?.index)) || false,
@@ -446,12 +460,12 @@ function curateOptions(options) {
}
// function curateFacet(results, field, count = 'count(distinct id)') {
function curateFacet(results, field) {
function curateFacet(results, field, { includeZero = false } = {}) {
return results
.find((result) => result.columns[0][field] && (result.columns[1]['count(distinct id)'] || result.columns[1]['count(*)']))
?.data
.map((row) => ({ key: row[field], doc_count: row['count(distinct id)'] || row['count(*)'] }))
.filter((row) => !!row.key)
.filter((row) => !!row.key || (includeZero && row.key === 0)) // zero is a meaningful actor count, e.g. no men
|| [];
}
@@ -492,7 +506,11 @@ async function queryManticoreSql(filters, options, _reqUser) {
:actorsFacet:
:tagsFacet:
:channelsFacet:
:studiosFacet:;
:studiosFacet:
:groupFacet:
:femalesFacet:
:malesFacet:
:transFacet:;
show meta;
`, {
query: knexManticore(filters.stashId ? 'scenes_stashed' : 'scenes')
@@ -502,6 +520,11 @@ async function queryManticoreSql(filters, options, _reqUser) {
scenes.id as id,
scenes.title as title,
scenes.actor_ids as actor_ids,
scenes.actor_count as actor_count,
scenes.female_count as female_count,
scenes.male_count as male_count,
scenes.trans_count as trans_count,
scenes.unidentified_count as unidentified_count,
scenes.entity_ids as entity_ids,
scenes.tag_ids as tag_ids,
scenes.channel_id as channel_id,
@@ -562,6 +585,23 @@ async function queryManticoreSql(filters, options, _reqUser) {
builder.whereNotIn('actor_ids', filters.notActorIds);
}
// actor counts include unidentified actors, gender counts only actors with a known gender
Object.entries({
actor_count: filters.actorCount,
female_count: filters.femaleCount,
male_count: filters.maleCount,
trans_count: filters.transCount,
unidentified_count: filters.unidentifiedCount,
}).forEach(([field, range]) => {
if (typeof range?.min === 'number') {
builder.where(`scenes.${field}`, '>=', range.min);
}
if (typeof range?.max === 'number') {
builder.where(`scenes.${field}`, '<=', range.max);
}
});
if (filters.entityId) {
builder.whereRaw('any(entity_ids) = ?', filters.entityId);
@@ -644,6 +684,10 @@ async function queryManticoreSql(filters, options, _reqUser) {
tagsFacet: options.aggregateTags ? knex.raw('facet scenes.assigned_tag_ids as tags_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
channelsFacet: options.aggregateChannels ? knex.raw('facet scenes.channel_id as channels_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
studiosFacet: options.aggregateChannels ? knex.raw('facet scenes.studio_id as studios_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
groupFacet: options.aggregateActorCounts ? knex.raw('facet scenes.actor_count as group_facet order by actor_count asc limit ?', [aggSize]) : null,
femalesFacet: options.aggregateActorCounts ? knex.raw('facet scenes.female_count as females_facet order by female_count asc limit ?', [aggSize]) : null,
malesFacet: options.aggregateActorCounts ? knex.raw('facet scenes.male_count as males_facet order by male_count asc limit ?', [aggSize]) : null,
transFacet: options.aggregateActorCounts ? knex.raw('facet scenes.trans_count as trans_facet order by trans_count asc limit ?', [aggSize]) : null,
maxMatches: config.database.manticore.maxMatches,
maxQueryTime: config.database.manticore.maxQueryTime,
}).toString();
@@ -667,6 +711,13 @@ async function queryManticoreSql(filters, options, _reqUser) {
const channelIds = curateFacet(results, 'channels_facet');
const studioIds = curateFacet(results, 'studios_facet');
const actorCounts = {
group: curateFacet(results, 'group_facet', { includeZero: true }),
females: curateFacet(results, 'females_facet', { includeZero: true }),
males: curateFacet(results, 'males_facet', { includeZero: true }),
trans: curateFacet(results, 'trans_facet', { includeZero: true }),
};
const allTagIds = mergePackedTags(tagIds);
const actorTagIds = mergePackedTags(tagIds.filter((tag) => {
@@ -696,6 +747,7 @@ async function queryManticoreSql(filters, options, _reqUser) {
actorTagIds,
channelIds,
studioIds,
actorCounts,
},
};
}
@@ -708,6 +760,14 @@ function countAggregations(buckets) {
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
}
// URL parameter and aggregation key to filter key
const actorCountFilters = {
group: 'actorCount',
females: 'femaleCount',
males: 'maleCount',
trans: 'transCount',
};
export async function fetchScenes(filters, rawOptions, reqUser, context) {
const options = curateOptions(rawOptions);
@@ -716,15 +776,42 @@ export async function fetchScenes(filters, rawOptions, reqUser, context) {
console.log('options', options);
}
// an active count filter would narrow its own range to the filtered value, get the unfiltered ranges from a lightweight parallel query (~10ms vs 40-500ms for the main query with its actor and tag facets)
const activeCountKeys = options.aggregateActorCounts
? Object.keys(actorCountFilters).filter((key) => filters[actorCountFilters[key]])
: [];
const countsFilters = Object.fromEntries(Object.entries(filters).filter(([key]) => !Object.values(actorCountFilters).includes(key)));
const countsOptions = {
...options,
limit: 1,
page: 1,
aggregateYears: false,
aggregateActors: false,
aggregateTags: false,
aggregateChannels: false,
};
console.time('manticore sql');
const result = await queryManticoreSql(filters, options, reqUser);
const [result, countsResult] = await Promise.all([
queryManticoreSql(filters, options, reqUser),
activeCountKeys.length > 0 ? queryManticoreSql(countsFilters, countsOptions, reqUser) : null,
]);
console.timeEnd('manticore sql');
const aggYears = options.aggregateYears && result.aggregations.years.map((bucket) => ({ year: bucket.key, count: bucket.doc_count }));
const entityIds = options.aggregateChannels && [...(result.aggregations.channelIds || []), ...(result.aggregations.studioIds || [])];
// keyed by URL parameter, e.g. { group: [{ value: 2, count: 1234 }, ...], females: [...] }, sorted by value so the first and last bucket give the range
// filtered counts take their range from the query without count filters, so with several count filters, each of their ranges ignores the others
const aggActorCounts = options.aggregateActorCounts && Object.fromEntries(Object.keys(actorCountFilters).map((key) => [
key,
(activeCountKeys.includes(key) ? countsResult : result).aggregations.actorCounts[key].map((bucket) => ({ value: bucket.key, count: bucket.doc_count })),
]));
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
const channelCounts = options.aggregateChannels && countAggregations(entityIds);
@@ -752,6 +839,7 @@ export async function fetchScenes(filters, rawOptions, reqUser, context) {
aggTags,
aggActorTags,
aggChannels,
aggActorCounts,
total: result.total,
limit: options.limit,
};
@@ -809,7 +897,7 @@ export async function fetchSceneRevisions(revisionId, filters = {}, reqUser) {
...(revision.deltas.find((delta) => delta.key === 'actors')?.value || []),
...revision.base.tags.map((tag) => tag.actorId),
...revision.deltas.find((delta) => delta.key === 'tags')?.value.map((tag) => tag.actorId) || [],
].filter(Boolean))));
].filter((actorId) => typeof actorId === 'number')))); // unidentified actors have string IDs
const tagIds = Array.from(new Set(revisions.flatMap((revision) => [...revision.base.tags, ...(revision.deltas.find((delta) => delta.key === 'tags')?.value || [])].map((tag) => tag.id))));
const movieIds = Array.from(new Set(revisions.flatMap((revision) => [...revision.base.movies, ...(revision.deltas.find((delta) => delta.key === 'movies')?.value || [])])));
@@ -855,10 +943,22 @@ async function applySceneActorsDelta(sceneId, delta, trx) {
if (delta.value.length > 0) {
await knexOwner('releases_actors')
.insert(delta.value.map((actorId) => ({
release_id: sceneId,
actor_id: actorId,
})))
.insert(delta.value.map((actorId) => {
if (typeof actorId === 'number') {
return {
release_id: sceneId,
actor_id: actorId,
};
}
const gender = actorId.match(/^([a-z]+)_/)?.[1];
return {
release_id: sceneId,
actor_id: null,
gender: ['male', 'female', 'transsexual'].includes(gender) ? gender : null,
};
}))
.transacting(trx);
}
}
+11
View File
@@ -117,6 +117,12 @@ export async function syncManticoreScenes(sceneIds) {
grandparents.id as parent_network_id,
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(JSON_AGG(DISTINCT (actors_aliases.id, actors_aliases.name)) FILTER (WHERE actors_aliases.id IS NOT NULL), '[]') as actors_aliases,
-- count release actor rows rather than actors to include unidentified actors, distinct because the other joins multiply rows
COUNT(DISTINCT local_actors.id) as actor_count,
COUNT(DISTINCT local_actors.id) FILTER (WHERE COALESCE(actors.gender, local_actors.gender) = 'female') as female_count,
COUNT(DISTINCT local_actors.id) FILTER (WHERE COALESCE(actors.gender, local_actors.gender) = 'male') as male_count,
COUNT(DISTINCT local_actors.id) FILTER (WHERE COALESCE(actors.gender, local_actors.gender) = 'transsexual') as trans_count,
COUNT(DISTINCT local_actors.id) FILTER (WHERE local_actors.actor_id IS NULL) as unidentified_count,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
@@ -240,6 +246,11 @@ export async function syncManticoreScenes(sceneIds) {
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
actor_ids: scene.actors.map((actor) => actor.f1), // don't include aliases in ID or they would show up in filters
actors: Array.from(new Set([...scene.actors.map((actor) => actor.f2), ...scene.actors_aliases.map((actor) => actor.f2)])).join(),
actor_count: Number(scene.actor_count), // postgres returns counts as bigint strings
female_count: Number(scene.female_count),
male_count: Number(scene.male_count),
trans_count: Number(scene.trans_count),
unidentified_count: Number(scene.unidentified_count),
tag_ids: Array.from(new Set(scene.tags.map((tag) => tag.f1))),
assigned_tag_ids: assignedTagIds,
tags: flatTags.join(' '), // only make top tags searchable to minimize cluttered results
+6 -5
View File
@@ -1,6 +1,8 @@
import { Router } from 'express';
import omit from 'object.omit';
import { parseLetterRange, parseRange } from '../../utils/parse-range.js';
import {
createActor,
createActorRevision,
@@ -10,21 +12,20 @@ import {
mergeActors,
reviewActorRevision,
} from '../actors.js';
import { fetchStashByUsernameAndSlug } from '../stashes.js';
export function curateActorsQuery(query) {
return {
query: query.q,
gender: query.gender,
age: query.age?.split(',').map((age) => Number(age)),
age: parseRange(query.age),
dateOfBirth: query.dob && new Date(query.dob),
dobType: ({ dob: 'dateOfBirth', bd: 'birthday' })[query.dobt] || 'birthday',
cup: query.cup?.split(','),
cup: parseLetterRange(query.cup),
country: query.c,
naturalBoobs: query.nb,
height: query.height?.split(',').map((height) => Number(height)),
weight: query.weight?.split(',').map((weight) => Number(weight)),
height: parseRange(query.height),
weight: parseRange(query.weight),
requireAvatar: query.avatar,
stashId: Number(query.stashId) || null,
isGlobal: Object.hasOwn(query, 'global'),
+9 -1
View File
@@ -1,7 +1,8 @@
import { Router } from 'express';
import promiseProps from '../../utils/promise-props.js';
import { parseRange } from '../../utils/parse-range.js';
import promiseProps from '../../utils/promise-props.js';
import slugify from '../../utils/slugify.js';
import { getIdsBySlug } from '../cache.js';
@@ -41,6 +42,11 @@ export async function curateScenesQuery(query) {
years: splitYears.map((year) => Number(year)).filter(Boolean) || [],
actorIds: [query.actorId, ...splitActors.filter((actor) => actor.charAt(0) !== '!').map((identifier) => parseActorIdentifier(identifier)?.id)].filter(Boolean),
notActorIds: splitActors.filter((actor) => actor.charAt(0) === '!').map((identifier) => parseActorIdentifier(identifier.slice(1))?.id).filter(Boolean),
actorCount: parseRange(query.group),
femaleCount: parseRange(query.females),
maleCount: parseRange(query.males),
transCount: parseRange(query.trans),
unidentifiedCount: parseRange(query.unidentified), // for curators, e.g. unidentified=1- finds scenes with unidentified actors
tagIds,
notTagIds: notTagIds.filter((tagId) => !tagIds.includes(tagId)), // included tags get priority over excluded tags
onlyActorTags: !!query.at,
@@ -61,6 +67,7 @@ async function fetchScenesApi(req, res) {
aggTags,
aggActorTags,
aggChannels,
aggActorCounts,
limit,
total,
} = await fetchScenes(await curateScenesQuery({
@@ -80,6 +87,7 @@ async function fetchScenesApi(req, res) {
aggTags,
aggActorTags,
aggChannels,
aggActorCounts,
limit,
total,
});