From 76ccd3729fa3de795fdf3883ede91c19f1e1121b Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Nov 2026 22:23:50 +0100 Subject: [PATCH] Added unidentified actors and actor count filters. --- assets/css/theme.css | 1 + components/actors/actors.vue | 28 +++-- components/actors/search.vue | 58 +++++++++++ components/actors/tile.vue | 12 ++- components/edit/actors.vue | 12 ++- components/edit/revisions.vue | 15 ++- components/edit/tags.vue | 2 +- components/filters/groups.vue | 149 +++++++++++++++++++++++++++ components/scenes/scenes.vue | 18 ++++ components/scenes/tile.vue | 20 +++- pages/actors/edit/+Page.vue | 10 +- pages/scenes/@sceneId/edit/+Page.vue | 8 +- pages/search/+onBeforeRender.js | 2 + pages/tags/+onBeforeRender.js | 1 + src/actors.js | 85 ++++++++++----- src/scenes.js | 128 ++++++++++++++++++++--- src/sync.js | 11 ++ src/web/actors.js | 11 +- src/web/scenes.js | 10 +- tools/manticore-actors.js | 7 +- tools/manticore-movies.js | 4 +- tools/manticore-scenes.js | 9 +- tools/manticore-stashes.js | 4 +- utils/parse-range.js | 40 +++++++ 24 files changed, 575 insertions(+), 70 deletions(-) create mode 100644 components/filters/groups.vue create mode 100644 utils/parse-range.js diff --git a/assets/css/theme.css b/assets/css/theme.css index 1ab0ca2..99390e3 100644 --- a/assets/css/theme.css +++ b/assets/css/theme.css @@ -69,6 +69,7 @@ --male: #0af; --female: #f0a; + --trans: #a0f; --enabled: #5c2; --enabled-background: rgba(0, 255, 0, .1); diff --git a/components/actors/actors.vue b/components/actors/actors.vue index 7de8aed..0f3c298 100644 --- a/components/actors/actors.vue +++ b/components/actors/actors.vue @@ -16,6 +16,18 @@ import Pagination from '#/components/pagination/pagination.vue'; import { get } from '#/src/api.js'; import events from '#/src/events.js'; import navigate from '#/src/navigate.js'; +import { formatRange, parseLetterRange, parseRange } from '#/utils/parse-range.js'; + +// sliders need both ends, fill open ends with the slider limits +function getSliderRange(rangeString, [defaultMin, defaultMax], parse = parseRange) { + const range = parse(rangeString); + + return [range?.min ?? defaultMin, range?.max ?? defaultMax]; +} + +function formatSliderRange([min, max]) { + return formatRange({ min, max }); +} const pageContext = inject('pageContext'); const { pageProps, urlParsed, routeParams } = pageContext; @@ -35,18 +47,18 @@ const order = ref(routeParams.order || urlParsed.search.order || 'likes.desc'); const filters = ref({ gender: urlParsed.search.gender, ageRequired: !!urlParsed.search.age, - age: urlParsed.search.age?.split(',').map((age) => Number(age)) || [18, 100], + age: getSliderRange(urlParsed.search.age, [18, 100]), dobRequired: !!urlParsed.search.dob, dobType: urlParsed.search.dobt ? ({ bd: 'birthday', dob: 'dob' })[urlParsed.search.dobt] : 'birthday', dob: urlParsed.search.dob || format(subYears(new Date(), 21), 'yyyy-MM-dd'), country: urlParsed.search.c, braSizeRequired: !!urlParsed.search.cup, - braSize: urlParsed.search.cup?.split(',') || ['A', 'Z'], + braSize: getSliderRange(urlParsed.search.cup, ['A', 'Z'], parseLetterRange), naturalBoobs: urlParsed.search.nb ? urlParsed.search.nb === 'true' : undefined, heightRequired: !!urlParsed.search.height, - height: urlParsed.search.height?.split(',').map((height) => Number(height)) || [50, 220], + height: getSliderRange(urlParsed.search.height, [50, 220]), weightRequired: !!urlParsed.search.weight, - weight: urlParsed.search.weight?.split(',').map((weight) => Number(weight)) || [30, 200], + weight: getSliderRange(urlParsed.search.weight, [30, 200]), avatarRequired: !!urlParsed.search.avatar, }); @@ -85,14 +97,14 @@ async function search(options = {}) { q: q.value || undefined, order: order.value, gender: filters.value.gender || undefined, - age: filters.value.ageRequired ? filters.value.age.join(',') : undefined, + age: filters.value.ageRequired ? formatSliderRange(filters.value.age) : undefined, dob: filters.value.dobRequired ? filters.value.dob : undefined, dobt: filters.value.dobRequired ? ({ birthday: 'bd', dob: 'dob' })[filters.value.dobType] : undefined, - cup: filters.value.braSizeRequired ? filters.value.braSize.join(',') : undefined, + cup: filters.value.braSizeRequired ? formatSliderRange(filters.value.braSize) : undefined, c: filters.value.country || undefined, nb: filters.value.naturalBoobs, - height: filters.value.heightRequired ? filters.value.height.join(',') : undefined, - weight: filters.value.weightRequired ? filters.value.weight.join(',') : undefined, + height: filters.value.heightRequired ? formatSliderRange(filters.value.height) : undefined, + weight: filters.value.weightRequired ? formatSliderRange(filters.value.weight) : undefined, avatar: filters.value.avatarRequired || undefined, stashId: pageStash?.id, }; diff --git a/components/actors/search.vue b/components/actors/search.vue index a478649..ef91c70 100644 --- a/components/actors/search.vue +++ b/components/actors/search.vue @@ -1,4 +1,5 @@ + + + + diff --git a/components/scenes/scenes.vue b/components/scenes/scenes.vue index 5b032ea..6194c8f 100644 --- a/components/scenes/scenes.vue +++ b/components/scenes/scenes.vue @@ -12,6 +12,7 @@ import Campaign from '#/components/campaigns/campaign.vue'; import ActorsFilter from '#/components/filters/actors.vue'; import ChannelsFilter from '#/components/filters/channels.vue'; import Filters from '#/components/filters/filters.vue'; +import GroupsFilter from '#/components/filters/groups.vue'; import TagsFilter from '#/components/filters/tags.vue'; import YearsFilter from '#/components/filters/years.vue'; @@ -23,6 +24,7 @@ import { get } from '#/src/api.js'; import entityPrefixes from '#/src/entities-prefixes.js'; import navigate from '#/src/navigate.js'; import { getActorIdentifier, parseActorIdentifier } from '#/src/query.js'; +import { formatRange, parseRange } from '#/utils/parse-range.js'; const props = defineProps({ showFilters: { @@ -64,6 +66,7 @@ const aggActors = ref(pageProps.aggActors || []); const aggTags = ref(pageProps.aggTags || []); const aggActorTags = ref(pageProps.aggActorTags || []); const aggChannels = ref(pageProps.aggChannels || []); +const aggActorCounts = ref(pageProps.aggActorCounts || null); const currentPage = ref(Number(routeParams.page)); const scope = ref(routeParams.scope || props.defaultScope); @@ -88,6 +91,10 @@ const filters = ref({ onlyActorTags: Object.hasOwn(urlParsed.search, 'at'), entity: queryEntity, actors: queryActors, + group: parseRange(urlParsed.search.group), + females: parseRange(urlParsed.search.females), + males: parseRange(urlParsed.search.males), + trans: parseRange(urlParsed.search.trans), }); const sceneCampaign = campaigns?.scenes; @@ -132,6 +139,10 @@ async function search(options = {}) { const query = { q: filters.value.search || undefined, + group: formatRange(filters.value.group), + females: formatRange(filters.value.females), + males: formatRange(filters.value.males), + trans: formatRange(filters.value.trans), }; const entity = filters.value.entity || pageEntity; @@ -168,6 +179,7 @@ async function search(options = {}) { aggTags.value = res.aggTags; aggActorTags.value = res.aggActorTags; aggChannels.value = res.aggChannels; + aggActorCounts.value = res.aggActorCounts; total.value = res.total; loading.value = false; @@ -248,6 +260,12 @@ function setView(newView) { :actors="aggActors" @update="updateFilter" /> + + diff --git a/components/scenes/tile.vue b/components/scenes/tile.vue index 217f554..278727e 100644 --- a/components/scenes/tile.vue +++ b/components/scenes/tile.vue @@ -92,16 +92,23 @@ const favorited = ref(props.scene.stashes.some((sceneStash) => sceneStash.id === @@ -237,7 +244,7 @@ const favorited = ref(props.scene.stashes.some((sceneStash) => sceneStash.id === color: var(--shadow); } -.actor:hover, +.actor:not(.unknown):hover, .tag:hover { color: var(--primary); } @@ -255,6 +262,13 @@ const favorited = ref(props.scene.stashes.some((sceneStash) => sceneStash.id === } } +.actor.unknown { + .actor-name, + &:not(:last-child)::after { + color: var(--glass); + } +} + .tags { height: 1.25rem; overflow: hidden; diff --git a/pages/actors/edit/+Page.vue b/pages/actors/edit/+Page.vue index 2c5c092..83a010b 100644 --- a/pages/actors/edit/+Page.vue +++ b/pages/actors/edit/+Page.vue @@ -786,7 +786,7 @@ async function submit() { .row { display: flex; align-items: center; - padding: .25rem 1rem; + padding: .75rem 1rem; margin-bottom: .25rem; overflow: hidden; @@ -797,10 +797,14 @@ async function submit() { width: auto; } } + + &:not(:last-child) { + border-bottom: solid 1px var(--glass-weak-40); + } } .key { - width: 10rem; + width: 9rem; display: inline-flex; align-items: center; text-transform: capitalize; @@ -945,7 +949,7 @@ async function submit() { .item-actions { .icon { - padding: .25rem 1rem; + padding: .25rem 1.5rem .25rem 1rem; fill: var(--glass); overflow: hidden; diff --git a/pages/scenes/@sceneId/edit/+Page.vue b/pages/scenes/@sceneId/edit/+Page.vue index ce19c32..9d47413 100644 --- a/pages/scenes/@sceneId/edit/+Page.vue +++ b/pages/scenes/@sceneId/edit/+Page.vue @@ -451,7 +451,11 @@ async function submit() { .row { display: flex; align-items: center; - padding: .25rem 1rem; + padding: .75rem 1rem; + + &:not(:last-child) { + border-bottom: solid 1px var(--glass-weak-40); + } .value.disabled { color: var(--glass); @@ -512,7 +516,7 @@ async function submit() { .item-actions { .icon { - padding: .25rem 1rem; + padding: .25rem 1.5rem .25rem 1rem; fill: var(--glass); overflow: hidden; diff --git a/pages/search/+onBeforeRender.js b/pages/search/+onBeforeRender.js index 3fb3f08..2ee3b54 100644 --- a/pages/search/+onBeforeRender.js +++ b/pages/search/+onBeforeRender.js @@ -43,6 +43,7 @@ export async function onBeforeRender(pageContext) { aggActors, aggTags, aggChannels, + aggActorCounts, total: sceneTotal, } = searchScenes; @@ -68,6 +69,7 @@ export async function onBeforeRender(pageContext) { aggActors, aggTags, aggChannels, + aggActorCounts, sceneTotal, actorTotal, movieTotal, diff --git a/pages/tags/+onBeforeRender.js b/pages/tags/+onBeforeRender.js index 385710e..516828b 100644 --- a/pages/tags/+onBeforeRender.js +++ b/pages/tags/+onBeforeRender.js @@ -89,6 +89,7 @@ const tagSlugs = { 'free-use', 'latex', 'blindfold', + 'body-writing', ], manual: [ 'handjob', diff --git a/src/actors.js b/src/actors.js index 9404dde..4bfa499 100644 --- a/src/actors.js +++ b/src/actors.js @@ -24,7 +24,7 @@ import { curateMedia } from './media.js'; import redis from './redis.js'; import { curateRevision } from './revisions.js'; import { curateStash } from './stashes.js'; -import { syncActors, syncScenes, syncStashes } from './sync.js'; +import { syncActors, syncManticoreScenes, syncScenes, syncStashes } from './sync.js'; const logger = initLogger(); const mj = new MerkleJson(); @@ -235,15 +235,17 @@ export function sortActorsByGender(actors, context = {}) { return actors; } - const alphaActors = actors.sort((actorA, actorB) => actorA.name.localeCompare(actorB.name, 'en')); - const genderActors = ['transsexual', 'female', undefined, null, 'male'].flatMap((gender) => alphaActors.filter((actor) => actor.gender === gender)); - + const alphaActors = actors.toSorted((actorA, actorB) => actorA.name?.localeCompare(actorB.name, 'en')); const titleSlug = slugify(context.title); const titleActors = titleSlug - ? genderActors.sort((actorA, actorB) => { - const actorASlug = actorA.slug.split('-')[0]; - const actorBSlug = actorB.slug.split('-')[0]; + ? alphaActors.sort((actorA, actorB) => { + const actorASlug = actorA.slug?.split('-')[0]; + const actorBSlug = actorB.slug?.split('-')[0]; + + if (!actorASlug || !actorBSlug) { + return 0; + } if (titleSlug.includes(actorASlug) && !titleSlug.includes(actorBSlug)) { return -1; @@ -257,7 +259,10 @@ export function sortActorsByGender(actors, context = {}) { }) : alphaActors; - return titleActors; + const genderActors = ['transsexual', 'female', undefined, null, 'male'].flatMap((gender) => titleActors.filter((actor) => actor.gender === gender)); + const identifiedActors = genderActors.toSorted((actorA, actorB) => !!actorA.isUnknown - !!actorB.isUnknown); + + return identifiedActors; } export async function fetchActorsById(actorIds, options = {}, reqUser) { @@ -465,22 +470,21 @@ async function queryManticoreSql(filters, options, _reqUser) { builder.select('if(date_of_birth, floor((now() - date_of_birth) / 31556952), 0) as age'); } - // range filters - ['age', 'height'].forEach((attribute) => { - if (filters[attribute]) { - builder - .where(attribute, '>=', filters[attribute][0]) - .where(attribute, '<=', filters[attribute][1]); + // range filters, either end may be open, weight is a reserved keyword in manticore + Object.entries({ age: 'age', height: 'height', weight: 'mass' }).forEach(([attribute, column]) => { + if (typeof filters[attribute]?.min === 'number') { + builder.where(column, '>=', filters[attribute].min); + } + + if (typeof filters[attribute]?.max === 'number') { + builder.where(column, '<=', filters[attribute].max); + + if (typeof filters[attribute].min !== 'number') { + builder.where(column, '>', 0); // unknown values are stored as 0, an open lower bound should not match them + } } }); - if (filters.weight) { - // weight is a reserved keyword in manticore - builder - .where('mass', '>=', filters.weight[0]) - .where('mass', '<=', filters.weight[1]); - } - if (filters.dateOfBirth && filters.dobType === 'dateOfBirth') { builder.where('date_of_birth', Math.floor(filters.dateOfBirth.getTime() / 1000)); } @@ -497,7 +501,8 @@ async function queryManticoreSql(filters, options, _reqUser) { } if (filters.cup) { - builder.select(`regex(actors.cup, '^[${filters.cup[0]}-${filters.cup[1]}]') as cup_in_range`); + // cup range is validated to single letters by the query parser + builder.select(`regex(actors.cup, '^[${filters.cup.min || 'A'}-${filters.cup.max || 'Z'}]') as cup_in_range`); builder.where('cup_in_range', 1); } @@ -1195,6 +1200,36 @@ async function fetchMainProfile(actorId, wasCreated = false) { return fetchMainProfile(actorId, true); } +async function fetchActorGenders(actorIds) { + const actors = await knex('actors') + .select('id', 'gender') + .whereIn('id', actorIds); + + return Object.fromEntries(actors.map((actor) => [actor.id, actor.gender])); +} + +// scene search documents count actors per gender, so scenes need a resync when an actor's gender changes +// syncs the search documents directly, the materialized views don't depend on actor genders +async function syncGenderChangedScenes(actorIds, gendersBefore) { + const gendersAfter = await fetchActorGenders(actorIds); + const changedActorIds = actorIds.filter((actorId) => gendersAfter[actorId] !== gendersBefore[actorId]); + + if (changedActorIds.length === 0) { + return; + } + + const sceneIds = await knex('releases_actors') + .distinct('release_id') + .whereIn('actor_id', changedActorIds) + .pluck('release_id'); + + logger.info(`Gender changed for ${changedActorIds.length} actors, resyncing ${sceneIds.length} scenes`); + + if (sceneIds.length > 0) { // without IDs, all scenes are synced + await syncManticoreScenes(sceneIds); + } +} + async function applyActorRevision(revisionIds, reqUser) { const revisions = await knex('actors_revisions') .whereIn('id', revisionIds) @@ -1276,6 +1311,7 @@ async function applyActorRevision(revisionIds, reqUser) { }, Promise.resolve()); const actorIds = Array.from(new Set(revisions.map((revision) => revision.actor_id))); + const gendersBefore = await fetchActorGenders(actorIds); await interpolateProfiles(actorIds, { knex, @@ -1285,7 +1321,10 @@ async function applyActorRevision(revisionIds, reqUser) { omit, }, { refreshView: false }); - await syncActors(actorIds); + await Promise.all([ + syncActors(actorIds), + syncGenderChangedScenes(actorIds, gendersBefore), + ]); } export async function reviewActorRevision(revisionId, isApproved, { feedback }, reqUser) { diff --git a/src/scenes.js b/src/scenes.js index dda5414..1a16dd7 100644 --- a/src/scenes.js +++ b/src/scenes.js @@ -79,9 +79,12 @@ function curateScene(rawScene, assets, reqUser, context) { entityId: assets.channel.affiliate.entity_id, } : null, - actors: sortActorsByGender(assets.actors.map((actor) => curateActor(actor, { - sceneDate: rawScene.effective_date, - stashes: assets.actorStashes.filter((actorStash) => actorStash.actor_id === actor.id), + actors: sortActorsByGender(assets.actors.map((actor) => ({ + ...curateActor(actor, { + sceneDate: rawScene.effective_date, + stashes: assets.actorStashes.filter((actorStash) => actorStash.actor_id === actor.id), + }), + isUnknown: !!actor.isUnknown, })), { title: rawScene.title }), directors: assets.directors.map((director) => ({ id: director.id, @@ -228,6 +231,8 @@ export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) { 'countries.name as birth_country_name', 'countries.alias as birth_country_alias', 'releases_actors.release_id', + 'releases_actors.id as release_actor_id', + 'releases_actors.gender as release_actor_gender', ) .leftJoin('actors', 'actors.id', 'releases_actors.actor_id') .leftJoin('actors as aliases', 'aliases.id', 'releases_actors.alias_id') @@ -236,7 +241,7 @@ export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) { .leftJoin('media_biometrics', 'media_biometrics.media_id', 'actors.avatar_media_id') .leftJoin('countries', 'countries.alpha2', 'actors.birth_country_alpha2') .whereIn('release_id', sceneIds) - .groupBy('actors.id', 'aliases.id', 'releases_actors.release_id', 'avatars.id', 'media_biometrics.id', 'countries.name', 'countries.alias', 'sfw_media.id'), + .groupBy('releases_actors.id', 'actors.id', 'aliases.id', 'releases_actors.release_id', 'avatars.id', 'media_biometrics.id', 'countries.name', 'countries.alias', 'sfw_media.id'), directors: knex('releases_directors') .whereIn('release_id', sceneIds) .leftJoin('actors as directors', 'directors.id', 'releases_directors.director_id'), @@ -385,7 +390,15 @@ export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) { const sceneChannel = channels.find((entity) => entity.id === scene.entity_id); const sceneStudio = studios.find((entity) => entity.id === scene.studio_id); - const sceneActors = actors.filter((actor) => actor.release_id === sceneId); + const sceneActors = actors.filter((actor) => actor.release_id === sceneId).map((actor) => (actor.id + ? actor + : { + ...actor, + // unidentified actor, needs a unique ID for editing, encodes gender for storage + id: `${actor.release_actor_gender || 'other'}_${actor.release_actor_id}`, + gender: actor.release_actor_gender, + isUnknown: true, + })); const sceneDirectors = directors.filter((director) => director.release_id === sceneId); const sceneTags = tags.filter((tag) => tag.release_id === sceneId); const sceneChapters = chapters.filter((chapter) => chapter.release_id === sceneId); @@ -439,6 +452,7 @@ function curateOptions(options) { aggregateActors: (options?.aggregate ?? true) && (options?.aggregateActors ?? true), aggregateTags: (options?.aggregate ?? true) && (options?.aggregateTags ?? true), aggregateChannels: (options?.aggregate ?? true) && (options?.aggregateChannels ?? true), + aggregateActorCounts: (options?.aggregate ?? true) && (options?.aggregateActorCounts ?? true), dedupe: !!options?.dedupe, index: options?.index || 'scenes', useSql: options?.useSql || (typeof options?.useSql === 'undefined' && sqlImplied.includes(options?.index)) || false, @@ -446,12 +460,12 @@ function curateOptions(options) { } // function curateFacet(results, field, count = 'count(distinct id)') { -function curateFacet(results, field) { +function curateFacet(results, field, { includeZero = false } = {}) { return results .find((result) => result.columns[0][field] && (result.columns[1]['count(distinct id)'] || result.columns[1]['count(*)'])) ?.data .map((row) => ({ key: row[field], doc_count: row['count(distinct id)'] || row['count(*)'] })) - .filter((row) => !!row.key) + .filter((row) => !!row.key || (includeZero && row.key === 0)) // zero is a meaningful actor count, e.g. no men || []; } @@ -492,7 +506,11 @@ async function queryManticoreSql(filters, options, _reqUser) { :actorsFacet: :tagsFacet: :channelsFacet: - :studiosFacet:; + :studiosFacet: + :groupFacet: + :femalesFacet: + :malesFacet: + :transFacet:; show meta; `, { query: knexManticore(filters.stashId ? 'scenes_stashed' : 'scenes') @@ -502,6 +520,11 @@ async function queryManticoreSql(filters, options, _reqUser) { scenes.id as id, scenes.title as title, scenes.actor_ids as actor_ids, + scenes.actor_count as actor_count, + scenes.female_count as female_count, + scenes.male_count as male_count, + scenes.trans_count as trans_count, + scenes.unidentified_count as unidentified_count, scenes.entity_ids as entity_ids, scenes.tag_ids as tag_ids, scenes.channel_id as channel_id, @@ -562,6 +585,23 @@ async function queryManticoreSql(filters, options, _reqUser) { builder.whereNotIn('actor_ids', filters.notActorIds); } + // actor counts include unidentified actors, gender counts only actors with a known gender + Object.entries({ + actor_count: filters.actorCount, + female_count: filters.femaleCount, + male_count: filters.maleCount, + trans_count: filters.transCount, + unidentified_count: filters.unidentifiedCount, + }).forEach(([field, range]) => { + if (typeof range?.min === 'number') { + builder.where(`scenes.${field}`, '>=', range.min); + } + + if (typeof range?.max === 'number') { + builder.where(`scenes.${field}`, '<=', range.max); + } + }); + if (filters.entityId) { builder.whereRaw('any(entity_ids) = ?', filters.entityId); @@ -644,6 +684,10 @@ async function queryManticoreSql(filters, options, _reqUser) { tagsFacet: options.aggregateTags ? knex.raw('facet scenes.assigned_tag_ids as tags_facet distinct id order by count(*) desc limit ?', [aggSize]) : null, channelsFacet: options.aggregateChannels ? knex.raw('facet scenes.channel_id as channels_facet distinct id order by count(*) desc limit ?', [aggSize]) : null, studiosFacet: options.aggregateChannels ? knex.raw('facet scenes.studio_id as studios_facet distinct id order by count(*) desc limit ?', [aggSize]) : null, + groupFacet: options.aggregateActorCounts ? knex.raw('facet scenes.actor_count as group_facet order by actor_count asc limit ?', [aggSize]) : null, + femalesFacet: options.aggregateActorCounts ? knex.raw('facet scenes.female_count as females_facet order by female_count asc limit ?', [aggSize]) : null, + malesFacet: options.aggregateActorCounts ? knex.raw('facet scenes.male_count as males_facet order by male_count asc limit ?', [aggSize]) : null, + transFacet: options.aggregateActorCounts ? knex.raw('facet scenes.trans_count as trans_facet order by trans_count asc limit ?', [aggSize]) : null, maxMatches: config.database.manticore.maxMatches, maxQueryTime: config.database.manticore.maxQueryTime, }).toString(); @@ -667,6 +711,13 @@ async function queryManticoreSql(filters, options, _reqUser) { const channelIds = curateFacet(results, 'channels_facet'); const studioIds = curateFacet(results, 'studios_facet'); + const actorCounts = { + group: curateFacet(results, 'group_facet', { includeZero: true }), + females: curateFacet(results, 'females_facet', { includeZero: true }), + males: curateFacet(results, 'males_facet', { includeZero: true }), + trans: curateFacet(results, 'trans_facet', { includeZero: true }), + }; + const allTagIds = mergePackedTags(tagIds); const actorTagIds = mergePackedTags(tagIds.filter((tag) => { @@ -696,6 +747,7 @@ async function queryManticoreSql(filters, options, _reqUser) { actorTagIds, channelIds, studioIds, + actorCounts, }, }; } @@ -708,6 +760,14 @@ function countAggregations(buckets) { return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }])); } +// URL parameter and aggregation key to filter key +const actorCountFilters = { + group: 'actorCount', + females: 'femaleCount', + males: 'maleCount', + trans: 'transCount', +}; + export async function fetchScenes(filters, rawOptions, reqUser, context) { const options = curateOptions(rawOptions); @@ -716,15 +776,42 @@ export async function fetchScenes(filters, rawOptions, reqUser, context) { console.log('options', options); } + // an active count filter would narrow its own range to the filtered value, get the unfiltered ranges from a lightweight parallel query (~10ms vs 40-500ms for the main query with its actor and tag facets) + const activeCountKeys = options.aggregateActorCounts + ? Object.keys(actorCountFilters).filter((key) => filters[actorCountFilters[key]]) + : []; + + const countsFilters = Object.fromEntries(Object.entries(filters).filter(([key]) => !Object.values(actorCountFilters).includes(key))); + + const countsOptions = { + ...options, + limit: 1, + page: 1, + aggregateYears: false, + aggregateActors: false, + aggregateTags: false, + aggregateChannels: false, + }; + console.time('manticore sql'); - const result = await queryManticoreSql(filters, options, reqUser); + const [result, countsResult] = await Promise.all([ + queryManticoreSql(filters, options, reqUser), + activeCountKeys.length > 0 ? queryManticoreSql(countsFilters, countsOptions, reqUser) : null, + ]); console.timeEnd('manticore sql'); const aggYears = options.aggregateYears && result.aggregations.years.map((bucket) => ({ year: bucket.key, count: bucket.doc_count })); const entityIds = options.aggregateChannels && [...(result.aggregations.channelIds || []), ...(result.aggregations.studioIds || [])]; + // keyed by URL parameter, e.g. { group: [{ value: 2, count: 1234 }, ...], females: [...] }, sorted by value so the first and last bucket give the range + // filtered counts take their range from the query without count filters, so with several count filters, each of their ranges ignores the others + const aggActorCounts = options.aggregateActorCounts && Object.fromEntries(Object.keys(actorCountFilters).map((key) => [ + key, + (activeCountKeys.includes(key) ? countsResult : result).aggregations.actorCounts[key].map((bucket) => ({ value: bucket.key, count: bucket.doc_count })), + ])); + const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds); const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds); const channelCounts = options.aggregateChannels && countAggregations(entityIds); @@ -752,6 +839,7 @@ export async function fetchScenes(filters, rawOptions, reqUser, context) { aggTags, aggActorTags, aggChannels, + aggActorCounts, total: result.total, limit: options.limit, }; @@ -809,7 +897,7 @@ export async function fetchSceneRevisions(revisionId, filters = {}, reqUser) { ...(revision.deltas.find((delta) => delta.key === 'actors')?.value || []), ...revision.base.tags.map((tag) => tag.actorId), ...revision.deltas.find((delta) => delta.key === 'tags')?.value.map((tag) => tag.actorId) || [], - ].filter(Boolean)))); + ].filter((actorId) => typeof actorId === 'number')))); // unidentified actors have string IDs const tagIds = Array.from(new Set(revisions.flatMap((revision) => [...revision.base.tags, ...(revision.deltas.find((delta) => delta.key === 'tags')?.value || [])].map((tag) => tag.id)))); const movieIds = Array.from(new Set(revisions.flatMap((revision) => [...revision.base.movies, ...(revision.deltas.find((delta) => delta.key === 'movies')?.value || [])]))); @@ -855,10 +943,22 @@ async function applySceneActorsDelta(sceneId, delta, trx) { if (delta.value.length > 0) { await knexOwner('releases_actors') - .insert(delta.value.map((actorId) => ({ - release_id: sceneId, - actor_id: actorId, - }))) + .insert(delta.value.map((actorId) => { + if (typeof actorId === 'number') { + return { + release_id: sceneId, + actor_id: actorId, + }; + } + + const gender = actorId.match(/^([a-z]+)_/)?.[1]; + + return { + release_id: sceneId, + actor_id: null, + gender: ['male', 'female', 'transsexual'].includes(gender) ? gender : null, + }; + })) .transacting(trx); } } diff --git a/src/sync.js b/src/sync.js index 63d34e0..e205dac 100644 --- a/src/sync.js +++ b/src/sync.js @@ -117,6 +117,12 @@ export async function syncManticoreScenes(sceneIds) { grandparents.id as parent_network_id, COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, COALESCE(JSON_AGG(DISTINCT (actors_aliases.id, actors_aliases.name)) FILTER (WHERE actors_aliases.id IS NOT NULL), '[]') as actors_aliases, + -- count release actor rows rather than actors to include unidentified actors, distinct because the other joins multiply rows + COUNT(DISTINCT local_actors.id) as actor_count, + COUNT(DISTINCT local_actors.id) FILTER (WHERE COALESCE(actors.gender, local_actors.gender) = 'female') as female_count, + COUNT(DISTINCT local_actors.id) FILTER (WHERE COALESCE(actors.gender, local_actors.gender) = 'male') as male_count, + COUNT(DISTINCT local_actors.id) FILTER (WHERE COALESCE(actors.gender, local_actors.gender) = 'transsexual') as trans_count, + COUNT(DISTINCT local_actors.id) FILTER (WHERE local_actors.actor_id IS NULL) as unidentified_count, COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags, COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies, COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series, @@ -240,6 +246,11 @@ export async function syncManticoreScenes(sceneIds) { entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN actor_ids: scene.actors.map((actor) => actor.f1), // don't include aliases in ID or they would show up in filters actors: Array.from(new Set([...scene.actors.map((actor) => actor.f2), ...scene.actors_aliases.map((actor) => actor.f2)])).join(), + actor_count: Number(scene.actor_count), // postgres returns counts as bigint strings + female_count: Number(scene.female_count), + male_count: Number(scene.male_count), + trans_count: Number(scene.trans_count), + unidentified_count: Number(scene.unidentified_count), tag_ids: Array.from(new Set(scene.tags.map((tag) => tag.f1))), assigned_tag_ids: assignedTagIds, tags: flatTags.join(' '), // only make top tags searchable to minimize cluttered results diff --git a/src/web/actors.js b/src/web/actors.js index 780fb4f..6f1684a 100644 --- a/src/web/actors.js +++ b/src/web/actors.js @@ -1,6 +1,8 @@ import { Router } from 'express'; import omit from 'object.omit'; +import { parseLetterRange, parseRange } from '../../utils/parse-range.js'; + import { createActor, createActorRevision, @@ -10,21 +12,20 @@ import { mergeActors, reviewActorRevision, } from '../actors.js'; - import { fetchStashByUsernameAndSlug } from '../stashes.js'; export function curateActorsQuery(query) { return { query: query.q, gender: query.gender, - age: query.age?.split(',').map((age) => Number(age)), + age: parseRange(query.age), dateOfBirth: query.dob && new Date(query.dob), dobType: ({ dob: 'dateOfBirth', bd: 'birthday' })[query.dobt] || 'birthday', - cup: query.cup?.split(','), + cup: parseLetterRange(query.cup), country: query.c, naturalBoobs: query.nb, - height: query.height?.split(',').map((height) => Number(height)), - weight: query.weight?.split(',').map((weight) => Number(weight)), + height: parseRange(query.height), + weight: parseRange(query.weight), requireAvatar: query.avatar, stashId: Number(query.stashId) || null, isGlobal: Object.hasOwn(query, 'global'), diff --git a/src/web/scenes.js b/src/web/scenes.js index ec935a0..cf627bf 100644 --- a/src/web/scenes.js +++ b/src/web/scenes.js @@ -1,7 +1,8 @@ import { Router } from 'express'; -import promiseProps from '../../utils/promise-props.js'; +import { parseRange } from '../../utils/parse-range.js'; +import promiseProps from '../../utils/promise-props.js'; import slugify from '../../utils/slugify.js'; import { getIdsBySlug } from '../cache.js'; @@ -41,6 +42,11 @@ export async function curateScenesQuery(query) { years: splitYears.map((year) => Number(year)).filter(Boolean) || [], actorIds: [query.actorId, ...splitActors.filter((actor) => actor.charAt(0) !== '!').map((identifier) => parseActorIdentifier(identifier)?.id)].filter(Boolean), notActorIds: splitActors.filter((actor) => actor.charAt(0) === '!').map((identifier) => parseActorIdentifier(identifier.slice(1))?.id).filter(Boolean), + actorCount: parseRange(query.group), + femaleCount: parseRange(query.females), + maleCount: parseRange(query.males), + transCount: parseRange(query.trans), + unidentifiedCount: parseRange(query.unidentified), // for curators, e.g. unidentified=1- finds scenes with unidentified actors tagIds, notTagIds: notTagIds.filter((tagId) => !tagIds.includes(tagId)), // included tags get priority over excluded tags onlyActorTags: !!query.at, @@ -61,6 +67,7 @@ async function fetchScenesApi(req, res) { aggTags, aggActorTags, aggChannels, + aggActorCounts, limit, total, } = await fetchScenes(await curateScenesQuery({ @@ -80,6 +87,7 @@ async function fetchScenesApi(req, res) { aggTags, aggActorTags, aggChannels, + aggActorCounts, limit, total, }); diff --git a/tools/manticore-actors.js b/tools/manticore-actors.js index 1173651..fdec725 100644 --- a/tools/manticore-actors.js +++ b/tools/manticore-actors.js @@ -2,6 +2,7 @@ import argv from '../src/argv.js'; import { knexOwner as knex } from '../src/knex.js'; import { utilsApi } from '../src/manticore.js'; +import redis from '../src/redis.js'; import { syncManticoreActors } from '../src/sync.js'; async function init() { @@ -34,12 +35,14 @@ async function init() { console.log('data', data); - knex.destroy(); + await knex.destroy(); + await redis.quit(); // sync imports connect to redis, which keeps the process alive return; } - knex.destroy(); + await knex.destroy(); + await redis.quit(); // sync imports connect to redis, which keeps the process alive } init(); diff --git a/tools/manticore-movies.js b/tools/manticore-movies.js index 0397cc7..5143742 100644 --- a/tools/manticore-movies.js +++ b/tools/manticore-movies.js @@ -1,6 +1,7 @@ import argv from '../src/argv.js'; import { knexOwner as knex } from '../src/knex.js'; import { utilsApi } from '../src/manticore.js'; +import redis from '../src/redis.js'; import { syncManticoreMovies } from '../src/sync.js'; async function init() { @@ -39,7 +40,8 @@ async function init() { console.log('data', data); } - knex.destroy(); + await knex.destroy(); + await redis.quit(); // sync imports connect to redis, which keeps the process alive } init(); diff --git a/tools/manticore-scenes.js b/tools/manticore-scenes.js index a61ad2c..03d9854 100644 --- a/tools/manticore-scenes.js +++ b/tools/manticore-scenes.js @@ -1,6 +1,7 @@ import argv from '../src/argv.js'; import { knexOwner as knex } from '../src/knex.js'; import { utilsApi } from '../src/manticore.js'; +import redis from '../src/redis.js'; import { syncManticoreScenes } from '../src/sync.js'; async function init() { @@ -24,6 +25,11 @@ async function init() { entity_ids multi, actor_ids multi, actors text, + actor_count int, + female_count int, + male_count int, + trans_count int, + unidentified_count int, tag_ids multi, assigned_tag_ids multi64, tags text, @@ -57,7 +63,8 @@ async function init() { await syncManticoreScenes(); } - knex.destroy(); + await knex.destroy(); + await redis.quit(); // sync imports connect to redis, which keeps the process alive } init(); diff --git a/tools/manticore-stashes.js b/tools/manticore-stashes.js index 766870e..99bf0a5 100644 --- a/tools/manticore-stashes.js +++ b/tools/manticore-stashes.js @@ -1,5 +1,6 @@ import { knexOwner as knex } from '../src/knex.js'; import { utilsApi } from '../src/manticore.js'; +import redis from '../src/redis.js'; import { syncStashes } from '../src/sync.js'; async function init() { @@ -36,7 +37,8 @@ async function init() { await syncStashes('actor'); await syncStashes('movie'); - knex.destroy(); + await knex.destroy(); + await redis.quit(); // sync imports connect to redis, which keeps the process alive } init(); diff --git a/utils/parse-range.js b/utils/parse-range.js new file mode 100644 index 0000000..4cd7b83 --- /dev/null +++ b/utils/parse-range.js @@ -0,0 +1,40 @@ +const numberPattern = /^\d+$/; +const letterPattern = /^[a-z]$/i; + +// ranges: 3 is exactly 3, 2- is at least 2, -4 is at most 4, 2-4 is between 2 and 4 inclusive +// comma is accepted as a legacy separator for closed ranges, e.g. age=22,30 from older links +// values not matching the pattern invalidate the range, which keeps them safe to use in queries +export function parseRange(rangeString, { pattern = numberPattern, parse = Number } = {}) { + const parts = String(rangeString ?? '').trim().split(/[-,]/); + + if (parts.length > 2 || parts.every((part) => part === '') || parts.some((part) => part !== '' && !pattern.test(part))) { + return null; + } + + const [minString, maxString = minString] = parts; // no separator means an exact value + + return { + min: minString ? parse(minString) : null, + max: maxString ? parse(maxString) : null, + }; +} + +// single letter cup sizes, matching the cup filter's regex character class +export function parseLetterRange(rangeString) { + return parseRange(rangeString, { + pattern: letterPattern, + parse: (letter) => letter.toUpperCase(), + }); +} + +export function formatRange(range) { + if (!range || (range.min === null && range.max === null)) { + return undefined; + } + + if (range.min === range.max) { + return String(range.min); + } + + return `${range.min ?? ''}-${range.max ?? ''}`; +}