Aggregating channels, filter inoperable.

This commit is contained in:
DebaucheryLibrarian 2024-01-09 02:26:32 +01:00
parent 58f7ca0d89
commit d242eb3b73
17 changed files with 381 additions and 68 deletions

View File

@ -0,0 +1,5 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
<title>arrow-up4</title>
<path d="M0 10.5l1 1 7-7 7 7 1-1-8-8-8 8z"></path>
</svg>

After

Width:  |  Height:  |  Size: 214 B

View File

@ -0,0 +1,5 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
<title>enter5</title>
<path d="M13 1v7h-6v-4.5l-6 6 6 6v-4.5h9v-10z"></path>
</svg>

After

Width:  |  Height:  |  Size: 215 B

6
assets/img/icons/opt.svg Normal file
View File

@ -0,0 +1,6 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
<title>opt</title>
<path d="M14.5 13h-4c-0.198 0-0.377-0.116-0.457-0.297l-3.868-8.703h-4.675c-0.276 0-0.5-0.224-0.5-0.5s0.224-0.5 0.5-0.5h5c0.198 0 0.377 0.116 0.457 0.297l3.868 8.703h3.675c0.276 0 0.5 0.224 0.5 0.5s-0.224 0.5-0.5 0.5z"></path>
<path d="M14.5 4h-5c-0.276 0-0.5-0.224-0.5-0.5s0.224-0.5 0.5-0.5h5c0.276 0 0.5 0.224 0.5 0.5s-0.224 0.5-0.5 0.5z"></path>
</svg>

After

Width:  |  Height:  |  Size: 505 B

View File

@ -54,8 +54,8 @@
v-for="(actor, index) in group"
:key="`filter-actor-${actor.id}`"
class="filter-item"
:class="{ selected: filters.actors.includes(actor.id), first: groupKey === 'available' && index === 0 && filters.actors.length > 0 }"
@click="emit('update', 'actors', [actor.id])"
:class="{ selected: filters.actors.some((filterActor) => filterActor.id === actor.id), first: groupKey === 'available' && index === 0 && filters.actors.length > 0 }"
@click="emit('update', 'actors', [actor])"
>
<div
class="filter-include"
@ -73,7 +73,10 @@
</div>
<span class="filter-name actor-name">
{{ actor.name }}
<span
class="filter-text"
:title="actor.name"
>{{ actor.name }}</span>
<span class="actor-details">
<div class="actor-gender">
@ -121,9 +124,9 @@ const { pageProps } = inject('pageContext');
const { actor: pageActor } = pageProps;
const groupedActors = computed(() => ({
selected: props.filters.actors.map((actorId) => props.actors.find((actor) => actor.id === actorId)).filter(Boolean),
selected: props.filters.actors.map((filterActor) => props.actors.find((actor) => actor.id === filterActor.id)).filter(Boolean),
available: props.actors
.filter((actor) => !props.filters.actors.includes(actor.id)
.filter((actor) => !props.filters.actors.some((filterActor) => filterActor.id === actor.id)
&& actor.id !== pageActor?.id
&& searchRegexp.value.test(actor.name)
&& (!selectedGender.value || actor.gender === selectedGender.value))
@ -139,12 +142,12 @@ const groupedActors = computed(() => ({
const genders = computed(() => [null, ...['female', 'male', 'transsexual', 'other'].filter((gender) => props.actors.some((actor) => actor.gender === gender))]);
function toggleActor(actor) {
if (props.filters.actors.includes(actor.id)) {
emit('update', 'actors', props.filters.actors.filter((actorId) => actorId !== actor.id));
if (props.filters.actors.some((filterActor) => filterActor.id === actor.id)) {
emit('update', 'actors', props.filters.actors.filter((filterActor) => filterActor.id !== actor.id));
return;
}
emit('update', 'actors', props.filters.actors.concat(actor.id));
emit('update', 'actors', props.filters.actors.concat(actor));
}
function selectGender() {
@ -160,12 +163,6 @@ function selectGender() {
</script>
<style scoped>
.gender-unselected {
.icon {
fill: var(--shadow);
}
}
.filter {
padding: 0;
}
@ -175,9 +172,7 @@ function selectGender() {
}
.filter-name {
height: 1rem;
align-items: stretch;
padding: .25rem 0 .25rem .25rem;
}
.actor-name {

View File

@ -0,0 +1,151 @@
<template>
<div class="filter channels-container">
<div class="filters-sort">
<input
v-model="search"
type="search"
placeholder="Filter channels"
class="input input-inline filters-search"
>
<div
v-show="order === 'name'"
class="filter-sort order noselect"
@click="order = 'count'"
>
<Icon
icon="sort-alpha-asc"
/>
</div>
<div
v-show="order === 'count'"
class="filter-sort order noselect"
@click="order = 'priority'"
>
<Icon
icon="sort-numeric-desc"
/>
</div>
</div>
<ul
class="filter-items nolist"
>
<template v-for="network in networks">
<li
v-for="(channel) in [network, ...(network.children || [])]"
:key="`filter-channel-${channel.id}`"
class="filter-item"
:class="{ channel: !channel.isIndependent && channel.type !== 'network', selected: filters.channel?.id === channel.id }"
@click="emit('update', 'channel', channel)"
>
<span class="filter-name">
<span
class="filter-text"
:title="channel.name"
>
<img
v-if="channel.isIndependent || channel.type === 'network'"
:src="`/logos/${channel.slug}/favicon_dark.png`"
class="favicon"
>
<Icon
v-else
icon="arrow-up4"
/>
{{ channel.name }}
</span>
<span class="channel-details">
<span
v-if="channel.count"
class="filter-count"
>{{ channel.count }}</span>
</span>
</span>
</li>
</template>
</ul>
</div>
</template>
<script setup>
import { ref, computed, inject } from 'vue';
const props = defineProps({
filters: {
type: Object,
default: null,
},
channels: {
type: Array,
default: () => [],
},
});
const emit = defineEmits(['update']);
const search = ref('');
const searchRegexp = computed(() => new RegExp(search.value, 'i'));
const order = ref('name');
const { pageProps } = inject('pageContext');
const { channel: pageChannel } = pageProps;
const networks = computed(() => {
const filteredChannels = props.channels.filter((channel) => channel.id !== pageChannel?.id
&& (searchRegexp.value.test(channel.name)
|| searchRegexp.value.test(channel.slug)
|| (channel.parent && searchRegexp.value.test(channel.parent.name))
|| (channel.parent && searchRegexp.value.test(channel.parent.slug))));
return Object.values(filteredChannels.reduce((acc, channel) => {
if (!channel.parent || channel.isIndependent) {
acc[channel.id] = channel;
return acc;
}
if (!acc[channel.parent.id]) {
acc[channel.parent.id] = {
...channel.parent,
children: [],
};
}
acc[channel.parent.id].children.push(channel);
return acc;
}, {}));
});
</script>
<style scoped>
.filter-items {
max-height: 10rem;
overflow-y: auto;
}
.filter {
padding: 0;
}
.filter-item.channel {
.icon {
width: 1.5rem;
height: 1rem;
transform: rotate(-135deg);
fill: var(--shadow-weak-30);
}
}
.favicon {
width: 1rem;
height: 1rem;
margin-right: .5rem;
object-fit: contain;
}
</style>

View File

@ -186,11 +186,17 @@ function toggleFilters(state) {
justify-content: space-between;
align-items: center;
flex-grow: 1;
padding: .25rem .75rem .25rem 1rem;
padding: .25rem 0 .25rem .25rem;
color: var(--text);
text-decoration: none;
}
.filter-text {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.filter-include {
display: flex;
align-items: center;
@ -222,13 +228,13 @@ function toggleFilters(state) {
align-items: center;
align-self: stretch;
justify-content: center;
padding: 0 .5rem;
padding: 0 .25rem;
cursor: pointer;
font-weight: bold;
color: var(--shadow);
&.order {
padding: 0 1rem 0 .25rem;
padding: 0 .5rem 0 .25rem;
}
.icon {

View File

@ -75,6 +75,3 @@ defineProps({
const emit = defineEmits(['change', 'input', 'enable']);
</script>
<style scoped>
</style>

View File

@ -67,13 +67,18 @@
/>
</div>
<span class="filter-name tag-name">{{ tag.name }}</span>
<span class="tag-details">
<span class="filter-name tag-name">
<span
v-if="tag.count"
class="filter-count"
>{{ tag.count }}</span>
class="filter-text"
:title="tag.name"
>{{ tag.name }}</span>
<span class="tag-details">
<span
v-if="tag.count"
class="filter-count"
>{{ tag.count }}</span>
</span>
</span>
</li>
</ul>
@ -170,7 +175,7 @@ function toggleTag(tag) {
<style scoped>
.filter-items {
max-height: 15rem;
max-height: 10rem;
overflow-y: auto;
}

View File

@ -205,6 +205,7 @@ async function setRange(prop, value) {
}
.label {
height: 100%;
padding: 0 .5rem;
&:hover:not(.disabled) {

View File

@ -7,6 +7,12 @@
@update="updateFilter"
/>
<ChannelsFilter
:filters="filters"
:channels="aggChannels"
@update="updateFilter"
/>
<ActorsFilter
:filters="filters"
:actors="aggActors"
@ -63,10 +69,12 @@ import { parse } from 'path-to-regexp';
import navigate from '#/src/navigate.js';
import { get } from '#/src/api.js';
import events from '#/src/events.js';
import { getActorIdentifier, parseActorIdentifier } from '#/src/query.js';
import Filters from '#/components/filters/filters.vue';
import ActorsFilter from '#/components/filters/actors.vue';
import TagsFilter from '#/components/filters/tags.vue';
import ChannelsFilter from '#/components/filters/channels.vue';
import Scene from './tile.vue';
import Pagination from '../pagination/pagination.vue';
@ -85,19 +93,31 @@ const { pageProps, routeParams, urlParsed } = inject('pageContext');
const { scope } = routeParams;
const {
actor,
actor: pageActor,
tag: pageTag,
channel: pageChannel,
} = pageProps;
const scenes = ref(pageProps.scenes);
const aggActors = ref(pageProps.aggActors);
const aggTags = ref(pageProps.aggTags);
const aggChannels = ref(pageProps.aggChannels);
const currentPage = ref(Number(routeParams.page));
const total = ref(Number(pageProps.total));
const actorIds = urlParsed.search.actors?.split(',').map((identifier) => parseActorIdentifier(identifier)?.id).filter(Boolean) || [];
const queryActors = actorIds.map((urlActorId) => aggActors.value.find((aggActor) => aggActor.id === urlActorId)).filter(Boolean);
const networks = Object.fromEntries(aggChannels.value.map((channel) => (channel.type === 'network' ? channel : channel.parent)).filter(Boolean).map((parent) => [`_${parent.slug}`, parent]));
const channels = Object.fromEntries(aggChannels.value.filter((channel) => channel.type === 'channel').map((channel) => [channel.slug, channel]));
const queryChannel = networks[urlParsed.search.e] || channels[urlParsed.search.e];
const filters = ref({
actors: urlParsed.search.actors?.split(',').filter(Boolean).map((actorId) => Number(actorId)) || [],
tags: urlParsed.search.tags?.split(',').filter(Boolean) || [],
channel: queryChannel,
actors: queryActors,
});
function getPath(targetScope, preserveQuery) {
@ -127,15 +147,16 @@ async function search(resetPage = true) {
currentPage.value = 1;
}
const query = {
tags: filters.value.tags.join(','),
};
const query = {};
console.log('actor id', actor?.id);
const entity = filters.value.channel || pageChannel;
const entitySlug = entity?.type === 'network' ? `_${entity.slug}` : entity?.slug;
const res = await get('/scenes', {
...query,
actors: [actor?.id, filters.value.actors].filter(Boolean).join(','), // if we're on an actor page, that actor ID needs to be included
actors: [pageActor, ...filters.value.actors].filter(Boolean).map((filterActor) => getActorIdentifier(filterActor)).join(','), // if we're on an actor page, that actor ID needs to be included
tags: [pageTag?.slug, ...filters.value.tags].filter(Boolean).join(','),
entity: entitySlug,
scope,
page: currentPage.value, // client uses param rather than query pagination
});
@ -145,21 +166,19 @@ async function search(resetPage = true) {
aggTags.value = res.aggTags;
total.value = res.total;
console.log(scenes.value);
events.emit('scrollUp');
navigate(getPath(scope, false), {
...query,
actors: filters.value.actors.join(',') || undefined, // don't include actor page ID in query, already a parameter
actors: filters.value.actors.map((filterActor) => getActorIdentifier(filterActor)).join(',') || undefined, // don't include page actor ID in query, already a parameter
tags: filters.value.tags.join(',') || undefined,
e: filters.value.channel?.type === 'network' ? `_${filters.value.channel.slug}` : (filters.value.channel?.slug || undefined),
}, { redirect: false });
}
function updateFilter(prop, value, reload = true) {
filters.value[prop] = value;
console.log(prop, value);
if (reload) {
search();
}

View File

@ -20,6 +20,7 @@ export async function onBeforeRender(pageContext) {
scenes,
aggActors,
aggTags,
aggChannels,
total,
limit,
} = actorScenes;
@ -32,6 +33,7 @@ export async function onBeforeRender(pageContext) {
scenes,
aggActors,
aggTags,
aggChannels,
total,
limit,
},

View File

@ -1,8 +1,12 @@
import initServer from './web/server.js';
import { cacheTagIds } from './tags.js';
import { cacheEntityIds } from './entities.js';
async function init() {
await cacheTagIds();
await Promise.all([
cacheTagIds(),
cacheEntityIds(),
]);
initServer();
}

63
src/entities.js Normal file
View File

@ -0,0 +1,63 @@
import knex from './knex.js';
import redis from './redis.js';
import initLogger from './logger.js';
const logger = initLogger();
function curateEntity(entity, context) {
if (!entity) {
return null;
}
return {
id: entity.id,
name: entity.name,
slug: entity.slug,
type: entity.type,
isIndependent: entity.independent,
hasLogo: entity.has_logo,
parent: curateEntity(entity.parent, context),
...context?.append?.[entity.id],
};
}
export async function fetchEntitiesById(entityIds, options = {}) {
const [entities] = await Promise.all([
knex('entities')
.select('entities.*', knex.raw('row_to_json(parents) as parent'))
.whereIn('entities.id', entityIds)
.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
.modify((builder) => {
if (options.order) {
builder.orderBy(...options.order);
}
})
.groupBy('entities.id', 'parents.id'),
]);
if (options.order) {
return entities.map((entityEntry) => curateEntity(entityEntry, { append: options.append }));
}
const curatedEntities = entityIds.map((entityId) => {
const entity = entities.find((entityEntry) => entityEntry.id === entityId);
if (!entity) {
console.warn(`Can't match entity ${entityId}`);
return null;
}
return curateEntity(entity, { append: options.append });
}).filter(Boolean);
return curatedEntities;
}
export async function cacheEntityIds() {
const entities = await knex('entities').select('id', 'slug', 'type');
await redis.del('traxxx:entities:id_by_slug');
await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [entity.type === 'network' ? `_${entity.slug}` : entity.slug, entity.id]));
logger.info('Cached entity IDs by slug');
}

22
src/query.js Normal file
View File

@ -0,0 +1,22 @@
export function getActorIdentifier(actor) {
if (!actor) {
return null;
}
return `${actor.slug}:${actor.id}`;
}
export function parseActorIdentifier(identifier) {
if (!identifier) {
return null;
}
const [slug, idString] = identifier.split(':');
const id = Number(idString);
if (!id) {
return null;
}
return { slug, id };
}

View File

@ -3,6 +3,7 @@ import { searchApi } from './manticore.js';
import { HttpError } from './errors.js';
import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js';
import { fetchTagsById } from './tags.js';
import { fetchEntitiesById } from './entities.js';
function curateMedia(media) {
if (!media) {
@ -151,6 +152,7 @@ function curateOptions(options) {
aggregate: options.aggregate ?? true,
aggregateActors: (options.aggregate ?? true) && (options.aggregateActors ?? true),
aggregateTags: (options.aggregate ?? true) && (options.aggregateTags ?? true),
aggregateChannels: (options.aggregate ?? true) && (options.aggregateChannels ?? true),
};
}
@ -224,11 +226,54 @@ function buildQuery(filters = {}) {
return { query, sort };
}
function buildAggregates(options) {
const aggregates = {};
if (options.aggregateActors) {
aggregates.actorIds = {
terms: {
field: 'actor_ids',
size: 5000,
},
// sort: [{ doc_count: { order: 'asc' } }],
};
}
if (options.aggregateTags) {
aggregates.tagIds = {
terms: {
field: 'tag_ids',
size: 1000,
},
};
}
if (options.aggregateChannels) {
aggregates.channelIds = {
terms: {
field: 'channel_id',
size: 1000,
},
};
}
return aggregates;
}
function countAggregations(buckets) {
if (!buckets) {
return null;
}
return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
}
export async function fetchScenes(filters, rawOptions) {
const options = curateOptions(rawOptions);
const { query, sort } = buildQuery(filters);
console.log(filters);
console.log('filters', filters);
console.log('options', options);
const result = await searchApi.search({
index: 'scenes',
@ -236,34 +281,17 @@ export async function fetchScenes(filters, rawOptions) {
limit: options.limit,
offset: (options.page - 1) * options.limit,
sort,
aggs: {
...(options.aggregateActors && {
actorIds: {
terms: {
field: 'actor_ids',
size: 5000,
},
// sort: [{ doc_count: { order: 'asc' } }],
},
}),
...(options.aggregateTags && {
tagIds: {
terms: {
field: 'tag_ids',
size: 1000,
},
// sort: [{ doc_count: { order: 'asc' } }],
},
}),
},
aggs: buildAggregates(options),
});
const actorCounts = options.aggregateActors && Object.fromEntries(result.aggregations?.actorIds?.buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
const tagCounts = options.aggregateTags && Object.fromEntries(result.aggregations?.tagIds?.buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
const [aggActors, aggTags] = await Promise.all([
const [aggActors, aggTags, aggChannels] = await Promise.all([
options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
]);
const sceneIds = result.hits.hits.map((hit) => Number(hit._id));
@ -273,6 +301,7 @@ export async function fetchScenes(filters, rawOptions) {
scenes,
aggActors,
aggTags,
aggChannels,
total: result.hits.total,
limit: options.limit,
};

View File

@ -52,5 +52,5 @@ export async function cacheTagIds() {
await redis.del('traxxx:tags:id_by_slug');
await redis.hSet('traxxx:tags:id_by_slug', tags.map((tag) => [tag.slug, tag.id]));
logger.info('Cached tags IDs by slug');
logger.info('Cached tag IDs by slug');
}

View File

@ -1,6 +1,7 @@
import { stringify } from '@brillout/json-serializer/stringify'; /* eslint-disable-line import/extensions */
import { fetchScenes } from '../scenes.js';
import { parseActorIdentifier } from '../query.js';
import redis from '../redis.js';
async function getTagIdsBySlug(tagSlugs) {
@ -24,7 +25,7 @@ async function getTagIdsBySlug(tagSlugs) {
export async function curateScenesQuery(query) {
return {
scope: query.scope || 'latest',
actorIds: [query.actorId, ...(query.actors?.split(',') || [])].filter(Boolean).map((actorId) => Number(actorId)),
actorIds: [query.actorId, ...(query.actors?.split(',') || []).map((identifier) => parseActorIdentifier(identifier)?.id)].filter(Boolean),
tagIds: await getTagIdsBySlug([query.tagId, ...(query.tags?.split(',') || [])]),
};
}
@ -34,6 +35,7 @@ export async function fetchScenesApi(req, res) {
scenes,
aggActors,
aggTags,
aggChannels,
limit,
total,
} = await fetchScenes(await curateScenesQuery(req.query), {
@ -45,6 +47,7 @@ export async function fetchScenesApi(req, res) {
scenes,
aggActors,
aggTags,
aggChannels,
limit,
total,
}));