From 0e5724533fe132450575b8e46d47dd5cdc630b74 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Fri, 13 Mar 2026 04:52:22 +0100 Subject: [PATCH] Removed Manticore tools to prevent confusion. --- src/tools/manticore-joins.js | 82 ---------------- src/tools/manticore-scenes.js | 174 ---------------------------------- src/tools/sync-stashes.js | 77 --------------- 3 files changed, 333 deletions(-) delete mode 100644 src/tools/manticore-joins.js delete mode 100644 src/tools/manticore-scenes.js delete mode 100644 src/tools/sync-stashes.js diff --git a/src/tools/manticore-joins.js b/src/tools/manticore-joins.js deleted file mode 100644 index 6e29f4c..0000000 --- a/src/tools/manticore-joins.js +++ /dev/null @@ -1,82 +0,0 @@ -import { indexApi, utilsApi } from '../manticore.js'; -import rawvideos from './movies.json' with { type: 'json' }; - -async function fetchvideos() { - const videos = rawvideos - .filter((video) => video.cast.length > 0 - && video.genres.length > 0 - && video.cast.every((actor) => actor.charCodeAt(0) >= 65)) // throw out videos with non-alphanumerical actor names - .map((video, index) => ({ id: index + 1, ...video })); - - const actors = Array.from(new Set(videos.flatMap((video) => video.cast))).sort(); - const genres = Array.from(new Set(videos.flatMap((video) => video.genres))); - - return { - videos, - actors, - genres, - }; -} - -async function init() { - await utilsApi.sql('drop table if exists videos'); - await utilsApi.sql('drop table if exists videos_liked'); - - await utilsApi.sql(`create table videos ( - id int, - title text, - actor_ids multi, - actors text, - genre_ids multi, - genres text - )`); - - await utilsApi.sql(`create table videos_liked ( - id int, - user_id int, - video_id int - )`); - - const { videos, actors, genres } = await fetchvideos(); - - const likedvideoIds = Array.from(new Set(Array.from({ length: 10.000 }, () => videos[Math.round(Math.random() * videos.length)].id))); - - const docs = videos - .map((video) => ({ - replace: { - index: 'videos', - id: video.id, - doc: { - title: video.title, - actor_ids: video.cast.map((actor) => actors.indexOf(actor)), - actors: video.cast.join(','), - genre_ids: video.genres.map((genre) => genres.indexOf(genre)), - genres: video.genres.join(','), - }, - }, - })) - .concat(likedvideoIds.map((videoId, index) => ({ - replace: { - index: 'videos_liked', - id: index + 1, - doc: { - user_id: Math.floor(Math.random() * 51), - video_id: videoId, - }, - }, - }))); - - const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')); - - console.log('data', data); - - const result = await utilsApi.sql(` - select * from videos_liked - limit 10 - `); - - console.log(result[0].data); - console.log(result[1]); -} - -init(); diff --git a/src/tools/manticore-scenes.js b/src/tools/manticore-scenes.js deleted file mode 100644 index 6f29f0e..0000000 --- a/src/tools/manticore-scenes.js +++ /dev/null @@ -1,174 +0,0 @@ -// import config from 'config'; -import { format } from 'date-fns'; -import { faker } from '@faker-js/faker'; - -import { indexApi } from '../manticore.js'; - -import { knexOwner as knex } from '../knex.js'; -import slugify from '../utils/slugify.js'; -import chunk from '../utils/chunk.js'; - -async function fetchScenes() { - const scenes = await knex.raw(` - SELECT - releases.id AS id, - releases.title, - releases.created_at, - releases.date, - releases.shoot_id, - scenes_meta.stashed, - entities.id as channel_id, - entities.slug as channel_slug, - entities.name as channel_name, - parents.id as network_id, - parents.slug as network_slug, - parents.name as network_name, - COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, - COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags - FROM releases - LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id - LEFT JOIN entities ON releases.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id - LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id - LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id - LEFT JOIN actors ON local_actors.actor_id = actors.id - LEFT JOIN actors AS directors ON local_directors.director_id = directors.id - LEFT JOIN tags ON local_tags.tag_id = tags.id - LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true - GROUP BY - releases.id, - releases.title, - releases.created_at, - releases.date, - releases.shoot_id, - scenes_meta.stashed, - entities.id, - entities.name, - entities.slug, - entities.alias, - parents.id, - parents.name, - parents.slug, - parents.alias; - `); - - const actors = Object.fromEntries(scenes.rows.flatMap((row) => row.actors.map((actor) => [actor.f1, faker.person.fullName()]))); - const tags = Object.fromEntries(scenes.rows.flatMap((row) => row.tags.map((tag) => [tag.f1, faker.word.adjective()]))); - - return scenes.rows.map((row) => { - const title = faker.lorem.lines(1); - - const channelName = faker.company.name(); - const channelSlug = slugify(channelName, ''); - - const networkName = faker.company.name(); - const networkSlug = slugify(networkName, ''); - - const rowActors = row.actors.map((actor) => ({ f1: actor.f1, f2: actors[actor.f1] })); - const rowTags = row.tags.map((tag) => ({ f1: tag.f1, f2: tags[tag.f1], f3: tag.f3 })); - - return { - ...row, - title, - actors: rowActors, - tags: rowTags, - channel_name: channelName, - channel_slug: channelSlug, - network_name: networkName, - network_slug: networkSlug, - }; - }); -} - -async function updateStashed(docs) { - await chunk(docs, 1000).reduce(async (chain, docsChunk) => { - await chain; - - const sceneIds = docsChunk.map((doc) => doc.replace.id); - - const stashes = await knex('stashes_scenes') - .select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes.id as stash_id', 'stashes.user_id as user_id') - .leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id') - .whereIn('scene_id', sceneIds); - - if (stashes.length > 0) { - console.log(stashes); - } - - const stashDocs = docsChunk.flatMap((doc) => { - const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id); - - if (sceneStashes.length === 0) { - return []; - } - - const stashDoc = sceneStashes.map((stash) => ({ - replace: { - index: 'scenes_stashed', - id: stash.stashed_id, - doc: { - // ...doc.replace.doc, - scene_id: doc.replace.id, - user_id: stash.user_id, - }, - }, - })); - - return stashDoc; - }); - - console.log(stashDocs); - - if (stashDocs.length > 0) { - await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n')); - } - }, Promise.resolve()); -} - -async function init() { - const scenes = await fetchScenes(); - - const docs = scenes.map((scene) => { - const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc. - const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results - const filteredTitle = scene.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'i'), ''), scene.title).trim().replace(/\s{2,}/, ' '); - - return { - replace: { - index: 'scenes', - id: scene.id, - doc: { - title: scene.title || undefined, - title_filtered: filteredTitle || undefined, - date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined, - created_at: Math.round(scene.created_at.getTime() / 1000), - effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000), - // shoot_id: scene.shoot_id || undefined, - channel_id: scene.channel_id, - channel_slug: scene.channel_slug, - channel_name: scene.channel_name, - network_id: scene.network_id || undefined, - network_slug: scene.network_slug || undefined, - network_name: scene.network_name || undefined, - actor_ids: scene.actors.map((actor) => actor.f1), - actors: scene.actors.map((actor) => actor.f2).join(), - tag_ids: scene.tags.map((tag) => tag.f1), - tags: flatTags.join(' '), - meta: scene.date ? format(scene.date, 'y yy M MMM MMMM d') : undefined, - liked: scene.stashed || 0, - }, - }, - }; - }); - - const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')); - - await updateStashed(docs); - - console.log('data', data); - - knex.destroy(); -} - -init(); diff --git a/src/tools/sync-stashes.js b/src/tools/sync-stashes.js deleted file mode 100644 index 8de6351..0000000 --- a/src/tools/sync-stashes.js +++ /dev/null @@ -1,77 +0,0 @@ -import { indexApi, utilsApi } from '../manticore.js'; -import { knexOwner as knex } from '../knex.js'; -import chunk from '../utils/chunk.js'; - -async function syncStashes(domain = 'scene') { - await utilsApi.sql(`truncate table ${domain}s_stashed`); - - const stashes = await knex(`stashes_${domain}s`) - .select( - `stashes_${domain}s.id as stashed_id`, - `stashes_${domain}s.${domain}_id`, - 'stashes.id as stash_id', - 'stashes.user_id as user_id', - `stashes_${domain}s.created_at as created_at`, - ) - .leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`); - - await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => { - await chain; - - const stashDocs = stashChunk.map((stash) => ({ - replace: { - index: `${domain}s_stashed`, - id: stash.stashed_id, - doc: { - [`${domain}_id`]: stash[`${domain}_id`], - stash_id: stash.stash_id, - user_id: stash.user_id, - created_at: Math.round(stash.created_at.getTime() / 1000), - }, - }, - })); - - await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n')); - - console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`); - }, Promise.resolve()); -} - -async function init() { - await utilsApi.sql('drop table if exists scenes_stashed'); - - await utilsApi.sql(`create table if not exists scenes_stashed ( - scene_id int, - stash_id int, - user_id int, - created_at timestamp - )`); - - await utilsApi.sql('drop table if exists movies_stashed'); - - await utilsApi.sql(`create table if not exists movies_stashed ( - movie_id int, - stash_id int, - user_id int, - created_at timestamp - )`); - - await utilsApi.sql('drop table if exists actors_stashed'); - - await utilsApi.sql(`create table if not exists actors_stashed ( - actor_id int, - stash_id int, - user_id int, - created_at timestamp - )`); - - await syncStashes('scene'); - await syncStashes('actor'); - await syncStashes('movie'); - - console.log('Done!'); - - knex.destroy(); -} - -init();