diff --git a/config/default.js b/config/default.js index 41b59afc8..6309173b0 100755 --- a/config/default.js +++ b/config/default.js @@ -68,6 +68,9 @@ module.exports = { usernameLength: [2, 24], usernamePattern: /^[a-zA-Z0-9_-]+$/, }, + stashes: { + viewRefreshCooldown: 60, // minutes + }, exclude: { channels: [ // 21sextreme, no longer updated diff --git a/docs/puppeteer.md b/docs/puppeteer.md new file mode 100644 index 000000000..4ce0ca04f --- /dev/null +++ b/docs/puppeteer.md @@ -0,0 +1,4 @@ +# Puppeteer +Puppeteer has several dependencies that may not be available in Debian 12 by default: + +`apt install libasound2 libatk-bridge2.0-0 libcairo2 libcups2 libdrm2 libgbm-dev libpango-1.0-0 libxcomposite1 libxdamage1 libxfixes3 libxkbcommon-x11-0 libxrandr2 ` diff --git a/migrations/20240104234936_actors_metadata.js b/migrations/20240104234936_actors_metadata.js new file mode 100644 index 000000000..2f7b815c1 --- /dev/null +++ b/migrations/20240104234936_actors_metadata.js @@ -0,0 +1,34 @@ +exports.up = async function up(knex) { + await knex.raw(` + CREATE MATERIALIZED VIEW actors_meta AS ( + SELECT + actors.*, + COUNT(DISTINCT stashes_actors) as stashed, + COUNT(DISTINCT releases_actors) as scenes, + row_to_json(avatars) as avatar + FROM actors + LEFT JOIN stashes_actors ON stashes_actors.actor_id = actors.id + LEFT JOIN releases_actors ON releases_actors.actor_id = actors.id + LEFT JOIN media AS avatars ON avatars.id = actors.avatar_media_id + GROUP BY + actors.id, + avatars.id + ); + + CREATE MATERIALIZED VIEW scenes_meta AS ( + SELECT + releases.*, + COUNT(DISTINCT stashes_scenes) as stashed + FROM releases + LEFT JOIN stashes_scenes ON stashes_scenes.scene_id = releases.id + GROUP BY releases.id + ); + `); +}; + +exports.down = async function down(knex) { + await knex.raw(` + DROP MATERIALIZED VIEW IF EXISTS actors_meta; + DROP MATERIALIZED VIEW IF EXISTS scenes_meta; + `); +}; diff --git a/public/img/logos/dfxtra/dfextra.png b/public/img/logos/dfxtra/dfxtra.png similarity index 100% rename from public/img/logos/dfxtra/dfextra.png rename to public/img/logos/dfxtra/dfxtra.png diff --git a/public/img/logos/dfxtra/dfxtraoriginals.png b/public/img/logos/dfxtra/dfxtraoriginals.png new file mode 100644 index 000000000..8e747cea7 Binary files /dev/null and b/public/img/logos/dfxtra/dfxtraoriginals.png differ diff --git a/public/img/logos/dfxtra/lazy/dfextra.png b/public/img/logos/dfxtra/lazy/dfextra.png new file mode 100644 index 000000000..ec5b8f790 Binary files /dev/null and b/public/img/logos/dfxtra/lazy/dfextra.png differ diff --git a/public/img/logos/dfxtra/lazy/dfxtra.png b/public/img/logos/dfxtra/lazy/dfxtra.png new file mode 100644 index 000000000..fcf8699e8 Binary files /dev/null and b/public/img/logos/dfxtra/lazy/dfxtra.png differ diff --git a/public/img/logos/dfxtra/lazy/dfxtraoriginals.png b/public/img/logos/dfxtra/lazy/dfxtraoriginals.png new file mode 100644 index 000000000..4dfd7bc39 Binary files /dev/null and b/public/img/logos/dfxtra/lazy/dfxtraoriginals.png differ diff --git a/public/img/logos/dfxtra/lazy/favicon.png b/public/img/logos/dfxtra/lazy/favicon.png index 75739a711..6d610a6bc 100644 Binary files a/public/img/logos/dfxtra/lazy/favicon.png and b/public/img/logos/dfxtra/lazy/favicon.png differ diff --git a/public/img/logos/dfxtra/lazy/favicon_dark.png b/public/img/logos/dfxtra/lazy/favicon_dark.png index 5159a515e..6cb5b4f73 100644 Binary files a/public/img/logos/dfxtra/lazy/favicon_dark.png and b/public/img/logos/dfxtra/lazy/favicon_dark.png differ diff --git a/public/img/logos/dfxtra/lazy/favicon_light.png b/public/img/logos/dfxtra/lazy/favicon_light.png index 50b00e5f2..9fa49d628 100644 Binary files a/public/img/logos/dfxtra/lazy/favicon_light.png and b/public/img/logos/dfxtra/lazy/favicon_light.png differ diff --git a/public/img/logos/dfxtra/lazy/network.png b/public/img/logos/dfxtra/lazy/network.png index aac63cde2..41847e368 100644 Binary files a/public/img/logos/dfxtra/lazy/network.png and b/public/img/logos/dfxtra/lazy/network.png differ diff --git a/public/img/logos/dfxtra/thumbs/dfextra.png b/public/img/logos/dfxtra/thumbs/dfextra.png new file mode 100644 index 000000000..786155ab5 Binary files /dev/null and b/public/img/logos/dfxtra/thumbs/dfextra.png differ diff --git a/public/img/logos/dfxtra/thumbs/dfxtra.png b/public/img/logos/dfxtra/thumbs/dfxtra.png new file mode 100644 index 000000000..029bc6fe2 Binary files /dev/null and b/public/img/logos/dfxtra/thumbs/dfxtra.png differ diff --git a/public/img/logos/dfxtra/thumbs/dfxtraoriginals.png b/public/img/logos/dfxtra/thumbs/dfxtraoriginals.png new file mode 100644 index 000000000..55f21fbf1 Binary files /dev/null and b/public/img/logos/dfxtra/thumbs/dfxtraoriginals.png differ diff --git a/public/img/logos/dfxtra/thumbs/favicon.png b/public/img/logos/dfxtra/thumbs/favicon.png index 75739a711..6d610a6bc 100644 Binary files a/public/img/logos/dfxtra/thumbs/favicon.png and b/public/img/logos/dfxtra/thumbs/favicon.png differ diff --git a/public/img/logos/dfxtra/thumbs/favicon_dark.png b/public/img/logos/dfxtra/thumbs/favicon_dark.png index 5159a515e..6cb5b4f73 100644 Binary files a/public/img/logos/dfxtra/thumbs/favicon_dark.png and b/public/img/logos/dfxtra/thumbs/favicon_dark.png differ diff --git a/public/img/logos/dfxtra/thumbs/favicon_light.png b/public/img/logos/dfxtra/thumbs/favicon_light.png index 50b00e5f2..9fa49d628 100644 Binary files a/public/img/logos/dfxtra/thumbs/favicon_light.png and b/public/img/logos/dfxtra/thumbs/favicon_light.png differ diff --git a/public/img/logos/dfxtra/thumbs/network.png b/public/img/logos/dfxtra/thumbs/network.png index 5ce5a7ce5..f45b8f692 100644 Binary files a/public/img/logos/dfxtra/thumbs/network.png and b/public/img/logos/dfxtra/thumbs/network.png differ diff --git a/public/img/logos/misc/i-love-lupe.png b/public/img/logos/misc/i-love-lupe.png new file mode 100644 index 000000000..54919e16b Binary files /dev/null and b/public/img/logos/misc/i-love-lupe.png differ diff --git a/public/img/logos/misc/i-love-lupe_notld.png b/public/img/logos/misc/i-love-lupe_notld.png new file mode 100644 index 000000000..7de43b50d Binary files /dev/null and b/public/img/logos/misc/i-love-lupe_notld.png differ diff --git a/public/img/logos/misc/little-lupe.png b/public/img/logos/misc/little-lupe.png new file mode 100644 index 000000000..1e43cad31 Binary files /dev/null and b/public/img/logos/misc/little-lupe.png differ diff --git a/public/img/logos/misc/little-lupe_watermark.png b/public/img/logos/misc/little-lupe_watermark.png new file mode 100644 index 000000000..8548f93b6 Binary files /dev/null and b/public/img/logos/misc/little-lupe_watermark.png differ diff --git a/seeds/00_tags.js b/seeds/00_tags.js index 1010ffd2f..289f3fda5 100755 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -1085,8 +1085,8 @@ const tags = [ slug: 'toys', }, { - name: 'toy anal', - slug: 'toy-anal', + name: 'anal toy', + slug: 'anal-toy', description: 'Stuffing a toy, such as a dildo or buttplug, into the ass', }, { @@ -1359,8 +1359,8 @@ const aliases = [ secondary: true, }, { - name: 'mfm', - for: 'mmf', + name: 'mmf', + for: 'mfm', }, { name: 'fmf', @@ -1499,18 +1499,22 @@ const aliases = [ name: 'brunettes', for: 'brunette', }, + { + name: 'anal toys', + for: 'anal-toy', + }, { name: 'buttplug', - for: 'anal-toys', + for: 'anal-toy', secondary: true, }, { name: 'butt plug', - for: 'anal-toys', + for: 'anal-toy', }, { name: 'butt plugs', - for: 'anal-toys', + for: 'anal-toy', }, { name: 'caning', @@ -2430,7 +2434,7 @@ const aliases = [ }, { name: 'strip pole dancing', - for: 'strip-pole-dancing', + for: 'pole-dancing', }, { name: 'anal gangbangs', diff --git a/seeds/01_networks.js b/seeds/01_networks.js index f0ce0e786..4c66a1bff 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -433,6 +433,7 @@ const networks = [ name: 'Kink Men', url: 'https://www.kinkmen.com', parent: 'kink', + tags: ['gay'], parameters: { interval: 1000, concurrency: 1, diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 167fab908..9b1443f01 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -2955,11 +2955,32 @@ const sites = [ parent: 'digitalplayground', }, // DFXTRA DOGFART NETWORK + { + slug: 'dfxtra', + name: 'DFXtra', + url: 'https://www.dfxtra.com', + parent: 'dfxtra', + parameters: { + layout: 'api', + }, + }, { slug: 'dfxtraoriginals', name: 'DFXtra Originals', url: 'https://www.dfxtra.com', - parent: 'dogfartnetwork', + parent: 'dfxtra', + parameters: { + layout: 'api', + }, + }, + { + slug: 'dfxtracompilations', + name: 'DFXtra Compilations', + url: 'https://www.dfxtra.com', + parent: 'dfxtra', + parameters: { + layout: 'api', + }, }, { slug: 'blacksonblondes', diff --git a/src/actors.js b/src/actors.js index 2848bef1c..f379bb9c2 100755 --- a/src/actors.js +++ b/src/actors.js @@ -1074,6 +1074,7 @@ async function associatePeople(releases, batchId, type = 'actor') { } await bulkInsert(`releases_${type}`, validReleaseActorAssociations, false); + await knex.schema.refreshMaterializedView('actors_meta'); logger.verbose(`Associated ${releaseActorAssociations.length} actors to ${releases.length} scenes`); diff --git a/src/stashes.js b/src/stashes.js index 7d02fa411..a4f1dbbc2 100755 --- a/src/stashes.js +++ b/src/stashes.js @@ -1,8 +1,13 @@ 'use strict'; +const config = require('config'); + const knex = require('./knex'); const { HttpError } = require('./errors'); const slugify = require('./utils/slugify'); +const logger = require('./logger')(__filename); + +let lastActorsViewRefresh = 0; function curateStash(stash) { if (!stash) { @@ -119,6 +124,21 @@ async function removeStash(stashId, sessionUser) { } } +async function refreshActorsView() { + if (new Date() - lastActorsViewRefresh > config.stashes.viewRefreshCooldown * 60000) { + // don't refresh actors view more than once an hour + lastActorsViewRefresh = new Date(); + + logger.debug('Refreshing actors view'); + + return knex.schema.refreshMaterializedView('actors_meta'); + } + + logger.silly('Skipping actors view refresh'); + + return false; +} + async function stashActor(actorId, stashId, sessionUser) { const stash = await fetchStash(stashId, sessionUser); @@ -128,6 +148,8 @@ async function stashActor(actorId, stashId, sessionUser) { actor_id: actorId, }); + refreshActorsView(); + return fetchStashes('actor', actorId, sessionUser); } @@ -166,6 +188,8 @@ async function unstashActor(actorId, stashId, sessionUser) { .where('stashes.user_id', sessionUser.id)) .delete(); + refreshActorsView(); + return fetchStashes('actor', actorId, sessionUser); } diff --git a/src/tools/manticore-actors.js b/src/tools/manticore-actors.js index daf67f974..00fd172d3 100644 --- a/src/tools/manticore-actors.js +++ b/src/tools/manticore-actors.js @@ -20,9 +20,10 @@ const update = args.update; async function fetchActors() { // manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC... const actors = await knex.raw(` - SELECT actors.*, date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob - FROM actors - GROUP BY actors.id; + SELECT + actors_meta.*, + date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob + FROM actors_meta; `); return actors.rows; @@ -44,7 +45,9 @@ async function init() { cup string, natural_boobs int, penis_length int, - penis_girth int + penis_girth int, + stashed int, + scenes int )`); const actors = await fetchActors(); @@ -66,6 +69,8 @@ async function init() { natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown penis_length: actor.penis_length || undefined, penis_girth: actor.penis_girth || undefined, + stashed: actor.stashed || 0, + scenes: actor.scenes || 0, }, }, })); diff --git a/src/tools/manticore.js b/src/tools/manticore.js index 764fe1b58..c6936c335 100644 --- a/src/tools/manticore.js +++ b/src/tools/manticore.js @@ -20,11 +20,12 @@ const update = args.update; async function fetchScenes() { const scenes = await knex.raw(` SELECT - releases.id AS id, - releases.title, - releases.created_at, - releases.date, - releases.entry_id, + scenes_meta.id AS id, + scenes_meta.title, + scenes_meta.created_at, + scenes_meta.date, + scenes_meta.entry_id, + scenes_meta.stashed, entities.id as channel_id, entities.slug as channel_slug, entities.name as channel_name, @@ -33,17 +34,31 @@ async function fetchScenes() { parents.name as network_name, COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags - FROM releases - LEFT JOIN entities ON releases.entity_id = entities.id + FROM scenes_meta + LEFT JOIN entities ON scenes_meta.entity_id = entities.id LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id - LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id - LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id + LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = scenes_meta.id + LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = scenes_meta.id + LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = scenes_meta.id LEFT JOIN actors ON local_actors.actor_id = actors.id LEFT JOIN actors AS directors ON local_directors.director_id = directors.id LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6 LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true - GROUP BY releases.id, entities.id, entities.name, entities.slug, entities.alias, parents.id, parents.name, parents.slug, parents.alias; + GROUP BY + scenes_meta.id, + scenes_meta.title, + scenes_meta.created_at, + scenes_meta.date, + scenes_meta.entry_id, + scenes_meta.stashed, + entities.id, + entities.name, + entities.slug, + entities.alias, + parents.id, + parents.name, + parents.slug, + parents.alias; `); return scenes.rows; @@ -68,13 +83,14 @@ async function init() { tags text, date timestamp, created_at timestamp, - effective_date timestamp + effective_date timestamp, + stashed int )`); const scenes = await fetchScenes(); const docs = scenes.map((scene) => ({ - insert: { + replace: { index: 'scenes', id: scene.id, doc: { @@ -93,6 +109,7 @@ async function init() { actors: scene.actors.map((actor) => actor.f2).join(), tag_ids: scene.tags.map((tag) => tag.f1), tags: scene.tags.map((tag) => tag.f2).join(), + stashed: scene.stashed || 0, }, }, })); diff --git a/src/update-search.js b/src/update-search.js index bf03dd5d4..3ef08f832 100644 --- a/src/update-search.js +++ b/src/update-search.js @@ -1,10 +1,99 @@ 'use strict'; +const manticore = require('manticoresearch'); + const knex = require('./knex'); const logger = require('./logger')(__filename); const bulkInsert = require('./utils/bulk-insert'); -async function updateSceneSearch(releaseIds) { +const mantiClient = new manticore.ApiClient(); +const indexApi = new manticore.IndexApi(mantiClient); + +async function updateManticoreSearch(releaseIds) { + const scenes = await knex.raw(` + SELECT + scenes_meta.id AS id, + scenes_meta.title, + scenes_meta.created_at, + scenes_meta.date, + scenes_meta.entry_id, + scenes_meta.stashed, + entities.id as channel_id, + entities.slug as channel_slug, + entities.name as channel_name, + parents.id as network_id, + parents.slug as network_slug, + parents.name as network_name, + COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, + COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags + FROM scenes_meta + LEFT JOIN entities ON scenes_meta.entity_id = entities.id + LEFT JOIN entities AS parents ON parents.id = entities.parent_id + LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = scenes_meta.id + LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = scenes_meta.id + LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = scenes_meta.id + LEFT JOIN actors ON local_actors.actor_id = actors.id + LEFT JOIN actors AS directors ON local_directors.director_id = directors.id + LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6 + LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true + ${releaseIds ? 'WHERE scenes_meta.id = ANY(?)' : ''} + GROUP BY + scenes_meta.id, + scenes_meta.title, + scenes_meta.created_at, + scenes_meta.date, + scenes_meta.entry_id, + scenes_meta.stashed, + entities.id, + entities.name, + entities.slug, + entities.alias, + parents.id, + parents.name, + parents.slug, + parents.alias; + `, releaseIds && [releaseIds]); + + console.log(releaseIds); + console.log(scenes); + + const docs = scenes.rows.map((scene) => ({ + replace: { + index: 'scenes', + id: scene.id, + doc: { + title: scene.title || undefined, + date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined, + created_at: Math.round(scene.created_at.getTime() / 1000), + effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000), + entry_id: scene.entry_id, + channel_id: scene.channel_id, + channel_slug: scene.channel_slug, + channel_name: scene.channel_name, + network_id: scene.network_id || undefined, + network_slug: scene.network_slug || undefined, + network_name: scene.network_name || undefined, + actor_ids: scene.actors.map((actor) => actor.f1), + actors: scene.actors.map((actor) => actor.f2).join(), + tag_ids: scene.tags.map((tag) => tag.f1), + tags: scene.tags.map((tag) => tag.f2).join(), + stashed: scene.stashed || 0, + }, + }, + })); + + console.log('docs', docs); + + if (docs.length === 0) { + return; + } + + const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')); + + console.log('data', data); +} + +async function updateSqlSearch(releaseIds) { logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`); const documents = await knex.raw(` @@ -48,6 +137,13 @@ async function updateSceneSearch(releaseIds) { await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;'); } +async function updateSceneSearch(releaseIds) { + await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;'); + + await updateSqlSearch(releaseIds); + await updateManticoreSearch(releaseIds); +} + async function updateMovieSearch(movieIds, target = 'movie') { logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);