diff --git a/assets/components/entities/entity.vue b/assets/components/entities/entity.vue index 705e70cc..4007c035 100644 --- a/assets/components/entities/entity.vue +++ b/assets/components/entities/entity.vue @@ -108,6 +108,7 @@ :fetch-releases="fetchEntity" :items-total="totalCount" :items-per-page="limit" + :available-tags="entity.tags" />
diff --git a/assets/components/releases/movies.vue b/assets/components/releases/movies.vue index 37bc929e..3ff57609 100644 --- a/assets/components/releases/movies.vue +++ b/assets/components/releases/movies.vue @@ -3,12 +3,6 @@
- -
tag); + if (entity.sceneTags) curatedEntity.sceneTags = entity.sceneTags; if (entity.children) { if (entity.children.nodes) { diff --git a/assets/js/entities/actions.js b/assets/js/entities/actions.js index d5ab782a..889e85e0 100644 --- a/assets/js/entities/actions.js +++ b/assets/js/entities/actions.js @@ -41,6 +41,11 @@ function initEntitiesActions(store, router) { slug } } + sceneTags { + id + name + slug + } children: childEntitiesConnection( orderBy: [PRIORITY_DESC, NAME_ASC], filter: { diff --git a/migrations/20220209010315_movies_tags.js b/migrations/20220209010315_movies_tags.js deleted file mode 100644 index 92aeccb8..00000000 --- a/migrations/20220209010315_movies_tags.js +++ /dev/null @@ -1,8 +0,0 @@ -exports.up = async (knex) => knex.raw(` - CREATE VIEW movies_tagged AS - SELECT * FROM movies; -`); - -exports.down = async (knex) => knex.raw(` - DROP VIEW IF EXISTS movies_tagged; -`); diff --git a/migrations/20220227215315_entity_filters.js b/migrations/20220227215315_entity_filters.js new file mode 100644 index 00000000..736ad412 --- /dev/null +++ b/migrations/20220227215315_entity_filters.js @@ -0,0 +1,23 @@ +exports.up = async (knex) => knex.raw(` + CREATE FUNCTION entities_scene_tags(entity entities, selectable_tags text[]) RETURNS SETOF tags AS $$ + SELECT tags.* + FROM releases + LEFT JOIN + releases_tags ON releases_tags.release_id = releases.id + LEFT JOIN + tags ON tags.id = releases_tags.tag_id + WHERE + releases.entity_id = entity.id + AND + CASE WHEN array_length(selectable_tags, 1) IS NOT NULL + THEN tags.slug = ANY(selectable_tags) + ELSE true + END + GROUP BY tags.id + ORDER BY tags.name; + $$ LANGUAGE SQL STABLE; +`); + +exports.down = async (knex) => knex.raw(` + DROP FUNCTION IF EXISTS entities_tags; +`); diff --git a/src/actors.js b/src/actors.js index f8c189dd..7af1d6fe 100644 --- a/src/actors.js +++ b/src/actors.js @@ -20,6 +20,7 @@ const scrapers = require('./scrapers/scrapers').actors; const argv = require('./argv'); const include = require('./utils/argv-include')(argv); const bulkInsert = require('./utils/bulk-insert'); +const chunk = require('./utils/chunk'); const logger = require('./logger')(__filename); const { toBaseReleases } = require('./deep'); @@ -1048,33 +1049,42 @@ async function flushProfiles(actorIdsOrNames) { logger.info(`Removed ${deleteCount} profiles`); } -async function deleteActors(actorIdsOrNames) { - const actors = await knex('actors') - .whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number')) - .orWhere((builder) => { - builder - .whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string')) - .whereNull('entity_id'); - }); +async function deleteActors(allActorIdsOrNames) { + const deleteCounts = await Promise.map(chunk(allActorIdsOrNames), async (actorIdsOrNames) => { + const actors = await knex('actors') + .whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number')) + .orWhere((builder) => { + builder + .whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string')) + .whereNull('entity_id'); + }); - const actorIds = actors.map((actor) => actor.id); + const actorIds = actors.map((actor) => actor.id); - const sceneIds = await knex('releases_actors') - .select('releases.id') - .whereIn('actor_id', actorIds) - .leftJoin('releases', 'releases.id', 'releases_actors.release_id') - .pluck('id'); + const sceneIds = await knex('releases_actors') + .select('releases.id') + .whereIn('actor_id', actorIds) + .leftJoin('releases', 'releases.id', 'releases_actors.release_id') + .pluck('id'); - const [deletedScenesCount, deletedActorsCount] = await Promise.all([ - deleteScenes(sceneIds), - knex('actors') - .whereIn('id', actorIds) - .delete(), - ]); + const [deletedScenesCount, deletedActorsCount] = await Promise.all([ + deleteScenes(sceneIds), + knex('actors') + .whereIn('id', actorIds) + .delete(), + ]); + + return { deletedScenesCount, deletedActorsCount }; + }, { concurrency: 10 }); + + const deletedActorsCount = deleteCounts.reduce((acc, count) => acc + count.deletedActorsCount, 0); + const deletedScenesCount = deleteCounts.reduce((acc, count) => acc + count.deletedScenesCount, 0); await flushOrphanedMedia(); logger.info(`Removed ${deletedActorsCount} actors with ${deletedScenesCount} scenes`); + + return deletedActorsCount; } async function flushActors() { diff --git a/src/media.js b/src/media.js index 1f0b9789..eaf8e6f0 100644 --- a/src/media.js +++ b/src/media.js @@ -961,9 +961,12 @@ async function flushOrphanedMedia() { await deleteS3Objects(orphanedMedia.filter((media) => media.is_s3)); } - await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true }); - - logger.info('Cleared temporary media directory'); + try { + await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true }); + logger.info('Cleared temporary media directory'); + } catch (error) { + logger.warn(`Failed to clear temporary media directory: ${error.message}`); + } } module.exports = { diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index 4fc89b09..8c9e3a4c 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -11,6 +11,12 @@ const slugify = require('../utils/slugify'); const http = require('../utils/http'); const { inchesToCm, lbsToKg } = require('../utils/convert'); +function getBasePath(channel, path = '/scene') { + return channel.parameters?.scene + || ((channel.parameters?.native || channel.type === 'network') && `${channel.url}${path}`) + || `${channel.parent.url}${path}`; +} + function getThumbs(scene) { if (scene.images.poster) { return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array @@ -18,7 +24,7 @@ function getThumbs(scene) { .map((image) => image.xl.url); } - if (scene.images.card_main_rect) { + if (Array.isArray(scene.images.card_main_rect)) { return scene.images.card_main_rect .concat(scene.images.card_secondary_rect || []) .map((image) => image.xl.url.replace('.thumb', '')); @@ -27,6 +33,20 @@ function getThumbs(scene) { return []; } +function getCovers(images) { + return [ + [ + images.cover[0].md?.url, + images.cover[0].sm?.url, + images.cover[0].xs?.url, + // bigger but usually upscaled + images.cover[0].xx?.url, + images.cover[0].xl?.url, + images.cover[0].lg?.url, + ], + ]; +} + function getVideos(data) { const teaserSources = data.videos.mediabook?.files; const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files; @@ -51,9 +71,7 @@ function scrapeLatestX(data, site, filterChannel) { description: data.description, }; - const basepath = site.parameters?.scene - || (site.parameters?.native && `${site.url}/scene`) - || `${site.parent.url}/scene`; + const basepath = getBasePath(site); release.url = `${basepath}/${release.entryId}/${slugify(release.title)}`; release.date = new Date(data.dateReleased); @@ -96,7 +114,7 @@ async function scrapeLatest(items, site, filterChannel) { }; } -function scrapeScene(data, url, _site, networkName) { +function scrapeRelease(data, url, channel, networkName) { const release = {}; const { id: entryId, title, description } = data; @@ -129,6 +147,29 @@ function scrapeScene(data, url, _site, networkName) { release.url = url || `https://www.${networkName || data.brand}.com/scene/${entryId}/`; + if (data.parent?.type === 'movie') { + release.movie = { + entryId: data.parent.id, + url: `${getBasePath(channel, '/movie')}/${data.parent.id}/${slugify(data.parent.title, '-', { removePunctuation: true })}`, + title: data.parent.title, + description: data.parent.description, + date: new Date(data.parent.dateReleased), + channel: slugify(data.parent.collections?.name || data.parent.brand), + covers: getCovers(data.parent.images), + shallow: true, + }; + } + + if (data.type === 'movie') { + release.covers = getCovers(data.images); + release.scenes = data.children?.map((scene) => ({ + entryId: scene.id, + url: `${getBasePath(channel)}/${scene.id}/${slugify(scene.title)}`, + title: scene.title, + shallow: true, + })); + } + return release; } @@ -230,7 +271,7 @@ function scrapeProfile(data, html, releases = [], networkName) { profile.naturalBoobs = false; } - profile.releases = releases.map((release) => scrapeScene(release, null, null, networkName)); + profile.releases = releases.map((release) => scrapeRelease(release, null, null, networkName)); return profile; } @@ -292,8 +333,8 @@ async function fetchUpcoming(site, page, options) { return res.statusCode; } -async function fetchScene(url, site, baseScene, options) { - if (baseScene?.entryId) { +async function fetchRelease(url, site, baseScene, options) { + if (baseScene?.entryId && !baseScene.shallow) { // overview and deep data is the same, don't hit server unnecessarily return baseScene; } @@ -312,7 +353,7 @@ async function fetchScene(url, site, baseScene, options) { if (res.status === 200 && res.body.result) { return { - scene: scrapeScene(res.body.result, url, site), + scene: scrapeRelease(res.body.result, url, site), }; } @@ -374,6 +415,7 @@ module.exports = { scrapeLatestX, fetchLatest, fetchUpcoming, - fetchScene, + fetchScene: fetchRelease, + fetchMovie: fetchRelease, fetchProfile, }; diff --git a/src/scrapers/vixen.js b/src/scrapers/vixen.js index c945a52f..ce4d95db 100644 --- a/src/scrapers/vixen.js +++ b/src/scrapers/vixen.js @@ -142,6 +142,7 @@ async function getTrailer(scene, channel, url) { return null; } +/* async function getPhotosLegacy(url) { const htmlRes = await http.get(url, { extract: { @@ -169,6 +170,7 @@ async function getPhotosLegacy(url) { return []; } } +*/ async function getPhotos(url) { const htmlRes = await http.get(url, { diff --git a/src/store-releases.js b/src/store-releases.js index b94a5bb0..c4a9e53a 100644 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -392,7 +392,8 @@ async function associateMovieScenes(movies, movieScenes) { return null; } - const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]; + const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId] + || moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId]; if (sceneMovie?.id) { return {