diff --git a/assets/components/releases/movies.vue b/assets/components/releases/movies.vue
index 37bc929e..3ff57609 100644
--- a/assets/components/releases/movies.vue
+++ b/assets/components/releases/movies.vue
@@ -3,12 +3,6 @@
-
-
tag);
+ if (entity.sceneTags) curatedEntity.sceneTags = entity.sceneTags;
if (entity.children) {
if (entity.children.nodes) {
diff --git a/assets/js/entities/actions.js b/assets/js/entities/actions.js
index d5ab782a..889e85e0 100644
--- a/assets/js/entities/actions.js
+++ b/assets/js/entities/actions.js
@@ -41,6 +41,11 @@ function initEntitiesActions(store, router) {
slug
}
}
+ sceneTags {
+ id
+ name
+ slug
+ }
children: childEntitiesConnection(
orderBy: [PRIORITY_DESC, NAME_ASC],
filter: {
diff --git a/migrations/20220209010315_movies_tags.js b/migrations/20220209010315_movies_tags.js
deleted file mode 100644
index 92aeccb8..00000000
--- a/migrations/20220209010315_movies_tags.js
+++ /dev/null
@@ -1,8 +0,0 @@
-exports.up = async (knex) => knex.raw(`
- CREATE VIEW movies_tagged AS
- SELECT * FROM movies;
-`);
-
-exports.down = async (knex) => knex.raw(`
- DROP VIEW IF EXISTS movies_tagged;
-`);
diff --git a/migrations/20220227215315_entity_filters.js b/migrations/20220227215315_entity_filters.js
new file mode 100644
index 00000000..736ad412
--- /dev/null
+++ b/migrations/20220227215315_entity_filters.js
@@ -0,0 +1,23 @@
+exports.up = async (knex) => knex.raw(`
+ CREATE FUNCTION entities_scene_tags(entity entities, selectable_tags text[]) RETURNS SETOF tags AS $$
+ SELECT tags.*
+ FROM releases
+ LEFT JOIN
+ releases_tags ON releases_tags.release_id = releases.id
+ LEFT JOIN
+ tags ON tags.id = releases_tags.tag_id
+ WHERE
+ releases.entity_id = entity.id
+ AND
+ CASE WHEN array_length(selectable_tags, 1) IS NOT NULL
+ THEN tags.slug = ANY(selectable_tags)
+ ELSE true
+ END
+ GROUP BY tags.id
+ ORDER BY tags.name;
+ $$ LANGUAGE SQL STABLE;
+`);
+
+exports.down = async (knex) => knex.raw(`
+ DROP FUNCTION IF EXISTS entities_tags;
+`);
diff --git a/src/actors.js b/src/actors.js
index f8c189dd..7af1d6fe 100644
--- a/src/actors.js
+++ b/src/actors.js
@@ -20,6 +20,7 @@ const scrapers = require('./scrapers/scrapers').actors;
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const bulkInsert = require('./utils/bulk-insert');
+const chunk = require('./utils/chunk');
const logger = require('./logger')(__filename);
const { toBaseReleases } = require('./deep');
@@ -1048,33 +1049,42 @@ async function flushProfiles(actorIdsOrNames) {
logger.info(`Removed ${deleteCount} profiles`);
}
-async function deleteActors(actorIdsOrNames) {
- const actors = await knex('actors')
- .whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
- .orWhere((builder) => {
- builder
- .whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
- .whereNull('entity_id');
- });
+async function deleteActors(allActorIdsOrNames) {
+ const deleteCounts = await Promise.map(chunk(allActorIdsOrNames), async (actorIdsOrNames) => {
+ const actors = await knex('actors')
+ .whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
+ .orWhere((builder) => {
+ builder
+ .whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
+ .whereNull('entity_id');
+ });
- const actorIds = actors.map((actor) => actor.id);
+ const actorIds = actors.map((actor) => actor.id);
- const sceneIds = await knex('releases_actors')
- .select('releases.id')
- .whereIn('actor_id', actorIds)
- .leftJoin('releases', 'releases.id', 'releases_actors.release_id')
- .pluck('id');
+ const sceneIds = await knex('releases_actors')
+ .select('releases.id')
+ .whereIn('actor_id', actorIds)
+ .leftJoin('releases', 'releases.id', 'releases_actors.release_id')
+ .pluck('id');
- const [deletedScenesCount, deletedActorsCount] = await Promise.all([
- deleteScenes(sceneIds),
- knex('actors')
- .whereIn('id', actorIds)
- .delete(),
- ]);
+ const [deletedScenesCount, deletedActorsCount] = await Promise.all([
+ deleteScenes(sceneIds),
+ knex('actors')
+ .whereIn('id', actorIds)
+ .delete(),
+ ]);
+
+ return { deletedScenesCount, deletedActorsCount };
+ }, { concurrency: 10 });
+
+ const deletedActorsCount = deleteCounts.reduce((acc, count) => acc + count.deletedActorsCount, 0);
+ const deletedScenesCount = deleteCounts.reduce((acc, count) => acc + count.deletedScenesCount, 0);
await flushOrphanedMedia();
logger.info(`Removed ${deletedActorsCount} actors with ${deletedScenesCount} scenes`);
+
+ return deletedActorsCount;
}
async function flushActors() {
diff --git a/src/media.js b/src/media.js
index 1f0b9789..eaf8e6f0 100644
--- a/src/media.js
+++ b/src/media.js
@@ -961,9 +961,12 @@ async function flushOrphanedMedia() {
await deleteS3Objects(orphanedMedia.filter((media) => media.is_s3));
}
- await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true });
-
- logger.info('Cleared temporary media directory');
+ try {
+ await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true });
+ logger.info('Cleared temporary media directory');
+ } catch (error) {
+ logger.warn(`Failed to clear temporary media directory: ${error.message}`);
+ }
}
module.exports = {
diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js
index 4fc89b09..8c9e3a4c 100644
--- a/src/scrapers/mindgeek.js
+++ b/src/scrapers/mindgeek.js
@@ -11,6 +11,12 @@ const slugify = require('../utils/slugify');
const http = require('../utils/http');
const { inchesToCm, lbsToKg } = require('../utils/convert');
+function getBasePath(channel, path = '/scene') {
+ return channel.parameters?.scene
+ || ((channel.parameters?.native || channel.type === 'network') && `${channel.url}${path}`)
+ || `${channel.parent.url}${path}`;
+}
+
function getThumbs(scene) {
if (scene.images.poster) {
return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
@@ -18,7 +24,7 @@ function getThumbs(scene) {
.map((image) => image.xl.url);
}
- if (scene.images.card_main_rect) {
+ if (Array.isArray(scene.images.card_main_rect)) {
return scene.images.card_main_rect
.concat(scene.images.card_secondary_rect || [])
.map((image) => image.xl.url.replace('.thumb', ''));
@@ -27,6 +33,20 @@ function getThumbs(scene) {
return [];
}
+function getCovers(images) {
+ return [
+ [
+ images.cover[0].md?.url,
+ images.cover[0].sm?.url,
+ images.cover[0].xs?.url,
+ // bigger but usually upscaled
+ images.cover[0].xx?.url,
+ images.cover[0].xl?.url,
+ images.cover[0].lg?.url,
+ ],
+ ];
+}
+
function getVideos(data) {
const teaserSources = data.videos.mediabook?.files;
const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
@@ -51,9 +71,7 @@ function scrapeLatestX(data, site, filterChannel) {
description: data.description,
};
- const basepath = site.parameters?.scene
- || (site.parameters?.native && `${site.url}/scene`)
- || `${site.parent.url}/scene`;
+ const basepath = getBasePath(site);
release.url = `${basepath}/${release.entryId}/${slugify(release.title)}`;
release.date = new Date(data.dateReleased);
@@ -96,7 +114,7 @@ async function scrapeLatest(items, site, filterChannel) {
};
}
-function scrapeScene(data, url, _site, networkName) {
+function scrapeRelease(data, url, channel, networkName) {
const release = {};
const { id: entryId, title, description } = data;
@@ -129,6 +147,29 @@ function scrapeScene(data, url, _site, networkName) {
release.url = url || `https://www.${networkName || data.brand}.com/scene/${entryId}/`;
+ if (data.parent?.type === 'movie') {
+ release.movie = {
+ entryId: data.parent.id,
+ url: `${getBasePath(channel, '/movie')}/${data.parent.id}/${slugify(data.parent.title, '-', { removePunctuation: true })}`,
+ title: data.parent.title,
+ description: data.parent.description,
+ date: new Date(data.parent.dateReleased),
+ channel: slugify(data.parent.collections?.name || data.parent.brand),
+ covers: getCovers(data.parent.images),
+ shallow: true,
+ };
+ }
+
+ if (data.type === 'movie') {
+ release.covers = getCovers(data.images);
+ release.scenes = data.children?.map((scene) => ({
+ entryId: scene.id,
+ url: `${getBasePath(channel)}/${scene.id}/${slugify(scene.title)}`,
+ title: scene.title,
+ shallow: true,
+ }));
+ }
+
return release;
}
@@ -230,7 +271,7 @@ function scrapeProfile(data, html, releases = [], networkName) {
profile.naturalBoobs = false;
}
- profile.releases = releases.map((release) => scrapeScene(release, null, null, networkName));
+ profile.releases = releases.map((release) => scrapeRelease(release, null, null, networkName));
return profile;
}
@@ -292,8 +333,8 @@ async function fetchUpcoming(site, page, options) {
return res.statusCode;
}
-async function fetchScene(url, site, baseScene, options) {
- if (baseScene?.entryId) {
+async function fetchRelease(url, site, baseScene, options) {
+ if (baseScene?.entryId && !baseScene.shallow) {
// overview and deep data is the same, don't hit server unnecessarily
return baseScene;
}
@@ -312,7 +353,7 @@ async function fetchScene(url, site, baseScene, options) {
if (res.status === 200 && res.body.result) {
return {
- scene: scrapeScene(res.body.result, url, site),
+ scene: scrapeRelease(res.body.result, url, site),
};
}
@@ -374,6 +415,7 @@ module.exports = {
scrapeLatestX,
fetchLatest,
fetchUpcoming,
- fetchScene,
+ fetchScene: fetchRelease,
+ fetchMovie: fetchRelease,
fetchProfile,
};
diff --git a/src/scrapers/vixen.js b/src/scrapers/vixen.js
index c945a52f..ce4d95db 100644
--- a/src/scrapers/vixen.js
+++ b/src/scrapers/vixen.js
@@ -142,6 +142,7 @@ async function getTrailer(scene, channel, url) {
return null;
}
+/*
async function getPhotosLegacy(url) {
const htmlRes = await http.get(url, {
extract: {
@@ -169,6 +170,7 @@ async function getPhotosLegacy(url) {
return [];
}
}
+*/
async function getPhotos(url) {
const htmlRes = await http.get(url, {
diff --git a/src/store-releases.js b/src/store-releases.js
index b94a5bb0..c4a9e53a 100644
--- a/src/store-releases.js
+++ b/src/store-releases.js
@@ -392,7 +392,8 @@ async function associateMovieScenes(movies, movieScenes) {
return null;
}
- const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId];
+ const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]
+ || moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId];
if (sceneMovie?.id) {
return {