Merge branch 'master' into experimental

assets/components/container/container.vue

@@ -62,7 +62,7 @@ async function setConsent(consent, includeQueer) {
 	}
 
 	if (includeQueer) {
-		this.$store.dispatch('setTagFilter', this.$store.state.ui.tagFilter.filter(tag => !['gay', 'bisexual', 'transsexual'].includes(tag)));
+		this.$store.dispatch('setTagFilter', this.$store.state.ui.tagFilter.filter((tag) => !['gay', 'bisexual', 'transsexual'].includes(tag)));
 		return;
 	}
 

assets/components/entities/entity.vue

@@ -108,6 +108,7 @@
 				:fetch-releases="fetchEntity"
 				:items-total="totalCount"
 				:items-per-page="limit"
+				:available-tags="entity.tags"
 			/>
 
 			<div class="releases">

assets/components/entities/tile.vue

@@ -102,6 +102,8 @@ export default {
 }
 
 .name {
+	display: flex;
+	align-items: center;
 	color: var(--text-light);
     font-size: 1.25rem;
     font-weight: bold;

assets/components/icon/icon.vue

@@ -28,7 +28,7 @@ export default {
 		};
 	},
 	beforeMount() {
-		this.svg = require(`../../img/icons/${this.icon}.svg`).default;
+		this.svg = require(`../../img/icons/${this.icon}.svg`).default; // eslint-disable-line global-require, import/no-dynamic-require
 	},
 };
 </script>

assets/components/networks/networks.vue

@@ -11,8 +11,22 @@
 				class="empty"
 			>No results for "{{ $route.query.query }}"</span>
 
+			<template v-else>
+				<h2 class="heading">Popular</h2>
+
+				<div
+					class="entity-tiles"
+				>
+					<Entity
+						v-for="entity in popularEntities"
+						:key="entity.parent ? `entity-tile-${entity.parent.slug}-${entity.slug}` : `entity-tile-${entity.slug}`"
+						:entity="entity"
+					/>
+				</div>
+
+				<h2 class="heading">All networks</h2>
+
 				<div
-				v-else
 					class="entity-tiles"
 				>
 					<Entity
@@ -21,6 +35,7 @@
 						:entity="entity"
 					/>
 				</div>
+			</template>
 		</div>
 
 		<Footer />
@@ -58,6 +73,45 @@ async function searchEntities() {
 	this.done = true;
 }
 
+function popularEntities() {
+	const entitiesBySlug = Object.fromEntries(this.entities.map((entity) => [entity.slug, entity]));
+
+	return [
+		'21sextury',
+		'amateurallure',
+		'analvids',
+		'bamvisions',
+		'bang',
+		'bangbros',
+		'blowpass',
+		'brazzers',
+		'burningangel',
+		'digitalplayground',
+		'dogfartnetwork',
+		'dorcel',
+		'elegantangel',
+		'evilangel',
+		'fakehub',
+		'girlsway',
+		'hookuphotshot',
+		'hussiepass',
+		'insex',
+		'julesjordan',
+		'kellymadison',
+		'kink',
+		'mofos',
+		'naughtyamerica',
+		'newsensations',
+		'pervcity',
+		'pornpros',
+		'private',
+		'realitykings',
+		'twistys',
+		'vixen',
+		'xempire',
+	].map((slug) => entitiesBySlug[slug]).filter(Boolean);
+}
+
 async function mounted() {
 	this.pageTitle = 'Channels';
 
@@ -82,6 +136,7 @@ export default {
 	},
 	computed: {
 		channelCount,
+		popularEntities,
 	},
 	watch: {
 		$route: fetchEntities,
@@ -130,6 +185,10 @@ export default {
 	font-weight: bold;
 }
 
+.heading {
+	margin: 1rem 0 0 0;
+}
+
 @media(max-width: $breakpoint2) {
     .entity-tiles {
         grid-gap: .5rem;

assets/components/releases/banner.vue

@@ -2,7 +2,7 @@
 	<div class="media-container">
 		<div
 			class="media"
-			:class="{ center: release.photos.length < 2, preview: !me }"
+			:class="{ center: (release.photos?.length || 0) + (release.scenesPhotos?.length || 0) < 2, preview: !me }"
 		>
 			<div
 				v-if="release.trailer || release.teaser"
@@ -71,10 +71,13 @@
 				</span>
 			</div>
 
-			<template v-if="release.covers && release.covers.length > 0">
-				<a
+			<template v-if="release.covers?.length > 0">
+				<div
 					v-for="cover in release.covers"
 					:key="`cover-${cover.id}`"
+					class="item-container"
+				>
+					<a
 						:href="getPath(cover)"
 						target="_blank"
 						rel="noopener noreferrer"
@@ -89,6 +92,7 @@
 							@load="$emit('load', $event)"
 						>
 					</a>
+				</div>
 			</template>
 
 			<div
@@ -164,8 +168,8 @@ function poster() {
 function photos() {
 	const clips = this.release.clips || [];
 	const clipPostersById = clips.reduce((acc, clip) => ({ ...acc, [clip.poster.id]: clip.poster }), {});
-	const uniqueClipPosters = Array.from(new Set(clips.map(clip => clip.poster.id) || [])).map(posterId => clipPostersById[posterId]);
-	const photosWithClipPosters = (this.release.photos || []).concat(uniqueClipPosters);
+	const uniqueClipPosters = Array.from(new Set(clips.map((clip) => clip.poster.id) || [])).map((posterId) => clipPostersById[posterId]);
+	const photosWithClipPosters = (this.release.photos || []).concat(this.release.scenesPhotos || []).concat(uniqueClipPosters);
 
 	if (this.release.trailer || (this.release.teaser && this.release.teaser.mime !== 'image/gif')) {
 		// poster will be on trailer video

assets/components/releases/movies.vue

@@ -3,12 +3,6 @@
 		<div class="content-inner">
 			<SearchBar :placeholder="`Search ${totalCount} movies`" />
 
-			<TagFilter
-				class="filters-filter"
-				:filter="filter"
-				:available-tags="availableTags"
-			/>
-
 			<div
 				ref="tiles"
 				class="tiles"
@@ -36,7 +30,6 @@
 import MovieTile from './movie-tile.vue';
 import SearchBar from '../search/bar.vue';
 import Pagination from '../pagination/pagination.vue';
-import TagFilter from '../filters/tag-filter.vue';
 
 async function fetchMovies() {
 	if (this.$route.query.query) {
@@ -80,7 +73,6 @@ export default {
 		MovieTile,
 		SearchBar,
 		Pagination,
-		TagFilter,
 	},
 	data() {
 		return {

assets/components/releases/release.vue

@@ -21,14 +21,14 @@
 		<Details :release="release" />
 
 		<button
-			v-if="release.photos.length > 0"
+			v-if="release.photos?.length > 0 || release.scenesPhotos?.length > 0"
 			class="album-toggle"
 			@click="$router.push({ hash: '#album' })"
 		><Icon icon="grid3" />View album</button>
 
 		<Album
 			v-if="showAlbum"
-			:items="[release.poster, ...release.photos]"
+			:items="[release.poster, ...(release.photos || []), ...(release.scenesPhotos || [])]"
 			:title="release.title"
 			:path="config.media.mediaPath"
 			@close="$router.replace({ hash: undefined })"
@@ -79,22 +79,23 @@
 			/>
 
 			<div
-				v-if="release.movies && release.movies.length > 0"
+				v-if="release.movies?.length > 0 || release.series?.length > 0"
 				class="row"
 			>
 				<span class="row-label">Part of</span>
 
 				<div class="movies">
 					<router-link
-						v-for="movie in release.movies"
+						v-for="movie in [...release.movies, ...release.series]"
 						:key="`movie-${movie.id}`"
-						:to="{ name: 'movie', params: { releaseId: movie.id, releaseSlug: movie.slug } }"
+						:to="{ name: movie.type || 'movie', params: { releaseId: movie.id, releaseSlug: movie.slug } }"
 						class="movie"
 					>
 						<span class="movie-title">{{ movie.title }}</span>
+
 						<img
-							v-if="movie.covers.length > 0"
-							:src="getPath(movie.covers[0], 'thumbnail')"
+							v-if="movie.covers.length > 0 || movie.poster"
+							:src="getPath(movie.covers[0] || movie.poster, 'thumbnail')"
 							class="movie-cover"
 						>
 					</router-link>
@@ -243,6 +244,10 @@ async function fetchRelease(scroll = true) {
 		this.release = await this.$store.dispatch('fetchMovieById', this.$route.params.releaseId);
 	}
 
+	if (this.$route.name === 'serie') {
+		this.release = await this.$store.dispatch('fetchSerieById', this.$route.params.releaseId);
+	}
+
 	if (scroll && this.$refs.content) {
 		this.$refs.content.scrollTop = 0;
 	}
@@ -282,7 +287,7 @@ function pageTitle() {
 }
 
 function showAlbum() {
-	return this.release.photos?.length > 0 && this.$route.hash === '#album';
+	return (this.release.photos?.length > 0 || this.release.scenesPhotos?.length > 0) && this.$route.hash === '#album';
 }
 
 export default {

assets/components/tags/tags.vue

@@ -87,7 +87,6 @@ const tagSlugsByCategory = {
 		'titty-fucking',
 		'fisting',
 		'anal-fisting',
-		'fisting-dp',
 	],
 	group: [
 		'mfm',
@@ -108,16 +107,6 @@ const tagSlugsByCategory = {
 		'bukkake',
 		'fake-cum',
 	],
-	toys: [
-		'toys',
-		'toy-anal',
-		'toy-dp',
-		'double-dildo',
-		'double-dildo-blowjob',
-		'double-dildo-kiss',
-		'double-dildo-anal',
-		'double-dildo-dp',
-	],
 	roleplay: [
 		'family',
 		'parody',
@@ -126,6 +115,15 @@ const tagSlugsByCategory = {
 		'maid',
 		'nun',
 	],
+	extreme: [
+		'dp',
+		'airtight',
+		'dap',
+		'dvp',
+		'triple-penetration',
+		'tap',
+		'tvp',
+	],
 	fetish: [
 		'bdsm',
 		'femdom',
@@ -134,15 +132,15 @@ const tagSlugsByCategory = {
 		'latex',
 		'blindfold',
 	],
-	extreme: [
-		'dp',
-		'airtight',
-		'dap',
-		'dvp',
-		'da-tp',
-		'dv-tp',
-		'tap',
-		'tvp',
+	toys: [
+		'toys',
+		'toy-anal',
+		'toy-dp',
+		'double-dildo',
+		'double-dildo-blowjob',
+		'double-dildo-kiss',
+		'double-dildo-anal',
+		'double-dildo-dp',
 	],
 	misc: [
 		'gaping',

assets/js/curate.js

@@ -65,19 +65,23 @@ function curateActor(actor, release) {
 	return curatedActor;
 }
 
-function curateRelease(release) {
+function curateRelease(release, type = 'scene') {
 	const curatedRelease = {
 		...release,
+		type: release.type || type,
 		actors: [],
 		poster: release.poster && release.poster.media,
 		tags: release.tags ? release.tags.map((tag) => tag.tag || tag) : [],
 	};
 
-	if (release.scenes) curatedRelease.scenes = release.scenes.filter(Boolean).map(({ scene }) => curateRelease(scene));
-	if (release.movies) curatedRelease.movies = release.movies.filter(Boolean).map(({ movie }) => curateRelease(movie));
-	if (release.chapters) curatedRelease.chapters = release.chapters.filter(Boolean).map((chapter) => curateRelease(chapter));
-	if (release.photos) curatedRelease.photos = release.photos.filter(Boolean).map((photo) => photo.media || photo);
-	if (release.covers) curatedRelease.covers = release.covers.filter(Boolean).map(({ media }) => media);
+	curatedRelease.scenes = release.scenes?.filter(Boolean).map(({ scene }) => curateRelease(scene, 'scene')) || [];
+	curatedRelease.movies = release.movies?.filter(Boolean).map(({ movie }) => curateRelease(movie, 'movie')) || [];
+	curatedRelease.series = release.series?.filter(Boolean).map(({ serie }) => curateRelease(serie, 'serie')) || [];
+	curatedRelease.chapters = release.chapters?.filter(Boolean).map((chapter) => curateRelease(chapter)) || [];
+	curatedRelease.photos = release.photos?.filter(Boolean).map((photo) => photo.media || photo) || [];
+	curatedRelease.scenesPhotos = release.scenesPhotos?.filter(Boolean).map((photo) => photo.media || photo) || [];
+	curatedRelease.covers = release.covers?.filter(Boolean).map(({ media }) => media) || [];
+
 	if (release.trailer) curatedRelease.trailer = release.trailer.media;
 	if (release.teaser) curatedRelease.teaser = release.teaser.media;
 	if (release.actors) curatedRelease.actors = release.actors.filter(Boolean).map((actor) => curateActor(actor.actor || actor, curatedRelease));
@@ -105,6 +109,7 @@ function curateEntity(entity, parent, releases) {
 	};
 
 	if (entity.tags) curatedEntity.tags = entity.tags.map(({ tag }) => tag);
+	if (entity.sceneTags) curatedEntity.sceneTags = entity.sceneTags;
 
 	if (entity.children) {
 		if (entity.children.nodes) {

assets/js/entities/actions.js

@@ -41,6 +41,11 @@ function initEntitiesActions(store, router) {
 							slug
 						}
 					}
+					sceneTags {
+						id
+						name
+						slug
+					}
 					children: childEntitiesConnection(
                         orderBy: [PRIORITY_DESC, NAME_ASC],
 						filter: {

assets/js/fragments.js

@@ -442,6 +442,29 @@ const releasesFragment = `
     }
 `;
 
+const mediaFields = `
+	id
+	index
+	path
+	thumbnail
+	lazy
+	isS3
+	comment
+	sfw: sfwMedia {
+		id
+		thumbnail
+		lazy
+		path
+		comment
+	}
+`;
+
+const mediaFragment = `
+	media {
+		${mediaFields}
+	}
+`;
+
 const releaseFragment = `
   release(id: $releaseId) {
     id
@@ -536,6 +559,19 @@ const releaseFragment = `
 			}
 		}
 	}
+	series: seriesScenesBySceneId {
+	    serie {
+			id
+			title
+			slug
+			covers: seriesCoversBySerieId(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) {
+				${mediaFragment}
+			}
+			poster: seriesPosterBySerieId {
+				${mediaFragment}
+			}
+		}
+	}
 	isFavorited
 	isStashed(includeFavorites: false)
 	stashes: stashesScenesBySceneId(
@@ -624,6 +660,8 @@ export {
 	actorFields,
 	actorStashesFields,
 	campaignsFragment,
+	mediaFields,
+	mediaFragment,
 	movieFields,
 	releaseActorsFragment,
 	releaseFields,

assets/js/releases/actions.js

@@ -4,6 +4,8 @@ import {
 	releaseFragment,
 	releaseFields,
 	movieFields,
+	mediaFragment,
+	mediaFields,
 } from '../fragments';
 import { curateRelease } from '../curate';
 import getDateRange from '../get-date-range';
@@ -141,16 +143,16 @@ function initReleasesActions(store, router) {
 		};
 	}
 
-	async function fetchMovieById({ _commit }, movieId) {
+	async function fetchCollectionById({ _commit }, movieId, type = 'movie') {
 		// const release = await get(`/releases/${releaseId}`);
 
-		const { movie } = await graphql(`
+		const result = await graphql(`
 			query Movie(
 				$movieId: Int!
 				$hasAuth: Boolean!
 				$userId: Int
 			) {
-				movie(id: $movieId) {
+				${type}(id: $movieId) {
 					id
 					title
 					description
@@ -182,7 +184,7 @@ function initReleasesActions(store, router) {
 							isS3
 						}
 					}
-					poster: moviesPoster {
+					poster: ${type === 'series' ? 'seriesPosterBySerieId' : 'moviesPoster'} {
 						media {
 							id
 							path
@@ -195,7 +197,7 @@ function initReleasesActions(store, router) {
 							isS3
 						}
 					}
-					covers: moviesCovers(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) {
+					covers: ${type === 'series' ? 'seriesCoversBySerieId' : 'moviesCovers'}(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) {
 						media {
 							id
 							path
@@ -208,14 +210,14 @@ function initReleasesActions(store, router) {
 							isS3
 						}
 					}
-					trailer: moviesTrailer {
+					trailer: ${type === 'series' ? 'seriesTrailerBySerieId' : 'moviesTrailer'} {
 						media {
 							id
 							path
 							isS3
 						}
 					}
-					scenes: moviesScenes {
+					scenes: ${type === 'series' ? 'seriesScenesBySerieId' : 'moviesScenes'} {
 						scene {
 							${releaseFields}
 						}
@@ -225,25 +227,11 @@ function initReleasesActions(store, router) {
 						slug
 						name
 					}
-					photos {
-						id
-						index
-						path
-						thumbnail
-						lazy
-						width
-						height
-						thumbnailWidth
-						thumbnailHeight
-						isS3
-						comment
-						sfw: sfwMedia {
-							id
-							thumbnail
-							lazy
-							path
-							comment
+					photos: ${type === 'series' ? 'seriesPhotosBySerieId' : 'moviesPhotos'} {
+						${mediaFragment}
 					}
+					scenesPhotos {
+						${mediaFields}
 					}
 					entity {
 						id
@@ -259,7 +247,7 @@ function initReleasesActions(store, router) {
 							hasLogo
 						}
 					}
-					stashes: stashesMovies(
+					stashes: ${type === 'series' ? 'stashesSeriesBySerieId' : 'stashesMovies'}(
 						filter: {
 							stash: {
 								userId: {
@@ -283,12 +271,20 @@ function initReleasesActions(store, router) {
 			userId: store.state.auth.user?.id,
 		});
 
-		if (!movie) {
+		if (!result[type]) {
 			router.replace('/not-found');
 			return null;
 		}
 
-		return curateRelease(movie);
+		return curateRelease(result[type]);
+	}
+
+	async function fetchMovieById(context, movieId) {
+		return fetchCollectionById(context, movieId, 'movie');
+	}
+
+	async function fetchSerieById(context, serieId) {
+		return fetchCollectionById(context, serieId, 'series');
 	}
 
 	return {
@@ -296,6 +292,7 @@ function initReleasesActions(store, router) {
 		fetchReleaseById,
 		fetchMovies,
 		fetchMovieById,
+		fetchSerieById,
 		searchMovies,
 	};
 }

assets/js/router.js

@@ -71,6 +71,11 @@ const routes = [
 		component: Release,
 		name: 'movie',
 	},
+	{
+		path: '/serie/:releaseId/:releaseSlug?',
+		component: Release,
+		name: 'serie',
+	},
 	{
 		path: '/actor/:actorId/:actorSlug',
 		name: 'actor',

migrations/20220126132955_scene_counts.js

@@ -1,5 +1,5 @@
 exports.up = async (knex) => knex.raw(`
-	CREATE OR REPLACE FUNCTION entities_scene_total(entity entities) RETURNS integer AS $$
+	CREATE OR REPLACE FUNCTION entities_scene_total(entity entities) RETURNS bigint AS $$
 		SELECT COUNT(id)
 		FROM releases
 		WHERE releases.entity_id = entity.id;

migrations/20220209010315_movies_tags.js

@@ -1,8 +0,0 @@
-exports.up = async (knex) => knex.raw(`
-	CREATE VIEW movies_tagged AS
-		SELECT * FROM movies;
-`);
-
-exports.down = async (knex) => knex.raw(`
-	DROP VIEW IF EXISTS movies_tagged;
-`);

migrations/20220227215315_entity_filters.js

@@ -0,0 +1,23 @@
+exports.up = async (knex) => knex.raw(`
+	CREATE FUNCTION entities_scene_tags(entity entities, selectable_tags text[]) RETURNS SETOF tags AS $$
+		SELECT tags.*
+		FROM releases
+		LEFT JOIN
+			releases_tags ON releases_tags.release_id = releases.id
+		LEFT JOIN
+			tags ON tags.id = releases_tags.tag_id
+		WHERE
+			releases.entity_id = entity.id
+		AND
+			CASE WHEN array_length(selectable_tags, 1) IS NOT NULL
+			THEN tags.slug = ANY(selectable_tags)
+			ELSE true
+			END
+		GROUP BY tags.id
+		ORDER BY tags.name;
+	$$ LANGUAGE SQL STABLE;
+`);
+
+exports.down = async (knex) => knex.raw(`
+	DROP FUNCTION IF EXISTS entities_scene_tags;
+`);

migrations/20220304233846_series.js

@@ -0,0 +1,215 @@
+const config = require('config');
+
+exports.up = async (knex) => Promise.resolve()
+	.then(() => knex.schema.createTable('series', (table) => {
+		table.increments('id', 16);
+
+		table.integer('entity_id', 12)
+			.references('id')
+			.inTable('entities')
+			.notNullable();
+
+		table.integer('studio_id', 12)
+			.references('id')
+			.inTable('entities');
+
+		table.text('entry_id');
+		table.unique(['entity_id', 'entry_id']);
+
+		table.text('url', 1000);
+		table.text('title');
+		table.text('slug');
+
+		table.timestamp('date');
+		table.index('date');
+
+		table.enum('date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
+			.defaultTo('day');
+
+		table.text('description');
+
+		table.boolean('deep');
+		table.text('deep_url', 1000);
+
+		table.text('comment');
+
+		table.integer('created_batch_id', 12)
+			.references('id')
+			.inTable('batches')
+			.onDelete('cascade');
+
+		table.integer('updated_batch_id', 12)
+			.references('id')
+			.inTable('batches')
+			.onDelete('cascade');
+
+		table.datetime('created_at')
+			.defaultTo(knex.fn.now());
+	}))
+	.then(() => knex.schema.createTable('series_scenes', (table) => {
+		table.integer('serie_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('series')
+			.onDelete('cascade');
+
+		table.integer('scene_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('releases')
+			.onDelete('cascade');
+
+		table.unique(['serie_id', 'scene_id']);
+
+		table.datetime('created_at')
+			.defaultTo(knex.fn.now());
+	}))
+	.then(() => knex.schema.createTable('series_trailers', (table) => {
+		table.integer('serie_id', 16)
+			.unique()
+			.notNullable()
+			.references('id')
+			.inTable('series')
+			.onDelete('cascade');
+
+		table.text('media_id', 21)
+			.notNullable()
+			.references('id')
+			.inTable('media');
+	}))
+	.then(() => knex.schema.createTable('series_posters', (table) => {
+		table.integer('serie_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('series')
+			.onDelete('cascade');
+
+		table.text('media_id', 21)
+			.notNullable()
+			.references('id')
+			.inTable('media')
+			.onDelete('cascade');
+
+		table.unique('serie_id');
+	}))
+	.then(() => knex.schema.createTable('series_covers', (table) => {
+		table.integer('serie_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('series')
+			.onDelete('cascade');
+
+		table.text('media_id', 21)
+			.notNullable()
+			.references('id')
+			.inTable('media');
+
+		table.unique(['serie_id', 'media_id']);
+	}))
+	.then(() => knex.schema.createTable('series_search', (table) => {
+		table.integer('serie_id', 16)
+			.references('id')
+			.inTable('series')
+			.onDelete('cascade');
+	}))
+	.then(() => knex.schema.createTable('stashes_series', (table) => {
+		table.integer('stash_id')
+			.notNullable()
+			.references('id')
+			.inTable('stashes')
+			.onDelete('cascade');
+
+		table.integer('serie_id')
+			.notNullable()
+			.references('id')
+			.inTable('series')
+			.onDelete('cascade');
+
+		table.unique(['stash_id', 'serie_id']);
+
+		table.string('comment');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	}))
+	.then(() => knex.raw(`
+		ALTER TABLE series_search ADD COLUMN document tsvector;
+
+		CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
+		CREATE INDEX series_search_index ON series_search USING GIN (document);
+
+		CREATE FUNCTION series_actors(serie series) RETURNS SETOF actors AS $$
+			SELECT actors.*
+			FROM series_scenes
+			LEFT JOIN
+				releases ON releases.id = series_scenes.scene_id
+			LEFT JOIN
+				releases_actors ON releases_actors.release_id = releases.id
+			LEFT JOIN
+				actors ON actors.id = releases_actors.actor_id
+			WHERE series_scenes.serie_id = serie.id
+			AND actors.id IS NOT NULL
+			GROUP BY actors.id
+			ORDER BY actors.name, actors.gender
+		$$ LANGUAGE SQL STABLE;
+
+		CREATE FUNCTION series_tags(serie series) RETURNS SETOF tags AS $$
+			SELECT tags.*
+			FROM series_scenes
+			LEFT JOIN
+				releases ON releases.id = series_scenes.scene_id
+			LEFT JOIN
+				releases_tags ON releases_tags.release_id = releases.id
+			LEFT JOIN
+				tags ON tags.id = releases_tags.tag_id
+			WHERE series_scenes.serie_id = serie.id
+			AND tags.id IS NOT NULL
+			GROUP BY tags.id
+			ORDER BY tags.priority DESC
+		$$ LANGUAGE SQL STABLE;
+
+		CREATE FUNCTION series_photos(serie series) RETURNS SETOF media AS $$
+			SELECT media.*
+			FROM series_scenes
+			LEFT JOIN
+				releases ON releases.id = series_scenes.scene_id
+			INNER JOIN
+				releases_photos ON releases_photos.release_id = releases.id
+			LEFT JOIN
+				media ON media.id = releases_photos.media_id
+			WHERE series_scenes.serie_id = serie.id
+			GROUP BY media.id
+			ORDER BY media.index ASC
+		$$ LANGUAGE SQL STABLE;
+
+		GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor;
+		GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO :visitor;
+
+		ALTER TABLE stashes_series ENABLE ROW LEVEL SECURITY;
+
+		CREATE POLICY stashes_policy ON stashes_series
+			USING (EXISTS (
+				SELECT *
+				FROM stashes
+				WHERE stashes.id = stashes_series.stash_id
+				AND (stashes.user_id = current_user_id() OR stashes.public)
+			));
+	`, {
+		visitor: knex.raw(config.database.query.user),
+	}));
+
+exports.down = async (knex) => Promise.resolve()
+	.then(() => knex.raw(`
+		DROP FUNCTION IF EXISTS series_actors;
+		DROP FUNCTION IF EXISTS series_tags;
+		DROP FUNCTION IF EXISTS series_photos;
+
+		DROP TABLE IF EXISTS stashes_series CASCADE;
+		DROP TABLE IF EXISTS series_scenes CASCADE;
+		DROP TABLE IF EXISTS series_trailers CASCADE;
+		DROP TABLE IF EXISTS series_posters CASCADE;
+		DROP TABLE IF EXISTS series_covers CASCADE;
+		DROP TABLE IF EXISTS series_search CASCADE;
+		DROP TABLE IF EXISTS series CASCADE;
+	`));

migrations/20220327230125_movie_photos.js

@@ -0,0 +1,49 @@
+const config = require('config');
+
+exports.up = async (knex) => Promise.resolve()
+	.then(() => knex.raw(`
+		ALTER FUNCTION movies_photos(movie movies) RENAME TO movies_scenes_photos;
+		ALTER FUNCTION series_photos(serie series) RENAME TO series_scenes_photos;
+	`))
+	.then(() => knex.schema.createTable('movies_photos', (table) => {
+		table.integer('movie_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('movies')
+			.onDelete('cascade');
+
+		table.text('media_id', 21)
+			.notNullable()
+			.references('id')
+			.inTable('media');
+
+		table.unique(['movie_id', 'media_id']);
+	}))
+	.then(() => knex.schema.createTable('series_photos', (table) => {
+		table.integer('serie_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('series')
+			.onDelete('cascade');
+
+		table.text('media_id', 21)
+			.notNullable()
+			.references('id')
+			.inTable('media');
+
+		table.unique(['serie_id', 'media_id']);
+	}))
+	.then(() => knex.raw(`
+		GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor;
+		GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO :visitor;
+	`, {
+		visitor: knex.raw(config.database.query.user),
+	}));
+
+exports.down = async (knex) => knex.raw(`
+	DROP TABLE IF EXISTS movies_photos CASCADE;
+	DROP TABLE IF EXISTS series_photos CASCADE;
+
+	ALTER FUNCTION movies_scenes_photos(movie movies) RENAME TO movies_photos;
+	ALTER FUNCTION series_scenes_photos(serie series) RENAME TO series_photos;
+`);

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.209.4",
+    "version": "1.212.9",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.209.4",
+            "version": "1.212.9",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",
@@ -11650,6 +11650,25 @@
                 "webidl-conversions": "^3.0.0"
             }
         },
+        "node_modules/node-fetch/node_modules/tr46": {
+            "version": "0.0.3",
+            "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+            "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+        },
+        "node_modules/node-fetch/node_modules/webidl-conversions": {
+            "version": "3.0.1",
+            "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+            "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+        },
+        "node_modules/node-fetch/node_modules/whatwg-url": {
+            "version": "5.0.0",
+            "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+            "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+            "dependencies": {
+                "tr46": "~0.0.3",
+                "webidl-conversions": "^3.0.0"
+            }
+        },
         "node_modules/node-gyp": {
             "version": "7.1.2",
             "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-7.1.2.tgz",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.209.4",
+    "version": "1.212.9",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

seeds/01_networks.js

@@ -225,6 +225,7 @@ const networks = [
 		description: 'DigitalPlayground.com is the leader in high quality adult blockbuster movies and award winning sex parodies that feature the most exclusive pornstars online! Adult Film Database of adult movies.',
 		parameters: {
 			actorPath: 'modelprofile',
+			forceDeep: true, // Digital Playground has movie and series information not available in the latest updates API
 		},
 		parent: 'mindgeek',
 	},
@@ -411,6 +412,9 @@ const networks = [
 		url: 'https://www.milehighmedia.com',
 		description: 'MileHighMedia.com is the only niche porn network you need! Watch lesbian sex, hardcore fucking and family porn stories with the hottest teens & MILFs!',
 		parent: 'mindgeek',
+		parameters: {
+			forceDeep: true, // Mile High Media has movie and series information not available in the latest updates API
+		},
 	},
 	{
 		slug: 'mofos',

seeds/02_sites.js

@@ -810,6 +810,13 @@ const sites = [
 		parameters: { siteId: 3261 },
 		parent: 'bang',
 	},
+	{
+		name: 'Bang! Podcast',
+		slug: 'bangpodcast',
+		url: 'https://www.bang.com/videos?in=bang!%20podcast',
+		parameters: { siteId: 6305 },
+		parent: 'bang',
+	},
 	// BANGBROS
 	{
 		name: 'Ass Parade',
@@ -2646,7 +2653,7 @@ const sites = [
 	{
 		slug: 'digitalplayground',
 		name: 'Digital Playground',
-		url: 'https://www.digitalplayground.com/scenes',
+		url: 'https://www.digitalplayground.com',
 		description: '',
 		parameters: { extract: true },
 		parent: 'digitalplayground',
@@ -2680,6 +2687,13 @@ const sites = [
 		description: '',
 		parent: 'digitalplayground',
 	},
+	{
+		slug: 'dpstarsexchallenges',
+		name: 'DP Star Sex Challenges',
+		url: 'https://www.digitalplayground.com/scenes?site=210',
+		parent: 'digitalplayground',
+		hasLogo: false,
+	},
 	{
 		slug: 'blockbuster',
 		name: 'Blockbuster',
@@ -5264,7 +5278,7 @@ const sites = [
 	{
 		slug: 'bigdicksatschool',
 		name: 'Big Dicks At School',
-		url: 'https://www.bigdicksatschool.com',
+		url: 'https://www.men.com/scenes?site=252',
 		description: '',
 		parameters: { siteId: 252 },
 		tags: ['gay'],
@@ -5273,7 +5287,7 @@ const sites = [
 	{
 		slug: 'drillmyhole',
 		name: 'Drill My Hole',
-		url: 'https://www.drillmyhole.com',
+		url: 'https://www.men.com/scenes?site=253',
 		description: '',
 		parameters: { siteId: 253 },
 		tags: ['gay'],
@@ -5282,7 +5296,7 @@ const sites = [
 	{
 		slug: 'str8togay',
 		name: 'Str8 to Gay',
-		url: 'https://www.str8togay.com',
+		url: 'https://www.men.com/scenes?site=254',
 		tags: ['gay'],
 		parameters: { siteId: 254 },
 		parent: 'men',
@@ -5290,7 +5304,7 @@ const sites = [
 	{
 		slug: 'thegayoffice',
 		name: 'The Gay Office',
-		url: 'https://www.thegayoffice.com',
+		url: 'https://www.men.com/scenes?site=255',
 		tags: ['gay'],
 		parameters: { siteId: 255 },
 		parent: 'men',
@@ -5298,7 +5312,7 @@ const sites = [
 	{
 		slug: 'jizzorgy',
 		name: 'Jizz Orgy',
-		url: 'https://www.jizzorgy.com',
+		url: 'https://www.men.com/scenes?site=256',
 		tags: ['gay'],
 		parameters: { siteId: 256 },
 		parent: 'men',
@@ -5306,7 +5320,7 @@ const sites = [
 	{
 		slug: 'menofuk',
 		name: 'Men of UK',
-		url: 'https://www.menofuk.com',
+		url: 'https://www.men.com/scenes?site=258',
 		tags: ['gay'],
 		parameters: { siteId: 258 },
 		parent: 'men',
@@ -5314,7 +5328,7 @@ const sites = [
 	{
 		slug: 'toptobottom',
 		name: 'Top to Bottom',
-		url: 'https://www.toptobottom.com',
+		url: 'https://www.men.com/scenes?site=259',
 		tags: ['gay'],
 		parameters: { siteId: 259 },
 		parent: 'men',
@@ -5322,7 +5336,7 @@ const sites = [
 	{
 		slug: 'godsofmen',
 		name: 'Gods of Men',
-		url: 'https://www.godsofmen.com',
+		url: 'https://www.men.com/scenes?site=260',
 		tags: ['gay'],
 		parameters: { siteId: 260 },
 		parent: 'men',
@@ -5413,7 +5427,10 @@ const sites = [
 		name: 'Doghouse Digital',
 		url: 'https://www.doghousedigital.com',
 		alias: ['dhd'],
-		parameters: { siteId: 321 },
+		parameters: {
+			siteId: 321,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5429,7 +5446,10 @@ const sites = [
 		name: 'Reality Junkies',
 		url: 'https://www.realityjunkies.com',
 		alias: ['rj'],
-		parameters: { siteId: 324 },
+		parameters: {
+			siteId: 324,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5437,7 +5457,10 @@ const sites = [
 		name: 'Sweetheart Video',
 		url: 'https://www.sweetheartvideo.com',
 		alias: ['shv'],
-		parameters: { siteId: 325 },
+		parameters: {
+			siteId: 325,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5445,7 +5468,10 @@ const sites = [
 		name: 'Sweet Sinner',
 		url: 'https://www.sweetsinner.com',
 		alias: ['ss'],
-		parameters: { siteId: 326 },
+		parameters: {
+			siteId: 326,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5454,7 +5480,10 @@ const sites = [
 		alias: ['fs'],
 		tags: ['family'],
 		url: 'https://www.familysinners.com',
-		parameters: { siteId: 317 },
+		parameters: {
+			siteId: 317,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5463,7 +5492,10 @@ const sites = [
 		url: 'https://www.iconmale.com',
 		alias: ['im'],
 		tags: ['gay'],
-		parameters: { native: true, siteId: 328 },
+		parameters: {
+			siteId: 328,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	// MOFOS
@@ -8579,7 +8611,10 @@ const sites = [
 		name: 'Dane Jones',
 		alias: ['dnj'],
 		url: 'https://www.danejones.com/',
-		parameters: { siteId: 290 },
+		parameters: {
+			siteId: 290,
+			native: true,
+		},
 		parent: 'sexyhub',
 	},
 	{
@@ -8587,7 +8622,10 @@ const sites = [
 		name: 'Lesbea',
 		alias: ['lsb'],
 		url: 'https://www.lesbea.com',
-		parameters: { siteId: 291 },
+		parameters: {
+			siteId: 291,
+			native: true,
+		},
 		tags: ['lesbian'],
 		parent: 'sexyhub',
 	},

seeds/04_media.js

@@ -727,8 +727,6 @@ const tagMedia = [
 	['da-tp', 7, 'Polly Petrova in YE069', 'analvids'],
 	['da-tp', 5, 'Venera Maxima in GIO1287'],
 	['da-tp', 6, 'Adriana Chechik in "Gangbang Me"', 'hardx'],
-	['da-tp', 0, 'Natasha Teen in SZ2164'],
-	['da-tp', 1, 'Francys Belle in SZ1702', 'analvids'],
 	['dap', 7, 'Adriana Chechik in "DP Masters 6"', 'julesjordan'],
 	['dap', 10, 'Kira Noir', 'hardx'],
 	['dap', 'emily_pink_legalporno', 'Emily Pink', 'analvids'],
@@ -1049,6 +1047,7 @@ const tagMedia = [
 	['toy-dp', 0, 'Marley Brinx, Ivy Lebelle and Lyra Law in "Marley Brinx First GGDP"', 'lesbianx'],
 	['toys', 1, 'Chloe Lamour in "Curives In All The Right Places"', 'wetandpuffy'],
 	['toys', 'shawna_lenee_sunrisekings', 'Shawna Lenee', 'sunrisekings'],
+	['triple-penetration', 'lucky_bee_analvids', 'Lucky Bee', 'analvids'],
 	['triple-penetration', 'angela_white_julesjordan', 'Angela White in "Her Biggest Gangbang Ever"', 'julesjordan'],
 	['triple-penetration', 'ria_sunn_legalporno', 'Ria Sunn in SZ2082', 'analvids'],
 	['tvp', 'september_reign_wefuckblackgirls', 'September Reign in "Second Appearance"', 'wefuckblackgirls'],

src/.eslintrc

@@ -14,6 +14,6 @@
         "prefer-destructuring": "off",
         "template-curly-spacing": "off",
         "object-curly-newline": "off",
-        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}],
+        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}]
     }
 }

src/actors.js

@@ -20,6 +20,7 @@ const scrapers = require('./scrapers/scrapers').actors;
 const argv = require('./argv');
 const include = require('./utils/argv-include')(argv);
 const bulkInsert = require('./utils/bulk-insert');
+const chunk = require('./utils/chunk');
 const logger = require('./logger')(__filename);
 
 const { toBaseReleases } = require('./deep');
@@ -1048,7 +1049,8 @@ async function flushProfiles(actorIdsOrNames) {
 	logger.info(`Removed ${deleteCount} profiles`);
 }
 
-async function deleteActors(actorIdsOrNames) {
+async function deleteActors(allActorIdsOrNames) {
+	const deleteCounts = await Promise.map(chunk(allActorIdsOrNames), async (actorIdsOrNames) => {
 		const actors = await knex('actors')
 			.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
 			.orWhere((builder) => {
@@ -1072,9 +1074,17 @@ async function deleteActors(actorIdsOrNames) {
 				.delete(),
 		]);
 
+		return { deletedScenesCount, deletedActorsCount };
+	}, { concurrency: 10 });
+
+	const deletedActorsCount = deleteCounts.reduce((acc, count) => acc + count.deletedActorsCount, 0);
+	const deletedScenesCount = deleteCounts.reduce((acc, count) => acc + count.deletedScenesCount, 0);
+
 	await flushOrphanedMedia();
 
 	logger.info(`Removed ${deletedActorsCount} actors with ${deletedScenesCount} scenes`);
+
+	return deletedActorsCount;
 }
 
 async function flushActors() {

src/app.js

@@ -85,6 +85,23 @@ async function startMemorySample(snapshotTriggers = []) {
 	}, config.memorySampling.sampleDuration);
 }
 
+async function startMemorySample() {
+	await inspector.heap.enable();
+	await inspector.heap.startSampling();
+
+	// monitorMemory();
+
+	logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
+
+	setTimeout(async () => {
+		await stopMemorySample();
+
+		if (!done) {
+			await startMemorySample();
+		}
+	}, 30000);
+}
+
 async function init() {
 	try {
 		if (argv.server) {

src/argv.js

@@ -238,6 +238,7 @@ const { argv } = yargs
 		default: false,
 	})
 	.option('level', {
+		alias: 'log-level',
 		describe: 'Log level',
 		type: 'string',
 		default: process.env.NODE_ENV === 'development' ? 'silly' : 'info',

src/entities.js

@@ -6,7 +6,7 @@ const inquirer = require('inquirer');
 const logger = require('./logger')(__filename);
 const argv = require('./argv');
 const knex = require('./knex');
-const { deleteScenes, deleteMovies } = require('./releases');
+const { deleteScenes, deleteMovies, deleteSeries } = require('./releases');
 const { flushOrphanedMedia } = require('./media');
 const { resolveScraper, resolveLayoutScraper } = require('./scrapers/resolve');
 
@@ -359,29 +359,39 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) {
 		.leftJoin('movies', 'movies.entity_id', 'selected_entities.id')
 		.pluck('movies.id');
 
-	if (sceneIds.length === 0 && movieIds.length === 0) {
-		logger.info(`No scenes or movies found to remove for ${entitySlugs}`);
+	const serieIds = await entityQuery
+		.clone()
+		.select('series.id')
+		.distinct('series.id')
+		.whereNotNull('series.id')
+		.from('selected_entities')
+		.leftJoin('series', 'series.entity_id', 'selected_entities.id')
+		.pluck('series.id');
+
+	if (sceneIds.length === 0 && movieIds.length === 0 && serieIds.length === 0) {
+		logger.info(`No scenes, movies or series found to remove for ${entitySlugs}`);
 		return;
 	}
 
 	const confirmed = await inquirer.prompt([{
 		type: 'confirm',
 		name: 'flushEntities',
-		message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies for ${entitySlugs}. Are you sure?`,
+		message: `You are about to remove ${sceneIds.length} scenes, ${movieIds.length} movies and ${serieIds.length} series for ${entitySlugs}. Are you sure?`,
 		default: false,
 	}]);
 
 	if (!confirmed.flushEntities) {
-		logger.warn(`Confirmation rejected, not flushing scenes or movies for: ${entitySlugs}`);
+		logger.warn(`Confirmation rejected, not flushing scenes, movies or series for: ${entitySlugs}`);
 		return;
 	}
 
-	const [deletedScenesCount, deletedMoviesCount] = await Promise.all([
+	const [deletedScenesCount, deletedMoviesCount, deletedSeriesCount] = await Promise.all([
 		deleteScenes(sceneIds),
 		deleteMovies(movieIds),
+		deleteSeries(serieIds),
 	]);
 
-	logger.info(`Removed ${deletedScenesCount} scenes and ${deletedMoviesCount} movies for ${entitySlugs}`);
+	logger.info(`Removed ${deletedScenesCount} scenes, ${deletedMoviesCount} movies and ${deletedSeriesCount} series for ${entitySlugs}`);
 
 	await flushOrphanedMedia();
 }

src/media.js

@@ -961,9 +961,12 @@ async function flushOrphanedMedia() {
 		await deleteS3Objects(orphanedMedia.filter((media) => media.is_s3));
 	}
 
+	try {
 		await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true });
-
 		logger.info('Cleared temporary media directory');
+	} catch (error) {
+		logger.warn(`Failed to clear temporary media directory: ${error.message}`);
+	}
 }
 
 module.exports = {

src/releases.js

@@ -325,6 +325,24 @@ async function deleteMovies(movieIds) {
 	return deleteCount;
 }
 
+async function deleteSeries(serieIds) {
+	if (serieIds.length === 0) {
+		return 0;
+	}
+
+	await knex('series_scenes')
+		.whereIn('serie_id', serieIds)
+		.delete();
+
+	const deleteCount = await knex('series')
+		.whereIn('id', serieIds)
+		.delete();
+
+	logger.info(`Removed ${deleteCount}/${serieIds.length} series`);
+
+	return deleteCount;
+}
+
 async function flushScenes() {
 	const sceneIds = await knex('releases').select('id').pluck('id');
 
@@ -367,6 +385,27 @@ async function flushMovies() {
 	logger.info(`Removed ${deleteCount}/${movieIds.length} movies`);
 }
 
+async function flushSeries() {
+	const serieIds = await knex('series').select('id').pluck('id');
+
+	const confirmed = await inquirer.prompt([{
+		type: 'confirm',
+		name: 'flushSeries',
+		message: `You are about to remove ${serieIds.length} series. Are you sure?`,
+		default: false,
+	}]);
+
+	if (!confirmed.flushSeries) {
+		logger.warn('Confirmation rejected, not flushing series');
+		return;
+	}
+	const deleteCount = await deleteSeries(serieIds);
+
+	await flushOrphanedMedia();
+
+	logger.info(`Removed ${deleteCount}/${serieIds.length} series`);
+}
+
 async function flushBatches(batchIds) {
 	const [sceneIds, movieIds] = await Promise.all([
 		knex('releases')
@@ -407,8 +446,10 @@ module.exports = {
 	fetchScenes,
 	flushBatches,
 	flushMovies,
+	flushSeries,
 	flushScenes,
 	searchScenes,
 	deleteScenes,
 	deleteMovies,
+	deleteSeries,
 };

src/scrapers/mindgeek.js

@@ -11,6 +11,12 @@ const slugify = require('../utils/slugify');
 const http = require('../utils/http');
 const { inchesToCm, lbsToKg } = require('../utils/convert');
 
+function getBasePath(channel, path = '/scene') {
+	return channel.parameters?.scene
+		|| ((channel.parameters?.native || channel.type === 'network') && `${channel.url}${path}`)
+		|| `${channel.parent.url}${path}`;
+}
+
 function getThumbs(scene) {
 	if (scene.images.poster) {
 		return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
@@ -18,7 +24,7 @@ function getThumbs(scene) {
 			.map((image) => image.xl.url);
 	}
 
-	if (scene.images.card_main_rect) {
+	if (Array.isArray(scene.images.card_main_rect)) {
 		return scene.images.card_main_rect
 			.concat(scene.images.card_secondary_rect || [])
 			.map((image) => image.xl.url.replace('.thumb', ''));
@@ -27,6 +33,28 @@ function getThumbs(scene) {
 	return [];
 }
 
+function getCovers(images, target = 'cover') {
+	if (!images[target]) {
+		return [];
+	}
+
+	const covers = [
+		images[target][0].md?.url,
+		images[target][0].sm?.url,
+		images[target][0].xs?.url,
+		// bigger but usually upscaled
+		images[target][0].xx?.url,
+		images[target][0].xl?.url,
+		images[target][0].lg?.url,
+	];
+
+	if (target === 'poster') {
+		return covers;
+	}
+
+	return [covers];
+}
+
 function getVideos(data) {
 	const teaserSources = data.videos.mediabook?.files;
 	const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
@@ -51,9 +79,7 @@ function scrapeLatestX(data, site, filterChannel) {
 		description: data.description,
 	};
 
-	const basepath = site.parameters?.scene
-		|| (site.parameters?.native && `${site.url}/scene`)
-		|| `${site.parent.url}/scene`;
+	const basepath = getBasePath(site);
 
 	release.url = `${basepath}/${release.entryId}/${slugify(release.title)}`;
 	release.date = new Date(data.dateReleased);
@@ -84,6 +110,9 @@ function scrapeLatestX(data, site, filterChannel) {
 		};
 	}
 
+	const siteName = data.collections[0]?.name || data.brand;
+	release.channel = slugify(siteName, '');
+
 	return release;
 }
 
@@ -96,7 +125,7 @@ async function scrapeLatest(items, site, filterChannel) {
 	};
 }
 
-function scrapeScene(data, url, _site, networkName) {
+function scrapeRelease(data, url, channel, networkName) {
 	const release = {};
 
 	const { id: entryId, title, description } = data;
@@ -129,6 +158,29 @@ function scrapeScene(data, url, _site, networkName) {
 
 	release.url = url || `https://www.${networkName || data.brand}.com/scene/${entryId}/`;
 
+	if (data.parent?.type === 'movie' || data.parent?.type === 'serie') {
+		release[data.parent.type] = {
+			entryId: data.parent.id,
+			url: `${getBasePath(channel, data.parent.type === 'movie' ? '/movie' : '/series')}/${data.parent.id}/${slugify(data.parent.title, '-', { removePunctuation: true })}`,
+			title: data.parent.title,
+			description: data.parent.description,
+			date: new Date(data.parent.dateReleased),
+			channel: slugify(data.parent.collections?.name || data.parent.brand),
+			poster: getCovers(data.parent.images, 'poster'),
+			shallow: true,
+		};
+	}
+
+	if (data.type === 'movie') {
+		release.covers = getCovers(data.images);
+		release.scenes = data.children?.map((scene) => ({
+			entryId: scene.id,
+			url: `${getBasePath(channel)}/${scene.id}/${slugify(scene.title)}`,
+			title: scene.title,
+			shallow: true,
+		}));
+	}
+
 	return release;
 }
 
@@ -170,6 +222,9 @@ async function getSession(site, parameters, url) {
 
 	const res = await http.get(sessionUrl, {
 		session,
+		headers: {
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
+		},
 		interval: parameters?.interval,
 		concurrency: parameters?.concurrency,
 		parse: false,
@@ -230,7 +285,7 @@ function scrapeProfile(data, html, releases = [], networkName) {
 		profile.naturalBoobs = false;
 	}
 
-	profile.releases = releases.map((release) => scrapeScene(release, null, null, networkName));
+	profile.releases = releases.map((release) => scrapeRelease(release, null, null, networkName));
 
 	return profile;
 }
@@ -258,6 +313,7 @@ async function fetchLatest(site, page = 1, options) {
 			Instance: instanceToken,
 			Origin: site.url,
 			Referer: url,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
 		},
 	});
 
@@ -282,6 +338,7 @@ async function fetchUpcoming(site, page, options) {
 			Instance: instanceToken,
 			Origin: site.url,
 			Referer: url,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
 		},
 	});
 
@@ -292,8 +349,8 @@ async function fetchUpcoming(site, page, options) {
 	return res.statusCode;
 }
 
-async function fetchScene(url, site, baseScene, options) {
-	if (baseScene?.entryId) {
+async function fetchRelease(url, site, baseScene, options) {
+	if (baseScene?.entryId && !baseScene.shallow && !options.parameters.forceDeep) {
 		// overview and deep data is the same, don't hit server unnecessarily
 		return baseScene;
 	}
@@ -307,12 +364,13 @@ async function fetchScene(url, site, baseScene, options) {
 		concurrency: options.parameters.concurrency,
 		headers: {
 			Instance: instanceToken,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
 		},
 	});
 
 	if (res.status === 200 && res.body.result) {
 		return {
-			scene: scrapeScene(res.body.result, url, site),
+			scene: scrapeRelease(res.body.result, url, site),
 		};
 	}
 
@@ -329,6 +387,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
 		concurrency: parameters.concurrency,
 		headers: {
 			Instance: instanceToken,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
 		},
 	});
 
@@ -374,6 +433,7 @@ module.exports = {
 	scrapeLatestX,
 	fetchLatest,
 	fetchUpcoming,
-	fetchScene,
+	fetchScene: fetchRelease,
+	fetchMovie: fetchRelease,
 	fetchProfile,
 };

src/scrapers/vixen.js

@@ -142,6 +142,7 @@ async function getTrailer(scene, channel, url) {
 	return null;
 }
 
+/*
 async function getPhotosLegacy(url) {
 	const htmlRes = await http.get(url, {
 		extract: {
@@ -169,6 +170,7 @@ async function getPhotosLegacy(url) {
 		return [];
 	}
 }
+*/
 
 async function getPhotos(url) {
 	const htmlRes = await http.get(url, {

src/store-releases.js

@@ -146,7 +146,7 @@ function attachReleaseIds(releases, storedReleases, batchId) {
 
 	const releasesWithId = releases.map((release) => {
 		if (!release.entity) {
-			logger.error(`No entitity available for ${release.url}`);
+			logger.error(`No entity available for ${release.url}`);
 			return null;
 		}
 
@@ -315,6 +315,142 @@ async function storeChapters(releases) {
 	await associateReleaseMedia(chaptersWithId, 'chapter');
 }
 
+async function associateMovieScenes(movies, movieScenes) {
+	const moviesByEntityIdAndEntryId = movies.reduce((acc, movie) => ({
+		...acc,
+		[movie.entity.id]: {
+			...acc[movie.entity.id],
+			[movie.entryId]: movie,
+		},
+	}), {});
+
+	const associations = movieScenes.map((scene) => {
+		if (!scene.movie) {
+			return null;
+		}
+
+		const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]
+			|| moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId];
+
+		if (sceneMovie?.id) {
+			return {
+				movie_id: sceneMovie.id,
+				scene_id: scene.id,
+			};
+		}
+
+		return null;
+	}).filter(Boolean);
+
+	await bulkInsert('movies_scenes', associations, false);
+}
+
+async function associateSerieScenes(series, serieScenes) {
+	const seriesByEntityIdAndEntryId = series.reduce((acc, serie) => ({
+		...acc,
+		[serie.entity.id]: {
+			...acc[serie.entity.id],
+			[serie.entryId]: serie,
+		},
+	}), {});
+
+	const associations = serieScenes.map((scene) => {
+		if (!scene.serie) {
+			return null;
+		}
+
+		const sceneSerie = seriesByEntityIdAndEntryId[scene.entity.id]?.[scene.serie.entryId]
+			|| seriesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.serie.entryId];
+
+		if (sceneSerie?.id) {
+			return {
+				serie_id: sceneSerie.id,
+				scene_id: scene.id,
+			};
+		}
+
+		return null;
+	}).filter(Boolean);
+
+	await bulkInsert('series_scenes', associations, false);
+}
+
+async function updateMovieSearch(movieIds, target = 'movie') {
+	logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
+
+	const documents = await knex.raw(`
+		SELECT
+			${target}s.id AS ${target}_id,
+				TO_TSVECTOR(
+				'english',
+					COALESCE(${target}s.title, '') || ' ' ||
+					entities.name || ' ' ||
+					entities.slug || ' ' ||
+					COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
+					COALESCE(parents.name, '') || ' ' ||
+					COALESCE(parents.slug, '') || ' ' ||
+					COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
+					COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
+					STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
+					STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
+					STRING_AGG(COALESCE(tags.name, ''), ' ')
+			) as document
+		FROM ${target}s
+		LEFT JOIN entities ON ${target}s.entity_id = entities.id
+		LEFT JOIN entities AS parents ON parents.id = entities.parent_id
+		LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
+		LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
+		LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
+		LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
+		LEFT JOIN actors ON actors.id = releases_actors.actor_id
+		LEFT JOIN tags ON tags.id = releases_tags.tag_id
+        ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
+		GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
+    `, movieIds && [movieIds]);
+
+	if (documents.rows?.length > 0) {
+		await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
+	}
+}
+
+async function storeMovies(movies, useBatchId) {
+	if (!movies || movies.length === 0) {
+		return [];
+	}
+
+	const { uniqueReleases } = await filterDuplicateReleases(movies);
+	const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
+
+	const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
+
+	const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
+	const moviesWithId = attachReleaseIds(movies, storedMovies);
+
+	await updateMovieSearch(moviesWithId.map((movie) => movie.id));
+	await associateReleaseMedia(moviesWithId, 'movie');
+
+	return moviesWithId;
+}
+
+async function storeSeries(series, useBatchId) {
+	if (!series || series.length === 0) {
+		return [];
+	}
+
+	const { uniqueReleases } = await filterDuplicateReleases(series);
+	const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
+
+	const curatedSerieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'serie')));
+
+	const storedSeries = await bulkInsert('series', curatedSerieEntries, ['entity_id', 'entry_id'], true);
+	const seriesWithId = attachReleaseIds(series, storedSeries);
+
+	await updateMovieSearch(seriesWithId.map((serie) => serie.id), 'serie');
+	await associateReleaseMedia(seriesWithId, 'serie');
+
+	return seriesWithId;
+}
+
 async function storeScenes(releases, useBatchId) {
 	if (!releases || releases.length === 0) {
 		return [];
@@ -355,12 +491,14 @@ async function storeScenes(releases, useBatchId) {
 		scenes: JSON.stringify(duplicateReleasesWithId),
 	});
 
-	const [actors] = await Promise.all([
+	const [actors, storedSeries] = await Promise.all([
 		associateActors(releasesWithId, batchId),
+		storeSeries(releasesWithId.map((release) => release.serie && { ...release.serie, entity: release.entity }).filter(Boolean), batchId),
 		associateReleaseTags(releasesWithId),
 		storeChapters(releasesWithId),
 	]);
 
+	await associateSerieScenes(storedSeries, releasesWithId);
 	await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
 	await updateSceneSearch(releasesWithId.map((release) => release.id));
 
@@ -378,92 +516,6 @@ async function storeScenes(releases, useBatchId) {
 	return releasesWithId;
 }
 
-async function associateMovieScenes(movies, movieScenes) {
-	const moviesByEntityIdAndEntryId = movies.reduce((acc, movie) => ({
-		...acc,
-		[movie.entity.id]: {
-			...acc[movie.entity.id],
-			[movie.entryId]: movie,
-		},
-	}), {});
-
-	const associations = movieScenes.map((scene) => {
-		if (!scene.movie) {
-			return null;
-		}
-
-		const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId];
-
-		if (sceneMovie?.id) {
-			return {
-				movie_id: sceneMovie.id,
-				scene_id: scene.id,
-			};
-		}
-
-		return null;
-	}).filter(Boolean);
-
-	await bulkInsert('movies_scenes', associations, false);
-}
-
-async function updateMovieSearch(movieIds) {
-	logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } movies`);
-
-	const documents = await knex.raw(`
-		SELECT
-			movies.id AS movie_id,
-				TO_TSVECTOR(
-				'english',
-					COALESCE(movies.title, '') || ' ' ||
-					entities.name || ' ' ||
-					entities.slug || ' ' ||
-					COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
-					COALESCE(parents.name, '') || ' ' ||
-					COALESCE(parents.slug, '') || ' ' ||
-					COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
-					COALESCE(TO_CHAR(movies.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
-					STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
-					STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
-					STRING_AGG(COALESCE(tags.name, ''), ' ')
-			) as document
-		FROM movies
-		LEFT JOIN entities ON movies.entity_id = entities.id
-		LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-		LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
-		LEFT JOIN releases ON releases.id = movies_scenes.scene_id
-		LEFT JOIN releases_actors ON releases_actors.release_id = movies_scenes.scene_id
-		LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
-		LEFT JOIN actors ON actors.id = releases_actors.actor_id
-		LEFT JOIN tags ON tags.id = releases_tags.tag_id
-        ${movieIds ? 'WHERE movies.id = ANY(?)' : ''}
-		GROUP BY movies.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
-    `, movieIds && [movieIds]);
-
-	if (documents.rows?.length > 0) {
-		await bulkInsert('movies_search', documents.rows, ['movie_id']);
-	}
-}
-
-async function storeMovies(movies, useBatchId) {
-	if (!movies || movies.length === 0) {
-		return [];
-	}
-
-	const { uniqueReleases } = await filterDuplicateReleases(movies);
-	const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
-
-	const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
-
-	const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
-	const moviesWithId = attachReleaseIds(movies, storedMovies);
-
-	await updateMovieSearch(moviesWithId.map((movie) => movie.id));
-	await associateReleaseMedia(moviesWithId, 'movie');
-
-	return moviesWithId;
-}
-
 module.exports = {
 	associateMovieScenes,
 	storeScenes,

src/tools/knex-update.js

@@ -0,0 +1,18 @@
+'use strict';
+
+const knex = require('../knex');
+
+async function update() {
+	const query = knex('bans')
+		.update('type', {
+			type: 'mute',
+			username_original: 'charles',
+		})
+		.where('id', 2754);
+
+	console.log(query.toSQL());
+
+	await query;
+}
+
+update();