Refactored clips into chapters.

src/media.js

@@ -912,8 +912,8 @@ async function flushOrphanedMedia() {
 							knex('actors').select(knex.raw('avatar_media_id as media_id')),
 							knex('actors_profiles').select(knex.raw('avatar_media_id as media_id')),
 							knex('actors_photos').select('media_id'),
-							knex('clips_photos').select('media_id'),
-							knex('clips_posters').select('media_id'),
+							knex('chapters_photos').select('media_id'),
+							knex('chapters_posters').select('media_id'),
 						)
 						.as('associations'),
 				)

src/releases.js

@@ -45,6 +45,14 @@ function curateRelease(release, withMedia = false, withPoster = true) {
 			name: tag.name,
 			slug: tag.slug,
 		})),
+		chapters: (release.chapters || []).map(chapter => ({
+			id: chapter.id,
+			index: chapter.index,
+			time: chapter.time,
+			duration: chapter.duration,
+			title: chapter.title,
+			description: chapter.description,
+		})),
 		...((withMedia || withPoster) && {
 			poster: release.poster ? {
 				id: release.poster.id,
@@ -80,7 +88,8 @@ function withRelations(queryBuilder, withMedia = false, withPoster = true) {
 			row_to_json(entities) as entity,
 			row_to_json(parents) as parent,
 			COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
+			COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
+			COALESCE(json_agg(DISTINCT chapters) FILTER (WHERE chapters.id IS NOT NULL), '[]') as chapters
 		`))
 		.leftJoin('entities', 'entities.id', 'releases.entity_id')
 		.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
@@ -88,6 +97,7 @@ function withRelations(queryBuilder, withMedia = false, withPoster = true) {
 		.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
 		.leftJoin('releases_tags', 'releases_tags.release_id', 'releases.id')
 		.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
+		.leftJoin('chapters', 'chapters.release_id', 'releases.id')
 		.groupBy(knex.raw(`
 			releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
 			entities.id, parents.id
@@ -166,10 +176,6 @@ async function deleteScenes(sceneIds) {
 	}
 
 	// there can be too many scene IDs for where in, causing a stack depth error
-	await knex('movies_scenes')
-		.whereRaw('scene_id = ANY(:sceneIds)', { sceneIds })
-		.delete();
-
 	const deleteCount = await knex('releases')
 		.whereRaw('id = ANY(:sceneIds)', { sceneIds })
 		.delete();

src/scrapers/inthecrack.js

@@ -172,7 +172,7 @@ async function scrapeScene({ query, html }, url, channel) {
 
 	release.poster = qu.prefixUrl(html.match(/background-image: url\('(.*)'\)/)?.[1], channel.url);
 
-	release.clips = query.all('.ClipOuter').map((el) => {
+	release.chapters = query.all('.ClipOuter').map((el) => {
 		const chapter = {};
 
 		chapter.title = query.text(el, 'h4');

src/scrapers/vixen.js

@@ -213,6 +213,11 @@ async function scrapeScene(data, url, site, baseRelease) {
 	const trailer = await getTrailer(scene, site, url);
 	if (trailer) release.trailer = trailer;
 
+	release.chapters = data.video.chapters?.video.map(chapter => ({
+		tags: [chapter.title],
+		time: chapter.seconds,
+	}));
+
 	return release;
 }
 

src/store-releases.js

@@ -249,44 +249,47 @@ async function updateReleasesSearch(releaseIds) {
 	}
 }
 
-async function storeClips(releases) {
-	const clips = releases.map(release => release.clips?.map((clip, index) => ({
-		title: clip.title,
-		description: clip.description,
+async function storeChapters(releases) {
+	const chapters = releases.map(release => release.chapters?.map((chapter, index) => ({
 		releaseId: release.id,
-		clip: index + 1,
-		duration: clip.duration,
-		poster: clip.poster,
-		photos: clip.photos,
-		tags: clip.tags,
+		index: index + 1,
+		time: chapter.time,
+		duration: chapter.duration,
+		title: chapter.title,
+		description: chapter.description,
+		poster: chapter.poster,
+		photos: chapter.photos,
+		tags: chapter.tags,
 	}))).flat().filter(Boolean);
 
-	const curatedClipEntries = clips.map(clip => ({
-		title: clip.title,
-		description: clip.description,
-		duration: clip.duration,
-		release_id: clip.releaseId,
-		clip: clip.clip,
+	const curatedChapterEntries = chapters.map(chapter => ({
+		index: chapter.index,
+		time: chapter.time,
+		duration: chapter.duration,
+		title: chapter.title,
+		description: chapter.description,
+		release_id: chapter.releaseId,
 	}));
 
-	const storedClips = await bulkInsert('clips', curatedClipEntries, ['release_id', 'clip']);
-	const clipIdsByReleaseIdAndClip = storedClips.reduce((acc, clip) => ({
+	const storedChapters = await bulkInsert('chapters', curatedChapterEntries, ['release_id', 'index']);
+
+	const chapterIdsByReleaseIdAndChapter = storedChapters.reduce((acc, chapter) => ({
 		...acc,
-		[clip.release_id]: {
-			...acc[clip.release_id],
-			[clip.clip]: clip.id,
+		[chapter.release_id]: {
+			...acc[chapter.release_id],
+			[chapter.index]: chapter.id,
 		},
 	}), {});
 
-	const clipsWithId = clips.map(clip => ({
-		...clip,
-		id: clipIdsByReleaseIdAndClip[clip.releaseId][clip.clip],
+	const chaptersWithId = chapters.map(chapter => ({
+		...chapter,
+		id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.index],
 	}));
 
-	await associateReleaseTags(clipsWithId, 'clip');
+	await associateReleaseTags(chaptersWithId, 'chapter');
 
 	// media is more error-prone, associate separately
-	await associateReleaseMedia(clipsWithId, 'clip');
+	await associateReleaseMedia(chaptersWithId, 'chapter');
 }
 
 async function storeScenes(releases) {
@@ -313,7 +316,7 @@ async function storeScenes(releases) {
 	const [actors] = await Promise.all([
 		associateActors(releasesWithId, batchId),
 		associateReleaseTags(releasesWithId),
-		storeClips(releasesWithId),
+		storeChapters(releasesWithId),
 	]);
 
 	await updateReleasesSearch(releasesWithId.map(release => release.id));

src/utils/cf.js

@@ -0,0 +1,153 @@
+'use strict';
+
+const arrayEqual = require('array-equal');
+
+/* eslint-disable */
+function evaluateBranch(tree, modifiers) {
+	const result = tree.map((expression) => {
+		if (expression.type === 'group') {
+			return evaluateBranch(expression.values, expression.modifiers);
+		} if (expression.modifiers.length === 0) {
+			return ''; // This is a trigger to stringify the previous values
+		} if (arrayEqual(['plus'], expression.modifiers)) {
+			return 0;
+		} if (arrayEqual(['negate', 'negate'], expression.modifiers)) {
+			return true;
+		} if (arrayEqual(['negate', 'plus'], expression.modifiers)) {
+			return true;
+		} if (arrayEqual(['plus', 'plus', 'negate'], expression.modifiers)) {
+			return true;
+		} if (arrayEqual(['plus', 'negate', 'negate'], expression.modifiers)) {
+			return 1;
+		}
+		throw new Error(`Found unrecognized modifier pattern: ${expression.modifiers}`);
+	}).reduce((combined, value) => {
+		if (value === '') {
+			return combined.toString();
+		}
+		if (value === true) {
+			value = 1;
+		}
+
+		if (typeof combined === 'string') {
+			return combined + value.toString();
+		}
+		return combined + value;
+	}, 0);
+
+	if (modifiers == null) {
+		return result;
+	}
+	if (arrayEqual(['plus'], modifiers)) {
+		return parseInt(result);
+	}
+	return result;
+}
+
+function evaluate(tree) {
+	return evaluateBranch(tree);
+}
+
+function parse(string) {
+	const length = string.length;
+	let byte;
+
+	let stateFinishedItem = false;
+	const modifierStack = [[]];
+	const itemStack = [[]];
+	let currentStack = itemStack[0];
+	let currentModifiers = modifierStack[0];
+	let stackLevel = 0;
+
+	for (let pos = 0; pos < length; pos++) {
+		byte = string[pos];
+
+		switch (byte) {
+		case '+':
+			if (pos === 0 || stateFinishedItem === false) {
+				// Modifier / number-cast
+				currentModifiers.push('plus');
+				stateFinishedItem = false;
+			} else {
+				// Addition, we don't need to do anything here
+			}
+			break;
+		case '!':
+			stateFinishedItem = false;
+			currentModifiers.push('negate');
+			break;
+		case '(':
+			stateFinishedItem = false;
+			stackLevel++;
+			itemStack[stackLevel] = currentStack = [];
+			modifierStack[stackLevel] = currentModifiers = [];
+			break;
+		case ')':
+			if (stackLevel === 0) {
+				throw new Error('Encountered ) without matching (');
+			}
+
+			stackLevel--;
+			stateFinishedItem = true;
+
+			currentStack = itemStack[stackLevel];
+			currentStack.push({
+				type: 'group',
+				values: itemStack[stackLevel + 1],
+				modifiers: modifierStack[stackLevel],
+			});
+
+			currentModifiers = modifierStack[stackLevel] = [];
+			break;
+		case '[':
+			if (string[pos + 1] === ']') {
+				// Reached the brackets; end of the modifier sequence
+				currentStack.push({
+					type: 'brackets',
+					modifiers: currentModifiers,
+				});
+
+				currentModifiers = [];
+				pos += 1; // Skip over the closing bracket
+				stateFinishedItem = true;
+			} else {
+				throw new Error(`Invalid byte found; expected ] but got ${string[pos + 1]}`);
+			}
+		}
+	}
+
+	return itemStack[0];
+}
+
+function parseExpression(string) {
+	return evaluate(parse(string));
+}
+
+function findAll(regex, target) {
+	const results = []; let
+		match;
+
+	while (match = regex.exec(target)) {
+		results.push(match);
+	}
+
+	return results;
+}
+
+function cf(testcase) {
+	let [_, parent, child, initialExpression] = /var s,t,o,p,b,r,e,a,k,i,n,g,f,\s*([a-zA-Z]+)={"([a-zA-Z]+)":([^}]+)};/.exec(testcase);
+
+	const modifyingExpressions = findAll(new RegExp(`${parent}\.${child}\s*([*+-])=\s*([^;]+)`, 'g'), testcase).map(match => ({
+		operation: match[1],
+		expression: match[2],
+	})).map(({ operation, expression }) => ({
+		operation,
+		expression: parseExpression(expression),
+	}));
+
+	initialExpression = parseExpression(initialExpression);
+
+	return { parent, child, initialExpression, modifyingExpressions };
+};
+
+module.exports = cf;