Refactored alerts to use application code, added regex. Updated Jules Jordan for the Ass Factory relaunch.

2023-11-24 01:29:22 +01:00
parent 124ff3f5e3
commit 238dce78b5
79 changed files with 466 additions and 155 deletions
--- a/src/alerts.js
+++ b/src/alerts.js
@@ -1,5 +1,7 @@
 'use strict';

+const escapeRegexp = require('escape-string-regexp');
+
 const knex = require('./knex');
 const bulkInsert = require('./utils/bulk-insert');
 const { HttpError } = require('./errors');
@@ -9,10 +11,14 @@ async function addAlert(alert, sessionUser) {
 		throw new HttpError('You are not authenthicated', 401);
 	}

-	if (!alert.actors?.length > 0 && !alert.tags?.length > 0 && !alert.entity) {
+	if (!alert.actors?.length > 0 && !alert.tags?.length > 0 && !alert.entity && !alert.matches?.length > 0) {
 		throw new HttpError('Alert must contain at least one actor, tag or entity', 400);
 	}

+	if (alert.matches?.some((match) => !match.property || !match.expression)) {
+		throw new HttpError('Match must define a property and an expression', 400);
+	}
+
 	const [alertId] = await knex('alerts')
 		.insert({
 			user_id: sessionUser.id,
@@ -30,6 +36,11 @@ async function addAlert(alert, sessionUser) {
 				alert_id: alertId,
 				tag_id: tagId,
 			})), false),
+		alert.matches?.length > 0 && bulkInsert('alerts_matches', alert.matches.map((match) => ({
+				alert_id: alertId,
+				property: match.property,
+				expression: match.expression,
+			})), false),
 		alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map((stashId) => ({
 				alert_id: alertId,
 				stash_id: stashId,
@@ -48,88 +59,153 @@ async function removeAlert(alertId) {
 }

 async function notify(scenes) {
-	const releases = await knex.raw(`
-		SELECT alerts.id as alert_id, alerts.notify, alerts.email, releases.id as scene_id, users.id as user_id, COALESCE(json_agg(alerts_stashes.stash_id) FILTER (WHERE alerts_stashes.stash_id IS NOT NULL), '[]') as stashes
-		FROM releases
-		CROSS JOIN alerts
-		LEFT JOIN users ON users.id = alerts.user_id
-		LEFT JOIN alerts_stashes ON alerts_stashes.alert_id = alerts.id
-		/* match updated IDs from input */
-		WHERE (releases.id = ANY(:sceneIds))
-		/* match tags */
-		AND (NOT EXISTS (SELECT alerts_tags.alert_id
-			FROM alerts_tags
-			WHERE alerts_tags.alert_id = alerts.id)
-		OR (SELECT array_agg(releases_tags.tag_id)
-			FROM releases_tags
-			WHERE releases_tags.release_id = releases.id
-			GROUP BY releases_tags.release_id)
-		@> (SELECT array_agg(alerts_tags.tag_id)
-			FROM alerts_tags
-			WHERE alerts_tags.alert_id = alerts.id
-			GROUP BY alerts_tags.alert_id))
-		/* match actors */
-		AND (NOT EXISTS (SELECT alerts_actors.alert_id
-			FROM alerts_actors
-			WHERE alerts_actors.alert_id = alerts.id)
-		OR (SELECT array_agg(releases_actors.actor_id)
-			FROM releases_actors
-			WHERE releases_actors.release_id = releases.id
-			GROUP BY releases_actors.release_id)
-		@> (SELECT array_agg(alerts_actors.actor_id)
-			FROM alerts_actors
-			WHERE alerts_actors.alert_id = alerts.id
-			GROUP BY alerts_actors.alert_id))
-		/* match entity */
-		AND ((NOT EXISTS (SELECT alerts_entities.entity_id
-			FROM alerts_entities
-			WHERE alerts_entities.alert_id = alerts.id))
-		OR (releases.entity_id
-		= ANY(array(
-			/* include children of entities */
-			WITH RECURSIVE included AS (
-				SELECT entities.*
-				FROM alerts_entities
-				LEFT JOIN entities ON entities.id = alerts_entities.entity_id
-				WHERE alerts_entities.alert_id = alerts.id
+	const sceneIds = scenes.map((scene) => scene.id);

-				UNION ALL
+	const [
+		releasesActors,
+		releasesTags,
+		rawAlerts,
+		alertsActors,
+		alertsTags,
+		alertsEntities,
+		alertsMatches,
+		alertsStashes,
+	] = await Promise.all([
+		knex('releases_actors').whereIn('release_id', sceneIds),
+		knex('releases_tags').whereIn('release_id', sceneIds),
+		knex('alerts'),
+		knex('alerts_actors'),
+		knex('alerts_tags'),
+		knex('alerts_entities'),
+		knex('alerts_matches'),
+		knex('alerts_stashes'),
+	]);

-				SELECT entities.*
-				FROM entities
-				INNER JOIN included ON included.id = entities.parent_id
-			)
+	const actorIdsByReleaseId = releasesActors.reduce((acc, releaseActor) => {
+		if (!acc[releaseActor.release_id]) {
+			acc[releaseActor.release_id] = [];
+		}

-			SELECT included.id
-			FROM included
-			GROUP BY included.id
-		))))
-		GROUP BY releases.id, users.id, alerts.id;
-	`, {
-		sceneIds: scenes.map((scene) => scene.id),
+		acc[releaseActor.release_id].push(releaseActor.actor_id);
+
+		return acc;
+	}, {});
+
+	const tagIdsByReleaseId = releasesTags.reduce((acc, releaseTag) => {
+		if (!acc[releaseTag.release_id]) {
+			acc[releaseTag.release_id] = [];
+		}
+
+		acc[releaseTag.release_id].push(releaseTag.tag_id);
+
+		return acc;
+	}, {});
+
+	const alertsActorsByAlertId = alertsActors.reduce((acc, alertActor) => { if (!acc[alertActor.alert_id]) { acc[alertActor.alert_id] = []; } acc[alertActor.alert_id].push(alertActor.actor_id); return acc; }, {});
+	const alertsTagsByAlertId = alertsTags.reduce((acc, alertTag) => { if (!acc[alertTag.alert_id]) { acc[alertTag.alert_id] = []; } acc[alertTag.alert_id].push(alertTag.tag_id); return acc; }, {});
+	const alertsEntitiesByAlertId = alertsEntities.reduce((acc, alertEntity) => { if (!acc[alertEntity.alert_id]) { acc[alertEntity.alert_id] = []; } acc[alertEntity.alert_id].push(alertEntity.entity_id); return acc; }, {});
+	const alertsStashesByAlertId = alertsStashes.reduce((acc, alertStash) => { if (!acc[alertStash.alert_id]) { acc[alertStash.alert_id] = []; } acc[alertStash.alert_id].push(alertStash.stash_id); return acc; }, {});
+
+	const alertsMatchesByAlertId = alertsMatches.reduce((acc, alertMatch) => {
+		if (!acc[alertMatch.alert_id]) {
+			acc[alertMatch.alert_id] = [];
+		}
+
+		acc[alertMatch.alert_id].push({
+			property: alertMatch.property,
+			expression: /\/.*\//.test(alertMatch.expression)
+				? new RegExp(alertMatch.expression.slice(1, -1), 'ui')
+				: new RegExp(escapeRegexp(alertMatch.expression), 'ui'),
+		});
+
+		return acc;
+	}, {});
+
+	console.log(alertsStashesByAlertId);
+
+	const alerts = rawAlerts.map((alert) => ({
+		id: alert.id,
+		userId: alert.user_id,
+		notify: alert.notify,
+		email: alert.email,
+		all: alert.all,
+		actors: alertsActorsByAlertId[alert.id] || [],
+		tags: alertsTagsByAlertId[alert.id] || [],
+		entities: alertsEntitiesByAlertId[alert.id] || [],
+		matches: alertsMatchesByAlertId[alert.id] || [],
+		stashes: alertsStashesByAlertId[alert.id] || [],
+	}));
+
+	const curatedScenes = scenes.map((scene) => ({
+		id: scene.id,
+		title: scene.title,
+		description: scene.description,
+		actorIds: actorIdsByReleaseId[scene.id] || [],
+		tagIds: tagIdsByReleaseId[scene.id] || [],
+		entityId: scene.entity.id,
+		parentEntityId: scene.entity.parent?.id,
+	}));
+
+	const triggers = alerts.flatMap((alert) => {
+		const alertScenes = curatedScenes.filter((scene) => {
+			console.log(scene.title, alert.tags, scene.tagIds);
+
+			if (alert.actors.length > 0 && !alert.actors.every((actorId) => scene.actorIds.includes(actorId))) {
+				console.log('THROW ACTORS');
+				return false;
+			}
+
+			if (alert.tags.length > 0 && !alert.tags.every((tagId) => scene.tagIds.includes(tagId))) {
+				console.log('THROW TAGS');
+				return false;
+			}
+
+			// multiple entities can only be matched in OR mode
+			if (alert.entities.length > 0 && !alert.entities.some((alertEntityId) => alertEntityId === scene.entityId || alertEntityId === scene.parentEntityId)) {
+				console.log('THROW ENTITIES');
+				return false;
+			}
+
+			if (alert.matches.length > 0 && !alert.matches.every((match) => match.expression.test(scene[match.property]))) {
+				console.log('THROW MATCHES');
+				return false;
+			}
+
+			console.log('OK');
+
+			return true;
+		});
+
+		return alertScenes.map((scene) => ({
+			sceneId: scene.id,
+			alert,
+		}));
 	});

-	const notifications = releases.rows
-		.filter((alert) => alert.notify)
-		.map((notification) => ({
-			user_id: notification.user_id,
-			alert_id: notification.alert_id,
-			scene_id: notification.scene_id,
+	const notifications = Object.values(Object.fromEntries(triggers // prevent multiple notifications for the same scene
+		.filter((trigger) => trigger.alert.notify)
+		.map((trigger) => [`${trigger.alert.userId}:${trigger.sceneId}`, trigger])))
+		.map((trigger) => ({
+			user_id: trigger.alert.userId,
+			alert_id: trigger.alert.id,
+			scene_id: trigger.sceneId,
 		}));

-	const stashes = releases.rows
-		.filter((release) => release.stashes.length > 0)
-		.flatMap((release) => release.stashes.map((stash) => ({
-			scene_id: release.scene_id,
-			stash_id: stash,
-		})));
+	console.log('triggers', triggers);
+
+	const stashes = Object.values(Object.fromEntries(triggers.flatMap((trigger) => trigger.alert.stashes.map((stashId) => ({
+		scene_id: trigger.sceneId,
+		stash_id: stashId,
+	}))).map((stash) => [`${stash.stash_id}:${stash.scene_id}`, stash])));
+
+	console.log('stashes', stashes);

 	await Promise.all([
 		bulkInsert('notifications', notifications, false),
 		bulkInsert('stashes_scenes', stashes, false),
 	]);

-	return releases.rows;
+	return triggers;
 }

 async function updateNotification(notificationId, notification, sessionUser) {
--- a/src/scrapers/julesjordan.js
+++ b/src/scrapers/julesjordan.js
@@ -32,7 +32,7 @@ function scrapeAll(scenes, site, entryIdFromTitle) {

 		release.title = title?.slice(0, title.match(/starring:/i)?.index || Infinity).trim();
 		release.url = query.url('.content_img a, .dvd_info > a, a.update_title, a[title]');
-		release.date = query.date('.update_date', 'MM/DD/YYYY');
+		release.date = query.date('.update_date', ['MM/DD/YYYY', 'YYYY-MM-DD']);

 		release.entryId = (entryIdFromTitle && slugify(release.title)) || element.dataset.setid || query.element('.rating_box')?.dataset.id || query.attribute('a img', 'id')?.match(/set-target-(\d+)/)?.[1];

@@ -81,7 +81,7 @@ function scrapeUpcoming(scenes, channel) {
 		const release = {};

 		release.title = query.text('.overlay-text', { join: false })?.[0];
-		release.date = query.date('.overlay-text', 'MM/DD/YYYY');
+		release.date = query.date('.overlay-text', ['MM/DD/YYYY', 'YYYY-MM-DD']);

 		release.actors = query.all('.update_models a').map((actorEl) => ({
 			name: unprint.query.content(actorEl),
@@ -159,7 +159,7 @@ async function scrapeScene({ html, query }, context) {
 	release.description = query.content('.update_description') || query.text('//div[./span[contains(text(), "Description")]]');
 	release.entryId = context.entity.parameters?.entryIdFromTitle ? slugify(release.title) : getEntryId(html);

-	release.date = query.date(['.update_date', '//div[./span[contains(text(), "Date")]]'], 'MM/DD/YYYY');
+	release.date = query.date(['.update_date', '//div[./span[contains(text(), "Date")]]'], ['MM/DD/YYYY', 'YYYY-MM-DD']);

 	release.actors = query.all('.backgroundcolor_info > .update_models a, .item .update_models a, .player-scene-description .update_models a').map((actorEl) => ({
 		name: unprint.query.content(actorEl),