1.204.11

.gitignore

@@ -10,3 +10,4 @@ config/*
 assets/js/config/
 !assets/js/config/default.js
 *.heapprofile
+*.heapsnapshot

migrations/20190325001339_releases.js

@@ -1,6 +1,6 @@
 const config = require('config');
 
-exports.up = knex => Promise.resolve()
+exports.up = (knex) => Promise.resolve()
 	.then(() => knex.schema.createTable('countries', (table) => {
 		table.text('alpha2', 2)
 			.unique()

migrations/20211030162955_entity_scenes.js

@@ -1,4 +1,4 @@
-exports.up = async knex => knex.raw(`
+exports.up = async (knex) => knex.raw(`
 	CREATE FUNCTION entities_scenes(entity entities) RETURNS SETOF releases AS $$
 		WITH RECURSIVE children AS (
 			SELECT entities.id
@@ -19,6 +19,6 @@ exports.up = async knex => knex.raw(`
 	COMMENT ON FUNCTION entities_scenes IS E'@sortable';
 `);
 
-exports.down = async knex => knex.raw(`
+exports.down = async (knex) => knex.raw(`
 	DROP FUNCTION IF EXISTS entities_scenes;
 `);

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.204.10",
+    "version": "1.204.11",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.204.10",
+            "version": "1.204.11",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.204.10",
+    "version": "1.204.11",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

src/app.js

@@ -186,8 +186,8 @@ async function init() {
 		}
 
 		if (argv.save) {
-			const storedMovies = await storeMovies(deepMovies);
+			const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
-			const storedMovieScenes = await storeScenes(deepMovieScenes);
+			const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);
 
 			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
 		}

src/scrapers/mindgeek.js

@@ -179,8 +179,6 @@ async function getSession(site, parameters) {
 		const cookieString = await cookieJar.getCookieStringAsync(sessionUrl);
 		const { instance_token: instanceToken } = cookie.parse(cookieString);
 
-		console.log(site.name, instanceToken);
-
 		return { session, instanceToken };
 	}
 
@@ -242,8 +240,6 @@ async function fetchLatest(site, page = 1, options) {
 
 	const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters);
 
-	console.log('fetch', instanceToken);
-
 	const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
 	const limit = 24;
 	const apiUrl = site.parameters?.native || site.parameters?.extract

src/store-releases.js

@@ -136,7 +136,7 @@ async function attachStudios(releases) {
 	return releasesWithStudio;
 }
 
-function attachReleaseIds(releases, storedReleases) {
+function attachReleaseIds(releases, storedReleases, batchId) {
 	const storedReleaseIdsByEntityIdAndEntryId = storedReleases.reduce((acc, release) => {
 		if (!acc[release.entity_id]) acc[release.entity_id] = {};
 		acc[release.entity_id][release.entry_id] = release.id;
@@ -157,6 +157,7 @@ function attachReleaseIds(releases, storedReleases) {
 			return {
 				...release,
 				id,
+				batchId,
 			};
 		}
 
@@ -314,12 +315,12 @@ async function storeChapters(releases) {
 	await associateReleaseMedia(chaptersWithId, 'chapter');
 }
 
-async function storeScenes(releases) {
+async function storeScenes(releases, useBatchId) {
 	if (!releases || releases.length === 0) {
 		return [];
 	}
 
-	const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
+	const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
 
 	const releasesWithChannels = await attachChannelEntities(releases);
 	const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) }));
@@ -333,8 +334,8 @@ async function storeScenes(releases) {
 
 	const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
 
-	const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries);
+	const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries, batchId);
-	const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries);
+	const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries, batchId);
 	const releasesWithId = uniqueReleasesWithId.concat(duplicateReleasesWithId);
 
 	const updated = await knex.raw(`
@@ -444,13 +445,13 @@ async function updateMovieSearch(movieIds) {
 	}
 }
 
-async function storeMovies(movies) {
+async function storeMovies(movies, useBatchId) {
 	if (!movies || movies.length === 0) {
 		return [];
 	}
 
 	const { uniqueReleases } = await filterDuplicateReleases(movies);
-	const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
+	const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
 
 	const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));