Reusing batch ID for movies to preserve new-flag.

.gitignore

@@ -10,3 +10,4 @@ config/*
 assets/js/config/
 !assets/js/config/default.js
 *.heapprofile
+*.heapsnapshot

migrations/20190325001339_releases.js

@@ -1,6 +1,6 @@
 const config = require('config');
 
-exports.up = knex => Promise.resolve()
+exports.up = (knex) => Promise.resolve()
 	.then(() => knex.schema.createTable('countries', (table) => {
 		table.text('alpha2', 2)
 			.unique()

migrations/20211030162955_entity_scenes.js

@@ -1,4 +1,4 @@
-exports.up = async knex => knex.raw(`
+exports.up = async (knex) => knex.raw(`
 	CREATE FUNCTION entities_scenes(entity entities) RETURNS SETOF releases AS $$
 		WITH RECURSIVE children AS (
 			SELECT entities.id
@@ -19,6 +19,6 @@ exports.up = async knex => knex.raw(`
 	COMMENT ON FUNCTION entities_scenes IS E'@sortable';
 `);
 
-exports.down = async knex => knex.raw(`
+exports.down = async (knex) => knex.raw(`
 	DROP FUNCTION IF EXISTS entities_scenes;
 `);

src/app.js

@@ -186,8 +186,8 @@ async function init() {
 		}
 
 		if (argv.save) {
-			const storedMovies = await storeMovies(deepMovies);
+			const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
-			const storedMovieScenes = await storeScenes(deepMovieScenes);
+			const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);
 
 			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
 		}

src/scrapers/mindgeek.js

@@ -238,8 +238,7 @@ async function fetchLatest(site, page = 1, options) {
 	const { searchParams } = new URL(url);
 	const siteId = searchParams.get('site');
 
-	// const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters);
+	const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters);
-	const { session, instanceToken } = await getSession(site, options.parameters);
 
 	const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
 	const limit = 24;
@@ -365,7 +364,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
 }
 
 module.exports = {
-	// beforeNetwork: getSession,
+	beforeNetwork: getSession,
 	beforeFetchScenes: getSession,
 	scrapeLatestX,
 	fetchLatest,

src/store-releases.js

@@ -136,7 +136,7 @@ async function attachStudios(releases) {
 	return releasesWithStudio;
 }
 
-function attachReleaseIds(releases, storedReleases) {
+function attachReleaseIds(releases, storedReleases, batchId) {
 	const storedReleaseIdsByEntityIdAndEntryId = storedReleases.reduce((acc, release) => {
 		if (!acc[release.entity_id]) acc[release.entity_id] = {};
 		acc[release.entity_id][release.entry_id] = release.id;
@@ -157,6 +157,7 @@ function attachReleaseIds(releases, storedReleases) {
 			return {
 				...release,
 				id,
+				batchId,
 			};
 		}
 
@@ -314,12 +315,12 @@ async function storeChapters(releases) {
 	await associateReleaseMedia(chaptersWithId, 'chapter');
 }
 
-async function storeScenes(releases) {
+async function storeScenes(releases, useBatchId) {
 	if (!releases || releases.length === 0) {
 		return [];
 	}
 
-	const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
+	const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
 
 	const releasesWithChannels = await attachChannelEntities(releases);
 	const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) }));
@@ -333,8 +334,8 @@ async function storeScenes(releases) {
 
 	const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
 
-	const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries);
+	const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries, batchId);
-	const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries);
+	const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries, batchId);
 	const releasesWithId = uniqueReleasesWithId.concat(duplicateReleasesWithId);
 
 	const updated = await knex.raw(`
@@ -444,13 +445,13 @@ async function updateMovieSearch(movieIds) {
 	}
 }
 
-async function storeMovies(movies) {
+async function storeMovies(movies, useBatchId) {
 	if (!movies || movies.length === 0) {
 		return [];
 	}
 
 	const { uniqueReleases } = await filterDuplicateReleases(movies);
-	const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
+	const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
 
 	const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));