Added new bulk upsert utility.

src/scrapers/assylum.js

@@ -12,9 +12,11 @@ function extractActors(actorString) {
 }
 
 function matchActors(actorString, models) {
-	return models
-		.filter(model => new RegExp(model.name, 'i')
-			.test(actorString));
+	if (!actorString) {
+		return [];
+	}
+
+	return models.filter(model => new RegExp(model.name, 'i').test(actorString));
 }
 
 function scrapeLatest(scenes, site, models) {

src/store-releases.js

@@ -6,6 +6,7 @@ const argv = require('./argv');
 const logger = require('./logger')(__filename);
 const knex = require('./knex');
 const slugify = require('./utils/slugify');
+const bulkInsert = require('./utils/bulk-insert');
 const { formatDate } = require('./utils/qu');
 const { associateActors, scrapeActors } = require('./actors');
 const { associateReleaseTags } = require('./tags');
@@ -125,10 +126,23 @@ function attachReleaseIds(releases, storedReleases) {
 		return acc;
 	}, {});
 
-	const releasesWithId = releases.map(release => ({
-		...release,
-		id: storedReleaseIdsByEntityIdAndEntryId[release.entity.id][release.entryId],
-	}));
+	const releasesWithId = releases.map((release) => {
+		if (!release.entity) {
+			logger.error(`No entitity available for ${release.url}`);
+			return null;
+		}
+
+		const id = storedReleaseIdsByEntityIdAndEntryId[release.entity.id]?.[release.entryId];
+
+		if (id) {
+			return {
+				...release,
+				id,
+			};
+		}
+
+		return null;
+	}).filter(Boolean);
 
 	return releasesWithId;
 }
@@ -283,7 +297,7 @@ async function associateMovieScenes(movies, movieScenes) {
 		});
 	}).flat().filter(Boolean);
 
-	await knex.batchInsert('movies_scenes', associations);
+	await bulkInsert('movies_scenes', associations, ['movie_id', 'scene_id']);
 }
 
 async function storeMovies(movies, movieScenes) {
@@ -291,8 +305,8 @@ async function storeMovies(movies, movieScenes) {
 	const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
 
 	const curatedMovieEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie'));
-	const storedMovies = await knex.batchInsert('movies', curatedMovieEntries).returning('*');
 
+	const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
 	const moviesWithId = attachReleaseIds(movies, storedMovies);
 
 	await associateMovieScenes(moviesWithId, movieScenes);

src/utils/chunk.js

@@ -1,6 +1,6 @@
 'use strict';
 
-function chunk(array, chunkSize) {
+function chunk(array, chunkSize = 1000) {
 	return Array.from({ length: Math.ceil(array.length / chunkSize) })
 		.map((value, index) => array.slice(index * chunkSize, (index * chunkSize) + chunkSize));
 }