Added new bulk upsert utility.

README.md

@@ -55,7 +55,7 @@ To build traxxx, run the following command:
 
 `npm run build`
 
-To generate the thumbnails for logos and tag photos, run:
+To generate thumbnails for logos and tag photos, install ImageMagick and run:
 
 `npm run logos-thumbs`
 

assets/components/tags/tags.vue

@@ -71,14 +71,6 @@ async function mounted() {
 			'ass-eating',
 			'atm',
 		],
-		extreme: [
-			'airtight',
-			'dap',
-			'dvp',
-			'da-tp',
-			'dv-tp',
-			'tap',
-		],
 		cumshot: [
 			'facial',
 			'bukkake',
@@ -88,6 +80,14 @@ async function mounted() {
 			'oral-creampie',
 			'cum-on-butt',
 		],
+		extreme: [
+			'airtight',
+			'dap',
+			'dvp',
+			'da-tp',
+			'dv-tp',
+			'tap',
+		],
 		roleplay: [
 			'family',
 			'parody',

seeds/04_media.js

@@ -611,7 +611,7 @@ const tagPosters = [
 	['double-blowjob', 1, 'Veronica Rodriguez and Penny Pax in "Fucking Older Guys 5" for Penthouse'],
 	['double-dildo', 0, 'Kali Roses in "Double Dildo Party" for KaliRoses.com'],
 	['double-dildo-blowjob', 0, 'Adriana Chechik and Vicki Chase in "Anal Savages 1" for Jules Jordan'],
-	['dp', 2, 'Megan Rain in "DP Masters 4" for Jules Jordan'],
+	['dp', 3, 'Hime Marie in LegalPorno AA047'],
 	['dvp', 'poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
 	['dv-tp', 'poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'],
 	['ebony', 2, 'Nia Nacci for Sweetheart Video'],
@@ -683,21 +683,22 @@ const tagPhotos = [
 	['da-tp', 6, 'Adriana Chechik in "Gangbang Me" for HardX'],
 	['da-tp', 0, 'Natasha Teen in LegalPorno SZ2164'],
 	['da-tp', 1, 'Francys Belle in SZ1702 for LegalPorno'],
-	['dap', 2, 'Lana Rhoades in "Lana Rhoades Unleashed" for HardX'],
 	['dap', 6, 'Sheena Shaw in "Ass Worship 14" for Jules Jordan'],
+	['dap', 9, 'Polly Pons in LegalPorno GIO1520'],
+	['dap', 1, 'Ria Sunn in SZ1801 for LegalPorno'],
+	['dap', 2, 'Lana Rhoades in "Lana Rhoades Unleashed" for HardX'],
 	['dap', 5, 'Riley Reid in "The Gangbang of Riley Reid" for Jules Jordan'],
-	['dap', 8, 'Lady Gang in SZ2478 LegalPorno'],
+	// ['dap', 8, 'Lady Gang in SZ2478 LegalPorno'],
 	['dap', 'poster', 'Haley Reed in "Young Hot Ass" for Evil Angel'],
 	['dap', 0, 'Nicole Black doing double anal during a gangbang in GIO971 for LegalPorno'],
-	['dap', 1, 'Ria Sunn in SZ1801 for LegalPorno'],
 	['deepthroat', 1, 'Jynx Maze in "Slutty and Sluttier 13" for Evil Angel'],
 	['deepthroat', 0, 'Chanel Grey in "Deepthroating Is Fun" for Throated'],
 	['double-blowjob', 0, 'Kira Noir and Kali Roses for Brazzers'],
 	['double-dildo-blowjob', 1, 'Aidra Fox and Reena Sky in "Reena\'s Got A Staring Problem" for Brazzers'],
 	['double-dildo-dp', 0, 'u/LacyCrow "Sometimes you have to do it yourself"'],
-	['dp', 3, 'Hime Marie in LegalPorno AA047'],
+	['dp', 2, 'Megan Rain in "DP Masters 4" for Jules Jordan'],
+	['dp', 4, 'Rebecca Volpetti for Hands On Hardcore'],
 	['dp', 'poster', 'Mia Malkova in "DP Me 8" for HardX'],
-	['dp', 0, 'Zoey Monroe in "Slut Puppies 7" for Jules Jordan'],
 	['dvp', 0, 'Aaliyah Hadid in "Squirting From Double Penetration With Anal" for Bang Bros'],
 	['dv-tp', 1, 'Adriana Chechik in "Adriana\'s Triple Anal Penetration!"'],
 	['dv-tp', 0, 'Luna Rival in LegalPorno SZ1490'],

src/scrapers/assylum.js

@@ -12,9 +12,11 @@ function extractActors(actorString) {
 }
 
 function matchActors(actorString, models) {
-	return models
-		.filter(model => new RegExp(model.name, 'i')
-			.test(actorString));
+	if (!actorString) {
+		return [];
+	}
+
+	return models.filter(model => new RegExp(model.name, 'i').test(actorString));
 }
 
 function scrapeLatest(scenes, site, models) {

src/store-releases.js

@@ -6,6 +6,7 @@ const argv = require('./argv');
 const logger = require('./logger')(__filename);
 const knex = require('./knex');
 const slugify = require('./utils/slugify');
+const bulkInsert = require('./utils/bulk-insert');
 const { formatDate } = require('./utils/qu');
 const { associateActors, scrapeActors } = require('./actors');
 const { associateReleaseTags } = require('./tags');
@@ -125,10 +126,23 @@ function attachReleaseIds(releases, storedReleases) {
 		return acc;
 	}, {});
 
-	const releasesWithId = releases.map(release => ({
-		...release,
-		id: storedReleaseIdsByEntityIdAndEntryId[release.entity.id][release.entryId],
-	}));
+	const releasesWithId = releases.map((release) => {
+		if (!release.entity) {
+			logger.error(`No entitity available for ${release.url}`);
+			return null;
+		}
+
+		const id = storedReleaseIdsByEntityIdAndEntryId[release.entity.id]?.[release.entryId];
+
+		if (id) {
+			return {
+				...release,
+				id,
+			};
+		}
+
+		return null;
+	}).filter(Boolean);
 
 	return releasesWithId;
 }
@@ -283,7 +297,7 @@ async function associateMovieScenes(movies, movieScenes) {
 		});
 	}).flat().filter(Boolean);
 
-	await knex.batchInsert('movies_scenes', associations);
+	await bulkInsert('movies_scenes', associations, ['movie_id', 'scene_id']);
 }
 
 async function storeMovies(movies, movieScenes) {
@@ -291,8 +305,8 @@ async function storeMovies(movies, movieScenes) {
 	const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
 
 	const curatedMovieEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie'));
-	const storedMovies = await knex.batchInsert('movies', curatedMovieEntries).returning('*');
 
+	const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
 	const moviesWithId = attachReleaseIds(movies, storedMovies);
 
 	await associateMovieScenes(moviesWithId, movieScenes);

src/utils/chunk.js

@@ -1,6 +1,6 @@
 'use strict';
 
-function chunk(array, chunkSize) {
+function chunk(array, chunkSize = 1000) {
 	return Array.from({ length: Math.ceil(array.length / chunkSize) })
 		.map((value, index) => array.slice(index * chunkSize, (index * chunkSize) + chunkSize));
 }