1.250.7

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.250.6",
+    "version": "1.250.7",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.250.6",
+            "version": "1.250.7",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.250.6",
+    "version": "1.250.7",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

src/actors.js

@@ -22,6 +22,7 @@ const actorScrapers = require('./scrapers/scrapers').actors;
 const argv = require('./argv');
 const include = require('./utils/argv-include')(argv);
 const bulkInsert = require('./utils/bulk-insert');
+const batchInsert = require('./utils/batch-insert');
 const chunk = require('./utils/chunk');
 const logger = require('./logger')(__filename);
 
@@ -46,6 +47,7 @@ const commonContext = {
 	slugify,
 	omit,
 	unprint,
+	batchInsert,
 };
 
 const hairColors = {
@@ -544,7 +546,7 @@ async function curateProfile(profile, actor) {
 
 async function insertProfiles(newProfiles) {
 	if (newProfiles.length > 0) {
-		const entries = await bulkInsert('actors_profiles', newProfiles);
+		const entries = await batchInsert('actors_profiles', newProfiles);
 
 		logger.info(`Saved ${newProfiles.length} actor profiles`);
 
@@ -606,10 +608,7 @@ async function upsertProfiles(profiles) {
 		}));
 
 		if (avatars.length > 0) {
-			await knex('actors_avatars')
+			await batchInsert('actors_avatars', avatars, { conflict: false });
-				.insert(avatars)
-				.onConflict()
-				.ignore();
 		}
 	}
 }
@@ -759,7 +758,8 @@ function curateSocials(socials, platformsByHostname) {
 
 async function associateSocials(profiles) {
 	const { platformsByHostname } = await actorsCommon;
-	const profileEntries = await knex('actors_profiles').whereIn(['actor_id', 'entity_id'], profiles.map((profile) => [profile.actorId, profile.entity.id]));
+	const profileEntryChunks = await Promise.all(chunk(profiles).map((profilesChunk) => knex('actors_profiles').whereIn(['actor_id', 'entity_id'], profilesChunk.map((profile) => [profile.actorId, profile.entity.id]))));
+	const profileEntries = profileEntryChunks.flat();
 
 	const profileEntriesByActorIdAndEntityId = profileEntries.reduce((acc, profileEntry) => {
 		if (!acc[profileEntry.actor_id]) {
@@ -784,16 +784,14 @@ async function associateSocials(profiles) {
 			return;
 		}
 
-		await knex('actors_socials')
+		await batchInsert('actors_socials', curateSocials(profile.social, platformsByHostname).map((social) => ({
-			.insert(curateSocials(profile.social, platformsByHostname).map((social) => ({
+			platform: social.platform,
-				platform: social.platform,
+			handle: social.handle,
-				handle: social.handle,
+			url: social.url,
-				url: social.url,
+			actor_id: profile.actorId,
-				actor_id: profile.actorId,
+		})), {
-				// profile_id: profileId,
+			conflict: false,
-			})))
+		});
-			.onConflict()
-			.ignore();
 	}, Promise.resolve());
 }