Storing actor avatars. Using 1 second interval queue for location resolve as per OSM code of conduct.

src/actors.js

@@ -13,6 +13,7 @@ const logger = require('./logger')(__filename);
 const slugify = require('./utils/slugify');
 const capitalize = require('./utils/capitalize');
 const resolvePlace = require('./utils/resolve-place');
+const { associateAvatars } = require('./media');
 
 const { toBaseReleases } = require('./deep');
 
@@ -80,6 +81,7 @@ function curateProfileEntry(profile) {
 		has_piercings: profile.hasPiercings,
 		piercings: profile.piercings,
 		tattoos: profile.tattoos,
+		avatar_media_id: profile.avatarMediaId || null,
 	};
 
 	return curatedProfileEntry;
@@ -91,6 +93,7 @@ async function curateProfile(profile) {
 			id: profile.id,
 			name: profile.name,
 			avatar: profile.avatar,
+			scraper: profile.scraper,
 		};
 
 		curatedProfile.site = profile.site.isNetwork ? null : profile.site;
@@ -198,6 +201,7 @@ async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
 				return {
 					...actor,
 					...profile,
+					scraper: scraperSlug,
 					site: siteOrNetwork,
 				};
 			}), Promise.reject(new Error()));
@@ -256,16 +260,16 @@ async function upsertProfiles(curatedProfileEntries) {
 	}
 
 	if (argv.force && updatingProfileEntries.length > 0) {
-		knex.transaction(async (transaction) => {
-			const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles')
-				.where('id', profileEntry.id)
-				.update(profileEntry)
-				.transacting(transaction));
+		const transaction = await knex.transaction();
+		const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles')
+			.where('id', profileEntry.id)
+			.update(profileEntry)
+			.returning(['id', 'actor_id'])
+			.transacting(transaction));
 
-			return Promise.all(queries)
-				.then(transaction.commit)
-				.catch(transaction.rollback);
-		});
+		await Promise.all(queries)
+			.then(transaction.commit)
+			.catch(transaction.rollback);
 	}
 }
 
@@ -305,7 +309,9 @@ async function scrapeActors(actorNames) {
 	);
 
 	const profiles = await Promise.all(profilesPerActor.flat().map(profile => curateProfile(profile)));
-	const curatedProfileEntries = profiles.map(profile => curateProfileEntry(profile));
+	const profilesWithAvatarIds = await associateAvatars(profiles);
+
+	const curatedProfileEntries = profilesWithAvatarIds.map(profile => curateProfileEntry(profile));
 
 	await upsertProfiles(curatedProfileEntries);
 }
@@ -350,7 +356,7 @@ async function associateActors(releases, batchId) {
 	const baseActors = Object.values(baseActorsByReleaseId).flat();
 
 	if (baseActors.length === 0) {
-		return;
+		return null;
 	}
 
 	const baseActorsBySlugAndNetworkId = baseActors.reduce((acc, baseActor) => ({
@@ -382,6 +388,8 @@ async function associateActors(releases, batchId) {
 		.flat();
 
 	await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
+
+	return actors;
 }
 
 module.exports = {