Storing actor avatars. Using 1 second interval queue for location resolve as per OSM code of conduct.

assets/js/actors/actions.js

@@ -36,18 +36,15 @@ function curateActor(actor) {
 		updatedAt: new Date(actor.updatedAt),
 	};
 
-	if (actor.avatar) {
+	if (actor.profiles && actor.profiles.length > 0) {
-		curatedActor.avatar = actor.avatar.media;
+		curatedActor.avatar = actor.profiles.slice(0, 1)[0].avatar;
+		curatedActor.photos = actor.profiles.slice(1).map(profile => profile.avatar);
 	}
 
 	if (actor.releases) {
 		curatedActor.releases = actor.releases.map(release => curateRelease(release.release));
 	}
 
-	if (actor.photos) {
-		curatedActor.photos = actor.photos.map(photo => photo.media);
-	}
-
 	return curatedActor;
 }
 
@@ -99,21 +96,14 @@ function initActorActions(store, _router) {
                         name
                         slug
                     }
-                    actorsProfiles {
+					profiles: actorsProfiles {
-                        actorsAvatarByProfileId {
+						description
-                            media {
+						avatar: avatarMedia {
-                                path
-                                thumbnail
-                                copyright
-                            }
-                        }
-                    }
-                    photos: actorsPhotos {
-                        media {
 							id
-                            thumbnail
 							path
-                            index
+							thumbnail
+							lazy
+							comment
 							copyright
 						}
                     }
@@ -239,6 +229,7 @@ function initActorActions(store, _router) {
                     actorsProfiles {
                         actorsAvatarByProfileId {
                             media {
+								id
                                 path
                                 thumbnail
                                 copyright

migrations/20190325001339_releases.js

@@ -59,13 +59,16 @@ exports.up = knex => Promise.resolve()
 			.inTable('media')
 			.unique();
 	}))
-	.then(() => knex.raw(`
+	.then(() => { // eslint-disable-line arrow-body-style
+		// allow vim fold
+		return knex.raw(`
 			CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
 				SELECT media_id FROM media_sfw
 				ORDER BY random()
 				LIMIT 1;
 			$$ LANGUAGE sql STABLE;
-    `))
+		`);
+	})
 	.then(() => knex.schema.alterTable('media', (table) => {
 		table.string('sfw_media_id', 21)
 			.references('id')
@@ -140,6 +143,7 @@ exports.up = knex => Promise.resolve()
 		table.increments('id', 12);
 
 		table.string('name');
+		table.string('alias');
 		table.string('url');
 		table.text('description');
 		table.json('parameters');
@@ -322,6 +326,7 @@ exports.up = knex => Promise.resolve()
 		table.increments('id', 12);
 
 		table.integer('actor_id', 12)
+			.notNullable()
 			.references('id')
 			.inTable('actors');
 
@@ -371,6 +376,10 @@ exports.up = knex => Promise.resolve()
 		table.string('piercings');
 		table.string('tattoos');
 
+		table.string('avatar_media_id', 21)
+			.references('id')
+			.inTable('media');
+
 		table.datetime('scraped_at');
 		table.boolean('scrape_success');
 
@@ -386,7 +395,9 @@ exports.up = knex => Promise.resolve()
 
 		table.string('name');
 	}))
-	.then(() => knex('body').insert([
+	.then(() => { // eslint-disable-line arrow-body-style
+		// allow vim fold
+		return knex('body').insert([
 			// head
 			{ slug: 'head', name: 'head' },
 			{ slug: 'face', name: 'face' },
@@ -501,7 +512,8 @@ exports.up = knex => Promise.resolve()
 			{ slug: 'middle-toe', name: 'middle toe' },
 			{ slug: 'fourth-toe', name: 'fourth toe' },
 			{ slug: 'little-toe', name: 'little toe' },
-	]))
+		]);
+	})
 	.then(() => knex.schema.createTable('actors_tattoos', (table) => {
 		table.increments('id');
 
@@ -769,7 +781,9 @@ exports.up = knex => Promise.resolve()
 			.references('id')
 			.inTable('releases');
 	}))
-	.then(() => knex.raw(`
+	.then(() => { // eslint-disable-line arrow-body-style
+		// allow vim fold
+		return knex.raw(`
 			ALTER TABLE releases
 				ADD CONSTRAINT ensure_site_or_network CHECK (site_id IS NOT NULL OR network_id IS NOT NULL);
 
@@ -828,9 +842,12 @@ exports.up = knex => Promise.resolve()
 
 			COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many';
 			COMMENT ON COLUMN actors.weight IS E'@omit read,update,create,delete,all,many';
-    `));
+		`);
+	});
 
-exports.down = knex => knex.raw(`
+exports.down = (knex) => { // eslint-disable-line arrow-body-style
+	// allow vim fold
+	return knex.raw(`
 		DROP VIEW IF EXISTS movie_actors;
 		DROP VIEW IF EXISTS movie_tags;
 
@@ -880,3 +897,4 @@ exports.down = knex => knex.raw(`
 		DROP TEXT SEARCH CONFIGURATION IF EXISTS traxxx;
 		DROP TEXT SEARCH DICTIONARY IF EXISTS traxxx_dict;
 	`);
+};

public/img/logos/blowpass/misc/my-xxx-pass.svg

@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 18.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 547 112" enable-background="new 0 0 547 112" xml:space="preserve">
+<radialGradient id="SVGID_1_" cx="171.1188" cy="60.5308" r="34.3229" gradientUnits="userSpaceOnUse">
+	<stop  offset="0.1989" style="stop-color:#EE2724"/>
+	<stop  offset="0.3065" style="stop-color:#EE2424"/>
+	<stop  offset="1" style="stop-color:#CD2027"/>
+</radialGradient>
+<polygon fill="url(#SVGID_1_)" points="182.1,18.6 171.2,39 158.3,14.3 135.6,14.3 159.9,60.5 135.6,106.7 158.3,106.7 171.2,82.1 
+	184.2,106.7 191.1,106.7 199,91.7 182.6,60.5 193.5,39.6 "/>
+<radialGradient id="SVGID_2_" cx="257.2801" cy="60.5308" r="31.5337" gradientTransform="matrix(-1 0 0 1 540.7146 0)" gradientUnits="userSpaceOnUse">
+	<stop  offset="0.1989" style="stop-color:#EE2724"/>
+	<stop  offset="0.3065" style="stop-color:#EE2424"/>
+	<stop  offset="1" style="stop-color:#CD2027"/>
+</radialGradient>
+<polygon fill="url(#SVGID_2_)" points="272.4,18.4 283.3,39 296.3,14.3 319,14.3 294.7,60.5 319,106.7 296.3,106.7 283.3,82.1 
+	270.3,106.7 263.5,106.7 255.6,91.7 272,60.5 260.9,39.4 "/>
+<linearGradient id="SVGID_3_" gradientUnits="userSpaceOnUse" x1="43.4116" y1="15.5743" x2="43.4116" y2="380.5575">
+	<stop  offset="0" style="stop-color:#FFFFFF"/>
+	<stop  offset="1" style="stop-color:#000000"/>
+</linearGradient>
+<path fill="url(#SVGID_3_)" d="M9.3,35h18.8l15.3,27.6L58.7,35h18.8v71.2H59.8V65.3L43.5,93.2h-0.4L26.8,65.5v40.7H9.3V35z"/>
+<linearGradient id="SVGID_4_" gradientUnits="userSpaceOnUse" x1="108.1806" y1="15.5743" x2="108.1806" y2="380.5575">
+	<stop  offset="0" style="stop-color:#FFFFFF"/>
+	<stop  offset="1" style="stop-color:#000000"/>
+</linearGradient>
+<path fill="url(#SVGID_4_)" d="M99.3,79.5L74.9,35h20.2l13.2,25.9L121.6,35h19.9l-24.4,44.3v27H99.3V79.5z"/>
+<linearGradient id="SVGID_5_" gradientUnits="userSpaceOnUse" x1="349.3989" y1="15.5743" x2="349.3989" y2="380.5575">
+	<stop  offset="0" style="stop-color:#FFFFFF"/>
+	<stop  offset="1" style="stop-color:#000000"/>
+</linearGradient>
+<path fill="url(#SVGID_5_)" d="M322.3,35h27.5c4,0,7.7,0.6,11,1.7c3.3,1.1,6.1,2.7,8.4,4.9c2.3,2.1,4.1,4.8,5.4,7.9
+	c1.3,3.1,1.9,6.6,1.9,10.6v0.2c0,4.2-0.7,7.9-2.1,11.1c-1.4,3.2-3.3,5.9-5.7,8c-2.4,2.2-5.4,3.8-8.7,4.9c-3.4,1.1-7.1,1.6-11,1.6
+	h-8.8v20.4h-17.8V35z M348.5,70.4c3.2,0,5.7-0.8,7.5-2.5c1.8-1.7,2.7-3.9,2.7-6.8v-0.2c0-3.1-0.9-5.4-2.7-7
+	c-1.8-1.6-4.3-2.4-7.6-2.4h-8.3v18.9H348.5z"/>
+<linearGradient id="SVGID_6_" gradientUnits="userSpaceOnUse" x1="402.2677" y1="15.5743" x2="402.2677" y2="380.5575">
+	<stop  offset="0" style="stop-color:#FFFFFF"/>
+	<stop  offset="1" style="stop-color:#000000"/>
+</linearGradient>
+<path fill="url(#SVGID_6_)" d="M393.7,34.5h17.1l27.3,71.7h-19l-4.7-12.7h-24.7l-4.6,12.7h-18.7L393.7,34.5z M409.3,78.3L402.1,58
+	l-7.2,20.3H409.3z"/>
+<linearGradient id="SVGID_7_" gradientUnits="userSpaceOnUse" x1="460.045" y1="15.5743" x2="460.045" y2="380.5575">
+	<stop  offset="0" style="stop-color:#FFFFFF"/>
+	<stop  offset="1" style="stop-color:#000000"/>
+</linearGradient>
+<path fill="url(#SVGID_7_)" d="M462.1,107.5c-5.7,0-11.2-1-16.3-3c-5.2-2-9.7-5-13.7-8.9l9.9-13.2c6.4,5.7,13.3,8.5,20.8,8.5
+	c2.4,0,4.2-0.4,5.4-1.3c1.3-0.8,1.9-2,1.9-3.6v-0.2c0-0.7-0.2-1.4-0.5-2c-0.3-0.6-0.9-1.1-1.7-1.7c-0.8-0.5-1.9-1.1-3.3-1.6
+	c-1.4-0.5-3.2-1-5.4-1.6c-3.4-0.9-6.6-1.8-9.5-2.9c-2.9-1-5.5-2.4-7.6-4.1c-2.2-1.7-3.9-3.7-5.1-6.2c-1.3-2.5-1.9-5.5-1.9-9.2v-0.2
+	c0-3.3,0.6-6.4,1.7-9.1c1.1-2.8,2.8-5.2,4.9-7.2c2.2-2,4.8-3.6,7.8-4.7c3.1-1.1,6.5-1.7,10.3-1.7c5.5,0,10.4,0.8,14.7,2.4
+	c4.3,1.6,8.2,4,11.7,7.1l-8.9,14c-2.9-2.3-5.9-4.1-9-5.2c-3.1-1.2-6.1-1.8-8.9-1.8c-2.1,0-3.7,0.4-4.8,1.3c-1.1,0.9-1.6,2-1.6,3.2
+	v0.2c0,0.8,0.2,1.5,0.5,2.1c0.3,0.6,0.9,1.2,1.7,1.7c0.8,0.5,2,1,3.4,1.5c1.4,0.5,3.3,1,5.4,1.6c3.7,0.9,7,1.9,9.9,3.1
+	c3,1.2,5.5,2.6,7.6,4.3c2.1,1.7,3.7,3.7,4.8,6.1s1.6,5.3,1.6,8.6v0.2c0,3.7-0.6,6.9-1.9,9.8c-1.3,2.8-3,5.3-5.3,7.3
+	c-2.3,2-5,3.5-8.2,4.6C469.6,106.9,466,107.5,462.1,107.5z"/>
+<linearGradient id="SVGID_8_" gradientUnits="userSpaceOnUse" x1="512.3254" y1="15.5743" x2="512.3254" y2="380.5575">
+	<stop  offset="0" style="stop-color:#FFFFFF"/>
+	<stop  offset="1" style="stop-color:#000000"/>
+</linearGradient>
+<path fill="url(#SVGID_8_)" d="M514.4,107.5c-5.7,0-11.2-1-16.3-3c-5.2-2-9.7-5-13.7-8.9l9.9-13.2c6.4,5.7,13.3,8.5,20.8,8.5
+	c2.4,0,4.2-0.4,5.4-1.3c1.3-0.8,1.9-2,1.9-3.6v-0.2c0-0.7-0.2-1.4-0.5-2c-0.3-0.6-0.9-1.1-1.7-1.7c-0.8-0.5-1.9-1.1-3.3-1.6
+	c-1.4-0.5-3.2-1-5.4-1.6c-3.4-0.9-6.6-1.8-9.5-2.9s-5.5-2.4-7.6-4.1c-2.2-1.7-3.9-3.7-5.1-6.2c-1.3-2.5-1.9-5.5-1.9-9.2v-0.2
+	c0-3.3,0.6-6.4,1.7-9.1c1.1-2.8,2.8-5.2,4.9-7.2c2.2-2,4.8-3.6,7.8-4.7c3.1-1.1,6.5-1.7,10.3-1.7c5.5,0,10.4,0.8,14.7,2.4
+	c4.3,1.6,8.2,4,11.7,7.1l-8.9,14c-2.9-2.3-5.9-4.1-9-5.2c-3.1-1.2-6.1-1.8-8.9-1.8c-2.1,0-3.7,0.4-4.8,1.3c-1.1,0.9-1.6,2-1.6,3.2
+	v0.2c0,0.8,0.2,1.5,0.5,2.1c0.3,0.6,0.9,1.2,1.7,1.7c0.8,0.5,2,1,3.4,1.5c1.4,0.5,3.3,1,5.4,1.6c3.7,0.9,7,1.9,9.9,3.1
+	c3,1.2,5.5,2.6,7.6,4.3c2.1,1.7,3.7,3.7,4.8,6.1s1.6,5.3,1.6,8.6v0.2c0,3.7-0.6,6.9-1.9,9.8c-1.3,2.8-3,5.3-5.3,7.3
+	c-2.3,2-5,3.5-8.2,4.6C521.8,106.9,518.3,107.5,514.4,107.5z"/>
+<g>
+	<radialGradient id="SVGID_9_" cx="227.409" cy="56.389" r="44.9787" gradientUnits="userSpaceOnUse">
+		<stop  offset="0.5744" style="stop-color:#E82325"/>
+		<stop  offset="1" style="stop-color:#ED1C24"/>
+	</radialGradient>
+	<path fill="url(#SVGID_9_)" d="M215,56.4L188.5,6h24.7l14.1,26.9L241.5,6h24.7l-26.5,50.3l26.5,50.4h-24.7l-14.1-26.9l-14.1,26.9
+		h-24.7L215,56.4z"/>
+</g>
+<radialGradient id="SVGID_10_" cx="270.4181" cy="60.5308" r="41.2772" gradientTransform="matrix(-1 0 0 1 556.5228 0)" gradientUnits="userSpaceOnUse">
+	<stop  offset="0.3863" style="stop-color:#E81D25"/>
+	<stop  offset="0.5804" style="stop-color:#E21E25"/>
+	<stop  offset="0.7696" style="stop-color:#D51E25"/>
+	<stop  offset="0.9564" style="stop-color:#C12025"/>
+	<stop  offset="1" style="stop-color:#BB2025"/>
+</radialGradient>
+<polygon fill="url(#SVGID_10_)" points="261.9,14.3 256.2,25.2 256.2,25.2 261.9,14.3 "/>
+</svg>

seeds/01_networks.js

@@ -102,6 +102,7 @@ const networks = [
 	{
 		slug: 'blowpass',
 		name: 'Blowpass',
+		alias: ['myxxxpass'],
 		url: 'https://www.blowpass.com',
 		description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
 		parameters: {
@@ -435,6 +436,7 @@ exports.seed = knex => Promise.resolve()
 		const networksWithParent = networks.map(network => ({
 			slug: network.slug,
 			name: network.name,
+			alias: (network.alias || []).join(','),
 			url: network.url,
 			description: network.description,
 			parameters: network.parameters,

src/actors.js

@@ -13,6 +13,7 @@ const logger = require('./logger')(__filename);
 const slugify = require('./utils/slugify');
 const capitalize = require('./utils/capitalize');
 const resolvePlace = require('./utils/resolve-place');
+const { associateAvatars } = require('./media');
 
 const { toBaseReleases } = require('./deep');
 
@@ -80,6 +81,7 @@ function curateProfileEntry(profile) {
 		has_piercings: profile.hasPiercings,
 		piercings: profile.piercings,
 		tattoos: profile.tattoos,
+		avatar_media_id: profile.avatarMediaId || null,
 	};
 
 	return curatedProfileEntry;
@@ -91,6 +93,7 @@ async function curateProfile(profile) {
 			id: profile.id,
 			name: profile.name,
 			avatar: profile.avatar,
+			scraper: profile.scraper,
 		};
 
 		curatedProfile.site = profile.site.isNetwork ? null : profile.site;
@@ -198,6 +201,7 @@ async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
 				return {
 					...actor,
 					...profile,
+					scraper: scraperSlug,
 					site: siteOrNetwork,
 				};
 			}), Promise.reject(new Error()));
@@ -256,16 +260,16 @@ async function upsertProfiles(curatedProfileEntries) {
 	}
 
 	if (argv.force && updatingProfileEntries.length > 0) {
-		knex.transaction(async (transaction) => {
+		const transaction = await knex.transaction();
 		const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles')
 			.where('id', profileEntry.id)
 			.update(profileEntry)
+			.returning(['id', 'actor_id'])
 			.transacting(transaction));
 
-			return Promise.all(queries)
+		await Promise.all(queries)
 			.then(transaction.commit)
 			.catch(transaction.rollback);
-		});
 	}
 }
 
@@ -305,7 +309,9 @@ async function scrapeActors(actorNames) {
 	);
 
 	const profiles = await Promise.all(profilesPerActor.flat().map(profile => curateProfile(profile)));
-	const curatedProfileEntries = profiles.map(profile => curateProfileEntry(profile));
+	const profilesWithAvatarIds = await associateAvatars(profiles);
+
+	const curatedProfileEntries = profilesWithAvatarIds.map(profile => curateProfileEntry(profile));
 
 	await upsertProfiles(curatedProfileEntries);
 }
@@ -350,7 +356,7 @@ async function associateActors(releases, batchId) {
 	const baseActors = Object.values(baseActorsByReleaseId).flat();
 
 	if (baseActors.length === 0) {
-		return;
+		return null;
 	}
 
 	const baseActorsBySlugAndNetworkId = baseActors.reduce((acc, baseActor) => ({
@@ -382,6 +388,8 @@ async function associateActors(releases, batchId) {
 		.flat();
 
 	await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
+
+	return actors;
 }
 
 module.exports = {

src/media.js

@@ -106,10 +106,11 @@ function toBaseSource(rawSource) {
 	return null;
 }
 
-function baseSourceToBaseMedia(baseSource, role) {
+function baseSourceToBaseMedia(baseSource, role, metadata) {
 	if (Array.isArray(baseSource)) {
 		if (baseSource.length > 0) {
 			return {
+				...metadata,
 				id: nanoid(),
 				role,
 				sources: baseSource,
@@ -121,6 +122,7 @@ function baseSourceToBaseMedia(baseSource, role) {
 
 	if (baseSource) {
 		return {
+			...metadata,
 			id: nanoid(),
 			role,
 			sources: [baseSource],
@@ -130,15 +132,15 @@ function baseSourceToBaseMedia(baseSource, role) {
 	return null;
 }
 
-function fallbackMediaToBaseMedia(rawMedia, role) {
+function fallbackMediaToBaseMedia(rawMedia, role, metadata) {
 	const baseSources = rawMedia
 		.map(source => toBaseSource(source))
 		.filter(Boolean);
 
-	return baseSourceToBaseMedia(baseSources, role);
+	return baseSourceToBaseMedia(baseSources, role, metadata);
 }
 
-function toBaseMedias(rawMedias, role) {
+function toBaseMedias(rawMedias, role, metadata) {
 	if (!rawMedias || rawMedias.length === 0) {
 		return [];
 	}
@@ -150,12 +152,12 @@ function toBaseMedias(rawMedias, role) {
 
 		if (Array.isArray(rawMedia)) {
 			// fallback sources provided
-			return fallbackMediaToBaseMedia(rawMedia, role);
+			return fallbackMediaToBaseMedia(rawMedia, role, metadata);
 		}
 
 		const baseSource = toBaseSource(rawMedia);
 
-		return baseSourceToBaseMedia(baseSource, role);
+		return baseSourceToBaseMedia(baseSource, role, metadata);
 	}).filter(Boolean);
 
 	const sampledBaseMedias = sampleMedias(baseMedias);
@@ -604,6 +606,44 @@ async function associateReleaseMedia(releases) {
 	}, Promise.resolve());
 }
 
+async function associateAvatars(profiles) {
+	if (!argv.media) {
+		return profiles;
+	}
+
+	const profilesWithBaseMedias = profiles.map(profile => (profile.avatar
+		? {
+			...profile,
+			avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', {
+				copyright: profile.network?.name || null,
+				scraper: profile.scraper || null,
+			})[0],
+		}
+		: profile
+	));
+
+	const baseMedias = profilesWithBaseMedias.map(profile => profile.avatarBaseMedia).filter(Boolean);
+
+	const storedMedias = await storeMedias(baseMedias);
+	const storedMediasById = itemsByKey(storedMedias, 'id');
+
+	const profilesWithAvatarIds = profilesWithBaseMedias.map((profile) => {
+		const media = storedMediasById[profile.avatarBaseMedia?.id];
+
+		if (media) {
+			return {
+				...profile,
+				avatarMediaId: media.use || media.entry.id,
+			};
+		}
+
+		return profile;
+	});
+
+	return profilesWithAvatarIds;
+}
+
 module.exports = {
+	associateAvatars,
 	associateReleaseMedia,
 };

src/scrapers/gamma.js

@@ -365,8 +365,8 @@ function scrapeApiProfile(data, releases, siteSlug) {
 	if (data.attributes.eye_color) profile.eyes = data.attributes.eye_color;
 	if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
 
-	const avatarPath = Object.values(data.pictures).reverse()[0];
+	const avatarPaths = Object.values(data.pictures).reverse();
-	if (avatarPath) profile.avatar = `https://images01-evilangel.gammacdn.com/actors${avatarPath}`;
+	if (avatarPaths.length > 0) profile.avatar = avatarPaths.map(avatarPath => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
 
 	profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
 

src/scrapers/julesjordan.js

@@ -342,8 +342,6 @@ function scrapeProfile(html, url, actorName) {
 
 	profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href);
 
-	console.log(profile);
-
 	return profile;
 }
 

src/store-releases.js

@@ -5,7 +5,7 @@ const config = require('config');
 const logger = require('./logger')(__filename);
 const knex = require('./knex');
 const slugify = require('./utils/slugify');
-const { associateActors } = require('./actors');
+const { associateActors, scrapeActors } = require('./actors');
 const { associateReleaseTags } = require('./tags');
 const { curateSite } = require('./sites');
 const { associateReleaseMedia } = require('./media');
@@ -226,18 +226,19 @@ async function storeReleases(releases) {
 	const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
 	const releasesWithId = attachReleaseIds([].concat(uniqueReleases, duplicateReleases), [].concat(storedReleaseEntries, duplicateReleaseEntries));
 
-	await Promise.all([
+	const [actors] = await Promise.all([
 		associateActors(releasesWithId, batchId),
 		associateReleaseTags(releasesWithId),
 	]);
 
+	await updateReleasesSearch(releasesWithId.map(release => release.id));
+
 	// media is more error-prone, associate separately
 	await associateReleaseMedia(releasesWithId);
+	await scrapeActors(actors.map(actor => actor.name));
 
 	logger.info(`Stored ${storedReleaseEntries.length} releases`);
 
-	await updateReleasesSearch(releasesWithId.map(release => release.id));
-
 	return releasesWithId;
 }
 

src/utils/http.js

@@ -36,17 +36,13 @@ function useProxy(url) {
 
 const queue = taskQueue();
 
-queue.on('concurrencyReached:http', () => {
+async function handler({
-	logger.silly('Queueing requests');
-});
-
-queue.define('http', async ({
 	url,
 	method = 'GET',
 	body,
 	headers = {},
 	options = {},
-}) => {
+}) {
 	if (body) {
 		logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
 	} else {
@@ -55,12 +51,12 @@ queue.define('http', async ({
 
 	const reqOptions = {
 		headers: {
-			...(options.defaultHeaders !== false && defaultHeaders),
+			...(options?.defaultHeaders !== false && defaultHeaders),
 			...headers,
 		},
 		...defaultOptions,
 		...options,
-		...(options.timeout && { responseTimeout: options.timeout }),
+		...(options?.timeout && { responseTimeout: options?.timeout }),
 	};
 
 	if (useProxy(url)) {
@@ -71,8 +67,8 @@ queue.define('http', async ({
 		? await bhttp[method.toLowerCase()](url, body, reqOptions)
 		: await bhttp[method.toLowerCase()](url, reqOptions);
 
-	if (options.stream && options.destination) {
+	if (options?.stream && options?.destination) {
-		await pipeline(res, ...(options.transforms || []), options.destination);
+		await pipeline(res, ...(options?.transforms || []), options?.destination);
 	}
 
 	const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
@@ -88,12 +84,22 @@ queue.define('http', async ({
 		code: res.statusCode,
 		status: res.statusCode,
 	};
-}, {
+}
+
+queue.on('concurrencyReached:http', () => {
+	logger.silly('Queueing requests');
+});
+
+queue.define('20p', handler, {
 	concurrency: 20,
 });
 
-async function get(url, headers, options) {
+queue.define('1s', handler, {
-	return queue.push('http', {
+	interval: 1,
+});
+
+async function get(url, headers, options, queueMethod = '20p') {
+	return queue.push(queueMethod, {
 		method: 'GET',
 		url,
 		headers,
@@ -101,8 +107,8 @@ async function get(url, headers, options) {
 	});
 }
 
-async function post(url, body, headers, options) {
+async function post(url, body, headers, options, queueMethod = '20p') {
-	return queue.push('http', {
+	return queue.push(queueMethod, {
 		method: 'POST',
 		url,
 		body,

src/utils/resolve-place.js

@@ -12,7 +12,7 @@ async function resolvePlace(query) {
 		// https://operations.osmfoundation.org/policies/nominatim/
 		const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, {
 			'User-Agent': 'contact at moonloop.adult@protonmail.com',
-		});
+		}, null, '1s');
 
 		const [item] = res.body;
 

src/web/plugins/actors.js

@@ -12,7 +12,7 @@ const schemaExtender = makeExtendSchemaPlugin(_build => ({
         }
 
       extend type Actor {
-        age: Int @requires(columns: ["birthdate"])
+        age: Int @requires(columns: ["date_of_birth"])
         height(units:Units): String @requires(columns: ["height"])
         weight(units:Units): String @requires(columns: ["weight"])
       }