Merge branch 'experimental' into master

README.md

@@ -55,7 +55,7 @@ To build traxxx, run the following command:
 
 `npm run build`
 
-To generate thumbnails for logos and tag photos, install ImageMagick and run:
+To generate thumbnails for new logos and tag photos, install ImageMagick and run:
 
 `npm run logos-thumbs`
 

assets/components/releases/media.vue

@@ -105,18 +105,20 @@ function sfw() {
 }
 
 function photos() {
+	const photosWithChapterPosters = (this.release.photos || []).concat(this.release.chapters ? this.release.chapters.map(chapter => chapter.poster) : []);
+
 	if (this.release.trailer || this.release.teaser) {
 		// poster will be on trailer video
-		return this.release.photos;
+		return photosWithChapterPosters;
 	}
 
 	if (this.release.poster) {
 		// no trailer, add poster to photos
-		return [this.release.poster].concat(this.release.photos);
+		return [this.release.poster].concat(this.release.photos).concat(photosWithChapterPosters);
 	}
 
 	// no poster available
-	return this.release.photos;
+	return photosWithChapterPosters;
 }
 
 export default {

assets/components/releases/movies.vue

@@ -53,6 +53,7 @@ export default {
 	display: grid;
 	grid-template-columns: repeat(auto-fill, minmax(25rem, 1fr));
 	grid-gap: 1rem;
+	padding: 1rem;
 }
 
 @media(max-width: $breakpoint) {

assets/components/releases/scene.vue

@@ -99,6 +99,51 @@
 				<p class="description">{{ release.description }}</p>
 			</div>
 
+			<ul
+				v-if="release.chapters && release.chapters.length > 0"
+				class="chapters row nolist"
+			>
+				<span class="row-label">Chapters</span>
+				<li
+					v-for="chapter in release.chapters"
+					:key="`chapter-${chapter.id}`"
+					class="chapter"
+				>
+					<a
+						v-if="chapter.poster"
+						:href="`/media/${chapter.poster.path}`"
+						target="_blank"
+						rel="noopener noreferrer"
+						class="chapter-poster-link"
+					>
+						<img
+							:src="`/media/${chapter.poster.thumbnail}`"
+							class="chapter-poster"
+						>
+					</a>
+
+					<div class="chapter-info">
+						<div class="chapter-header">
+							<h3 class="chapter-title">{{ chapter.title }}</h3>
+							<span
+								v-if="chapter.duration"
+								class="chapter-duration"
+							>{{ chapter.duration }}</span>
+						</div>
+
+						<p
+							v-if="chapter.description"
+							class="chapter-description"
+						>{{ chapter.description }}</p>
+
+						<ul class="nolist"><li
+							v-for="tag in chapter.tags"
+							:key="`chapter-tag-${tag.id}`"
+						>{{ tag.name }}</li></ul>
+					</div>
+				</li>
+			</ul>
+
 			<div class="row row-tidbits">
 				<div
 					v-if="release.duration"
@@ -143,6 +188,32 @@
 					<span class="row-label">Shoot date</span>
 					{{ formatDate(release.productionDate, 'MMMM D, YYYY') }}
 				</div>
+
+				<div
+					v-if="release.productionLocation"
+					class="row-tidbit"
+				>
+					<span class="row-label">Location</span>
+					<span class="location">
+						<span
+							v-if="release.productionLocation.city"
+							class="location-segment"
+						>{{ release.productionLocation.city }}, </span>
+						<span
+							v-if="release.productionLocation.state"
+							class="location-segment"
+						>{{ release.productionLocation.state }}, </span>
+						<span
+							v-if="release.productionLocation.country"
+							class="location-segment"
+						>{{ release.productionLocation.country.alias || release.productionLocation.country.name }}
+							<img
+								class="flag"
+								:src="`/img/flags/${release.productionLocation.country.alpha2.toLowerCase()}.svg`"
+							>
+						</span>
+					</span>
+				</div>
 			</div>
 
 			<div
@@ -324,6 +395,41 @@ export default {
 	text-overflow: ellipsis;
 }
 
+.chapter {
+	display: flex;
+	background: var(--background);
+	box-shadow: 0 0 3px var(--shadow-weak);
+	margin: 0 0 .5rem 0;
+}
+
+.chapter-poster {
+	width: 12rem;
+	height: 100%;
+	object-fit: cover;
+	object-position: center;
+}
+
+.chapter-info {
+	flex-grow: 1;
+	padding: 1rem 1rem .5rem 1rem;
+}
+
+.chapter-header {
+	display: flex;
+	justify-content: space-between;
+}
+
+.chapter-title {
+	display: inline-block;
+	padding: 0;
+	margin: 0;
+}
+
+.flag {
+	height: 1rem;
+	margin: 0 0 -.15rem .1rem;
+}
+
 .link {
     display: inline-flex;
     color: var(--link);

assets/js/curate.js

@@ -69,6 +69,7 @@ function curateRelease(release) {
 
 	if (release.scenes) curatedRelease.scenes = release.scenes.map(({ scene }) => curateRelease(scene));
 	if (release.movies) curatedRelease.movies = release.movies.map(({ movie }) => curateRelease(movie));
+	if (release.chapters) curatedRelease.chapters = release.chapters.map(chapter => curateRelease(chapter));
 	if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
 	if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media);
 	if (release.trailer) curatedRelease.trailer = release.trailer.media;
@@ -77,6 +78,15 @@ function curateRelease(release) {
 	if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.map(({ tag }) => tag);
 	if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.map(({ actor }) => curateActor(actor, curatedRelease));
 
+	if (release.productionLocation) {
+		curatedRelease.productionLocation = {
+			raw: release.productionLocation,
+			city: release.productionCity,
+			state: release.productionState,
+			country: release.productionCountry,
+		};
+	}
+
 	return curatedRelease;
 }
 

assets/js/fragments.js

@@ -237,6 +237,14 @@ const releaseFragment = `
     createdAt
     shootId
 	productionDate
+	productionLocation
+	productionCity
+	productionState
+	productionCountry: countryByProductionCountryAlpha2 {
+		alpha2
+		name
+		alias
+	}
 	comment
     url
     ${releaseActorsFragment}
@@ -247,6 +255,35 @@ const releaseFragment = `
     ${releaseTrailerFragment}
     ${releaseTeaserFragment}
     ${siteFragment}
+	chapters {
+		id
+		title
+		description
+		duration
+		tags: chaptersTags {
+			tag {
+				id
+				name
+				slug
+			}
+		}
+		poster: chaptersPosterByChapterId {
+			media {
+				index
+				path
+				thumbnail
+				lazy
+				comment
+				sfw: sfwMedia {
+					id
+					thumbnail
+					lazy
+					path
+					comment
+				}
+			}
+		}
+	}
     studio {
         id
         name
@@ -258,7 +295,7 @@ const releaseFragment = `
 			id
 			title
 			slug
-			covers: moviesCoversByReleaseId {
+			covers: moviesCovers {
 				media {
 					index
 					path

assets/js/releases/actions.js

@@ -7,8 +7,6 @@ function initReleasesActions(store, _router) {
 	async function fetchReleases({ _commit }, { limit = 10, pageNumber = 1, range = 'latest' }) {
 		const { before, after, orderBy } = getDateRange(range);
 
-		console.log(after, before, orderBy);
-
 		const { connection: { releases, totalCount } } = await graphql(`
             query Releases(
                 $limit:Int = 1000,
@@ -89,7 +87,7 @@ function initReleasesActions(store, _router) {
 								type
 							}
 						}
-						covers: moviesCoversByReleaseId {
+						covers: moviesCovers {
 							media {
 								id
 								path
@@ -139,14 +137,14 @@ function initReleasesActions(store, _router) {
 							lazy
 						}
 					}
-					covers: moviesCoversByReleaseId {
+					covers: moviesCovers {
 						media {
 							id
 							path
 							thumbnail
 						}
 					}
-					trailer: moviesTrailerByReleaseId {
+					trailer: moviesTrailerByMovieId {
 						media {
 							id
 							path

config/default.js

@@ -26,8 +26,6 @@ module.exports = {
 			'amberathome',
 			'marycarey',
 			'racqueldevonshire',
-			// boobpedia
-			'boobpedia',
 			// blowpass
 			'sunlustxxx',
 			// ddfnetwork

migrations/20190325001339_releases.js

@@ -614,6 +614,7 @@ exports.up = knex => Promise.resolve()
 
 		table.text('shoot_id');
 		table.text('entry_id');
+
 		table.unique(['entity_id', 'entry_id']);
 
 		table.text('url', 1000);
@@ -625,6 +626,13 @@ exports.up = knex => Promise.resolve()
 
 		table.date('production_date');
 
+		table.text('production_location');
+		table.text('production_city');
+		table.text('production_state');
+		table.text('production_country_alpha2', 2)
+			.references('alpha2')
+			.inTable('countries');
+
 		table.enum('date_precision', ['year', 'month', 'day', 'hour', 'minute', 'second'])
 			.defaultTo('day');
 
@@ -821,7 +829,7 @@ exports.up = knex => Promise.resolve()
 			.defaultTo(knex.fn.now());
 	}))
 	.then(() => knex.schema.createTable('movies_covers', (table) => {
-		table.integer('release_id', 16)
+		table.integer('movie_id', 16)
 			.notNullable()
 			.references('id')
 			.inTable('movies');
@@ -831,10 +839,10 @@ exports.up = knex => Promise.resolve()
 			.references('id')
 			.inTable('media');
 
-		table.unique(['release_id', 'media_id']);
+		table.unique(['movie_id', 'media_id']);
 	}))
 	.then(() => knex.schema.createTable('movies_trailers', (table) => {
-		table.integer('release_id', 16)
+		table.integer('movie_id', 16)
 			.unique()
 			.notNullable()
 			.references('id')
@@ -845,6 +853,74 @@ exports.up = knex => Promise.resolve()
 			.references('id')
 			.inTable('media');
 	}))
+	.then(() => knex.schema.createTable('chapters', (table) => {
+		table.increments('id', 16);
+
+		table.integer('release_id', 12)
+			.references('id')
+			.inTable('releases')
+			.notNullable();
+
+		table.integer('chapter', 6);
+
+		table.unique(['release_id', 'chapter']);
+
+		table.text('title');
+		table.text('description');
+
+		table.integer('duration')
+			.unsigned();
+
+		table.integer('created_batch_id', 12)
+			.references('id')
+			.inTable('batches');
+
+		table.integer('updated_batch_id', 12)
+			.references('id')
+			.inTable('batches');
+
+		table.datetime('created_at')
+			.defaultTo(knex.fn.now());
+	}))
+	.then(() => knex.schema.createTable('chapters_posters', (table) => {
+		table.integer('chapter_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('chapters');
+
+		table.text('media_id', 21)
+			.notNullable()
+			.references('id')
+			.inTable('media');
+
+		table.unique('chapter_id');
+	}))
+	.then(() => knex.schema.createTable('chapters_photos', (table) => {
+		table.integer('chapter_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('chapters');
+
+		table.text('media_id', 21)
+			.notNullable()
+			.references('id')
+			.inTable('media');
+
+		table.unique(['chapter_id', 'media_id']);
+	}))
+	.then(() => knex.schema.createTable('chapters_tags', (table) => {
+		table.integer('tag_id', 12)
+			.notNullable()
+			.references('id')
+			.inTable('tags');
+
+		table.integer('chapter_id', 16)
+			.notNullable()
+			.references('id')
+			.inTable('chapters');
+
+		table.unique(['tag_id', 'chapter_id']);
+	}))
 	// SEARCH
 	.then(() => { // eslint-disable-line arrow-body-style
 		// allow vim fold
@@ -1024,6 +1100,10 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS movies_scenes CASCADE;
 		DROP TABLE IF EXISTS movies_trailers CASCADE;
 
+		DROP TABLE IF EXISTS chapters_tags CASCADE;
+		DROP TABLE IF EXISTS chapters_posters CASCADE;
+		DROP TABLE IF EXISTS chapters_photos CASCADE;
+
 		DROP TABLE IF EXISTS batches CASCADE;
 
 		DROP TABLE IF EXISTS actors_avatars CASCADE;
@@ -1042,6 +1122,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS tags_posters CASCADE;
 		DROP TABLE IF EXISTS tags_photos CASCADE;
 		DROP TABLE IF EXISTS movies CASCADE;
+		DROP TABLE IF EXISTS chapters CASCADE;
 		DROP TABLE IF EXISTS releases CASCADE;
 		DROP TABLE IF EXISTS actors CASCADE;
 		DROP TABLE IF EXISTS directors CASCADE;

seeds/00_tags.js

@@ -57,6 +57,11 @@ const groups = [
 ];
 
 const tags = [
+	{
+		name: '3d',
+		slug: '3d',
+		description: 'Available in 3D.',
+	},
 	{
 		name: '4K',
 		slug: '4k',

seeds/02_sites.js

@@ -2645,6 +2645,12 @@ const sites = [
 			accFilter: true,
 		},
 	},
+	// IN THE CRACK
+	{
+		slug: 'inthecrack',
+		name: 'InTheCrack',
+		url: 'https://inthecrack.com/',
+	},
 	// INTERRACIAL PASS
 	{
 		slug: '2bigtobetrue',

src/actors.js

@@ -608,7 +608,7 @@ async function scrapeActors(argNames) {
 
 	logger.info(`Scraping profiles for ${actorNames.length} actors`);
 
-	const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
+	const sources = argv.actorsSources || config.profiles || Object.keys(scrapers.actors);
 	const entitySlugs = sources.flat();
 
 	const [entities, existingActorEntries] = await Promise.all([

src/media.js

@@ -617,7 +617,7 @@ async function storeMedias(baseMedias) {
 	return [...newMediaWithEntries, ...existingHashMedias];
 }
 
-async function associateReleaseMedia(releases, type = 'releases') {
+async function associateReleaseMedia(releases, type = 'release') {
 	if (!argv.media) {
 		return;
 	}
@@ -664,7 +664,7 @@ async function associateReleaseMedia(releases, type = 'releases') {
 
 					if (media) {
 						acc.push({
-							release_id: releaseId,
+							[`${type}_id`]: releaseId,
 							media_id: media.use || media.entry.id,
 						});
 					}
@@ -675,7 +675,7 @@ async function associateReleaseMedia(releases, type = 'releases') {
 			.filter(Boolean);
 
 		if (associations.length > 0) {
-			await bulkInsert(`${type}_${role}`, associations, false);
+			await bulkInsert(`${type}s_${role}`, associations, false);
 		}
 	}, Promise.resolve());
 }

src/scrapers/inthecrack.js

@@ -0,0 +1,124 @@
+'use strict';
+
+const moment = require('moment');
+
+const qu = require('../utils/q');
+const slugify = require('../utils/slugify');
+
+function scrapeAll(scenes, channel) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.url = query.url('a', 'href', { origin: channel.url });
+		release.entryId = new URL(release.url).pathname.match(/\/Collection\/(\d+)/)[1];
+
+		release.shootId = query.cnt('a span:nth-of-type(1)').match(/^\d+/)?.[0];
+		release.date = query.date('a span:nth-of-type(2)', 'YYYY-MM-DD');
+
+		release.actors = (query.q('a img', 'alt') || query.cnt('a span:nth-of-type(1)'))?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g);
+
+		release.poster = release.shootId
+			? `https://inthecrack.com/assets/images/posters/collections/${release.shootId}.jpg`
+			: query.img('a img', 'src', { origin: channel.url });
+
+		return release;
+	});
+}
+
+function scrapeScene({ query, html }, url, channel) {
+	const release = {};
+
+	release.entryId = new URL(url).pathname.match(/\/Collection\/(\d+)/)[1];
+	release.shootId = query.cnt('h2 span').match(/^\d+/)?.[0];
+
+	release.actors = query.cnt('h2 span')?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g);
+
+	release.description = query.cnt('p#CollectionDescription');
+	release.productionLocation = query.cnt('.modelCollectionHeader p')?.match(/Shoot Location: (.*)/)?.[1];
+
+	release.poster = qu.prefixUrl(html.match(/background-image: url\('(.*)'\)/)?.[1], channel.url);
+
+	release.chapters = query.all('.ClipOuter').map((el) => {
+		const chapter = {};
+
+		chapter.title = query.text(el, 'h4');
+		chapter.description = query.cnt(el, 'p');
+		chapter.duration = query.dur(el, '.InlineDuration');
+
+		const posterStyle = query.style(el, '.clipImage', 'background-image');
+		const poster = qu.prefixUrl(posterStyle.match(/url\((.*)\)/)?.[1], channel.url);
+
+		if (poster) {
+			const { origin, pathname } = new URL(poster);
+
+			chapter.poster = [
+				`${origin}${pathname}`, // full size
+				poster,
+			];
+		}
+
+		if (query.exists(el, '.ThreeDInfo')) {
+			chapter.tags = ['3d'];
+		}
+
+		return chapter;
+	});
+
+	return release;
+}
+
+function scrapeProfile({ query, el }, actorName, entity, include) {
+	const profile = {};
+
+	profile.description = query.cnt('.bio-text');
+	profile.birthPlace = query.cnt('.birth-place span');
+
+	profile.avatar = query.img('.actor-photo img');
+
+	if (include.releases) {
+		return scrapeAll(qu.initAll(el, '.scene'));
+	}
+
+	console.log(profile);
+	return profile;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const year = moment().subtract(page - 1, ' year').year();
+
+	const url = `${channel.url}/Collections/Date/${year}`;
+	const res = await qu.getAll(url, '.collectionGridLayout li');
+
+	if (res.ok) {
+		return scrapeAll(res.items, channel);
+	}
+
+	return res.status;
+}
+
+async function fetchScene(url, channel) {
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		return scrapeScene(res.item, url, channel);
+	}
+
+	return res.status;
+}
+
+async function fetchProfile({ name: actorName }, entity, include) {
+	const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		return scrapeProfile(res.item, actorName, entity, include);
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchScene,
+	// fetchProfile,
+};

src/scrapers/scrapers.js

@@ -27,6 +27,7 @@ const hitzefrei = require('./hitzefrei');
 const hush = require('./hush');
 const iconmale = require('./iconmale');
 const insex = require('./insex');
+const inthecrack = require('./inthecrack');
 const jayrock = require('./jayrock');
 const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials');
 const julesjordan = require('./julesjordan');
@@ -108,6 +109,7 @@ module.exports = {
 		hushpass: hush,
 		insex,
 		interracialpass: hush,
+		inthecrack,
 		jayrock,
 		jesseloadsmonsterfacials,
 		julesjordan,

src/store-releases.js

@@ -7,13 +7,14 @@ const logger = require('./logger')(__filename);
 const knex = require('./knex');
 const slugify = require('./utils/slugify');
 const bulkInsert = require('./utils/bulk-insert');
+const resolvePlace = require('./utils/resolve-place');
 const { formatDate } = require('./utils/qu');
 const { associateActors, scrapeActors } = require('./actors');
 const { associateReleaseTags } = require('./tags');
 const { curateEntity } = require('./entities');
 const { associateReleaseMedia } = require('./media');
 
-function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
+async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
 	const slugBase = release.title
 		|| (release.actors?.length && `${release.entity.slug} ${release.actors.map(actor => actor.name).join(' ')}`)
 		|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
@@ -50,6 +51,20 @@ function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
 		curatedRelease.duration = release.duration;
 	}
 
+	if (release.productionLocation) {
+		curatedRelease.production_location = release.productionLocation;
+
+		if (argv.resolvePlace) {
+			const productionLocation = await resolvePlace(release.productionLocation);
+
+			if (productionLocation) {
+				curatedRelease.production_city = productionLocation.city;
+				curatedRelease.production_state = productionLocation.state;
+				curatedRelease.production_country_alpha2 = productionLocation.country;
+			}
+		}
+	}
+
 	if (!existingRelease && !release.id) {
 		curatedRelease.created_batch_id = batchId;
 	}
@@ -228,6 +243,46 @@ async function updateReleasesSearch(releaseIds) {
 	}
 }
 
+async function storeChapters(releases) {
+	const chapters = releases.map(release => release.chapters?.map((chapter, index) => ({
+		title: chapter.title,
+		description: chapter.description,
+		releaseId: release.id,
+		chapter: index + 1,
+		duration: chapter.duration,
+		poster: chapter.poster,
+		photos: chapter.photos,
+		tags: chapter.tags,
+	}))).flat().filter(Boolean);
+
+	const curatedChapterEntries = chapters.map(chapter => ({
+		title: chapter.title,
+		description: chapter.description,
+		duration: chapter.duration,
+		release_id: chapter.releaseId,
+		chapter: chapter.chapter,
+	}));
+
+	const storedChapters = await bulkInsert('chapters', curatedChapterEntries);
+	const chapterIdsByReleaseIdAndChapter = storedChapters.reduce((acc, chapter) => ({
+		...acc,
+		[chapter.release_id]: {
+			...acc[chapter.release_id],
+			[chapter.chapter]: chapter.id,
+		},
+	}), {});
+
+	const chaptersWithId = chapters.map(chapter => ({
+		...chapter,
+		id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.chapter],
+	}));
+
+	await associateReleaseTags(chaptersWithId, 'chapter');
+
+	// media is more error-prone, associate separately
+	await associateReleaseMedia(chaptersWithId, 'chapter');
+}
+
 async function storeScenes(releases) {
 	if (releases.length === 0) {
 		return [];
@@ -241,7 +296,7 @@ async function storeScenes(releases) {
 	// uniqueness is entity ID + entry ID, filter uniques after adding entities
 	const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
 
-	const curatedNewReleaseEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId));
+	const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId)));
 
 	const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries);
 	// TODO: update duplicate releases
@@ -263,6 +318,8 @@ async function storeScenes(releases) {
 		await scrapeActors(actors.map(actor => actor.name));
 	}
 
+	await storeChapters(releasesWithId);
+
 	logger.info(`Stored ${storedReleaseEntries.length} releases`);
 
 	return releasesWithId;
@@ -303,13 +360,13 @@ async function storeMovies(movies, movieScenes) {
 	const { uniqueReleases } = await filterDuplicateReleases(movies);
 	const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
 
-	const curatedMovieEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie'));
+	const curatedMovieEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie')));
 
 	const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
 	const moviesWithId = attachReleaseIds(movies, storedMovies);
 
 	await associateMovieScenes(moviesWithId, movieScenes);
-	await associateReleaseMedia(moviesWithId, 'movies');
+	await associateReleaseMedia(moviesWithId, 'movie');
 
 	return storedMovies;
 }

src/tags.js

@@ -2,6 +2,7 @@
 
 const knex = require('./knex');
 const slugify = require('./utils/slugify');
+const bulkInsert = require('./utils/bulk-insert');
 
 async function matchReleaseTags(releases) {
 	const rawTags = releases
@@ -28,7 +29,7 @@ async function matchReleaseTags(releases) {
 }
 
 async function getEntityTags(releases) {
-	const entityIds = releases.map(release => release.entity.id);
+	const entityIds = releases.map(release => release.entity?.id).filter(Boolean);
 	const entityTags = await knex('entities_tags').whereIn('entity_id', entityIds);
 
 	const entityTagIdsByEntityId = entityTags.reduce((acc, entityTag) => {
@@ -44,10 +45,10 @@ async function getEntityTags(releases) {
 	return entityTagIdsByEntityId;
 }
 
-function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId) {
+function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type) {
 	const tagAssociations = releases
 		.map((release) => {
-			const entityTagIds = entityTagIdsByEntityId[release.entity.id];
+			const entityTagIds = entityTagIdsByEntityId[release.entity?.id] || [];
 			const releaseTags = release.tags || [];
 
 			const releaseTagIds = releaseTags.every(tag => typeof tag === 'number')
@@ -61,7 +62,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit
 					.filter(Boolean),
 			)]
 				.map(tagId => ({
-					release_id: release.id,
+					[`${type}_id`]: release.id,
 					tag_id: tagId,
 				}));
 
@@ -72,34 +73,13 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit
 	return tagAssociations;
 }
 
-async function filterUniqueAssociations(tagAssociations) {
+async function associateReleaseTags(releases, type = 'release') {
-	const duplicateAssociations = await knex('releases_tags')
-		.whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id]));
-
-	const duplicateAssociationsByReleaseIdAndTagId = duplicateAssociations.reduce((acc, association) => {
-		if (!acc[association.release_id]) {
-			acc[association.release_id] = {};
-		}
-
-		acc[association.release_id][association.tag_id] = true;
-
-		return acc;
-	}, {});
-
-	const uniqueAssociations = tagAssociations
-		.filter(association => !duplicateAssociationsByReleaseIdAndTagId[association.release_id]?.[association.tag_id]);
-
-	return uniqueAssociations;
-}
-
-async function associateReleaseTags(releases) {
 	const tagIdsBySlug = await matchReleaseTags(releases);
 	const EntityTagIdsByEntityId = await getEntityTags(releases);
 
-	const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, EntityTagIdsByEntityId);
+	const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, EntityTagIdsByEntityId, type);
-	const uniqueAssociations = await filterUniqueAssociations(tagAssociations);
 
-	await knex('releases_tags').insert(uniqueAssociations);
+	await bulkInsert(`${type}s_tags`, tagAssociations, false);
 }
 
 module.exports = {

src/utils/bulk-insert.js

@@ -4,6 +4,10 @@ const knex = require('../knex');
 const chunk = require('./chunk');
 
 async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
+	if (items.length === 0) {
+		return [];
+	}
+
 	const updated = (conflict === false && ':query ON CONFLICT DO NOTHING RETURNING *;')
 	|| (conflict && update && `
 		:query ON CONFLICT (${conflict})

src/utils/resolve-place.js

@@ -20,7 +20,15 @@ async function resolvePlace(query) {
 			const rawPlace = item.address;
 			const place = {};
 
-			if (rawPlace.city) place.city = rawPlace.city;
+			if (item.class === 'place' || item.class === 'boundary') {
+				const location = rawPlace[item.type] || rawPlace.city || rawPlace.place;
+
+				if (location) {
+					place.place = location;
+					place.city = rawPlace.city || location;
+				}
+			}
+
 			if (rawPlace.state) place.state = rawPlace.state;
 			if (rawPlace.country_code) place.country = rawPlace.country_code.toUpperCase();
 			if (rawPlace.continent) place.continent = rawPlace.continent;