Querying infinite parent depth for deep release entities.

src/entities.js

@@ -66,6 +66,21 @@ async function curateEntities(entities, includeParameters) {
 	return Promise.all(entities.map(async entity => curateEntity(entity, includeParameters)));
 }
 
+function urlToSiteSlug(url) {
+	try {
+		const slug = new URL(url)
+			.hostname
+			.match(/([\w-]+)\.\w+$/)?.[1]
+			.replace(/[-_]+/g, '');
+
+		return slug;
+	} catch (error) {
+		logger.warn(`Failed to derive entity slug from '${url}': ${error.message}`);
+
+		return null;
+	}
+}
+
 async function fetchIncludedEntities() {
 	const include = {
 		includeAll: !argv.networks && !argv.channels && !config.include?.networks && !config.include?.channels,
@@ -139,6 +154,46 @@ async function fetchIncludedEntities() {
 	return curatedNetworks;
 }
 
+async function fetchReleaseEntities(baseReleases) {
+	const baseReleasesWithoutEntity = baseReleases.filter(release => release.url && !release.site && !release.entity);
+
+	const entitySlugs = Array.from(new Set(
+		baseReleasesWithoutEntity
+			.map(baseRelease => urlToSiteSlug(baseRelease.url))
+			.filter(Boolean),
+	));
+
+	const entities = await knex.raw(`
+		WITH RECURSIVE tree as (
+			SELECT to_jsonb(entities) as entity,
+				parent_id,
+				array['parent'] as parent_path,
+				0 as depth
+			FROM entities
+			WHERE slug = ANY(:entitySlugs)
+
+			UNION ALL
+
+			SELECT jsonb_set(tree.entity, tree.parent_path, to_jsonb(entities)),
+				   entities.parent_id,
+				   tree.parent_path || array['parent'],
+				   depth + 1
+			FROM tree
+			JOIN entities ON tree.parent_id = entities.id
+		)
+		SELECT entity FROM tree WHERE parent_id is null
+		ORDER BY entity->'type' ASC;
+	`, { entitySlugs });
+
+	// channel entity will overwrite network entity
+	const entitiesBySlug = entities.rows.reduce((accEntities, { entity }) => ({
+		...accEntities,
+		[entity.slug]: accEntities[entity.slug] || curateEntity(entity, true),
+	}), {});
+
+	return entitiesBySlug;
+}
+
 async function fetchEntity(entityId, type) {
 	const entity = await knex('entities')
 		.select(knex.raw(`
@@ -290,8 +345,10 @@ module.exports = {
 	curateEntity,
 	curateEntities,
 	fetchIncludedEntities,
+	fetchReleaseEntities,
 	fetchEntity,
 	fetchEntities,
 	searchEntities,
 	flushEntities,
+	urlToSiteSlug,
 };