Added Teen Core Club. Changed network to entity in GraphQL query.

src/argv.js

@@ -22,7 +22,12 @@ const { argv } = yargs
 	.option('sites', {
 		describe: 'Sites to scrape (overrides configuration)',
 		type: 'array',
-		alias: 'entities',
+		alias: 'site',
+	})
+	.option('entities', {
+		describe: 'Networks or sites to scrape (overrides configuration)',
+		type: 'array',
+		alias: 'entity',
 	})
 	.option('actors', {
 		describe: 'Scrape actors by name or slug',

src/entities.js

@@ -0,0 +1,144 @@
+'use strict';
+
+const config = require('config');
+
+const logger = require('./logger')(__filename);
+const argv = require('./argv');
+const knex = require('./knex');
+const whereOr = require('./utils/where-or');
+
+function curateEntity(entity, includeParameters = false) {
+	const curatedEntity = {
+		id: entity.id,
+		name: entity.name,
+		url: entity.url,
+		description: entity.description,
+		slug: entity.slug,
+		independent: !!entity.parameters && entity.parameters.independent,
+		parameters: includeParameters ? entity.parameters : null,
+		network: {
+			id: entity.network_id,
+			name: entity.network_name,
+			description: entity.network_description,
+			slug: entity.network_slug,
+			url: entity.network_url,
+			parameters: includeParameters ? entity.network_parameters : null,
+		},
+	};
+
+	return curatedEntity;
+}
+
+async function curateEntities(entities, includeParameters) {
+	return Promise.all(entities.map(async entity => curateEntity(entity, includeParameters)));
+}
+
+async function findSiteByUrl(url) {
+	const { origin, hostname, pathname } = new URL(url);
+	// const domain = hostname.replace(/www.|tour./, '');
+	const dirUrl = `${origin}${pathname.split('/').slice(0, 2).join('/')}`; // allow for sites on URI directory
+
+	const site = await knex('sites')
+		.leftJoin('networks', 'sites.network_id', 'networks.id')
+		.select(
+			'sites.*',
+			'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
+		)
+		.where('sites.url', url)
+		.orWhere('sites.url', origin)
+		.orWhere('sites.url', origin.replace(/www\.|tour\./, ''))
+		.orWhere('sites.url', `https://www.${hostname}`)
+		.orWhere('sites.url', `http://www.${hostname}`)
+		.orWhere('sites.url', dirUrl)
+	// .orWhere('sites.url', 'like', `%${domain}`)
+		.first();
+
+	if (site) {
+		const curatedSite = curateSite(site, true, false);
+
+		return curatedSite;
+	}
+
+	return null;
+}
+
+async function fetchEntitiesFromArgv() {
+	const rawEntities = await knex.raw(`
+		WITH RECURSIVE temp AS (
+			SELECT id, parent_id, name, slug, type FROM entities WHERE slug IN (?)
+			UNION
+				SELECT entities.id, entities.parent_id, entities.name, entities.slug, entities.type FROM entities
+				INNER JOIN temp ON temp.id = entities.parent_id
+		) SELECT * FROM temp;
+	`, argv.sites || argv.networks || argv.entities);
+
+	console.log(rawEntities.rows);
+
+	const curatedEntities = await curateEntities(rawEntities.rows, true);
+	logger.info(`Found ${curatedEntities.length} entities in database`);
+
+	console.log(curatedEntities);
+
+	return curatedEntities;
+}
+
+async function fetchEntitiesFromConfig() {
+	const rawSites = await knex('entities')
+		.select('entities.*')
+		.leftJoin('entities as entities_parents', 'entities_parents.id', 'entities.id')
+		.where((builder) => {
+			if (config.include) {
+				builder.whereIn('entities.slug', config.include);
+			}
+		})
+		.whereNot((builder) => {
+			builder.whereIn('entities.slug', config.exclude || []);
+		});
+
+	const curatedSites = await curateSites(rawSites, true);
+	logger.info(`Found ${curatedSites.length} sites in database`);
+
+	return curatedSites;
+}
+
+async function fetchIncludedEntities() {
+	if (argv.networks || argv.sites) {
+		return fetchEntitiesFromArgv();
+	}
+
+	return fetchEntitiesFromConfig();
+}
+
+async function fetchSites(queryObject) {
+	const sites = await knex('sites')
+		.where(builder => whereOr(queryObject, 'sites', builder))
+		.select(
+			'sites.*',
+			'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
+		)
+		.leftJoin('networks', 'sites.network_id', 'networks.id')
+		.limit(100);
+
+	return curateSites(sites);
+}
+
+async function fetchSitesFromReleases() {
+	const sites = await knex('releases')
+		.select('site_id', '')
+		.leftJoin('sites', 'sites.id', 'releases.site_id')
+		.groupBy('sites.id')
+		.limit(100);
+
+	return curateSites(sites);
+}
+
+module.exports = {
+	curateEntity,
+	curateEntities,
+	fetchIncludedEntities,
+	fetchSites,
+	fetchEntitiesFromConfig,
+	fetchEntitiesFromArgv,
+	fetchSitesFromReleases,
+	findSiteByUrl,
+};