145 lines
4.1 KiB
JavaScript
145 lines
4.1 KiB
JavaScript
|
'use strict';
|
||
|
|
||
|
const config = require('config');
|
||
|
|
||
|
const logger = require('./logger')(__filename);
|
||
|
const argv = require('./argv');
|
||
|
const knex = require('./knex');
|
||
|
const whereOr = require('./utils/where-or');
|
||
|
|
||
|
function curateEntity(entity, includeParameters = false) {
|
||
|
const curatedEntity = {
|
||
|
id: entity.id,
|
||
|
name: entity.name,
|
||
|
url: entity.url,
|
||
|
description: entity.description,
|
||
|
slug: entity.slug,
|
||
|
independent: !!entity.parameters && entity.parameters.independent,
|
||
|
parameters: includeParameters ? entity.parameters : null,
|
||
|
network: {
|
||
|
id: entity.network_id,
|
||
|
name: entity.network_name,
|
||
|
description: entity.network_description,
|
||
|
slug: entity.network_slug,
|
||
|
url: entity.network_url,
|
||
|
parameters: includeParameters ? entity.network_parameters : null,
|
||
|
},
|
||
|
};
|
||
|
|
||
|
return curatedEntity;
|
||
|
}
|
||
|
|
||
|
async function curateEntities(entities, includeParameters) {
|
||
|
return Promise.all(entities.map(async entity => curateEntity(entity, includeParameters)));
|
||
|
}
|
||
|
|
||
|
async function findSiteByUrl(url) {
|
||
|
const { origin, hostname, pathname } = new URL(url);
|
||
|
// const domain = hostname.replace(/www.|tour./, '');
|
||
|
const dirUrl = `${origin}${pathname.split('/').slice(0, 2).join('/')}`; // allow for sites on URI directory
|
||
|
|
||
|
const site = await knex('sites')
|
||
|
.leftJoin('networks', 'sites.network_id', 'networks.id')
|
||
|
.select(
|
||
|
'sites.*',
|
||
|
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
|
||
|
)
|
||
|
.where('sites.url', url)
|
||
|
.orWhere('sites.url', origin)
|
||
|
.orWhere('sites.url', origin.replace(/www\.|tour\./, ''))
|
||
|
.orWhere('sites.url', `https://www.${hostname}`)
|
||
|
.orWhere('sites.url', `http://www.${hostname}`)
|
||
|
.orWhere('sites.url', dirUrl)
|
||
|
// .orWhere('sites.url', 'like', `%${domain}`)
|
||
|
.first();
|
||
|
|
||
|
if (site) {
|
||
|
const curatedSite = curateSite(site, true, false);
|
||
|
|
||
|
return curatedSite;
|
||
|
}
|
||
|
|
||
|
return null;
|
||
|
}
|
||
|
|
||
|
async function fetchEntitiesFromArgv() {
|
||
|
const rawEntities = await knex.raw(`
|
||
|
WITH RECURSIVE temp AS (
|
||
|
SELECT id, parent_id, name, slug, type FROM entities WHERE slug IN (?)
|
||
|
UNION
|
||
|
SELECT entities.id, entities.parent_id, entities.name, entities.slug, entities.type FROM entities
|
||
|
INNER JOIN temp ON temp.id = entities.parent_id
|
||
|
) SELECT * FROM temp;
|
||
|
`, argv.sites || argv.networks || argv.entities);
|
||
|
|
||
|
console.log(rawEntities.rows);
|
||
|
|
||
|
const curatedEntities = await curateEntities(rawEntities.rows, true);
|
||
|
logger.info(`Found ${curatedEntities.length} entities in database`);
|
||
|
|
||
|
console.log(curatedEntities);
|
||
|
|
||
|
return curatedEntities;
|
||
|
}
|
||
|
|
||
|
async function fetchEntitiesFromConfig() {
|
||
|
const rawSites = await knex('entities')
|
||
|
.select('entities.*')
|
||
|
.leftJoin('entities as entities_parents', 'entities_parents.id', 'entities.id')
|
||
|
.where((builder) => {
|
||
|
if (config.include) {
|
||
|
builder.whereIn('entities.slug', config.include);
|
||
|
}
|
||
|
})
|
||
|
.whereNot((builder) => {
|
||
|
builder.whereIn('entities.slug', config.exclude || []);
|
||
|
});
|
||
|
|
||
|
const curatedSites = await curateSites(rawSites, true);
|
||
|
logger.info(`Found ${curatedSites.length} sites in database`);
|
||
|
|
||
|
return curatedSites;
|
||
|
}
|
||
|
|
||
|
async function fetchIncludedEntities() {
|
||
|
if (argv.networks || argv.sites) {
|
||
|
return fetchEntitiesFromArgv();
|
||
|
}
|
||
|
|
||
|
return fetchEntitiesFromConfig();
|
||
|
}
|
||
|
|
||
|
async function fetchSites(queryObject) {
|
||
|
const sites = await knex('sites')
|
||
|
.where(builder => whereOr(queryObject, 'sites', builder))
|
||
|
.select(
|
||
|
'sites.*',
|
||
|
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
|
||
|
)
|
||
|
.leftJoin('networks', 'sites.network_id', 'networks.id')
|
||
|
.limit(100);
|
||
|
|
||
|
return curateSites(sites);
|
||
|
}
|
||
|
|
||
|
async function fetchSitesFromReleases() {
|
||
|
const sites = await knex('releases')
|
||
|
.select('site_id', '')
|
||
|
.leftJoin('sites', 'sites.id', 'releases.site_id')
|
||
|
.groupBy('sites.id')
|
||
|
.limit(100);
|
||
|
|
||
|
return curateSites(sites);
|
||
|
}
|
||
|
|
||
|
module.exports = {
|
||
|
curateEntity,
|
||
|
curateEntities,
|
||
|
fetchIncludedEntities,
|
||
|
fetchSites,
|
||
|
fetchEntitiesFromConfig,
|
||
|
fetchEntitiesFromArgv,
|
||
|
fetchSitesFromReleases,
|
||
|
findSiteByUrl,
|
||
|
};
|