Refactoring deep scrape. Added tag posters.

This commit is contained in:
2020-03-16 04:10:52 +01:00
parent c8ebe7892a
commit 0f09fd53eb
31 changed files with 851 additions and 589 deletions

View File

@@ -7,19 +7,13 @@ const argv = require('./argv');
const knex = require('./knex');
const whereOr = require('./utils/where-or');
async function curateSite(site, includeParameters = false) {
const tags = await knex('sites_tags')
.select('tags.*', 'sites_tags.inherit')
.where('site_id', site.id)
.join('tags', 'tags.id', 'sites_tags.tag_id');
return {
async function curateSite(site, includeParameters = false, includeTags = true) {
const curatedSite = {
id: site.id,
name: site.name,
url: site.url,
description: site.description,
slug: site.slug,
tags,
independent: !!site.parameters && site.parameters.independent,
parameters: includeParameters ? site.parameters : null,
network: {
@@ -31,6 +25,15 @@ async function curateSite(site, includeParameters = false) {
parameters: includeParameters ? site.network_parameters : null,
},
};
if (includeTags) {
curatedSite.tags = await knex('sites_tags')
.select('tags.*', 'sites_tags.inherit')
.where('site_id', site.id)
.join('tags', 'tags.id', 'sites_tags.tag_id');
}
return curatedSite;
}
function curateSites(sites, includeParameters) {
@@ -78,7 +81,7 @@ async function findSiteByUrl(url) {
.first();
if (site) {
const curatedSite = curateSite(site, true);
const curatedSite = curateSite(site, true, false);
return curatedSite;
}
@@ -182,6 +185,7 @@ async function fetchSitesFromReleases() {
}
module.exports = {
curateSite,
curateSites,
fetchIncludedSites,
fetchSites,