Integrated Blowpass into generic Gamma scraper.

src/sites.js

@@ -39,7 +39,7 @@ function curateSites(sites, includeParameters) {
     return Promise.all(sites.map(async site => curateSite(site, includeParameters)));
 }
 
-function destructConfigNetworks(networks) {
+function destructConfigNetworks(networks = []) {
     return networks.reduce((acc, network) => {
         if (Array.isArray(network)) {
             // network specifies sites
@@ -119,6 +119,7 @@ async function fetchSitesFromArgv() {
 
 async function fetchSitesFromConfig() {
     const included = destructConfigNetworks(config.include);
+    const excluded = destructConfigNetworks(config.exclude);
 
     const rawSites = await knex('sites')
         .select(
@@ -126,8 +127,18 @@ async function fetchSitesFromConfig() {
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
         .leftJoin('networks', 'sites.network_id', 'networks.id')
-        .whereIn('sites.slug', included.sites || [])
-        .orWhereIn('networks.slug', included.networks || []);
+        .where((builder) => {
+            if (config.include) {
+                builder
+                    .whereIn('sites.slug', included.sites)
+                    .orWhereIn('networks.slug', included.networks);
+            }
+        })
+        .whereNot((builder) => {
+            builder
+                .whereIn('sites.slug', excluded.sites)
+                .orWhereIn('networks.slug', excluded.networks);
+        });
 
     const curatedSites = await curateSites(rawSites, true);
     logger.info(`Found ${curatedSites.length} sites in database`);