Improved update runner. Improved HTTP module API, added default user agent. Added PornCZ and Czechav logos.

src/deep.js

@@ -3,6 +3,7 @@
 const Promise = require('bluebird');
 
 const argv = require('./argv');
+const include = require('./utils/argv-include')(argv);
 const logger = require('./logger')(__filename);
 const knex = require('./knex');
 const scrapers = require('./scrapers/scrapers');
@@ -32,7 +33,11 @@ async function findSites(baseReleases) {
             .filter(Boolean),
     ));
 
-    const siteEntries = await knex('sites').whereIn('slug', siteSlugs);
+    const siteEntries = await knex('sites')
+        .leftJoin('networks', 'networks.id', 'sites.network_id')
+        .select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.parameters as network_parameters', 'networks.description as network_description')
+        .whereIn('sites.slug', siteSlugs);
+
     const networkEntries = await knex('networks').whereIn('slug', siteSlugs);
 
     const sites = await curateSites(siteEntries, true, false);
@@ -40,7 +45,7 @@ async function findSites(baseReleases) {
     const markedNetworks = networks.map(network => ({ ...network, isFallback: true }));
 
     const sitesBySlug = []
-        .concat(sites, markedNetworks)
+        .concat(markedNetworks, sites)
         .reduce((accSites, site) => ({ ...accSites, [site.slug]: site }), {});
 
     return sitesBySlug;
@@ -108,8 +113,8 @@ async function scrapeRelease(baseRelease, sites, type = 'scene') {
 
     try {
         const scrapedRelease = type === 'scene'
-            ? await scraper.fetchScene(baseRelease.url, site, baseRelease)
-            : await scraper.fetchMovie(baseRelease.url, site, baseRelease);
+            ? await scraper.fetchScene(baseRelease.url, site, baseRelease, null, include)
+            : await scraper.fetchMovie(baseRelease.url, site, baseRelease, null, include);
 
         const mergedRelease = {
             ...baseRelease,
@@ -129,21 +134,33 @@ async function scrapeRelease(baseRelease, sites, type = 'scene') {
     }
 }
 
-async function scrapeReleases(baseReleases, sites) {
+async function scrapeReleases(baseReleases, sites, type) {
     return Promise.map(
         baseReleases,
-        async baseRelease => scrapeRelease(baseRelease, sites),
+        async baseRelease => scrapeRelease(baseRelease, sites, type),
         { concurrency: 10 },
     );
 }
 
-async function fetchReleases(baseReleasesOrUrls) {
+async function fetchReleases(baseReleasesOrUrls, type = 'scene') {
     const baseReleases = toBaseReleases(baseReleasesOrUrls);
     const sites = await findSites(baseReleases);
 
-    const deepReleases = await scrapeReleases(baseReleases, sites);
+    const deepReleases = await scrapeReleases(baseReleases, sites, type);
 
     return deepReleases;
 }
 
-module.exports = fetchReleases;
+async function fetchScenes(baseReleasesOrUrls) {
+    return fetchReleases(baseReleasesOrUrls, 'scene');
+}
+
+async function fetchMovies(baseReleasesOrUrls) {
+    return fetchReleases(baseReleasesOrUrls, 'movie');
+}
+
+module.exports = {
+    fetchReleases,
+    fetchScenes,
+    fetchMovies,
+};