Added include and exclude config options. Moved network definitions to dedicated file. Scraping all XEmpire sites. Added definitions for Perv City.

src/app.js

@@ -1,45 +1,99 @@
 'use strict';
 
-const bhttp = require('bhttp');
-const cheerio = require('cheerio');
-const { parse, format } = require('date-fns');
+const util = require('util');
+const config = require('config');
+const networks = require('../networks.js');
+const scrapers = require('./scrapers');
+
+function accumulateIncludedSites() {
+    return config.include.reduce((acc, network) => {
+        // network included with specific sites, only include specified sites
+        if (Array.isArray(network)) {
+            const [networkId, siteIds] = network;
+
+            return [
+                ...acc,
+                ...siteIds.map(siteId => ({
+                    id: siteId,
+                    network: networkId,
+                    ...networks[networkId].sites[siteId],
+                })),
+            ];
+        }
+
+        // network included without further specification, include all sites
+        return [
+            ...acc,
+            ...Object.entries(networks[network].sites).map(([siteId, site]) => ({
+                id: siteId,
+                network,
+                ...site,
+            })),
+        ];
+    }, []);
+}
+
+function accumulateExcludedSites() {
+    return Object.entries(networks).reduce((acc, [networkId, network]) => {
+        const excludedNetwork = config.exclude.find((excludedNetworkX) => {
+            if (Array.isArray(excludedNetworkX)) {
+                return excludedNetworkX[0] === networkId;
+            }
+
+            return excludedNetworkX === networkId;
+        });
+
+        // network excluded with specific sites, only exclude specified sites
+        if (excludedNetwork && Array.isArray(excludedNetwork)) {
+            const [, excludedSiteIds] = excludedNetwork;
+
+            return [
+                ...acc,
+                ...Object.entries(network.sites)
+                    .filter(([siteId]) => !excludedSiteIds.includes(siteId))
+                    .map(([siteId, site]) => ({
+                        id: siteId,
+                        network: networkId,
+                        ...site,
+                    })),
+            ];
+        }
+
+        // network excluded without further specification, exclude all its sites
+        if (excludedNetwork) {
+            return acc;
+        }
+
+        // network not excluded, include all its sites
+        return [
+            ...acc,
+            ...Object.entries(network.sites).map(([siteId, site]) => ({
+                id: siteId,
+                network: networkId,
+                ...site,
+            })),
+        ];
+    }, []);
+}
+
+function accumulateSites() {
+    return config.include ? accumulateIncludedSites() : accumulateExcludedSites();
+}
 
 async function init() {
-    const baseUrl = 'https://www.hardx.com';
-    const res = await bhttp.get(`${baseUrl}/en/videos`, {});
-    const $ = cheerio.load(res.body.toString(), { normalizeWhitespace: true });
-    const scenesElements = $('.sceneInfo').toArray();
+    const sites = accumulateSites();
 
-    const scenes = scenesElements.map((element) => {
-        const sceneLinkElement = $(element).find('.sceneTitle a');
-        const url = `${baseUrl}${sceneLinkElement.attr('href')}`;
-        const title = sceneLinkElement.attr('title');
+    const scenes = await Promise.all(sites.map(async (site) => {
+        const scraper = scrapers[site.id] || scrapers[site.network];
 
-        const [likes, dislikes] = $(element).find('.value')
-            .toArray()
-            .map(value => Number($(value).text()));
-        const stars = Math.floor(((likes * 5 + dislikes) / (likes + dislikes)) * 100) / 100;
+        if (scraper) {
+            return scraper(site);
+        }
 
-        const actors = $(element).find('.sceneActors a')
-            .map((actorIndex, actorElement) => $(actorElement).attr('title'))
-            .toArray();
+        return [];
+    }));
 
-        const date = parse($(element).find('.sceneDate').text(), 'MM-DD-YYYY');
-
-        return {
-            url,
-            title,
-            actors,
-            date,
-            rating: {
-                likes,
-                dislikes,
-                stars,
-            },
-        };
-    });
-
-    console.log(scenes);
+    console.log(util.inspect(scenes, { depth: 5 }));
 }
 
 init();