Added include and exclude config options. Moved network definitions to dedicated file. Scraping all XEmpire sites. Added definitions for Perv City.

src/scrapers/index.js

@@ -0,0 +1,9 @@
+'use strict';
+
+const pervcity = require('./pervcity');
+const xempire = require('./xempire');
+
+module.exports = {
+    pervcity,
+    xempire,
+};

src/scrapers/pervcity.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const bhttp = require('bhttp');
+const cheerio = require('cheerio');
+
+async function fetchReleases(site) {
+    // const session = bhttp.session();
+    return {};
+
+    console.log(site.url, site.parameters.tourId)
+
+    const res = await bhttp.get(`${site.url}/final_latestupdateview.php&limitstart=0&limitend=9&websiteid=0&deviceview=browser&tourId=${site.parameters.tourId}`);
+
+    console.log(res.body.toString());
+}
+
+module.exports = fetchReleases;

src/scrapers/xempire.js

@@ -0,0 +1,53 @@
+'use strict';
+
+const bhttp = require('bhttp');
+const cheerio = require('cheerio');
+const { parse, startOfDay } = require('date-fns');
+
+function scrape(html, site) {
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+    const scenesElements = $('.sceneInfo').toArray();
+
+    return scenesElements.map((element) => {
+        const sceneLinkElement = $(element).find('.sceneTitle a');
+        const url = `${site.url}${sceneLinkElement.attr('href')}`;
+        const title = sceneLinkElement.attr('title');
+
+        const [likes, dislikes] = $(element).find('.value')
+            .toArray()
+            .map(value => Number($(value).text()));
+        const stars = Math.floor(((likes * 5 + dislikes) / (likes + dislikes)) * 100) / 100;
+
+        const actors = $(element).find('.sceneActors a')
+            .map((actorIndex, actorElement) => $(actorElement).attr('title'))
+            .toArray();
+
+        const date = startOfDay(parse($(element).find('.sceneDate').text(), 'MM-DD-YYYY'));
+
+        return {
+            url,
+            title,
+            actors,
+            date,
+            rating: {
+                likes,
+                dislikes,
+                stars,
+            },
+        };
+    });
+}
+
+async function fetchReleases(site) {
+    const [latestRes, upcomingRes] = await Promise.all([
+        bhttp.get(`${site.url}/en/videos`),
+        bhttp.get(`${site.url}/en/videos/AllCategories/0/1/upcoming`),
+    ]);
+
+    return {
+        latest: scrape(latestRes.body.toString(), site),
+        upcoming: scrape(upcomingRes.body.toString(), site),
+    };
+}
+
+module.exports = fetchReleases;