Added generic Gamma photo and actor scraper for XEmpire, 21Sextury, Blowpass and Evil Angel.

src/scrapers/21sextury.js

@@ -1,51 +1,10 @@
 'use strict';
 
-const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-async function fetchPhotos(photoPath) {
-    const res = await bhttp.get(`https://21sextury.com${photoPath}`);
-
-    return res.body.toString();
-}
-
-function scrapePhotos(html) {
-    const $ = cheerio.load(html, { normalizeWhitespace: true });
-
-    const unlockedPhotos = $('.preview .imgLink.pgUnlocked')
-        .map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
-
-    const lockedThumbnails = $('.preview .imgLink.lockedPicture img')
-        .map((photoIndex, photoElement) => $(photoElement)
-            .attr('src'))
-        // .replace('_tb.jpg', '.jpg')) does not always work
-        .toArray();
-
-    return unlockedPhotos.concat(lockedThumbnails);
-}
-
-async function getPhotos(photoPath) {
-    if (!photoPath || photoPath.match('join')) {
-        return [];
-    }
-
-    const html = await fetchPhotos(photoPath);
-    const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const photos = scrapePhotos(html);
-    const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
-
-    const otherPhotos = await Promise.map(pages, async (pagePath) => {
-        const pageHtml = await fetchPhotos(pagePath);
-
-        return scrapePhotos(pageHtml);
-    }, {
-        concurrency: 2,
-    });
-
-    return photos.concat(otherPhotos.flat());
-}
+const { getPhotos, fetchProfile } = require('./gamma');
 
 function scrape(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -131,8 +90,7 @@ async function scrapeScene(html, url, site) {
     const poster = videoData.picPreview;
     const trailer = `${videoData.playerOptions.host}${videoData.url}`;
 
-    const photoPath = $('.picturesItem a').attr('href');
-    const photos = await getPhotos(photoPath, site);
+    const photos = await getPhotos($('.picturesItem a').attr('href'), '21sextury.com', site);
 
     const tags = data.keywords.split(', ');
     const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
@@ -181,8 +139,13 @@ async function fetchScene(url, site) {
     return scrapeScene(res.body.toString(), url, site);
 }
 
+async function networkFetchProfile(actorName) {
+    return fetchProfile(actorName, '21sextury', true);
+}
+
 module.exports = {
     fetchLatest,
+    fetchProfile: networkFetchProfile,
     fetchUpcoming,
     fetchScene,
 };