Added generic Gamma photo and actor scraper for XEmpire, 21Sextury, Blowpass and Evil Angel.

src/scrapers/xempire.js

@@ -1,79 +1,10 @@
 'use strict';
 
-const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
-const { JSDOM } = require('jsdom');
 const moment = require('moment');
 
-const defaultTags = {
-    hardx: [],
-    darkx: ['interracial'],
-    eroticax: [],
-    lesbianx: ['lesbian'],
-    allblackx: ['ebony', 'bbc'],
-};
-
-async function fetchPhotos(url) {
-    const res = await bhttp.get(url);
-
-    return res.body.toString();
-}
-
-function scrapePhotos(html) {
-    const $ = cheerio.load(html, { normalizeWhitespace: true });
-
-    return $('.preview .imgLink').toArray().map((linkEl) => {
-        const url = $(linkEl).attr('href');
-
-        if (url.match('/join')) {
-            // URL links to join page instead of full photo, extract thumbnail
-            const src = $(linkEl).find('img').attr('src');
-
-            if (src.match('previews/')) {
-                // resource often serves full photo at a modifier URL anyway, add as primary source
-                const highRes = src
-                    .replace('previews/', '')
-                    .replace('_tb.jpg', '.jpg');
-
-                // keep original thumbnail as fallback in case full photo is not available
-                return [highRes, src];
-            }
-
-            return src;
-        }
-
-        // URL links to full photo
-        return url;
-    });
-}
-
-async function getPhotos(albumPath, siteDomain) {
-    const albumUrl = `https://${siteDomain}${albumPath}`;
-
-    try {
-        const html = await fetchPhotos(albumUrl);
-        const $ = cheerio.load(html, { normalizeWhitespace: true });
-        const photos = scrapePhotos(html);
-
-        const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
-
-        const otherPhotos = await Promise.map(pages, async (page) => {
-            const pageUrl = `https://${siteDomain}${page}`;
-            const pageHtml = await fetchPhotos(pageUrl);
-
-            return scrapePhotos(pageHtml);
-        }, {
-            concurrency: 2,
-        });
-
-        return photos.concat(otherPhotos.flat());
-    } catch (error) {
-        console.error(`Failed to fetch XEmpire photos from ${albumPath}: ${error.message}`);
-
-        return [];
-    }
-}
+const { getPhotos, fetchProfile } = require('./gamma');
 
 function scrape(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -154,8 +85,7 @@ async function scrapeScene(html, url, site) {
 
     const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
 
-    const rawTags = data.keywords.split(', ');
-    const tags = [...defaultTags[siteSlug], ...rawTags];
+    const tags = data.keywords.split(', ');
 
     return {
         url: `${siteUrl}/en/video/${new URL(url).pathname.split('/').slice(-2).join('/')}`,
@@ -181,31 +111,6 @@ async function scrapeScene(html, url, site) {
     };
 }
 
-function scrapeActorSearch(html, url, actorName) {
-    const { document } = new JSDOM(html).window;
-    const actorLink = document.querySelector(`a[title="${actorName}" i]`);
-
-    return actorLink ? actorLink.href : null;
-}
-
-function scrapeProfile(html, url, actorName) {
-    const { document } = new JSDOM(html).window;
-
-    const avatarEl = document.querySelector('img.actorPicture');
-    const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
-
-    const profile = {
-        name: actorName,
-    };
-
-    if (avatarEl) profile.avatar = avatarEl.src;
-    if (descriptionEl) profile.description = descriptionEl.textContent.trim();
-
-    profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://xempire.com${el.href}`);
-
-    return profile;
-}
-
 async function fetchLatest(site, page = 1) {
     const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
 
@@ -224,34 +129,13 @@ async function fetchScene(url, site) {
     return scrapeScene(res.body.toString(), url, site);
 }
 
-async function fetchProfile(actorName) {
-    const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
-    const searchUrl = `https://www.xempire.com/en/search/xempire/actor/${actorSlug}`;
-    const searchRes = await bhttp.get(searchUrl);
-
-    if (searchRes.statusCode !== 200) {
-        return null;
-    }
-
-    const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
-
-    if (actorUrl) {
-        const url = `https://xempire.com${actorUrl}`;
-        const actorRes = await bhttp.get(url);
-
-        if (actorRes.statusCode !== 200) {
-            return null;
-        }
-
-        return scrapeProfile(actorRes.body.toString(), url, actorName);
-    }
-
-    return null;
+async function xEmpireFetchProfile(actorName) {
+    return fetchProfile(actorName, 'xempire');
 }
 
 module.exports = {
     fetchLatest,
-    fetchProfile,
+    fetchProfile: xEmpireFetchProfile,
     fetchUpcoming,
     fetchScene,
 };