Added Wicked network. Merged Evil Angel, XEmpire and Wicked into generic Gamma scraper.

src/scrapers/evilangel.js

@@ -1,235 +1,10 @@
 'use strict';
 
-const bhttp = require('bhttp');
-const cheerio = require('cheerio');
-const moment = require('moment');
-
-const { getPhotos } = require('./gamma');
-
-async function scrape(json, site) {
-    return Promise.all(json.map(async (scene) => {
-        const {
-            title,
-            description,
-            length,
-            master_categories: tags,
-            ratings_up: likes,
-            ratings_down: dislikes,
-        } = scene;
-
-        const entryId = scene.clip_id;
-        const url = `https://evilangel.com/en/video/${scene.url_title}/${entryId}`;
-        const date = moment(scene.release_date, 'YYYY-MM-DD').toDate();
-        const actors = scene.actors.map(({ name }) => name);
-        const director = scene.directors[0].name;
-
-        const poster = `https://images-evilangel.gammacdn.com/movies${scene.pictures.resized}`;
-        const movie = `https://evilangel.com/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
-
-        return {
-            url,
-            entryId,
-            title,
-            description,
-            length,
-            actors,
-            director,
-            date,
-            tags,
-            poster,
-            rating: {
-                likes,
-                dislikes,
-            },
-            movie,
-            site,
-        };
-    }));
-}
-
-async function scrapeScene(html, url, site) {
-    const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const json = $('script[type="application/ld+json"]').html();
-    const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
-
-    const [data, data2] = JSON.parse(json);
-    const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
-    const entryId = new URL(url).pathname.split('/').slice(-1)[0];
-
-    const {
-        name: title,
-        description,
-    } = data;
-    // date in data object is not the release date of the scene, but the date the entry was added
-    const date = moment.utc($('.updatedDate').first().text(), 'MM-DD-YYYY').toDate();
-
-    const actors = data.actor.map(actor => actor.name);
-    const hasTrans = data.actor.some(actor => actor.gender === 'shemale');
-
-    const director = (data.director && data.director[0].name) || (data2.director && data2.director[0].name) || null;
-    const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
-
-    const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
-
-    const rawTags = data.keywords.split(', ');
-    const tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
-
-    const poster = videoData.picPreview;
-    const trailer = `${videoData.playerOptions.host}${videoData.url}`;
-
-    const photos = await getPhotos($('.picturesItem a').attr('href'), 'evilangel.com', site);
-
-    return {
-        url,
-        entryId,
-        title,
-        date,
-        actors,
-        director,
-        description,
-        duration,
-        tags,
-        poster,
-        photos,
-        trailer: {
-            src: trailer,
-            quality: parseInt(videoData.sizeOnLoad, 10),
-        },
-        rating: {
-            stars,
-        },
-        site,
-    };
-}
-
-function scrapeActor(data, releases) {
-    const actor = {};
-
-    if (data.male === 1) actor.gender = 'male';
-    if (data.female === 1) actor.gender = 'female';
-    if (data.shemale === 1 || data.trans === 1) actor.gender = 'transsexual';
-
-    if (data.description) actor.description = data.description.trim();
-
-    if (data.attributes.ethnicity) actor.ethnicity = data.attributes.ethnicity;
-    if (data.attributes.eye_color) actor.eyes = data.attributes.eye_color;
-    if (data.attributes.hair_color) actor.hair = data.attributes.hair_color;
-
-    const avatarPath = Object.values(data.pictures).reverse()[0];
-    actor.avatar = `https://images01-evilangel.gammacdn.com/actors${avatarPath}`;
-
-    actor.releases = releases.map(release => `https://evilangel.com/en/video/${release.url_title}/${release.clip_id}`);
-
-    return actor;
-}
-
-async function fetchApiCredentials() {
-    const res = await bhttp.get('https://evilangel.com/en/videos');
-    const body = res.body.toString();
-
-    const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
-    const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
-    const apiData = JSON.parse(apiSerial);
-
-    const { applicationID: appId, apiKey } = apiData.api.algolia;
-    const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
-
-    const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
-
-    return {
-        appId,
-        apiKey,
-        userAgent,
-        apiUrl,
-    };
-}
-
-async function fetchLatest(site, page = 1, upcoming = false) {
-    const { apiUrl } = await fetchApiCredentials();
-
-    const res = await bhttp.post(apiUrl, {
-        requests: [
-            {
-                indexName: 'all_scenes',
-                params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]`,
-            },
-        ],
-    }, {
-        headers: {
-            Referer: 'https://www.evilangel.com/en/videos',
-        },
-        encodeJSON: true,
-    });
-
-    return scrape(res.body.results[0].hits, site);
-}
-
-async function fetchUpcoming(site) {
-    return fetchLatest(site, 1, true);
-}
-
-async function fetchScene(url, site) {
-    const res = await bhttp.get(url);
-
-    return scrapeScene(res.body.toString(), url, site);
-}
-
-async function fetchActorScenes(actorName, apiUrl) {
-    const res = await bhttp.post(apiUrl, {
-        requests: [
-            {
-                indexName: 'all_scenes',
-                params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=0&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["actors.name:${actorName}"]]`,
-            },
-        ],
-    }, {
-        headers: {
-            Referer: 'https://www.evilangel.com/en/videos',
-        },
-        encodeJSON: true,
-    });
-
-    if (res.statusCode === 200 && res.body.results[0].hits.length > 0) {
-        return res.body.results[0].hits;
-    }
-
-    return [];
-}
-
-async function fetchProfile(actorName) {
-    const { apiUrl } = await fetchApiCredentials();
-    const actorSlug = encodeURI(actorName);
-
-    const res = await bhttp.post(apiUrl, {
-        requests: [
-            {
-                indexName: 'all_actors',
-                params: `query=${actorSlug}`,
-            },
-        ],
-    }, {
-        headers: {
-            Referer: `https://www.evilangel.com/en/search?query=${actorSlug}&tab=actors`,
-        },
-        encodeJSON: true,
-    });
-
-    if (res.statusCode === 200 && res.body.results[0].hits.length > 0) {
-        const actorData = res.body.results[0].hits.find(actor => actor.name === actorName);
-
-        if (actorData) {
-            const actorScenes = await fetchActorScenes(actorName, apiUrl);
-
-            return scrapeActor(actorData, actorScenes);
-        }
-    }
-
-    return null;
-}
+const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
 
 module.exports = {
-    fetchLatest,
-    fetchProfile,
+    fetchLatest: fetchApiLatest,
+    fetchProfile: fetchApiProfile,
     fetchScene,
-    fetchUpcoming,
+    fetchUpcoming: fetchApiUpcoming,
 };