Completed Evil Angel scraper. Added trans tags.

src/fetch-releases.js

@@ -317,10 +317,13 @@ async function fetchReleases() {
                 }
 
                 console.log(`${site.id}: Failed to fetch releases`);
+
                 return [];
             }
         }
 
+        console.error(`Cound not find scraper for '${site.name}' (${site.slug})`);
+
         return [];
     }, {
         concurrency: 2,

src/scrapers/evilangel.js

@@ -0,0 +1,142 @@
+'use strict';
+
+const bhttp = require('bhttp');
+const cheerio = require('cheerio');
+const moment = require('moment');
+
+const { matchTags } = require('../tags');
+
+async function scrape(json, site) {
+    return Promise.all(json.map(async (scene) => {
+        const {
+            title,
+            description,
+            length,
+            master_categories: rawTags,
+            ratings_up: likes,
+            ratings_down: dislikes,
+        } = scene;
+
+        const entryId = scene.clip_id;
+        const url = `https://evilangel.com/en/video/${scene.url_title}/${entryId}`;
+        const date = moment(scene.release_date, 'YYYY-MM-DD').toDate();
+        const actors = scene.actors.map(({ name }) => name);
+        const director = scene.directors[0].name;
+
+        const tags = await matchTags(rawTags);
+        const poster = `https://images-evilangel.gammacdn.com/movies${scene.pictures.resized}`;
+
+        return {
+            url,
+            entryId,
+            title,
+            description,
+            length,
+            actors,
+            director,
+            date,
+            tags,
+            poster,
+            rating: {
+                likes,
+                dislikes,
+            },
+            site,
+        };
+    }));
+}
+
+async function scrapeScene(html, url, site) {
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+    const json = $('script[type="application/ld+json"]').html();
+    const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
+
+    const [data, data2] = JSON.parse(json);
+    const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
+    const entryId = new URL(url).pathname.split('/').slice(-1)[0];
+
+    const {
+        name: title,
+        description,
+    } = data;
+    // date in data object is not the release date of the scene, but the date the entry was added
+    const date = moment.utc($('.updatedDate').first().text(), 'MM-DD-YYYY').toDate();
+
+    const actors = data.actor.map(actor => actor.name);
+
+    const director = (data.director && data.director[0].name) || (data2.director && data2.director[0].name) || null;
+    const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
+
+    const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
+
+    const rawTags = data.keywords.split(', ');
+    const tags = await matchTags(rawTags);
+
+    const poster = videoData.picPreview;
+    const trailer = `${videoData.playerOptions.host}${videoData.url}`;
+
+    return {
+        url,
+        entryId,
+        title,
+        date,
+        actors,
+        director,
+        description,
+        duration,
+        tags,
+        poster,
+        trailer: {
+            src: trailer,
+            quality: parseInt(videoData.sizeOnLoad, 10),
+        },
+        rating: {
+            stars,
+        },
+        site,
+    };
+}
+
+async function fetchLatest(site, page = 1, upcoming = false) {
+    const res = await bhttp.get('https://evilangel.com/en/videos');
+    const body = res.body.toString();
+
+    const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
+    const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
+    const apiData = JSON.parse(apiSerial);
+
+    const { applicationID: appId, apiKey } = apiData.api.algolia;
+    const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
+
+    const apiRes = await bhttp.post(`https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`, {
+        requests: [
+            {
+                indexName: 'all_scenes',
+                params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? '' : 0}"]]`,
+            },
+        ],
+    }, {
+        headers: {
+            Referer: 'https://www.evilangel.com/en/videos',
+        },
+        encodeJSON: true,
+    });
+
+    return scrape(apiRes.body.results[0].hits, site);
+}
+
+async function fetchUpcoming(site) {
+    return fetchLatest(site, 1, true);
+}
+
+async function fetchScene(url, site) {
+    const res = await bhttp.get(url);
+
+    return scrapeScene(res.body.toString(), url, site);
+}
+
+module.exports = {
+    fetchLatest,
+    fetchUpcoming,
+    fetchScene,
+};

src/scrapers/index.js

@@ -5,6 +5,7 @@ const bangbros = require('./bangbros');
 const blowpass = require('./blowpass');
 const brazzers = require('./brazzers');
 const ddfnetwork = require('./ddfnetwork');
+const evilangel = require('./evilangel');
 const julesjordan = require('./julesjordan');
 const kink = require('./kink');
 const legalporno = require('./legalporno');
@@ -22,6 +23,7 @@ module.exports = {
     blowpass,
     brazzers,
     ddfnetwork,
+    evilangel,
     julesjordan,
     kink,
     legalporno,