Added 21Sextury scraper. Various improvements.

src/scrapers/21sextury.js

@@ -0,0 +1,141 @@
+'use strict';
+
+const bhttp = require('bhttp');
+const cheerio = require('cheerio');
+const moment = require('moment');
+
+const knex = require('../knex');
+const { matchTags } = require('../tags');
+
+function scrape(html, site) {
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+    const scenesElements = $('li[data-itemtype=scene]').toArray();
+
+    return scenesElements.reduce((accReleases, element) => {
+        const siteName = $(element).find('.studioName a').attr('title');
+
+        if (site.parameters && site.parameters.filter && siteName.toLowerCase() !== site.name.toLowerCase()) {
+            return accReleases;
+        }
+
+        const sceneLinkElement = $(element).find('.sceneTitle a');
+        const url = `${site.url}${sceneLinkElement.attr('href')}`;
+        const title = sceneLinkElement.attr('title').trim();
+
+        const entryId = $(element).attr('data-itemid');
+
+        const date = moment
+            .utc($(element).find('.sceneDate').text(), 'MM-DD-YYYY')
+            .toDate();
+
+        const actors = $(element).find('.sceneActors a')
+            .map((actorIndex, actorElement) => $(actorElement).attr('title'))
+            .toArray();
+
+        const [likes, dislikes] = $(element).find('.value')
+            .toArray()
+            .map(value => Number($(value).text()));
+
+        return [
+            ...accReleases,
+            {
+                url,
+                entryId,
+                title,
+                actors,
+                date,
+                rating: {
+                    likes,
+                    dislikes,
+                },
+                site,
+            },
+        ];
+    }, []);
+}
+
+async function scrapeScene(html, url, site) {
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+    const sceneElement = $('#videoWrapper');
+    const json = $('script[type="application/ld+json"]').html();
+
+    const data = JSON.parse(json)[0];
+    const entryId = new URL(url).pathname.split('/').slice(-1)[0];
+
+    const title = data.name;
+    const date = moment.utc(data.dateCreated, 'YYYY-MM-DD').toDate();
+
+    const actors = data.actor
+        .sort(({ genderA }, { genderB }) => {
+            if (genderA === 'female' && genderB === 'male') return 1;
+            if (genderA === 'male' && genderB === 'female') return -1;
+
+            return 0;
+        })
+        .map(actor => actor.name);
+
+    const description = data.description || null; // prevent empty string
+    const likes = Number(sceneElement.find('.rating .state_1 .value').text());
+    const dislikes = Number(sceneElement.find('#infoWrapper .rating .state_2 .value').text());
+
+    const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
+
+    const rawTags = data.keywords.split(', ');
+    const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
+    const siteId = siteName && siteName.replace(/\s+/g, '').toLowerCase();
+
+    const [channelSite, tags] = await Promise.all([
+        site.isFallback
+            ? knex('sites')
+                .where({ id: siteId })
+                .orWhereRaw('name = ? collate NOCASE', [siteName])
+                .first()
+            : site,
+        matchTags(rawTags),
+    ]);
+
+    // only replace generic URL with site URL if site is not marked to fetch scenes from generic site
+    const originalUrl = channelSite && !(channelSite.parameters && JSON.parse(channelSite.parameters).filter)
+        ? `${channelSite.url}/en/video/${new URL(url).pathname.split('/').slice(-2).join('/')}`
+        : url;
+
+    return {
+        url: originalUrl,
+        entryId,
+        title,
+        date,
+        actors,
+        description,
+        duration,
+        tags,
+        rating: {
+            likes,
+            dislikes,
+        },
+        site: channelSite || site,
+    };
+}
+
+async function fetchLatest(site, page = 1) {
+    const res = await bhttp.get(`${site.parameters.filter ? 'https://21sextury.com' : site.url}/en/videos/All-Categories/0/All-Pornstars/0/latest/${page}`);
+
+    return scrape(res.body.toString(), site);
+}
+
+async function fetchUpcoming(site) {
+    const res = await bhttp.get(`${site.parameters.filter ? 'https://21sextury.com' : site.url}/en/videos/All-Categories/0/All-Pornstars/0/upcoming`);
+
+    return scrape(res.body.toString(), site);
+}
+
+async function fetchScene(url, site) {
+    const res = await bhttp.get(url);
+
+    return scrapeScene(res.body.toString(), url, site);
+}
+
+module.exports = {
+    fetchLatest,
+    fetchUpcoming,
+    fetchScene,
+};