Added basic actor and network overview. Added DDF Network actor scraper. Various bug fixes and layout improvements.

2019-11-30 05:55:32 +01:00
parent 0dbe853f39
commit bead69de49
44 changed files with 1697 additions and 527 deletions
--- a/src/scrapers/brazzers.js
+++ b/src/scrapers/brazzers.js
@@ -130,7 +130,7 @@ function scrapeActorSearch(html, url, actorName) {
    const { document } = new JSDOM(html).window;
    const actorLink = document.querySelector(`a[title="${actorName}" i]`);

-    return actorLink;
+    return actorLink ? actorLink.href : null;
 }

 function scrapeProfile(html, url, actorName) {
@@ -157,6 +157,9 @@ function scrapeProfile(html, url, actorName) {
    if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]);
    if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();

+    if (bio['Tits Type'] && bio['Tits Type'].match('Natural')) profile.naturalBoobs = true;
+    if (bio['Tits Type'] && bio['Tits Type'].match('Enhanced')) profile.naturalBoobs = false;
+
    if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
    if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;

--- a/src/scrapers/ddfnetwork.js
+++ b/src/scrapers/ddfnetwork.js
@@ -2,9 +2,10 @@

 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
+const { JSDOM } = require('jsdom');
 const moment = require('moment');

-// const knex = require('../knex');
+const knex = require('../knex');
 const { matchTags } = require('../tags');

 /* eslint-disable newline-per-chained-call */
@@ -105,6 +106,67 @@ async function scrapeScene(html, url, site) {
    };
 }

+async function scrapeProfile(html, _url, actorName) {
+    const { document } = new JSDOM(html).window;
+
+    const keys = Array.from(document.querySelectorAll('.about-title'), el => el.textContent.trim().replace(':', ''));
+    const values = Array.from(document.querySelectorAll('.about-info'), (el) => {
+        if (el.children.length > 0) {
+            return Array.from(el.children, child => child.textContent.trim()).join(', ');
+        }
+
+        return el.textContent.trim();
+    });
+
+    const bio = keys.reduce((acc, key, index) => {
+        if (values[index] === '-') {
+            return acc;
+        }
+
+        return {
+            ...acc,
+            [key]: values[index],
+        };
+    }, {});
+
+    const descriptionEl = document.querySelector('.description-box');
+    const avatarEl = document.querySelector('.pornstar-details .card-img-top');
+
+    const country = await knex('countries')
+        .where('nationality', 'ilike', `%${bio.Nationality}%`)
+        .orderBy('priority', 'desc')
+        .first();
+
+    const profile = {
+        name: actorName,
+    };
+
+    profile.birthdate = moment.utc(bio.Birthday, 'MMMM DD, YYYY').toDate();
+    if (country) profile.birthPlace = country.name;
+
+    if (bio['Bra size']) [profile.bust] = bio['Bra size'].match(/\d+\w+/);
+    if (bio.Waist) profile.waist = Number(bio.Waist.match(/\d+/)[0]);
+    if (bio.Hips) profile.hip = Number(bio.Hips.match(/\d+/)[0]);
+
+    if (bio.Height) profile.height = Number(bio.Height.match(/\d{2,}/)[0]);
+
+    if (bio['Tit Style'] && bio['Tit Style'].match('Enhanced')) profile.naturalBoobs = false;
+    if (bio['Tit Style'] && bio['Tit Style'].match('Natural')) profile.naturalBoobs = true;
+
+    if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
+    if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
+
+    if (bio['Hair Style']) profile.hair = bio['Hair Style'].split(',')[0].trim().toLowerCase();
+    if (bio['Eye Color']) profile.eyes = bio['Eye Color'].match(/\w+/)[0].toLowerCase();
+
+    if (bio['Shoe size']) profile.shoes = Number(bio['Shoe size'].split('|')[1]);
+
+    if (descriptionEl) profile.description = descriptionEl.textContent.trim();
+    if (avatarEl) profile.avatar = `https:${avatarEl.dataset.src}`;
+
+    return profile;
+}
+
 async function fetchLatest(site, page = 1) {
    const res = await bhttp.get(`https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`);

@@ -117,7 +179,41 @@ async function fetchScene(url, site) {
    return scrapeScene(res.body.toString(), url, site);
 }

+async function fetchProfile(actorName) {
+    const resSearch = await bhttp.post('https://ddfnetwork.com/search/ajax',
+        {
+            type: 'hints',
+            word: actorName,
+        },
+        {
+            decodeJSON: true,
+            headers: {
+                'x-requested-with': 'XMLHttpRequest',
+            },
+        });
+
+    if (resSearch.statusCode !== 200 || Array.isArray(resSearch.body.list)) {
+        return null;
+    }
+
+    if (!resSearch.body.list.pornstarsName || resSearch.body.list.pornstarsName.length === 0) {
+        return null;
+    }
+
+    const [actor] = resSearch.body.list.pornstarsName;
+    const url = `https://ddfnetwork.com${actor.href}`;
+
+    const resActor = await bhttp.get(url);
+
+    if (resActor.statusCode !== 200) {
+        return null;
+    }
+
+    return scrapeProfile(resActor.body.toString(), url, actorName);
+}
+
 module.exports = {
    fetchLatest,
+    fetchProfile,
    fetchScene,
 };
--- a/src/scrapers/scrapers.js
+++ b/src/scrapers/scrapers.js
@@ -4,7 +4,6 @@
 const twentyonesextury = require('./21sextury');
 const bangbros = require('./bangbros');
 const blowpass = require('./blowpass');
-const ddfnetwork = require('./ddfnetwork');
 const dogfart = require('./dogfart');
 const evilangel = require('./evilangel');
 const kink = require('./kink');
@@ -15,12 +14,13 @@ const privateNetwork = require('./private'); // reserved keyword
 const naughtyamerica = require('./naughtyamerica');
 const realitykings = require('./realitykings');
 const vixen = require('./vixen');
-const xempire = require('./xempire');

 // releases and profiles
+const ddfnetwork = require('./ddfnetwork');
 const brazzers = require('./brazzers');
 const julesjordan = require('./julesjordan');
 const legalporno = require('./legalporno');
+const xempire = require('./xempire');

 // profiles
 const freeones = require('./freeones');
@@ -49,10 +49,13 @@ module.exports = {
        xempire,
    },
    actors: {
+        // ordered by data priority
+        xempire,
        brazzers,
        freeones,
        julesjordan,
        legalporno,
        pornhub,
+        ddfnetwork,
    },
 };
--- a/src/scrapers/xempire.js
+++ b/src/scrapers/xempire.js
@@ -3,9 +3,10 @@
 const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
+const { JSDOM } = require('jsdom');
 const moment = require('moment');

-const knex = require('../knex');
+const { fetchSites } = require('../sites');
 const { matchTags } = require('../tags');
 const pluckPhotos = require('../utils/pluck-photos');

@@ -142,7 +143,7 @@ async function scrapeScene(html, url, site) {
    const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();

    const siteDomain = $('meta[name="twitter:domain"]').attr('content');
-    const siteId = siteDomain && siteDomain.split('.')[0].toLowerCase();
+    const siteSlug = siteDomain && siteDomain.split('.')[0].toLowerCase();
    const siteUrl = siteDomain && `https://www.${siteDomain}`;

    const poster = videoData.picPreview;
@@ -152,14 +153,14 @@ async function scrapeScene(html, url, site) {

    const rawTags = data.keywords.split(', ');

-    const [channelSite, tags] = await Promise.all([
+    const [[channelSite], tags] = await Promise.all([
        site.isFallback
-            ? knex('sites')
-                .where({ url: siteUrl })
-                .orWhere({ slug: siteId })
-                .first()
-            : site,
-        matchTags([...defaultTags[siteId], ...rawTags]),
+            ? fetchSites({
+                url: siteUrl,
+                slug: siteSlug,
+            })
+            : [site],
+        matchTags([...defaultTags[siteSlug], ...rawTags]),
    ]);

    return {
@@ -185,6 +186,31 @@ async function scrapeScene(html, url, site) {
    };
 }

+function scrapeActorSearch(html, url, actorName) {
+    const { document } = new JSDOM(html).window;
+    const actorLink = document.querySelector(`a[title="${actorName}" i]`);
+
+    return actorLink ? actorLink.href : null;
+}
+
+function scrapeProfile(html, url, actorName) {
+    const { document } = new JSDOM(html).window;
+
+    const avatarEl = document.querySelector('img.actorPicture');
+    const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
+
+    const profile = {
+        name: actorName,
+    };
+
+    if (avatarEl) profile.avatar = avatarEl.src;
+    if (descriptionEl) profile.description = descriptionEl.textContent.trim();
+
+    profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://xempire.com${el.href}`);
+
+    return profile;
+}
+
 async function fetchLatest(site, page = 1) {
    const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);

@@ -203,8 +229,34 @@ async function fetchScene(url, site) {
    return scrapeScene(res.body.toString(), url, site);
 }

+async function fetchProfile(actorName) {
+    const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
+    const searchUrl = `https://www.xempire.com/en/search/xempire/actor/${actorSlug}`;
+    const searchRes = await bhttp.get(searchUrl);
+
+    if (searchRes.statusCode !== 200) {
+        return null;
+    }
+
+    const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
+
+    if (actorUrl) {
+        const url = `https://xempire.com${actorUrl}`;
+        const actorRes = await bhttp.get(url);
+
+        if (actorRes.statusCode !== 200) {
+            return null;
+        }
+
+        return scrapeProfile(actorRes.body.toString(), url, actorName);
+    }
+
+    return null;
+}
+
 module.exports = {
    fetchLatest,
+    fetchProfile,
    fetchUpcoming,
    fetchScene,
 };