Added MindGeek profile scraper for all MG sites.

assets/components/tile/actor.vue

@@ -5,8 +5,6 @@
     >
         <a
             :href="`/actor/${actor.slug}`"
-            target="_blank"
-            rel="noopener noreferrer"
             class="link"
         >
             <span

src/scrapers/babes.js

@@ -1,8 +1,13 @@
 'use strict';
 
-const { fetchLatest, fetchScene } = require('./mindgeek');
+const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
+
+async function networkFetchProfile(actorName) {
+    return fetchProfile(actorName, 'babes');
+}
 
 module.exports = {
     fetchLatest,
+    fetchProfile: networkFetchProfile,
     fetchScene,
 };

src/scrapers/ddfnetwork.js

@@ -148,7 +148,9 @@ async function scrapeProfile(html, _url, actorName) {
 }
 
 async function fetchLatest(site, page = 1) {
-    const res = await bhttp.get(`https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`);
+    const url = `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
+    console.log(url);
+    const res = await bhttp.get(url);
 
     return scrapeLatest(res.body.toString(), site);
 }

src/scrapers/digitalplayground.js

@@ -1,8 +1,13 @@
 'use strict';
 
-const { fetchScene, fetchLatest } = require('./mindgeek');
+const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
+
+async function networkFetchProfile(actorName) {
+    return fetchProfile(actorName, 'digitalplayground', 'modelprofile');
+}
 
 module.exports = {
     fetchLatest,
+    fetchProfile: networkFetchProfile,
     fetchScene,
 };

src/scrapers/fakehub.js

@@ -1,8 +1,13 @@
 'use strict';
 
-const { fetchScene, fetchLatest } = require('./mindgeek');
+const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
+
+async function networkFetchProfile(actorName) {
+    return fetchProfile(actorName, 'fakehub', 'modelprofile');
+}
 
 module.exports = {
     fetchLatest,
+    fetchProfile: networkFetchProfile,
     fetchScene,
 };

src/scrapers/milehighmedia.js

@@ -1,8 +1,13 @@
 'use strict';
 
-const { fetchScene, fetchLatest } = require('./mindgeek');
+const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
+
+async function networkFetchProfile(actorName) {
+    return fetchProfile(actorName, 'milehighmedia');
+}
 
 module.exports = {
     fetchLatest,
+    fetchProfile: networkFetchProfile,
     fetchScene,
 };

src/scrapers/mindgeek.js

@@ -7,6 +7,8 @@ const bhttp = require('bhttp');
 const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
 const moment = require('moment');
 
+const { ex } = require('../utils/q');
+const { inchesToCm, lbsToKg } = require('../utils/convert');
 const { cookieToData } = require('../utils/cookies');
 
 function getThumbs(scene) {
@@ -105,11 +107,7 @@ function getUrl(site) {
     throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`);
 }
 
-async function fetchLatest(site, page = 1) {
+async function getSession(url) {
-    const url = getUrl(site);
-    const { search } = new URL(url);
-    const siteId = new URLSearchParams(search).get('site');
-
     const cookieJar = new CookieJar();
     const session = bhttp.session({ cookieJar });
 
@@ -118,6 +116,47 @@ async function fetchLatest(site, page = 1) {
     const cookieString = await cookieJar.getCookieStringAsync(url);
     const { instance_token: instanceToken } = cookieToData(cookieString);
 
+    return { session, instanceToken };
+}
+
+function scrapeProfile(data, html) {
+    const { qa, qd } = ex(html);
+
+    const profile = {
+        gender: data.gender,
+        description: data.bio,
+        aliases: data.aliases,
+    };
+
+    const [bust, waist, hip] = data.measurements.split('-');
+
+    if (bust) profile.bust = bust.toUpperCase();
+    if (waist) profile.waist = waist;
+    if (hip) profile.hip = hip;
+
+    if (data.birthPlace) profile.birthPlace = data.birthPlace;
+    if (data.height) profile.height = inchesToCm(data.height);
+    if (data.weight) profile.weight = lbsToKg(data.weight);
+
+    profile.avatar = data.images.card_main_rect[0].xl?.url
+        || data.images.card_main_rect[0].lg?.url
+        || data.images.card_main_rect[0].md?.url
+        || data.images.card_main_rect[0].sm?.url
+        || data.images.card_main_rect[0].xs?.url;
+
+    const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent));
+    if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY');
+
+    return profile;
+}
+
+async function fetchLatest(site, page = 1) {
+    const url = getUrl(site);
+    const { search } = new URL(url);
+    const siteId = new URLSearchParams(search).get('site');
+
+    const { session, instanceToken } = await getSession(url);
+
     const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
     const limit = 10;
     const apiUrl = `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
@@ -132,14 +171,7 @@ async function fetchLatest(site, page = 1) {
 
 async function fetchScene(url, site) {
     const entryId = url.match(/\d+/)[0];
-
+    const { session, instanceToken } = await getSession(url);
-    const cookieJar = new CookieJar();
-    const session = bhttp.session({ cookieJar });
-
-    await session.get(url);
-
-    const cookieString = await cookieJar.getCookieStringAsync(url);
-    const { instance_token: instanceToken } = cookieToData(cookieString);
 
     const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
         headers: {
@@ -150,8 +182,31 @@ async function fetchScene(url, site) {
     return scrapeScene(res.body.result, url, site);
 }
 
+async function fetchProfile(actorName, networkName, actorPath = 'model') {
+    const url = `https://www.${networkName}.com`;
+    const { session, instanceToken } = await getSession(url);
+
+    const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    if (res.statusCode === 200) {
+        const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
+        const actorRes = await bhttp.get(`https://www.${networkName}.com/${actorPath}/${actorData.id}/`);
+
+        if (actorData && actorRes.statusCode === 200) {
+            return scrapeProfile(actorData, actorRes.body.toString());
+        }
+    }
+
+    return null;
+}
+
 module.exports = {
     scrapeLatestX,
     fetchLatest,
     fetchScene,
+    fetchProfile,
 };

src/scrapers/mofos.js

@@ -1,8 +1,13 @@
 'use strict';
 
-const { fetchScene, fetchLatest } = require('./mindgeek');
+const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
+
+async function networkFetchProfile(actorName) {
+    return fetchProfile(actorName, 'mofos');
+}
 
 module.exports = {
     fetchLatest,
     fetchScene,
+    fetchProfile: networkFetchProfile,
 };

src/scrapers/realitykings.js

@@ -3,7 +3,12 @@
 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 
-const { scrapeLatestX, fetchLatest, fetchScene } = require('./mindgeek');
+const {
+    scrapeLatestX,
+    fetchLatest,
+    fetchScene,
+    fetchProfile,
+} = require('./mindgeek');
 
 function scrapeLatestClassic(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -36,7 +41,12 @@ async function fetchLatestWrap(site, page = 1) {
     return fetchLatest(site, page);
 }
 
+async function networkFetchProfile(actorName) {
+    return fetchProfile(actorName, 'realitykings');
+}
+
 module.exports = {
     fetchLatest: fetchLatestWrap,
+    fetchProfile: networkFetchProfile,
     fetchScene,
 };

src/scrapers/scrapers.js

@@ -11,7 +11,6 @@ const jayrock = require('./jayrock');
 const kink = require('./kink');
 const mikeadriano = require('./mikeadriano');
 const milehighmedia = require('./milehighmedia');
-const mofos = require('./mofos');
 const perfectgonzo = require('./perfectgonzo');
 const pervcity = require('./pervcity');
 const pornpros = require('./pornpros');
@@ -29,6 +28,7 @@ const evilangel = require('./evilangel');
 const julesjordan = require('./julesjordan');
 const kellymadison = require('./kellymadison');
 const legalporno = require('./legalporno');
+const mofos = require('./mofos');
 const twentyonesextury = require('./21sextury');
 const xempire = require('./xempire');
 
@@ -76,6 +76,12 @@ module.exports = {
         evilangel,
         xempire,
         blowpass,
+        mofos,
+        realitykings,
+        digitalplayground,
+        fakehub,
+        babes,
+        milehighmedia,
         julesjordan,
         brazzers,
         boobpedia,

src/utils/convert.js

@@ -1,5 +1,9 @@
 'use strict';
 
+function inchesToCm(inches) {
+    return Math.round(Number(inches) * 2.54);
+}
+
 function feetInchesToCm(feet, inches) {
     return Math.round((Number(feet) * 30.48) + (Number(inches) * 2.54));
 }
@@ -33,6 +37,7 @@ module.exports = {
     cmToFeetInches,
     feetInchesToCm,
     heightToCm,
+    inchesToCm,
     lbsToKg,
     kgToLbs,
 };