Added MindGeek profile scraper for all MG sites.

This commit is contained in:
ThePendulum 2020-01-27 22:54:14 +01:00
parent 24fe61e064
commit 6d4fd5fd77
11 changed files with 124 additions and 23 deletions

View File

@ -5,8 +5,6 @@
>
<a
:href="`/actor/${actor.slug}`"
target="_blank"
rel="noopener noreferrer"
class="link"
>
<span

View File

@ -1,8 +1,13 @@
'use strict';
const { fetchLatest, fetchScene } = require('./mindgeek');
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'babes');
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
};

View File

@ -148,7 +148,9 @@ async function scrapeProfile(html, _url, actorName) {
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`);
const url = `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
console.log(url);
const res = await bhttp.get(url);
return scrapeLatest(res.body.toString(), site);
}

View File

@ -1,8 +1,13 @@
'use strict';
const { fetchScene, fetchLatest } = require('./mindgeek');
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'digitalplayground', 'modelprofile');
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
};

View File

@ -1,8 +1,13 @@
'use strict';
const { fetchScene, fetchLatest } = require('./mindgeek');
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'fakehub', 'modelprofile');
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
};

View File

@ -1,8 +1,13 @@
'use strict';
const { fetchScene, fetchLatest } = require('./mindgeek');
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'milehighmedia');
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
};

View File

@ -7,6 +7,8 @@ const bhttp = require('bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const moment = require('moment');
const { ex } = require('../utils/q');
const { inchesToCm, lbsToKg } = require('../utils/convert');
const { cookieToData } = require('../utils/cookies');
function getThumbs(scene) {
@ -105,11 +107,7 @@ function getUrl(site) {
throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`);
}
async function fetchLatest(site, page = 1) {
const url = getUrl(site);
const { search } = new URL(url);
const siteId = new URLSearchParams(search).get('site');
async function getSession(url) {
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
@ -118,6 +116,47 @@ async function fetchLatest(site, page = 1) {
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
return { session, instanceToken };
}
function scrapeProfile(data, html) {
const { qa, qd } = ex(html);
const profile = {
gender: data.gender,
description: data.bio,
aliases: data.aliases,
};
const [bust, waist, hip] = data.measurements.split('-');
if (bust) profile.bust = bust.toUpperCase();
if (waist) profile.waist = waist;
if (hip) profile.hip = hip;
if (data.birthPlace) profile.birthPlace = data.birthPlace;
if (data.height) profile.height = inchesToCm(data.height);
if (data.weight) profile.weight = lbsToKg(data.weight);
profile.avatar = data.images.card_main_rect[0].xl?.url
|| data.images.card_main_rect[0].lg?.url
|| data.images.card_main_rect[0].md?.url
|| data.images.card_main_rect[0].sm?.url
|| data.images.card_main_rect[0].xs?.url;
const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent));
if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY');
return profile;
}
async function fetchLatest(site, page = 1) {
const url = getUrl(site);
const { search } = new URL(url);
const siteId = new URLSearchParams(search).get('site');
const { session, instanceToken } = await getSession(url);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 10;
const apiUrl = `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
@ -132,14 +171,7 @@ async function fetchLatest(site, page = 1) {
async function fetchScene(url, site) {
const entryId = url.match(/\d+/)[0];
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
await session.get(url);
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
const { session, instanceToken } = await getSession(url);
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
headers: {
@ -150,8 +182,31 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.result, url, site);
}
async function fetchProfile(actorName, networkName, actorPath = 'model') {
const url = `https://www.${networkName}.com`;
const { session, instanceToken } = await getSession(url);
const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
headers: {
Instance: instanceToken,
},
});
if (res.statusCode === 200) {
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
const actorRes = await bhttp.get(`https://www.${networkName}.com/${actorPath}/${actorData.id}/`);
if (actorData && actorRes.statusCode === 200) {
return scrapeProfile(actorData, actorRes.body.toString());
}
}
return null;
}
module.exports = {
scrapeLatestX,
fetchLatest,
fetchScene,
fetchProfile,
};

View File

@ -1,8 +1,13 @@
'use strict';
const { fetchScene, fetchLatest } = require('./mindgeek');
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'mofos');
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile: networkFetchProfile,
};

View File

@ -3,7 +3,12 @@
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const { scrapeLatestX, fetchLatest, fetchScene } = require('./mindgeek');
const {
scrapeLatestX,
fetchLatest,
fetchScene,
fetchProfile,
} = require('./mindgeek');
function scrapeLatestClassic(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
@ -36,7 +41,12 @@ async function fetchLatestWrap(site, page = 1) {
return fetchLatest(site, page);
}
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'realitykings');
}
module.exports = {
fetchLatest: fetchLatestWrap,
fetchProfile: networkFetchProfile,
fetchScene,
};

View File

@ -11,7 +11,6 @@ const jayrock = require('./jayrock');
const kink = require('./kink');
const mikeadriano = require('./mikeadriano');
const milehighmedia = require('./milehighmedia');
const mofos = require('./mofos');
const perfectgonzo = require('./perfectgonzo');
const pervcity = require('./pervcity');
const pornpros = require('./pornpros');
@ -29,6 +28,7 @@ const evilangel = require('./evilangel');
const julesjordan = require('./julesjordan');
const kellymadison = require('./kellymadison');
const legalporno = require('./legalporno');
const mofos = require('./mofos');
const twentyonesextury = require('./21sextury');
const xempire = require('./xempire');
@ -76,6 +76,12 @@ module.exports = {
evilangel,
xempire,
blowpass,
mofos,
realitykings,
digitalplayground,
fakehub,
babes,
milehighmedia,
julesjordan,
brazzers,
boobpedia,

View File

@ -1,5 +1,9 @@
'use strict';
function inchesToCm(inches) {
return Math.round(Number(inches) * 2.54);
}
function feetInchesToCm(feet, inches) {
return Math.round((Number(feet) * 30.48) + (Number(inches) * 2.54));
}
@ -33,6 +37,7 @@ module.exports = {
cmToFeetInches,
feetInchesToCm,
heightToCm,
inchesToCm,
lbsToKg,
kgToLbs,
};