forked from DebaucheryLibrarian/traxxx
Added MindGeek profile scraper for all MG sites.
This commit is contained in:
parent
24fe61e064
commit
6d4fd5fd77
|
@ -5,8 +5,6 @@
|
|||
>
|
||||
<a
|
||||
:href="`/actor/${actor.slug}`"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="link"
|
||||
>
|
||||
<span
|
||||
|
|
|
@ -1,8 +1,13 @@
|
|||
'use strict';
|
||||
|
||||
const { fetchLatest, fetchScene } = require('./mindgeek');
|
||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||
|
||||
async function networkFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, 'babes');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
@ -148,7 +148,9 @@ async function scrapeProfile(html, _url, actorName) {
|
|||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const res = await bhttp.get(`https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`);
|
||||
const url = `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
|
||||
console.log(url);
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return scrapeLatest(res.body.toString(), site);
|
||||
}
|
||||
|
|
|
@ -1,8 +1,13 @@
|
|||
'use strict';
|
||||
|
||||
const { fetchScene, fetchLatest } = require('./mindgeek');
|
||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||
|
||||
async function networkFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, 'digitalplayground', 'modelprofile');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
@ -1,8 +1,13 @@
|
|||
'use strict';
|
||||
|
||||
const { fetchScene, fetchLatest } = require('./mindgeek');
|
||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||
|
||||
async function networkFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, 'fakehub', 'modelprofile');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
@ -1,8 +1,13 @@
|
|||
'use strict';
|
||||
|
||||
const { fetchScene, fetchLatest } = require('./mindgeek');
|
||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||
|
||||
async function networkFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, 'milehighmedia');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
@ -7,6 +7,8 @@ const bhttp = require('bhttp');
|
|||
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
|
||||
const moment = require('moment');
|
||||
|
||||
const { ex } = require('../utils/q');
|
||||
const { inchesToCm, lbsToKg } = require('../utils/convert');
|
||||
const { cookieToData } = require('../utils/cookies');
|
||||
|
||||
function getThumbs(scene) {
|
||||
|
@ -105,11 +107,7 @@ function getUrl(site) {
|
|||
throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`);
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = getUrl(site);
|
||||
const { search } = new URL(url);
|
||||
const siteId = new URLSearchParams(search).get('site');
|
||||
|
||||
async function getSession(url) {
|
||||
const cookieJar = new CookieJar();
|
||||
const session = bhttp.session({ cookieJar });
|
||||
|
||||
|
@ -118,6 +116,47 @@ async function fetchLatest(site, page = 1) {
|
|||
const cookieString = await cookieJar.getCookieStringAsync(url);
|
||||
const { instance_token: instanceToken } = cookieToData(cookieString);
|
||||
|
||||
return { session, instanceToken };
|
||||
}
|
||||
|
||||
function scrapeProfile(data, html) {
|
||||
const { qa, qd } = ex(html);
|
||||
|
||||
const profile = {
|
||||
gender: data.gender,
|
||||
description: data.bio,
|
||||
aliases: data.aliases,
|
||||
};
|
||||
|
||||
const [bust, waist, hip] = data.measurements.split('-');
|
||||
|
||||
if (bust) profile.bust = bust.toUpperCase();
|
||||
if (waist) profile.waist = waist;
|
||||
if (hip) profile.hip = hip;
|
||||
|
||||
if (data.birthPlace) profile.birthPlace = data.birthPlace;
|
||||
if (data.height) profile.height = inchesToCm(data.height);
|
||||
if (data.weight) profile.weight = lbsToKg(data.weight);
|
||||
|
||||
profile.avatar = data.images.card_main_rect[0].xl?.url
|
||||
|| data.images.card_main_rect[0].lg?.url
|
||||
|| data.images.card_main_rect[0].md?.url
|
||||
|| data.images.card_main_rect[0].sm?.url
|
||||
|| data.images.card_main_rect[0].xs?.url;
|
||||
|
||||
const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent));
|
||||
if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY');
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = getUrl(site);
|
||||
const { search } = new URL(url);
|
||||
const siteId = new URLSearchParams(search).get('site');
|
||||
|
||||
const { session, instanceToken } = await getSession(url);
|
||||
|
||||
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
|
||||
const limit = 10;
|
||||
const apiUrl = `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
|
||||
|
@ -132,14 +171,7 @@ async function fetchLatest(site, page = 1) {
|
|||
|
||||
async function fetchScene(url, site) {
|
||||
const entryId = url.match(/\d+/)[0];
|
||||
|
||||
const cookieJar = new CookieJar();
|
||||
const session = bhttp.session({ cookieJar });
|
||||
|
||||
await session.get(url);
|
||||
|
||||
const cookieString = await cookieJar.getCookieStringAsync(url);
|
||||
const { instance_token: instanceToken } = cookieToData(cookieString);
|
||||
const { session, instanceToken } = await getSession(url);
|
||||
|
||||
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
|
||||
headers: {
|
||||
|
@ -150,8 +182,31 @@ async function fetchScene(url, site) {
|
|||
return scrapeScene(res.body.result, url, site);
|
||||
}
|
||||
|
||||
async function fetchProfile(actorName, networkName, actorPath = 'model') {
|
||||
const url = `https://www.${networkName}.com`;
|
||||
const { session, instanceToken } = await getSession(url);
|
||||
|
||||
const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
|
||||
headers: {
|
||||
Instance: instanceToken,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.statusCode === 200) {
|
||||
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
|
||||
const actorRes = await bhttp.get(`https://www.${networkName}.com/${actorPath}/${actorData.id}/`);
|
||||
|
||||
if (actorData && actorRes.statusCode === 200) {
|
||||
return scrapeProfile(actorData, actorRes.body.toString());
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
scrapeLatestX,
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
};
|
||||
|
|
|
@ -1,8 +1,13 @@
|
|||
'use strict';
|
||||
|
||||
const { fetchScene, fetchLatest } = require('./mindgeek');
|
||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||
|
||||
async function networkFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, 'mofos');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile: networkFetchProfile,
|
||||
};
|
||||
|
|
|
@ -3,7 +3,12 @@
|
|||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
const { scrapeLatestX, fetchLatest, fetchScene } = require('./mindgeek');
|
||||
const {
|
||||
scrapeLatestX,
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
} = require('./mindgeek');
|
||||
|
||||
function scrapeLatestClassic(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
@ -36,7 +41,12 @@ async function fetchLatestWrap(site, page = 1) {
|
|||
return fetchLatest(site, page);
|
||||
}
|
||||
|
||||
async function networkFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, 'realitykings');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest: fetchLatestWrap,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
@ -11,7 +11,6 @@ const jayrock = require('./jayrock');
|
|||
const kink = require('./kink');
|
||||
const mikeadriano = require('./mikeadriano');
|
||||
const milehighmedia = require('./milehighmedia');
|
||||
const mofos = require('./mofos');
|
||||
const perfectgonzo = require('./perfectgonzo');
|
||||
const pervcity = require('./pervcity');
|
||||
const pornpros = require('./pornpros');
|
||||
|
@ -29,6 +28,7 @@ const evilangel = require('./evilangel');
|
|||
const julesjordan = require('./julesjordan');
|
||||
const kellymadison = require('./kellymadison');
|
||||
const legalporno = require('./legalporno');
|
||||
const mofos = require('./mofos');
|
||||
const twentyonesextury = require('./21sextury');
|
||||
const xempire = require('./xempire');
|
||||
|
||||
|
@ -76,6 +76,12 @@ module.exports = {
|
|||
evilangel,
|
||||
xempire,
|
||||
blowpass,
|
||||
mofos,
|
||||
realitykings,
|
||||
digitalplayground,
|
||||
fakehub,
|
||||
babes,
|
||||
milehighmedia,
|
||||
julesjordan,
|
||||
brazzers,
|
||||
boobpedia,
|
||||
|
|
|
@ -1,5 +1,9 @@
|
|||
'use strict';
|
||||
|
||||
function inchesToCm(inches) {
|
||||
return Math.round(Number(inches) * 2.54);
|
||||
}
|
||||
|
||||
function feetInchesToCm(feet, inches) {
|
||||
return Math.round((Number(feet) * 30.48) + (Number(inches) * 2.54));
|
||||
}
|
||||
|
@ -33,6 +37,7 @@ module.exports = {
|
|||
cmToFeetInches,
|
||||
feetInchesToCm,
|
||||
heightToCm,
|
||||
inchesToCm,
|
||||
lbsToKg,
|
||||
kgToLbs,
|
||||
};
|
||||
|
|
Loading…
Reference in New Issue