DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added basic actor and network overview. Added DDF Network actor scraper. Various bug fixes and layout improvements.

Browse Source
This commit is contained in:
ThePendulum
2019-11-30 05:55:32 +01:00
parent 0dbe853f39
commit bead69de49
44 changed files with 1697 additions and 527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
src/actors.js
View File

@@ -1,6 +1,7 @@
'use strict';
const Promise = require('bluebird');
const UrlPattern = require('url-pattern');
const knex = require('./knex');
const argv = require('./argv');
@@ -54,6 +55,7 @@ async function curateActor(actor) {
curatedActor.origin.country = {
alpha2: actor.birth_country_alpha2,
name: actor.birth_country_name,
alias: actor.birth_country_alias,
};
}
@@ -64,6 +66,7 @@ async function curateActor(actor) {
curatedActor.residence.country = {
alpha2: actor.residence_country_alpha2,
name: actor.residence_country_name,
alias: actor.residence_country_alias,
};
}
@@ -124,12 +127,54 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
}
function curateSocialEntry(url, actorId) {
const { hostname, origin, pathname } = new URL(url);
const platform = ['facebook', 'twitter', 'instagram', 'tumblr', 'snapchat', 'amazon', 'youtube', 'fancentro'].find(platformName => hostname.match(platformName));
const platforms = [
{
label: 'twitter',
pattern: 'http(s)\\://(*)twitter.com/:username(/)(?*)',
format: username => `https://www.twitter.com/${username}`,
},
{
label: 'instagram',
pattern: 'http(s)\\://(*)instagram.com/:username(/)(?*)',
format: username => `https://www.instagram.com/${username}`,
},
{
label: 'snapchat',
pattern: 'http(s)\\://(*)snapchat.com/add/:username(/)(?*)',
format: username => `https://www.snapchat.com/add/${username}`,
},
{
label: 'tumblr',
pattern: 'http(s)\\://:username.tumblr.com(*)',
format: username => `https://${username}.tumblr.com`,
},
{
label: 'fancentro',
pattern: 'http(s)\\://(www.)fancentro.com/:username(/)(?*)',
format: username => `https://www.fancentro.com/${username}`,
},
];
const match = platforms.reduce((acc, platform) => {
if (acc) return acc;
const patternMatch = new UrlPattern(platform.pattern).match(url);
if (patternMatch) {
return {
platform: platform.label,
original: url,
username: patternMatch.username,
url: platform.format ? platform.format(patternMatch.username) : url,
};
}
return null;
}, null) || { url };
return {
url: `${origin}${pathname}`,
platform,
url: match.url,
platform: match.platform,
domain: 'actors',
target_id: actorId,
};
@@ -148,7 +193,7 @@ async function curateSocialEntries(urls, actorId) {
return urls.reduce((acc, url) => {
const socialEntry = curateSocialEntry(url, actorId);
if (acc.some(entry => socialEntry.url === entry.url) || existingSocialLinks.some(entry => socialEntry.url === entry.url)) {
if (acc.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase()) || existingSocialLinks.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase())) {
// prevent duplicates
return acc;
}
@@ -161,11 +206,12 @@ async function fetchActors(queryObject) {
const releases = await knex('actors')
.select(
'actors.*',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name',
'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name', 'residence_countries.alias as residence_country_alias',
)
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('countries as residence_countries', 'actors.residence_country_alpha2', 'residence_countries.alpha2')
.orderBy(['actors.name', 'actors.gender'])
.where(builder => whereOr(queryObject, 'actors', builder))
.limit(100);
@@ -215,7 +261,7 @@ async function mergeProfiles(profiles, actor) {
return {
id: actor ? actor.id : null,
name: actor ? actor.name : profile.name,
name: actor ? actor.name : (prevProfile.name || profile.name),
description: prevProfile.description || profile.description,
gender: prevProfile.gender || profile.gender,
birthdate: Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
@@ -257,9 +303,10 @@ async function scrapeActors(actorNames) {
await Promise.map(actorNames || argv.actors, async (actorName) => {
try {
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
const profiles = await Promise.map(Object.entries(scrapers.actors), async ([scraperSlug, scraper]) => {
const sources = argv.sources ? argv.sources.map(source => [source, scrapers.actors[source]]) : Object.entries(scrapers.actors);
const profiles = await Promise.map(sources, async ([scraperSlug, scraper]) => {
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName);
return {

5
src/argv.js
View File

@@ -24,6 +24,11 @@ const { argv } = yargs
type: 'array',
alias: 'actor',
})
.option('sources', {
describe: 'Only use these scrapers for actor data',
type: 'array',
alias: 'source',
})
.option('deep', {
describe: 'Fetch details for all releases',
type: 'boolean',

3
src/scrape-sites.js
View File

@@ -117,6 +117,9 @@ async function scrapeReleases() {
try {
const siteReleases = await scrapeSiteReleases(scraper, site);
const siteActors = siteReleases.reduce((acc, release) => [...acc, ...release.actors], []);
console.log(siteActors);
if (argv.save) {
await storeReleases(siteReleases);

5
src/scrapers/brazzers.js
View File

@@ -130,7 +130,7 @@ function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
return actorLink;
return actorLink ? actorLink.href : null;
}
function scrapeProfile(html, url, actorName) {
@@ -157,6 +157,9 @@ function scrapeProfile(html, url, actorName) {
if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]);
if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();
if (bio['Tits Type'] && bio['Tits Type'].match('Natural')) profile.naturalBoobs = true;
if (bio['Tits Type'] && bio['Tits Type'].match('Enhanced')) profile.naturalBoobs = false;
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;

98
src/scrapers/ddfnetwork.js
View File

@@ -2,9 +2,10 @@
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const { JSDOM } = require('jsdom');
const moment = require('moment');
// const knex = require('../knex');
const knex = require('../knex');
const { matchTags } = require('../tags');
/* eslint-disable newline-per-chained-call */
@@ -105,6 +106,67 @@ async function scrapeScene(html, url, site) {
};
}
async function scrapeProfile(html, _url, actorName) {
const { document } = new JSDOM(html).window;
const keys = Array.from(document.querySelectorAll('.about-title'), el => el.textContent.trim().replace(':', ''));
const values = Array.from(document.querySelectorAll('.about-info'), (el) => {
if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', ');
}
return el.textContent.trim();
});
const bio = keys.reduce((acc, key, index) => {
if (values[index] === '-') {
return acc;
}
return {
...acc,
[key]: values[index],
};
}, {});
const descriptionEl = document.querySelector('.description-box');
const avatarEl = document.querySelector('.pornstar-details .card-img-top');
const country = await knex('countries')
.where('nationality', 'ilike', `%${bio.Nationality}%`)
.orderBy('priority', 'desc')
.first();
const profile = {
name: actorName,
};
profile.birthdate = moment.utc(bio.Birthday, 'MMMM DD, YYYY').toDate();
if (country) profile.birthPlace = country.name;
if (bio['Bra size']) [profile.bust] = bio['Bra size'].match(/\d+\w+/);
if (bio.Waist) profile.waist = Number(bio.Waist.match(/\d+/)[0]);
if (bio.Hips) profile.hip = Number(bio.Hips.match(/\d+/)[0]);
if (bio.Height) profile.height = Number(bio.Height.match(/\d{2,}/)[0]);
if (bio['Tit Style'] && bio['Tit Style'].match('Enhanced')) profile.naturalBoobs = false;
if (bio['Tit Style'] && bio['Tit Style'].match('Natural')) profile.naturalBoobs = true;
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
if (bio['Hair Style']) profile.hair = bio['Hair Style'].split(',')[0].trim().toLowerCase();
if (bio['Eye Color']) profile.eyes = bio['Eye Color'].match(/\w+/)[0].toLowerCase();
if (bio['Shoe size']) profile.shoes = Number(bio['Shoe size'].split('|')[1]);
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
if (avatarEl) profile.avatar = `https:${avatarEl.dataset.src}`;
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`);
@@ -117,7 +179,41 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName) {
const resSearch = await bhttp.post('https://ddfnetwork.com/search/ajax',
{
type: 'hints',
word: actorName,
},
{
decodeJSON: true,
headers: {
'x-requested-with': 'XMLHttpRequest',
},
});
if (resSearch.statusCode !== 200 || Array.isArray(resSearch.body.list)) {
return null;
}
if (!resSearch.body.list.pornstarsName || resSearch.body.list.pornstarsName.length === 0) {
return null;
}
const [actor] = resSearch.body.list.pornstarsName;
const url = `https://ddfnetwork.com${actor.href}`;
const resActor = await bhttp.get(url);
if (resActor.statusCode !== 200) {
return null;
}
return scrapeProfile(resActor.body.toString(), url, actorName);
}
module.exports = {
fetchLatest,
fetchProfile,
fetchScene,
};

7
src/scrapers/scrapers.js
View File

@@ -4,7 +4,6 @@
const twentyonesextury = require('./21sextury');
const bangbros = require('./bangbros');
const blowpass = require('./blowpass');
const ddfnetwork = require('./ddfnetwork');
const dogfart = require('./dogfart');
const evilangel = require('./evilangel');
const kink = require('./kink');
@@ -15,12 +14,13 @@ const privateNetwork = require('./private'); // reserved keyword
const naughtyamerica = require('./naughtyamerica');
const realitykings = require('./realitykings');
const vixen = require('./vixen');
const xempire = require('./xempire');
// releases and profiles
const ddfnetwork = require('./ddfnetwork');
const brazzers = require('./brazzers');
const julesjordan = require('./julesjordan');
const legalporno = require('./legalporno');
const xempire = require('./xempire');
// profiles
const freeones = require('./freeones');
@@ -49,10 +49,13 @@ module.exports = {
xempire,
},
actors: {
// ordered by data priority
xempire,
brazzers,
freeones,
julesjordan,
legalporno,
pornhub,
ddfnetwork,
},
};

70
src/scrapers/xempire.js
View File

@@ -3,9 +3,10 @@
const Promise = require('bluebird');
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const knex = require('../knex');
const { fetchSites } = require('../sites');
const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');
@@ -142,7 +143,7 @@ async function scrapeScene(html, url, site) {
const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
const siteDomain = $('meta[name="twitter:domain"]').attr('content');
const siteId = siteDomain && siteDomain.split('.')[0].toLowerCase();
const siteSlug = siteDomain && siteDomain.split('.')[0].toLowerCase();
const siteUrl = siteDomain && `https://www.${siteDomain}`;
const poster = videoData.picPreview;
@@ -152,14 +153,14 @@ async function scrapeScene(html, url, site) {
const rawTags = data.keywords.split(', ');
const [channelSite, tags] = await Promise.all([
const [[channelSite], tags] = await Promise.all([
site.isFallback
? knex('sites')
.where({ url: siteUrl })
.orWhere({ slug: siteId })
.first()
: site,
matchTags([...defaultTags[siteId], ...rawTags]),
? fetchSites({
url: siteUrl,
slug: siteSlug,
})
: [site],
matchTags([...defaultTags[siteSlug], ...rawTags]),
]);
return {
@@ -185,6 +186,31 @@ async function scrapeScene(html, url, site) {
};
}
function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
return actorLink ? actorLink.href : null;
}
function scrapeProfile(html, url, actorName) {
const { document } = new JSDOM(html).window;
const avatarEl = document.querySelector('img.actorPicture');
const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
const profile = {
name: actorName,
};
if (avatarEl) profile.avatar = avatarEl.src;
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://xempire.com${el.href}`);
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
@@ -203,8 +229,34 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName) {
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
const searchUrl = `https://www.xempire.com/en/search/xempire/actor/${actorSlug}`;
const searchRes = await bhttp.get(searchUrl);
if (searchRes.statusCode !== 200) {
return null;
}
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
if (actorUrl) {
const url = `https://xempire.com${actorUrl}`;
const actorRes = await bhttp.get(url);
if (actorRes.statusCode !== 200) {
return null;
}
return scrapeProfile(actorRes.body.toString(), url, actorName);
}
return null;
}
module.exports = {
fetchLatest,
fetchProfile,
fetchUpcoming,
fetchScene,
};

6
src/utils/where-or.js
View File

@@ -1,11 +1,17 @@
'use strict';
function whereOr(query, table, builder) {
if (!query) {
return {};
}
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined) {
builder.orWhere(`${table}.${key}`, value);
}
});
return builder;
}
module.exports = whereOr;

20
src/web/actors.js
View File

@@ -6,10 +6,22 @@ async function fetchActorsApi(req, res) {
const actorId = typeof req.params.actorId === 'number' ? req.params.actorId : null;
const actorSlug = typeof req.params.actorId === 'string' ? req.params.actorId : null;
const actors = await fetchActors({
id: actorId,
slug: actorSlug,
});
if (actorId || actorSlug) {
const actors = await fetchActors({
id: actorId,
slug: actorSlug,
});
if (actors.length > 0) {
res.send(actors[0]);
return;
}
res.status(404).send();
return;
}
const actors = await fetchActors();
res.send(actors);
}