boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed countries seed file. Updated MOFOS scraper. Improved Reality Kings scraper. Limiting photos for XEmpire scraper.

Browse Source
This commit is contained in:
ThePendulum
2019-11-27 04:58:38 +01:00
parent de36ed97e4
commit d113123778
61 changed files with 2182 additions and 2005 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/actors.js
View File

@@ -9,11 +9,13 @@ const whereOr = require('./utils/where-or');
const { createActorMediaDirectory, storeAvatars } = require('./media');
async function curateActor(actor) {
const [aliases, avatars] = await Promise.all([
const [aliases, avatars, social] = await Promise.all([
knex('actors').where({ alias_for: actor.id }),
knex('media')
.where({ domain: 'actors', target_id: actor.id })
.orderBy('index'),
knex('social')
.where({ domain: 'actors', target_id: actor.id }),
]);
return {
@@ -46,6 +48,7 @@ async function curateActor(actor) {
aliases: aliases.map(({ name }) => name),
slug: actor.slug,
avatars,
social,
};
}
@@ -94,6 +97,35 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
return curatedActor;
}
function curateSocialEntry(url, actor) {
const { hostname, origin, pathname } = new URL(url);
const platform = ['twitter', 'instagram', 'snapchat', 'modelhub', 'youtube'].find(platformName => hostname.match(platformName));
return {
url: `${origin}${pathname}`,
platform,
domain: 'actors',
target_id: actor.id,
};
}
function curateSocialEntries(urls, actor) {
if (!urls) {
return [];
}
return urls.reduce((acc, url) => {
const socialEntry = curateSocialEntry(url, actor);
if (acc.some(entry => socialEntry.url === entry.url)) {
// prevent duplicates
return acc;
}
return [...acc, socialEntry];
}, []);
}
async function fetchActors(queryObject) {
const releases = await knex('actors')
.select(
@@ -112,34 +144,30 @@ async function fetchActors(queryObject) {
async function storeActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const actorEntries = await knex('actors')
const [actorEntry] = await knex('actors')
.insert(curatedActor)
.returning('*');
if (actorEntries.length) {
const actorEntry = actorEntries[0];
await knex('social').insert(curateSocialEntries(actor.social, actor));
console.log(`Added new entry for actor '${actor.name}'`);
console.log(`Added new entry for actor '${actor.name}'`);
return actorEntry;
}
console.error(`Unable to save profile for '${actor.name}'`);
return null;
return actorEntry;
}
async function updateActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const actorEntries = await knex('actors')
const [actorEntry] = await knex('actors')
.where({ id: actor.id })
.update(curatedActor)
.returning('*');
await knex('social').insert(curateSocialEntries(actor.social, actor));
console.log(`Updated entry for actor '${actor.name}'`);
return actorEntries[0];
return actorEntry;
}
function mergeProfiles(profiles, actor) {
@@ -177,7 +205,6 @@ function mergeProfiles(profiles, actor) {
}, {
social: [],
avatars: [],
...actor,
});
}

2
src/scrapers/brazzers.js
View File

@@ -148,7 +148,7 @@ function scrapeProfile(html, url, actorName) {
};
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Measurements && bio.Measurements.match(/\w+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate();
if (bio['Birth Location']) profile.birthPlace = bio['Birth Location'];
if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase();

149
src/scrapers/mofos.js
View File

@@ -1,67 +1,84 @@
'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const moment = require('moment');
const knex = require('../knex');
const { fetchSites } = require('../sites');
const { cookieToData } = require('../utils/cookies');
const { matchTags } = require('../tags');
function getThumbs(scene) {
if (scene.images.poster) {
return scene.images.poster.map(image => image.xl.url);
}
if (scene.images.card_main_rect) {
return scene.images.card_main_rect
.concat(scene.images.card_secondary_rect || [])
.map(image => image.xl.url.replace('.thumb', ''));
}
return [];
}
/* eslint-disable newline-per-chained-call */
function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.widget-release-card').toArray();
async function scrapeLatest(items, site) {
return Promise.all(items.map(async (data) => {
const { id: entryId, title, description } = data;
const url = `https://www.mofos.com/scene/${entryId}/`;
const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name);
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('.title a');
const rawTags = data.tags.map(tag => tag.name);
const tags = await matchTags(rawTags);
const title = sceneLinkElement.text().trim();
const url = `https://www.mofos.com${sceneLinkElement.attr('href')}`;
const entryId = url.split('/').slice(-2, -1)[0];
const date = moment.utc($(element).find('.date-added').text(), 'MMM DD, YYYY').toDate();
const actors = $(element).find('.girls-name a').map((actorIndex, actorElement) => $(actorElement).attr('title').replace(/\s+/g, ' ')).toArray();
const stars = Number($(element).find('.rating').text().slice(0, -1).trim()) / 20;
const [poster, ...photos] = getThumbs(data);
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
return {
url,
entryId,
title,
description,
actors,
date,
rating: {
stars,
tags,
poster,
photos,
trailer: {
src: trailer.urls.view,
quality: parseInt(trailer.format, 10),
},
date,
site,
};
});
}));
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElement = $('.video-info');
async function scrapeScene(data, url, site) {
const { id: entryId, title, description } = data;
const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name);
const entryId = url.split('/').slice(-2, -1)[0];
const title = sceneElement.find('.title').text();
const description = sceneElement.find('.desc').text();
const actors = sceneElement.find('.girls-site-box a.model-name').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
const rawTags = data.tags.map(tag => tag.name);
const siteElement = sceneElement.find('.site-name');
const sitename = siteElement.text().trim();
const siteId = sitename.replace(/\s+/g, '').toLowerCase();
const siteUrl = siteElement.attr('href').split('/').slice(0, 4).join('/');
const [poster, ...photos] = getThumbs(data);
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
const stars = Number(sceneElement.find('.rating-box .rating').text().slice(0, -1).trim()) / 20;
const siteName = data.collections[0].name;
const siteId = data.collections[0].id;
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
const siteUrl = `https://www.mofos.com/scenes?site=${siteId}`;
const rawTags = sceneElement.find('.categories a').map((tagIndex, tagElement) => $(tagElement).text().trim()).toArray();
const [channelSite, tags] = await Promise.all([
knex('sites')
.where({ slug: siteId })
.orWhere({ url: `https://www.mofos.com${siteUrl}` })
.orWhere({ name: sitename })
.first(),
const [[channelSite], tags] = await Promise.all([
site.isFallback
? fetchSites({
slug: siteSlug,
name: siteName,
url: siteUrl,
})
: [site],
matchTags(rawTags),
]);
@@ -72,35 +89,61 @@ async function scrapeScene(html, url, site) {
description,
actors,
tags,
rating: {
stars,
poster,
photos,
trailer: {
src: trailer.urls.view,
quality: parseInt(trailer.format, 10),
},
site: channelSite || site,
date,
site: channelSite,
};
}
async function fetchLatest(site, page = 1) {
const res = page > 1
? await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/${page}/`)
: await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/`); // explicit page 1 redirects to homepage
const { search } = new URL(site.url);
const siteId = new URLSearchParams(search).get('site');
return scrape(res.body.toString(), site);
}
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.url}/all-models/all-categories/upcoming/bydate/`);
await session.get(site.url);
return scrape(res.body.toString(), site);
const cookieString = await cookieJar.getCookieStringAsync(site.url);
const { instance_token: instanceToken } = cookieToData(cookieString);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 10;
const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
headers: {
Instance: instanceToken,
},
});
return scrapeLatest(res.body.result, site);
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
const entryId = url.match(/\d+/)[0];
return scrapeScene(res.body.toString(), url, site);
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
await session.get(url);
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
headers: {
Instance: instanceToken,
},
});
return scrapeScene(res.body.result, url, site);
}
module.exports = {
fetchLatest,
fetchUpcoming,
fetchScene,
};

2
src/scrapers/pornhub.js
View File

@@ -59,7 +59,7 @@ async function scrapeProfile(html, _url, actorName) {
profile.residenceCountry = residenceCountryEntry ? residenceCountryEntry.alpha2 : null;
}
if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-').map(measurement => parseInt(measurement, 10) || null);
if (bio['Fake Boobs']) profile.naturalBoobs = bio['Fake Boobs'] === 'No';
if (bio.Height) profile.height = Number(bio.Height.match(/\(\d+/)[0].slice(1));

136
src/scrapers/realitykings.js
View File

@@ -4,8 +4,10 @@
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const { JSDOM } = require('jsdom');
const moment = require('moment');
const { fetchSites } = require('../sites');
const { cookieToData } = require('../utils/cookies');
const { matchTags } = require('../tags');
function getThumbs(scene) {
@@ -22,40 +24,18 @@ function getThumbs(scene) {
return [];
}
async function scrapeLatest(html, site) {
const { document } = new JSDOM(html).window;
const scriptString = document.querySelector('script').textContent;
const prefix = 'window.__JUAN.initialState = {';
const dataStart = scriptString.slice(scriptString.indexOf(prefix) + prefix.length - 1);
const dataString = dataStart.slice(0, dataStart.indexOf('};') + 1);
const data = JSON.parse(dataString);
const actorsMap = data.entities.actors;
const tagsMap = data.entities.tags;
const scenes = Object.values(data.entities.releases);
return Promise.all(scenes.map(async (scene) => {
const {
id: entryId,
title,
description,
} = scene;
async function scrapeLatest(items, site) {
return Promise.all(items.map(async (data) => {
const { id: entryId, title, description } = data;
const url = `https://www.realitykings.com/scene/${entryId}/`;
const date = new Date(scene.dateReleased);
const actors = scene.actors.map(actorId => actorsMap[actorId].name);
const duration = scene.videos.mediabook && scene.videos.mediabook.length;
const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name);
const rawTags = scene.tags.map(tagId => tagsMap[tagId].name);
const rawTags = data.tags.map(tag => tag.name);
const tags = await matchTags(rawTags);
const [poster, ...photos] = getThumbs(scene);
const trailer720p = scene.videos.mediabook && scene.videos.mediabook.files['720p'] && scene.videos.mediabook.files['720p'].urls.view;
const trailer360p = scene.videos.mediabook && scene.videos.mediabook.files['360p'] && scene.videos.mediabook.files['360p'].urls.view;
const { likes, dislikes } = scene.stats;
const [poster, ...photos] = getThumbs(data);
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
return {
url,
@@ -63,41 +43,44 @@ async function scrapeLatest(html, site) {
title,
description,
actors,
date,
tags,
duration,
poster,
photos,
trailer: {
src: trailer720p || trailer360p,
quality: trailer720p ? 720 : 360,
trailer: trailer && {
src: trailer.urls.view,
quality: parseInt(trailer.format, 10),
},
rating: { likes, dislikes },
date,
site,
};
}));
}
async function scrapeScene(data, url, site) {
const {
id: entryId,
title,
description,
} = data;
const { id: entryId, title, description } = data;
const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name);
const { likes, dislikes } = data.stats;
const rawTags = data.tags.map(tag => tag.name);
const tags = await matchTags(rawTags);
const [poster, ...photos] = getThumbs(data);
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
const duration = data.videos.mediabook && data.videos.mediabook.length;
const trailer720p = data.videos.mediabook && data.videos.mediabook.files['720p'] && data.videos.mediabook.files['720p'].urls.view;
const trailer360p = data.videos.mediabook && data.videos.mediabook.files['360p'] && data.videos.mediabook.files['360p'].urls.view;
const siteName = data.collections[0].name;
const siteId = data.collections[0].id;
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
const siteUrl = `https://www.realitykings.com/scenes?site=${siteId}`;
const [[channelSite], tags] = await Promise.all([
site.isFallback
? fetchSites({
slug: siteSlug,
name: siteName,
url: siteUrl,
})
: [site],
matchTags(rawTags),
]);
return {
url,
@@ -105,41 +88,56 @@ async function scrapeScene(data, url, site) {
title,
description,
actors,
date,
duration,
tags,
poster,
photos,
trailer: {
src: trailer720p || trailer360p,
quality: trailer720p ? 720 : 360,
trailer: trailer && {
src: trailer.urls.view,
quality: parseInt(trailer.format, 10),
},
rating: {
likes,
dislikes,
},
site,
date,
site: channelSite,
};
}
async function fetchLatest(site, page = 1) {
function getUrl(site) {
const { hostname, search } = new URL(site.url);
if (hostname.match(/(www\.)?realitykings\.com/) && search.match(/\?site=\d+/)) {
const res = await bhttp.get(`${site.url}&page=${page}`);
return scrapeLatest(res.body.toString(), site);
return site.url;
}
if (site.parameters && site.parameters.siteId) {
const res = await bhttp.get(`https://www.realitykings.com/scenes?site=${site.parameters.siteId}&page=${page}`);
return scrapeLatest(res.body.toString(), site);
return `https://www.realitykings.com/scenes?site=${site.parameters.siteId}`;
}
throw new Error(`Reality Kings site '${site.name}' (${site.url}) not supported`);
}
async function fetchLatest(site, page = 1) {
const url = getUrl(site);
const { search } = new URL(url);
const siteId = new URLSearchParams(search).get('site');
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
await session.get(url);
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 10;
const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
headers: {
Instance: instanceToken,
},
});
return scrapeLatest(res.body.result, site);
}
async function fetchScene(url, site) {
const entryId = url.match(/\d+/)[0];
@@ -148,8 +146,8 @@ async function fetchScene(url, site) {
await session.get(url);
const cookies = await cookieJar.getCookieStringAsync(url);
const instanceToken = cookies.split(';')[0].split('=')[1];
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
headers: {
@@ -157,7 +155,7 @@ async function fetchScene(url, site) {
},
});
return scrapeScene(res.body.result.parent || res.body.result, url, site);
return scrapeScene(res.body.result, url, site);
}
module.exports = {

14
src/scrapers/xempire.js
View File

@@ -7,6 +7,7 @@ const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');
const defaultTags = {
hardx: [],
@@ -36,7 +37,7 @@ function scrapePhotos(html) {
return unlockedPhotos.concat(lockedThumbnails);
}
async function getPhotos(albumPath, siteDomain) {
async function getPhotos(albumPath, siteDomain, site) {
const albumUrl = `https://${siteDomain}${albumPath}`;
const html = await fetchPhotos(albumUrl);
@@ -54,7 +55,14 @@ async function getPhotos(albumPath, siteDomain) {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
const allPhotos = photos.concat(otherPhotos.flat());
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
return pluckedPhotos;
}
function scrape(html, site) {
@@ -140,7 +148,7 @@ async function scrapeScene(html, url, site) {
const poster = videoData.picPreview;
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain);
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
const rawTags = data.keywords.split(', ');

16
src/utils/cookies.js Normal file
View File

@@ -0,0 +1,16 @@
'use strict';
function cookieToData(cookieString) {
return cookieString.split('; ').reduce((acc, cookie) => {
const [key, value] = cookie.split('=');
return {
...acc,
[key]: value,
};
}, {});
}
module.exports = {
cookieToData,
};

21
src/utils/mofos.js Normal file
View File

@@ -0,0 +1,21 @@
'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp');
const knex = require('../knex');
async function run() {
const network = await knex('networks').where('slug', 'mofos').first();
const sites = await knex('sites').where('network_id', network.id);
await Promise.map(sites, async (site) => {
const res = await bhttp.get(site.url);
console.log(site.url, res.statusCode);
}, {
concurrency: 5,
});
}
run();

2
src/utils/upsert.js
View File

@@ -35,7 +35,7 @@ async function upsert(table, items, duplicatesById, identifier = 'id', knex) {
return Promise.all([
knex(table).insert(insert),
knex.transaction(async trx => Promise.all(update.map(item => trx
.where({ id: item.id })
.where({ [identifier]: item[identifier] })
.update(item)
.into(table)))),
]);