DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Separated actor expand buttons. Refactored Brazzers scraper. Fixed actor releases not included in shallow scrape. Added number query and data-src default to qu img. Updated README. Removed post-install migrate and seed.

Browse Source
This commit is contained in:
ThePendulum
2020-07-09 02:00:54 +02:00
parent 17d46e804e
commit 44a8ced30c
10 changed files with 289 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/actors.js
View File

@@ -281,7 +281,7 @@ async function curateProfile(profile) {
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
curatedProfile.ethnicity = ethnicities[profile.ethnicity?.trim().toLowerCase()] || null;
curatedProfile.hairColor = hairColors[(profile.hairColor || profile.hair)?.trim().toLowerCase()] || null;
curatedProfile.hairColor = hairColors[(profile.hairColor || profile.hair)?.toLowerCase().replace('hair', '').trim()] || null;
curatedProfile.eyes = eyeColors[profile.eyes?.trim().toLowerCase()] || null;
curatedProfile.tattoos = profile.tattoos?.trim() || null;

4
src/app.js
View File

@@ -24,11 +24,11 @@ async function init() {
const actors = argv.actors && await scrapeActors(argv.actors);
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.releases).flat().filter(Boolean);
const updateBaseScenes = (argv.scrape || argv.channels || argv.networks) && await fetchUpdates();
const updateBaseScenes = (argv.all || argv.channels || argv.networks) && await fetchUpdates();
const deepScenes = argv.deep
? await fetchScenes([...(argv.scenes || []), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
: updateBaseScenes;
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const sceneMovies = deepScenes && argv.sceneMovies && deepScenes.map(scene => scene.movie).filter(Boolean);
const deepMovies = await fetchMovies([...(argv.movies || []), ...(sceneMovies || [])]);

55
src/entities.js
View File

@@ -85,7 +85,54 @@ async function fetchChannelsFromArgv() {
}
async function fetchChannelsFromConfig() {
const rawSites = await knex('entities')
const rawNetworks = await knex.raw(`
WITH RECURSIVE children AS (
SELECT
id, parent_id, name, slug, type, url, description, parameters
FROM
entities
WHERE
CASE WHEN array_length(?, 1) IS NOT NULL
THEN slug = ANY(?)
ELSE true
END
AND NOT
slug = ANY(?)
AND
entities.type = 'network'
UNION ALL
SELECT
entities.id, entities.parent_id, entities.name, entities.slug, entities.type, entities.url, entities.description, entities.parameters
FROM
entities
INNER JOIN
children ON children.id = entities.parent_id
)
SELECT
entities.*, row_to_json(parents) as parent, json_agg(children) as children
FROM
children
LEFT JOIN
entities ON entities.id = children.parent_id
LEFT JOIN
entities AS parents ON parents.id = entities.parent_id
WHERE
children.type = 'channel'
GROUP BY
children.parent_id, entities.id, entities.name, parents.id
`, [
config.include.networks,
config.include.networks,
config.exclude.networks,
]);
console.log(rawNetworks.rows);
/*
const curatedSites = await curateEntities(rawChannels, true);
logger.info(`Found ${curatedSites.length} entities in database`);
const rawChannels = await knex('entities')
.select(knex.raw('entities.*, row_to_json(parents) as parent'))
.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
.where((builder) => {
@@ -97,10 +144,10 @@ async function fetchChannelsFromConfig() {
builder.whereIn('entities.slug', config.exclude || []);
});
const curatedSites = await curateEntities(rawSites, true);
logger.info(`Found ${curatedSites.length} entities in database`);
console.log(rawChannels);
*/
return curatedSites;
// return curatedSites;
}
async function fetchIncludedEntities() {

221
src/scrapers/brazzers.js
View File

@@ -1,142 +1,111 @@
'use strict';
/* eslint-disable newline-per-chained-call */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const { get, ex } = require('../utils/q');
const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
const { heightToCm, lbsToKg } = require('../utils/convert');
const hairMap = {
Blonde: 'blonde',
Brunette: 'brown',
'Black Hair': 'black',
Redhead: 'red',
};
function scrapeAll(html, site, upcoming) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.release-card.scene').toArray();
return sceneElements.reduce((acc, element) => {
const isUpcoming = $(element).find('.icon-upcoming.active').length === 1;
function scrapeAll(items, channel, upcoming) {
return items.reduce((acc, { query }) => {
const isUpcoming = query.exists('.icon-upcoming.active');
if ((upcoming && !isUpcoming) || (!upcoming && isUpcoming)) {
return acc;
}
const sceneLinkElement = $(element).find('a');
const release = {};
const pathname = query.url('a');
const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title');
const entryId = url.split('/').slice(-3, -2)[0];
release.url = `https://www.brazzers.com${pathname}`;
release.entryId = pathname.match(/(\/view\/id\/|\/episode\/)(\d+)/)[2];
const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate();
const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
release.title = query.q('a', 'title');
release.date = query.date('time', 'MMMM DD, YYYY');
const likes = Number($(element).find('.label-rating .like-amount').text());
const dislikes = Number($(element).find('.label-rating .dislike-amount').text());
release.actors = query.all('.model-names a', 'title');
const poster = `https:${$(element).find('.card-main-img').attr('data-src')}`;
const photos = $(element).find('.card-overlay .image-under').map((photoIndex, photoElement) => `https:${$(photoElement).attr('data-src')}`).toArray();
release.likes = query.number('.label-rating .like-amount');
release.dislikes = query.number('.label-rating .dislike-amount');
const channel = slugify($(element).find('.collection').attr('title'), '');
release.poster = query.img('.card-main-img');
release.photos = query.imgs('.card-overlay .image-under');
return acc.concat({
url,
entryId,
title,
actors,
date,
poster,
photos,
rating: {
likes,
dislikes,
},
channel,
site,
});
release.channel = slugify(query.q('.collection', 'title'), '');
return acc.concat(release);
}, []);
}
async function scrapeScene(html, url, _site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
function getVideoData(html) {
try {
const videoScriptStart = html.indexOf('window.videoUiOptions');
const videoScript = html.slice(videoScriptStart, html.indexOf('};', videoScriptStart));
const videoString = videoScript.slice(videoScript.indexOf('{"stream_info"'), videoScript.lastIndexOf('},') + 1);
return JSON.parse(videoString);
} catch (error) {
return null;
}
}
async function scrapeScene({ query, html }, url, _site) {
const release = {};
const videoJson = $('script:contains("window.videoUiOptions")').html();
const videoString = videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('},') + 1);
const videoData = JSON.parse(videoString);
release.entryId = new URL(url).pathname.match(/(\/view\/id\/|\/episode\/)(\d+)/)[2];
[release.entryId] = url.split('/').slice(-3, -2);
release.title = $('.scene-title[itemprop="name"]').text();
release.title = query.q('.scene-title[itemprop="name"]', true);
release.description = query.text('#scene-description p[itemprop="description"]');
release.description = $('#scene-description p[itemprop="description"]')
.contents()
.first()
.text()
.trim();
release.date = query.date('.more-scene-info .scene-date', 'MMMM DD, YYYY');
release.duration = Number(query.q('#trailer-player-container', 'data-duration')) // more accurate
|| Number(query.q('.scene-length[itemprop="duration"]', 'content').slice(1, -1) * 60);
release.date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate();
release.duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60;
// actor cards have avatar, but truncated name
const actorImagesByActorId = query.imgs('.featured-model .card-image img').reduce((acc, img) => ({
...acc,
[img.match(/\/models\/(\d+)/)[1]]: [
img.replace('medium', 'large'),
img,
],
}), {});
const actorsFromCards = $('.featured-model .card-image a').map((actorIndex, actorElement) => {
const avatar = `https:${$(actorElement).find('img').attr('data-src')}`;
release.actors = query.all('.related-model a').map(actorEl => ({
name: query.q(actorEl, null, 'title'),
avatar: actorImagesByActorId[query.url(actorEl, null).match(/\/view\/id\/(\d+)/)?.[1]],
}));
return {
name: $(actorElement).attr('title'),
avatar: [avatar.replace('medium.jpg', 'large.jpg'), avatar],
};
}).toArray();
release.likes = query.number('.label-rating .like');
release.dislikes = query.number('.label-rating .dislike');
release.actors = actorsFromCards || $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.likes = Number($('.label-rating .like').text());
release.dislikes = Number($('.label-rating .dislike').text());
const siteElement = $('.niche-site-logo');
// const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
const siteName = siteElement.attr('title');
release.channel = slugify(siteName, '');
const tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const categories = $('.timeline a[href*="/categories"]').map((tagIndex, categoryElement) => $(categoryElement).attr('title')).toArray();
const tags = query.all('.tag-card-container a', true);
const categories = query.all('.timeline a[href*="/categories"]', 'title');
release.tags = tags.concat(categories);
release.photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray();
release.channel = slugify(query.q('.scene-site .label-text', true) || query.q('.niche-site-logo', 'title'), '');
const posterPath = videoData?.poster || $('meta[itemprop="thumbnailUrl"]').attr('content') || $('#trailer-player-container').attr('data-player-img');
if (posterPath) release.poster = `https:${posterPath}`;
const videoData = getVideoData(html);
const poster = videoData?.poster || query.meta('itemprop="thumbnailUrl"') || query.q('#trailer-player-container', 'data-player-img');
release.poster = qu.prefixUrl(poster);
release.photos = query.urls('.carousel-thumb a');
if (videoData) {
release.trailer = Object.entries(videoData.stream_info.http.paths).map(([quality, path]) => ({
src: `https:${path}`,
src: qu.prefixUrl(path),
quality: Number(quality.match(/\d{3,}/)[0]),
}));
}
console.log(release);
return release;
}
function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
return actorLink ? actorLink.href : null;
}
async function fetchActorReleases({ qu, html }, accReleases = []) {
const releases = scrapeAll(html);
const next = qu.url('.pagination .next a');
async function fetchActorReleases({ query }, accReleases = []) {
const releases = scrapeAll(qu.initAll(query.all('.release-card.scene')));
const next = query.url('.pagination .next a');
if (next) {
const url = `https://www.brazzers.com${next}`;
const res = await get(url);
const res = await qu.get(url);
if (res.ok) {
return fetchActorReleases(res.item, accReleases.concat(releases));
@@ -146,12 +115,9 @@ async function fetchActorReleases({ qu, html }, accReleases = []) {
return accReleases.concat(releases);
}
async function scrapeProfile(html, url, actorName, include) {
const qProfile = ex(html);
const { q, qa } = qProfile;
const bioKeys = qa('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim());
const bioValues = qa('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim());
async function scrapeProfile({ query }, url, actorName, include) {
const bioKeys = query.all('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim());
const bioValues = query.all('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim());
const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key]: bioValues[index] }), {});
@@ -159,17 +125,17 @@ async function scrapeProfile(html, url, actorName, include) {
name: actorName,
};
profile.description = q('.model-profile-specs p', true);
profile.description = query.q('.model-profile-specs p', true);
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate();
if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = qu.extractDate(bio['Date of Birth'], 'MMMM DD, YYYY');
if (bio['Birth Location']) profile.birthPlace = bio['Birth Location'];
if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase();
if (bio.Height) profile.height = heightToCm(bio.Height);
if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]);
if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();
if (bio['Hair Color']) profile.hair = bio['Hair Color'].toLowerCase();
if (bio['Tits Type'] && bio['Tits Type'].match('Natural')) profile.naturalBoobs = true;
if (bio['Tits Type'] && bio['Tits Type'].match('Enhanced')) profile.naturalBoobs = false;
@@ -177,49 +143,60 @@ async function scrapeProfile(html, url, actorName, include) {
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
const avatarEl = q('.big-pic-model-container img');
const avatarEl = query.q('.big-pic-model-container img');
if (avatarEl) profile.avatar = `https:${avatarEl.src}`;
if (include.releases) {
profile.releases = await fetchActorReleases(qProfile);
profile.releases = await fetchActorReleases({ query });
}
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/page/${page}/`);
async function fetchLatest(channel, page = 1) {
const res = await qu.getAll(`${channel.url}/page/${page}/`, '.release-card.scene');
return scrapeAll(res.body.toString(), site, false);
if (res.ok) {
return scrapeAll(res.items, channel, false);
}
return res.status;
}
async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.url}/`);
async function fetchUpcoming(channel) {
const res = await qu.getAll(`${channel.url}/page/1`, '.release-card.scene');
return scrapeAll(res.body.toString(), site, true);
if (res.ok) {
return scrapeAll(res.items, channel, true);
}
return res.status;
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
const res = await qu.get(url);
return scrapeScene(res.body.toString(), url, site);
if (res.ok) {
return scrapeScene(res.item, url, site);
}
return res.status;
}
async function fetchProfile(actorName, context, include) {
const searchUrl = 'https://brazzers.com/pornstars-search/';
const searchRes = await bhttp.get(searchUrl, {
headers: {
Cookie: `textSearch=${encodeURIComponent(actorName)};`,
},
const searchRes = await qu.get('https://brazzers.com/pornstars-search/', `a[title="${actorName}" i]`, {
Cookie: `textSearch=${encodeURIComponent(actorName)};`,
});
const actorLink = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
const actorLink = searchRes.ok && searchRes.item.qu.url(null);
if (actorLink) {
const url = `https://brazzers.com${actorLink}`;
const res = await bhttp.get(url);
const res = await qu.get(url);
return scrapeProfile(res.body.toString(), url, actorName, include);
if (res.ok) {
return scrapeProfile(res.item, url, actorName, include);
}
}
return null;

27
src/utils/qu.js
View File

@@ -103,6 +103,12 @@ function text(context, selector, applyTrim = true) {
return applyTrim ? trim(textValue) : textValue;
}
function number(context, selector, attr = true) {
const value = q(context, selector, attr);
return value ? Number(value) : null;
}
function meta(context, selector, attrArg = 'content', applyTrim = true) {
if (/meta\[.*\]/.test(selector)) {
return q(context, selector, attrArg, applyTrim);
@@ -119,17 +125,22 @@ function date(context, selector, format, match, attr = 'textContent') {
return extractDate(dateString, format, match);
}
function image(context, selector = 'img', attr = 'src', origin, protocol = 'https') {
const imageEl = q(context, selector, attr);
function image(context, selector = 'img', attr, origin, protocol = 'https') {
const imageEl = (attr && q(context, selector, attr))
|| q(context, selector, 'src')
|| q(context, selector, 'data-src');
// no attribute means q output will be HTML element
return attr ? prefixUrl(imageEl, origin, protocol) : imageEl;
return prefixUrl(imageEl, origin, protocol);
}
function images(context, selector = 'img', attr = 'src', origin, protocol = 'https') {
const imageEls = all(context, selector, attr);
function images(context, selector = 'img', attr, origin, protocol = 'https') {
const attribute = attr
|| (q(context, selector, 'src') && 'src')
|| (q(context, selector, 'data-src') && 'data-src');
return attr ? imageEls.map(imageEl => prefixUrl(imageEl, origin, protocol)) : imageEls;
const imageEls = all(context, selector, attribute);
return imageEls.map(imageEl => prefixUrl(imageEl, origin, protocol));
}
function url(context, selector = 'a', attr = 'href', origin, protocol = 'https') {
@@ -225,6 +236,8 @@ const quFuncs = {
imgs: images,
length: duration,
meta,
number,
num: number,
poster,
q,
text,