boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Major refactor, cleand up site scrape module, fixed and cleaned up release scrape module. Removed old CLI code

Browse Source
This commit is contained in:
ThePendulum
2019-11-16 03:33:36 +01:00
parent b07f88d023
commit b489c8fc33
35 changed files with 595 additions and 847 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/scrapers/brazzers.js
View File

@@ -5,7 +5,7 @@ const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const fetchSites = require('../sites');
const { fetchSites } = require('../sites');
const { matchTags } = require('../tags');
function scrape(html, site, upcoming) {
@@ -23,7 +23,7 @@ function scrape(html, site, upcoming) {
const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title');
const shootId = url.split('/').slice(-3, -2)[0];
const entryId = url.split('/').slice(-3, -2)[0];
const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate();
const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
@@ -36,7 +36,7 @@ function scrape(html, site, upcoming) {
return acc.concat({
url,
shootId,
entryId,
title,
actors,
date,
@@ -56,7 +56,7 @@ async function scrapeScene(html, url, site) {
const videoJson = $('script:contains("window.videoUiOptions")').html();
const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('"},') + 2));
const shootId = url.split('/').slice(-3, -2)[0];
const entryId = url.split('/').slice(-3, -2)[0];
const title = $('.scene-title[itemprop="name"]').text();
const description = $('#scene-description p[itemprop="description"]')
@@ -83,20 +83,20 @@ async function scrapeScene(html, url, site) {
const trailer = `https:${videoData.stream_info.http.paths.mp4_480_1500}`;
const photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray();
const [tags, channelSite] = await Promise.all([
const [tags, [channelSite]] = await Promise.all([
matchTags(rawTags),
site.isFallback
? [fetchSites({
? fetchSites({
slug: siteSlug,
name: siteName,
url: siteUrl,
})]
: site,
})
: [site],
]);
return {
url,
shootId,
entryId,
title,
description,
actors,

2
src/scrapers/dogfart.js
View File

@@ -5,8 +5,8 @@ const Promise = require('bluebird');
const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const knex = require('knex');
const knex = require('../knex');
const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');

14
src/scrapers/kink.js
View File

@@ -3,8 +3,8 @@
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('knex');
const { fetchSites } = require('../sites');
const { matchTags } = require('../tags');
function scrapeLatest(html, site) {
@@ -71,13 +71,17 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
const { average: stars } = ratingRes.body;
const sitename = $('.shoot-logo a').attr('href').split('/')[2];
const siteName = $('.shoot-logo a').attr('href').split('/')[2];
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [channelSite, tags] = await Promise.all([
const [[channelSite], tags] = await Promise.all([
site.isFallback
? knex('sites').where({ slug: sitename }).first()
: site,
? fetchSites({
slug: siteSlug,
name: siteName,
})
: [site],
matchTags(rawTags),
]);

4
src/scrapers/pervcity.js
View File

@@ -7,7 +7,7 @@ const moment = require('moment');
function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const shootId = $('li').attr('id');
const entryId = $('li').attr('id');
const sceneLinkElement = $('#scene_title_border a');
const url = `${site.url}/${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title').replace(/\u00E2\u0080\u0099/g, '\''); // replace weird apostrophes
@@ -22,7 +22,7 @@ function scrape(html, site) {
return {
url,
shootId,
entryId,
title,
actors,
date,

22
src/scrapers/private.js
View File

@@ -8,10 +8,10 @@ const moment = require('moment');
const fetchSites = require('../sites');
const { matchTags } = require('../tags');
async function getPhotos(shootId, site) {
async function getPhotos(entryId, site) {
const { hostname } = new URL(site.url);
const res = await bhttp.get(`https://${hostname}/gallery.php?type=highres&id=${shootId}`);
const res = await bhttp.get(`https://${hostname}/gallery.php?type=highres&id=${entryId}`);
const html = res.body.toString();
const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -50,7 +50,7 @@ function scrapeLatest(html, site) {
const url = sceneLinkElement.attr('href');
const title = sceneLinkElement.text();
const shootId = url.split('/').slice(-1)[0];
const entryId = url.split('/').slice(-1)[0];
const date = moment.utc($(element).find('.scene-date'), 'MM/DD/YYYY').toDate();
@@ -64,7 +64,7 @@ function scrapeLatest(html, site) {
const scene = {
url,
shootId,
entryId,
title,
actors,
date,
@@ -83,7 +83,7 @@ function scrapeLatest(html, site) {
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const shootId = url.split('/').slice(-1)[0];
const entryId = url.split('/').slice(-1)[0];
const title = $('.video-wrapper meta[itemprop="name"]').attr('content');
const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
@@ -93,8 +93,12 @@ async function scrapeScene(html, url, site) {
const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g);
const duration = Number(minutes) * 60 + Number(seconds);
const poster = $('meta[property="og:image"]').attr('content');
const trailer = $('meta[property="og:video"]').attr('content');
const posterScript = $('script:contains(poster)').html();
const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
const poster = $('meta[property="og:image"]').attr('content') || posterLink;
const trailerElementSrc = $('#videojs-trailer source').attr('src');
const trailer = $('meta[property="og:video"]').attr('content') || trailerElementSrc;
const likes = Number($('.content-desc #social-actions #likes').text());
@@ -102,13 +106,13 @@ async function scrapeScene(html, url, site) {
const [tags, photos, channelSite] = await Promise.all([
matchTags(rawTags),
getPhotos(shootId, site),
getPhotos(entryId, site),
getChannelSite($, site),
]);
const scene = {
url,
shootId,
entryId,
title,
date,
actors,

0
src/scrapers/index.js → src/scrapers/scrapers.js
View File

10
src/scrapers/vixen.js
View File

@@ -23,7 +23,7 @@ function scrapeLatest(html, site) {
const { videos: scenes } = JSON.parse(stateScript.slice(stateScript.indexOf('{'), stateScript.indexOf('};') + 1));
return scenes.map((scene) => {
const shootId = String(scene.newId);
const entryId = String(scene.newId);
const {
title,
@@ -40,7 +40,7 @@ function scrapeLatest(html, site) {
return {
url,
shootId,
entryId,
title,
actors,
date,
@@ -65,8 +65,8 @@ async function scrapeScene(html, url, site) {
const stateObject = $('script:contains("INITIAL_STATE")');
const data = JSON.parse(stateObject.html().trim().slice(27, -1));
const shootId = data.page.data[`${pathname}${search}`].data.video;
const scene = data.videos.find(video => video.newId === shootId);
const entryId = data.page.data[`${pathname}${search}`].data.video;
const scene = data.videos.find(video => video.newId === entryId);
const [poster, ...photos] = scene.rotatingThumbsUrlSizes.map(photo => photo['1040w']);
const trailer = scene.previews.listing.find(preview => preview.height === 353) || null;
@@ -86,7 +86,7 @@ async function scrapeScene(html, url, site) {
return {
url,
shootId,
entryId,
title,
description,
actors,

2
src/scrapers/xempire.js
View File

@@ -3,9 +3,9 @@
const Promise = require('bluebird');
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const knex = require('knex');
const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
const defaultTags = {