boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Added Fantasy Massage sites. Improved Private scraper, added movie link.

Browse Source
This commit is contained in:
ThePendulum
2020-02-08 04:52:32 +01:00
parent ff8ab2fe09
commit d2cb74a252
5 changed files with 143 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
src/scrapers/private.js
View File

@@ -5,9 +5,6 @@ const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const fetchSites = require('../sites');
const { matchTags } = require('../tags');
async function getPhotos(entryId, site) {
const { hostname } = new URL(site.url);
@@ -20,27 +17,6 @@ async function getPhotos(entryId, site) {
return photos;
}
async function getChannelSite($, site) {
const { hostname } = new URL(site.url);
if (site.isFallback && hostname.match('private.com')) {
const siteElement = $('.content-wrapper .logos-sites a');
const siteUrl = siteElement.attr('href').slice(0, -1);
const siteName = siteElement.text();
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
const channelSite = await fetchSites({
slug: siteSlug,
name: siteName,
url: siteUrl,
});
return channelSite;
}
return site;
}
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.content-wrapper .scene').toArray();
@@ -82,55 +58,40 @@ function scrapeLatest(html, site) {
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const release = {};
const entryId = url.split('/').slice(-1)[0];
const title = $('.video-wrapper meta[itemprop="name"]').attr('content');
[release.entryId] = url.split('/').slice(-1);
release.title = $('.video-wrapper meta[itemprop="name"]').attr('content');
release.description = $('.video-wrapper meta[itemprop="description"]').attr('content');
const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
const actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const description = $('.video-wrapper meta[itemprop="description"]').attr('content');
const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g);
const duration = Number(minutes) * 60 + Number(seconds);
const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content');
if (timestamp) {
const [minutes, seconds] = timestamp.match(/\d+/g);
release.duration = Number(minutes) * 60 + Number(seconds);
}
release.tags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
release.likes = Number($('.content-desc #social-actions #likes').text());
const posterScript = $('script:contains(poster)').html();
const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
const poster = $('meta[property="og:image"]').attr('content') || posterLink;
release.poster = $('meta[property="og:image"]').attr('content') || posterLink;
const trailerElementSrc = $('#videojs-trailer source').attr('src');
const trailer = $('meta[property="og:video"]').attr('content') || trailerElementSrc;
const likes = Number($('.content-desc #social-actions #likes').text());
const rawTags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [tags, photos, channelSite] = await Promise.all([
matchTags(rawTags),
getPhotos(entryId, site),
getChannelSite($, site),
]);
const scene = {
url,
entryId,
title,
date,
actors,
description,
duration,
tags,
poster,
photos,
trailer: {
src: trailer,
},
rating: {
likes,
},
site: channelSite || site,
release.trailer = {
src: $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'),
};
return scene;
release.photos = await getPhotos(release.entryId, site);
release.movie = $('a[data-track="FULL MOVIE"]').attr('href');
const siteElement = $('.content-wrapper .logos-sites a');
if (siteElement) release.channel = siteElement.text().replace(/\s+/g, '').toLowerCase();
return release;
}
async function fetchLatest(site, page = 1) {