DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Major refactor, cleand up site scrape module, fixed and cleaned up release scrape module. Removed old CLI code

Browse Source
This commit is contained in:
ThePendulum
2019-11-16 03:33:36 +01:00
parent b07f88d023
commit b489c8fc33
35 changed files with 595 additions and 847 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/scrapers/private.js
View File

@@ -8,10 +8,10 @@ const moment = require('moment');
const fetchSites = require('../sites');
const { matchTags } = require('../tags');
async function getPhotos(shootId, site) {
async function getPhotos(entryId, site) {
const { hostname } = new URL(site.url);
const res = await bhttp.get(`https://${hostname}/gallery.php?type=highres&id=${shootId}`);
const res = await bhttp.get(`https://${hostname}/gallery.php?type=highres&id=${entryId}`);
const html = res.body.toString();
const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -50,7 +50,7 @@ function scrapeLatest(html, site) {
const url = sceneLinkElement.attr('href');
const title = sceneLinkElement.text();
const shootId = url.split('/').slice(-1)[0];
const entryId = url.split('/').slice(-1)[0];
const date = moment.utc($(element).find('.scene-date'), 'MM/DD/YYYY').toDate();
@@ -64,7 +64,7 @@ function scrapeLatest(html, site) {
const scene = {
url,
shootId,
entryId,
title,
actors,
date,
@@ -83,7 +83,7 @@ function scrapeLatest(html, site) {
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const shootId = url.split('/').slice(-1)[0];
const entryId = url.split('/').slice(-1)[0];
const title = $('.video-wrapper meta[itemprop="name"]').attr('content');
const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
@@ -93,8 +93,12 @@ async function scrapeScene(html, url, site) {
const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g);
const duration = Number(minutes) * 60 + Number(seconds);
const poster = $('meta[property="og:image"]').attr('content');
const trailer = $('meta[property="og:video"]').attr('content');
const posterScript = $('script:contains(poster)').html();
const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
const poster = $('meta[property="og:image"]').attr('content') || posterLink;
const trailerElementSrc = $('#videojs-trailer source').attr('src');
const trailer = $('meta[property="og:video"]').attr('content') || trailerElementSrc;
const likes = Number($('.content-desc #social-actions #likes').text());
@@ -102,13 +106,13 @@ async function scrapeScene(html, url, site) {
const [tags, photos, channelSite] = await Promise.all([
matchTags(rawTags),
getPhotos(shootId, site),
getPhotos(entryId, site),
getChannelSite($, site),
]);
const scene = {
url,
shootId,
entryId,
title,
date,
actors,