forked from DebaucheryLibrarian/traxxx
171 lines
5.5 KiB
JavaScript
171 lines
5.5 KiB
JavaScript
|
'use strict';
|
||
|
|
||
|
/* eslint-disable newline-per-chained-call */
|
||
|
const Promise = require('bluebird');
|
||
|
const bhttp = require('bhttp');
|
||
|
const { JSDOM } = require('jsdom');
|
||
|
const moment = require('moment');
|
||
|
const knex = require('knex');
|
||
|
|
||
|
const { matchTags } = require('../tags');
|
||
|
|
||
|
async function getPhoto(url) {
|
||
|
const res = await bhttp.get(url);
|
||
|
const html = res.body.toString();
|
||
|
const { document } = new JSDOM(html).window;
|
||
|
|
||
|
const photoUrl = document.querySelector('.scenes-module img').src;
|
||
|
|
||
|
return photoUrl;
|
||
|
}
|
||
|
|
||
|
async function getPhotos(albumUrl, site, siteUrl) {
|
||
|
const res = await bhttp.get(albumUrl);
|
||
|
const html = res.body.toString();
|
||
|
const { document } = new JSDOM(html).window;
|
||
|
|
||
|
const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
|
||
|
const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
|
||
|
|
||
|
// dogfart has massive albums, pick 20 or specified photos: first, last and evenly inbetween
|
||
|
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
|
||
|
const photoIndexes = [1]
|
||
|
.concat(Array.from({ length: photoLimit - 2 }, (value, index) => Math.floor((index + 1) * (lastPhotoIndex / (photoLimit - 2)))))
|
||
|
.concat(lastPhotoIndex);
|
||
|
|
||
|
if (photoLimit > 25) {
|
||
|
console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`);
|
||
|
}
|
||
|
|
||
|
const photoUrls = await Promise.map(photoIndexes, async (index) => {
|
||
|
const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${index.toString().padStart(3, '0')}.jpg`)}`;
|
||
|
|
||
|
return getPhoto(pageUrl);
|
||
|
}, {
|
||
|
concurrency: 5,
|
||
|
});
|
||
|
|
||
|
return photoUrls;
|
||
|
}
|
||
|
|
||
|
function scrapeLatest(html, site) {
|
||
|
const { document } = new JSDOM(html).window;
|
||
|
const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));
|
||
|
|
||
|
return sceneElements.reduce((acc, element) => {
|
||
|
const siteUrl = element.querySelector('.help-block').textContent;
|
||
|
|
||
|
if (siteUrl.toLowerCase() !== new URL(site.url).host) {
|
||
|
// different dogfart site
|
||
|
return acc;
|
||
|
}
|
||
|
|
||
|
const sceneLinkElement = element.querySelector('.thumbnail');
|
||
|
const url = `https://dogfartnetwork.com${sceneLinkElement.href}`;
|
||
|
const { pathname } = new URL(url);
|
||
|
const entryId = `${site.slug}_${pathname.split('/')[4]}`;
|
||
|
|
||
|
const title = element.querySelector('.scene-title').textContent;
|
||
|
const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim());
|
||
|
|
||
|
const poster = `https:${element.querySelector('img').src}`;
|
||
|
const trailer = sceneLinkElement.dataset.preview_clip_url;
|
||
|
|
||
|
return [
|
||
|
...acc,
|
||
|
{
|
||
|
url,
|
||
|
entryId,
|
||
|
title,
|
||
|
actors,
|
||
|
poster,
|
||
|
trailer: {
|
||
|
src: trailer,
|
||
|
},
|
||
|
site,
|
||
|
},
|
||
|
];
|
||
|
}, []);
|
||
|
}
|
||
|
|
||
|
async function scrapeScene(html, url, site) {
|
||
|
const { document } = new JSDOM(html).window;
|
||
|
|
||
|
const title = document.querySelector('.description-title').textContent;
|
||
|
const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
|
||
|
const metaDescription = document.querySelector('meta[itemprop="description"]').content;
|
||
|
const description = metaDescription
|
||
|
? metaDescription.content
|
||
|
: document.querySelector('.description')
|
||
|
.textContent
|
||
|
.replace(/[ \t\n]{2,}/g, ' ')
|
||
|
.replace('...read more', '')
|
||
|
.trim();
|
||
|
|
||
|
const siteSlug = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
|
||
|
const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
|
||
|
const duration = moment
|
||
|
.duration(document
|
||
|
.querySelectorAll('.extra-info p')[1]
|
||
|
.textContent
|
||
|
.match(/\d+:\d+$/)[0])
|
||
|
.asSeconds();
|
||
|
|
||
|
const trailerElement = document.querySelector('.html5-video');
|
||
|
const poster = `https:${trailerElement.dataset.poster}`;
|
||
|
const { trailer } = trailerElement.dataset;
|
||
|
|
||
|
const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href;
|
||
|
const { origin, pathname } = new URL(url);
|
||
|
const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
|
||
|
|
||
|
const stars = Number(document.querySelector('span[itemprop="average"]').textContent) / 2;
|
||
|
const rawTags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
|
||
|
|
||
|
const [channelSite, tags] = await Promise.all([
|
||
|
site.isFallback
|
||
|
? knex('sites')
|
||
|
.where({ slug: siteSlug })
|
||
|
.orWhere({ url: `https://${siteSlug}.com` })
|
||
|
.first()
|
||
|
: site,
|
||
|
matchTags(rawTags),
|
||
|
]);
|
||
|
|
||
|
return {
|
||
|
url,
|
||
|
title,
|
||
|
description,
|
||
|
actors,
|
||
|
date,
|
||
|
duration,
|
||
|
poster,
|
||
|
photos,
|
||
|
trailer: {
|
||
|
src: trailer,
|
||
|
},
|
||
|
tags,
|
||
|
rating: {
|
||
|
stars,
|
||
|
},
|
||
|
site: channelSite || site,
|
||
|
};
|
||
|
}
|
||
|
|
||
|
async function fetchLatest(site, page = 1) {
|
||
|
const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
|
||
|
|
||
|
return scrapeLatest(res.body.toString(), site);
|
||
|
}
|
||
|
|
||
|
async function fetchScene(url, site) {
|
||
|
const res = await bhttp.get(url);
|
||
|
|
||
|
return scrapeScene(res.body.toString(), url, site);
|
||
|
}
|
||
|
|
||
|
module.exports = {
|
||
|
fetchLatest,
|
||
|
fetchScene,
|
||
|
};
|