Added support for release photo fallbacks. Limited photo fetching concurrency. Modifier XEmpire scraper for AllBlackX support and improved photo scraping. Added movie property to Evil Angel scraper.

This commit is contained in:
2019-12-12 03:12:05 +01:00
parent c26d5b8655
commit a310f9bb1d
9 changed files with 113 additions and 70 deletions

View File

@@ -7,7 +7,6 @@ const { JSDOM } = require('jsdom');
const moment = require('moment');
const { heightToCm } = require('../utils/convert');
const { matchTags } = require('../tags');
async function fetchPhotos(url) {
const res = await bhttp.get(url);
@@ -22,13 +21,8 @@ function scrapePhotos(html) {
.map((photoIndex, photoElement) => {
const src = $(photoElement).attr('src');
if (src.match(/dl\d+/)) {
// thumbnail URLs containing dl02/ or dl03/ don't appear to have
// a full photo available, fall back to thumbnail
return src;
}
return src.replace('thumbs/', 'photos/');
// high res often available in photos/ directory, but not always, provide original as fallback
return [src.replace('thumbs/', 'photos/'), src];
})
.toArray();
@@ -172,8 +166,8 @@ async function scrapeScene(html, url, site) {
const photos = await getPhotos(entryId, site);
const rawTags = $('.update_tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const tags = await matchTags(rawTags);
const tags = $('.update_tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const movie = $('.update_dvds a').href;
return {
url,
@@ -184,6 +178,7 @@ async function scrapeScene(html, url, site) {
description,
poster,
photos,
movie,
trailer: {
src: trailer,
quality: 720,