Replaced cheerio with qu in Gamma scraper.

This commit is contained in:
DebaucheryLibrarian 2021-11-29 02:49:07 +01:00
parent 53a1c697d0
commit 87094a9498
3 changed files with 81 additions and 115 deletions

View File

@ -3,7 +3,6 @@
const Promise = require('bluebird');
const util = require('util');
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
const format = require('template-format');
@ -62,21 +61,19 @@ function getAlbumUrl(albumPath, site) {
}
async function fetchPhotos(url) {
const res = await http.get(url);
const res = await qu.get(url);
return res.body.toString();
return res.item;
}
function scrapePhotos(html, includeThumbnails = true) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
return $('.preview .imgLink, .pgFooterThumb a').toArray().map((linkEl) => {
const url = $(linkEl).attr('href');
function scrapePhotos({ query }, includeThumbnails = true) {
return query.all('.preview .imgLink, .pgFooterThumb a').map((linkEl) => {
const url = linkEl.href;
if (/\/join|\/createaccount/.test(url)) {
// URL links to join page instead of full photo, extract thumbnail
// /createaccount is used by e.g. Tricky Spa native site
const src = $(linkEl).find('img').attr('src');
const src = query.img(linkEl);
if (/previews\//.test(src)) {
// resource often serves full photo at a modifier URL anyway, add as primary source
@ -106,20 +103,18 @@ async function getPhotos(albumPath, site, includeThumbnails = true) {
}
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html, includeThumbnails);
const item = await fetchPhotos(albumUrl);
const photos = scrapePhotos(item, includeThumbnails);
const lastPage = $('.Gamma_Paginator a.last').attr('href')?.match(/\d+$/)[0];
const lastPage = item.query.url('.Gamma_Paginator a.last')?.match(/\d+$/)[0];
if (lastPage) {
const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
const otherPhotos = await Promise.map(otherPages, async (page) => {
const pageUrl = `${albumUrl}/${page}`;
const pageHtml = await fetchPhotos(pageUrl);
const pageItem = await fetchPhotos(`${albumUrl}/${page}`);
return scrapePhotos(pageHtml, includeThumbnails);
return scrapePhotos(pageItem, includeThumbnails);
}, {
concurrency: 2,
});
@ -244,41 +239,21 @@ async function scrapeApiReleases(json, site) {
}).filter(Boolean);
}
function scrapeAll(html, site, networkUrl, hasTeaser = true) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const scenesElements = $('li[data-itemtype=scene], div[data-itemtype=scenes]').toArray();
return scenesElements.map((element) => {
function scrapeAll(scenes, site, networkUrl, hasTeaser = true) {
return scenes.map(({ query, el }) => {
const release = {};
const sceneLinkElement = $(element).find('.sceneTitle a, .tlcTitle a');
release.url = query.url('.sceneTitle a, .tlcTitle a', 'href', { origin: networkUrl ? site.parent.url : site.url });
if (site) release.url = `${networkUrl ? site.parent.url : site.url}${sceneLinkElement.attr('href')}`;
else release.url = `${networkUrl}${sceneLinkElement.attr('href')}`;
release.title = query.cnt('.sceneTitle a', 'tlcTitle a', 'title');
release.entryId = el.dataset.itemid;
release.title = sceneLinkElement.attr('title');
release.entryId = $(element).attr('data-itemid');
release.date = query.date('.sceneDate, .tlcSpecsDate .tlcDetailsValue', ['MM-DD-YYYY', 'YYYY-MM-DD']);
release.actors = query.cnts('.sceneActors a, .tlcActors a', ' title');
const dateEl = $(element).find('.sceneDate, .tlcSpecsDate .tlcDetailsValue').text() || null;
if (dateEl) {
release.date = moment
.utc(dateEl, ['MM-DD-YYYY', 'YYYY-MM-DD'])
.toDate();
}
[release.likes, release.dislikes] = query.all('.value').map((likeEl) => query.number(likeEl));
release.actors = $(element).find('.sceneActors a, .tlcActors a')
.map((actorIndex, actorElement) => $(actorElement).attr('title'))
.toArray();
[release.likes, release.dislikes] = $(element).find('.value')
.toArray()
.map((value) => Number($(value).text()));
const posterEl = $(element).find('.imgLink img, .tlcImageItem');
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
const channelEl = $(element).find('.fromSite a');
if (channelEl.attr('title')) release.channel = channelEl.attr('title').replace('.com', '');
release.poster = query.img('.imgLink img, .tlcImageItem', 'data-original') || query.img('.imgLink img, .tlcImageItem');
if (hasTeaser) {
release.teaser = [
@ -287,76 +262,66 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
];
}
release.channel = query.el('.fromSite a', 'title')?.replace('.com', '');
return release;
});
}
async function scrapeScene(html, url, site, baseRelease, mobileHtml, options) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const m$ = mobileHtml && cheerio.load(mobileHtml, { normalizeWhitespace: true });
const release = { $, url };
async function scrapeScene({ query }, url, channel, baseRelease, mobileItem, options) {
const release = { query }; // used by XEmpire scraper to resolve channel-specific details
const json = $('script[type="application/ld+json"]').html();
const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
const json = query.html('script[type="application/ld+json"]');
const videoJson = query.htmls('script').find((script) => /ScenePlayerOptions/i.test(script));
const [data, data2] = json ? JSON.parse(json) : [];
const videoData = videoJson && JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
release.entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name;
release.description = data?.description;
// date in data object is not the release date of the scene, but the date the entry was added; only use as fallback
const dateString = $('.updatedDate').first().text().trim();
const dateMatch = dateString.match(/\d{2,4}[-/]\d{2}[-/]\d{2,4}/)?.[0];
release.date = query.date('.updatedDate', ['MM-DD-YYYY', 'YYYY-MM-DD'])
|| qu.extractDate(data?.dateCreated, 'YYYY-MM-DD')
|| videoData?.playerOptions?.sceneInfos.sceneReleaseDate;
if (dateMatch) release.date = moment.utc(dateMatch, ['MM-DD-YYYY', 'YYYY-MM-DD']).toDate();
else if (data?.dateCreated) release.date = moment.utc(data.dateCreated, 'YYYY-MM-DD').toDate();
else release.date = videoData?.playerOptions?.sceneInfos.sceneReleaseDate;
if (data) {
release.description = data.description;
if (data.director?.[0]?.name) release.director = data.director[0].name;
else if (data2?.director?.[0]?.name) release.director = data2.director[0].name;
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
if (stars) release.rating = { stars };
release.duration = moment.duration(data.duration.slice(2)).asSeconds();
}
const actors = data?.actor || data2?.actor;
if (actors) {
release.actors = actors.map((actor) => ({
release.actors = (data?.actor || data2?.actor)?.map((actor) => ({
name: actor.name,
gender: actor.gender,
}));
})) || [];
release.duration = qu.durationToSeconds(data.duration);
release.director = data?.director?.[0]?.name || data2?.director?.[0]?.name;
release.tags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
release.stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5 || null;
release.channel = slugify(data?.productionCompany?.name
|| query.el('.studioLink a, .siteLink a', 'title')
|| query.cnt('.siteNameSpan')?.toLowerCase().replace('.com', '')
|| query.meta('meta[name="twitter:domain"]')?.replace('.com', ''), '');
if (videoData?.picPreview && new URL(videoData.picPreview).pathname.length > 1) {
// sometimes links to just https://images02-fame.gammacdn.com/
const poster = new URL(videoData.picPreview);
release.poster = [
`${poster.origin}${poster.pathname}`,
videoData.picPreview,
];
}
const hasTrans = release.actors?.some((actor) => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
const channel = data?.productionCompany?.name
|| $('.studioLink a, .siteLink a').attr('title')?.trim()
|| $('.siteNameSpan').text()
?.trim()
.toLowerCase()
.replace('.com', '')
|| $('meta[name="twitter:domain"]').attr('content')?.replace('.com', '');
if (channel) release.channel = slugify(channel, '');
if (videoData?.picPreview && new URL(videoData.picPreview).pathname.length > 1) release.poster = videoData.picPreview; // sometimes links to just https://images02-fame.gammacdn.com/
const photoLink = $('.picturesItem a').attr('href');
const mobilePhotos = m$ ? m$('.preview-displayer a img').map((photoIndex, photoEl) => $(photoEl).attr('src')).toArray() : [];
const photoLink = query.url('.picturesItem a');
const mobilePhotos = mobileItem?.query.imgs('.preview-displayer a img') || [];
if (photoLink && options.includePhotos) {
const photos = await getPhotos(photoLink, site, mobilePhotos.length < 3); // only get thumbnails when less than 3 mobile photos are available
const photos = await getPhotos(photoLink, channel, mobilePhotos.length < 3); // only get thumbnails when less than 3 mobile photos are available
if (photos.length < 7) release.photos = [...photos, ...mobilePhotos]; // probably only teaser photos available, supplement with mobile album
else release.photos = photos;
if (photos.length < 7) {
release.photos = [...photos, ...mobilePhotos]; // probably only teaser photos available, supplement with mobile album
} else {
release.photos = photos;
}
} else {
release.photos = mobilePhotos;
}
@ -397,15 +362,14 @@ async function scrapeScene(html, url, site, baseRelease, mobileHtml, options) {
];
}
const movie = $('.dvdLink');
const movieUrl = qu.prefixUrl(movie.attr('href'), site.url);
const movieUrl = query.url('.dvdLink', 'href', { origin: channel.url });
if (movieUrl) {
release.movie = {
url: movieUrl,
title: movie.attr('title'),
title: query.el('.dvdLink', 'title'),
entryId: movieUrl.match(/\/(\d+)(\/|$)/)?.[1],
covers: [movie.find('img').attr('src')],
covers: [qu.imgs('.dvdLink img')],
};
}
@ -547,9 +511,7 @@ async function fetchActorReleases(profileUrl, getActorReleasesUrl, page = 1, acc
return accReleases.concat(releases);
}
async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUrl, withReleases, context) {
const { query } = qu.extract(html);
async function scrapeProfile({ query }, url, actorName, _siteSlug, getActorReleasesUrl, withReleases, context) {
const avatar = query.el('img.actorPicture');
const hair = query.cnt('.actorProfile .attribute_hair_color');
const height = query.cnt('.actorProfile .attribute_height');
@ -699,10 +661,10 @@ function getUpcomingUrl(site) {
async function fetchLatest(site, page = 1) {
const url = getLatestUrl(site, page);
const res = await http.get(url);
const res = await qu.getAll(url, 'li[data-itemtype=scene], div[data-itemtype=scenes]');
if (res.ok) {
return scrapeAll(res.body.toString(), site);
return scrapeAll(res.items, site);
}
return res.status;
@ -710,10 +672,10 @@ async function fetchLatest(site, page = 1) {
async function fetchUpcoming(site) {
const url = getUpcomingUrl(site);
const res = await http.get(url);
const res = await qu.getAll(url, 'li[data-itemtype=scene], div[data-itemtype=scenes]');
if (res.ok) {
return scrapeAll(res.body.toString(), site, null, false);
return scrapeAll(res.items, site, null, false);
}
return res.status;
@ -753,8 +715,8 @@ async function fetchScene(url, site, baseRelease, options) {
if (deepUrl) {
const [res, mobileRes] = await Promise.all([
http.get(deepUrl),
mobileUrl && http.get(mobileUrl, {
qu.get(deepUrl),
mobileUrl && qu.get(mobileUrl, null, {
headers: {
// don't redirect to main site
'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Mobile Safari/537.36',
@ -763,8 +725,8 @@ async function fetchScene(url, site, baseRelease, options) {
]);
if (res.status === 200) {
const mobileBody = mobileRes?.status === 200 ? mobileRes.body.toString() : null;
const scene = await scrapeScene(res.body.toString(), url, site, baseRelease, mobileBody, options);
const mobileItem = mobileRes?.status === 200 ? mobileRes.item : null;
const scene = await scrapeScene(res.item, url, site, baseRelease, mobileItem, options);
return { ...scene, deepUrl };
}
@ -827,13 +789,13 @@ async function fetchProfile({ name: actorName }, context, include, altSearchUrl,
if (actorUrl) {
const url = `https://${siteSlug}.com${actorUrl}`;
const actorRes = await http.get(url);
const actorRes = await qu.get(url);
if (actorRes.status !== 200) {
return null;
}
return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug, getActorReleasesUrl, include.scenes, context);
return scrapeProfile(actorRes.item, url, actorName, siteSlug, getActorReleasesUrl, include.scenes, context);
}
return null;

View File

@ -1,14 +1,14 @@
'use strict';
const { fetchLatest, fetchUpcoming, scrapeScene, fetchProfile } = require('./gamma');
const http = require('../utils/http');
const qu = require('../utils/qu');
async function fetchScene(url, site, baseRelease, options) {
const res = await http.get(url);
const res = await qu.get(url);
const release = await scrapeScene(res.body.toString(), url, site, baseRelease, null, options);
const release = await scrapeScene(res.item, url, site, baseRelease, null, options);
const siteDomain = release.$('meta[name="twitter:domain"]').attr('content') || 'allblackx.com'; // only AllBlackX has no twitter domain, no other useful hints available
const siteDomain = release.query.el('meta[name="twitter:domain"]', 'content') || 'allblackx.com'; // only AllBlackX has no twitter domain, no other useful hints available
const siteSlug = siteDomain && siteDomain.split('.')[0].toLowerCase();
// const siteUrl = siteDomain && `https://www.${siteDomain}`;

View File

@ -16,6 +16,10 @@ function trim(str) {
}
function extractDate(dateString, format, match) {
if (!dateString) {
return null;
}
if (match) {
const dateStamp = trim(dateString).match(match);