DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Split Girlsway from Adult Time. Added Fantasy Massage. Using Gamma scraper for Pure Taboo. Added photo path parameter to Gamma scraper.

Browse Source
This commit is contained in:
ThePendulum
2020-02-08 02:49:39 +01:00
parent bfb26b717a
commit 1546e0836c
57 changed files with 2359 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
src/scrapers/gamma.js
View File

@@ -7,7 +7,7 @@ const cheerio = require('cheerio');
const moment = require('moment');
const argv = require('../argv');
const logger = require('../logger');
const logger = require('../logger')(__filename);
const { ex, get } = require('../utils/q');
const slugify = require('../utils/slugify');
@@ -20,11 +20,12 @@ async function fetchPhotos(url) {
function scrapePhotos(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
return $('.preview .imgLink').toArray().map((linkEl) => {
return $('.preview .imgLink, .pgFooterThumb a').toArray().map((linkEl) => {
const url = $(linkEl).attr('href');
if (url.match('/join')) {
if (/\/join|\/createaccount/.test(url)) {
// URL links to join page instead of full photo, extract thumbnail
// /createaccount is used by e.g. Tricky Spa
const src = $(linkEl).find('img').attr('src');
if (src.match('previews/')) {
@@ -46,12 +47,14 @@ function scrapePhotos(html) {
}
async function getPhotos(albumPath, site) {
const albumUrl = `${site.url}${albumPath}`;
const albumUrl = site.parameters?.photos
? `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${albumPath}`;
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html);
const photos = scrapePhotos(html, site);
const lastPage = $('.Gamma_Paginator a.last').attr('href')?.match(/\d+$/)[0];
@@ -62,7 +65,7 @@ async function getPhotos(albumPath, site) {
const pageUrl = `${site.url}/${albumPath}/${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml);
return scrapePhotos(pageHtml, site);
}, {
concurrency: 2,
});
@@ -97,9 +100,9 @@ async function scrapeApiReleases(json, site) {
release.actors = scene.actors.map(({ name }) => name);
release.director = scene.directors[0].name;
console.log(release.url);
release.tags = scene.master_categories.concat(scene.categories?.map(category => category.name));
release.tags = scene.master_categories
.concat(scene.categories?.map(category => category.name))
.filter(Boolean); // some categories don't have a name
const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
@@ -118,38 +121,35 @@ async function scrapeApiReleases(json, site) {
function scrapeAll(html, site, networkUrl, hasTeaser = true) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const scenesElements = $('li[data-itemtype=scene]').toArray();
const scenesElements = $('li[data-itemtype=scene], div[data-itemtype=scenes]').toArray();
return scenesElements.map((element) => {
const release = {};
const sceneLinkElement = $(element).find('.sceneTitle a');
const sceneLinkElement = $(element).find('.sceneTitle a, .tlcTitle a');
if (site) release.url = `${networkUrl ? site.network.url : site.url}${sceneLinkElement.attr('href')}`;
else release.url = `${networkUrl}${sceneLinkElement.attr('href')}`;
release.title = sceneLinkElement.attr('title');
release.entryId = $(element).attr('data-itemid');
const dateEl = $(element).find('.sceneDate').text() || null;
const dateEl = $(element).find('.sceneDate, .tlcSpecsDate .tlcDetailsValue').text() || null;
if (dateEl) {
release.date = moment
.utc($(element).find('.sceneDate').text(), ['MM-DD-YYYY', 'YYYY-MM-DD'])
.utc(dateEl, ['MM-DD-YYYY', 'YYYY-MM-DD'])
.toDate();
}
release.actors = $(element).find('.sceneActors a')
release.actors = $(element).find('.sceneActors a, .tlcActors a')
.map((actorIndex, actorElement) => $(actorElement).attr('title'))
.toArray();
const [likes, dislikes] = $(element).find('.value')
[release.likes, release.dislikes] = $(element).find('.value')
.toArray()
.map(value => Number($(value).text()));
release.rating = { likes, dislikes };
const posterEl = $(element).find('.imgLink img');
const posterEl = $(element).find('.imgLink img, .tlcImageItem');
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
if (hasTeaser) {
@@ -189,18 +189,18 @@ async function scrapeScene(html, url, site) {
release.director = data.director?.[0].name || data2?.director?.[0].name;
const actors = data?.actor || data2?.actor || [];
const hasTrans = actors.some(actor => actor.gender === 'shemale');
release.actors = actors.map(actor => actor.name);
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
if (stars) release.rating = { stars };
release.duration = moment.duration(data.duration.slice(2)).asSeconds();
const rawTags = data.keywords?.split(', ');
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
}
const hasTrans = release.actors.some(actor => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ');
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
const channel = data?.productionCompany?.name || $('.studioLink a').attr('title')?.trim();
if (channel) release.channel = slugify(channel, { delimiter: '' });
@@ -345,7 +345,7 @@ function getApiUrl(appId, apiKey) {
}
async function fetchApiCredentials(referer, site) {
if (site.parameters?.appId && site.parameters?.apiKey) {
if (site?.parameters?.appId && site?.parameters?.apiKey) {
return getApiUrl(site.parameters.appId, site.parameters.apiKey);
}
@@ -374,7 +374,7 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
requests: [
{
indexName: 'all_scenes',
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]&filters=sitename:${site.slug}`,
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]&filters=sitename:${site.slug} OR channels.id:${site.slug}`,
},
],
}, {
@@ -384,6 +384,8 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
encodeJSON: true,
});
console.log(res.body);
if (res.statusCode === 200 && res.body.results?.[0]?.hits) {
return scrapeApiReleases(res.body.results[0].hits, site);
}