DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added Wicked network. Merged Evil Angel, XEmpire and Wicked into generic Gamma scraper.

Browse Source
This commit is contained in:
ThePendulum
2020-02-01 01:15:40 +01:00
parent 37ab07356e
commit 94bf207397
17 changed files with 385 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

348
src/scrapers/gamma.js
View File

@@ -4,6 +4,7 @@ const Promise = require('bluebird');
const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
async function fetchPhotos(url) {
const res = await bhttp.get(url);
@@ -39,33 +40,191 @@ function scrapePhotos(html) {
});
}
async function getPhotos(albumPath, siteDomain) {
const albumUrl = `https://${siteDomain}${albumPath}`;
async function getPhotos(albumPath, site) {
const albumUrl = `${site.url}${albumPath}`;
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html);
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
const lastPage = $('.Gamma_Paginator a.last').attr('href')?.match(/\d+$/)[0];
const otherPhotos = await Promise.map(pages, async (page) => {
const pageUrl = `https://${siteDomain}${page}`;
const pageHtml = await fetchPhotos(pageUrl);
if (lastPage) {
const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
return scrapePhotos(pageHtml);
}, {
concurrency: 2,
});
const otherPhotos = await Promise.map(otherPages, async (page) => {
const pageUrl = `${site.url}/${albumPath}/${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return photos.concat(otherPhotos.flat());
return scrapePhotos(pageHtml);
}, {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
}
return photos;
} catch (error) {
console.error(`Failed to fetch ${siteDomain} photos from ${albumPath}: ${error.message}`);
console.error(`Failed to fetch ${site.name} photos from ${albumUrl}: ${error.message}`);
return [];
}
}
async function scrapeApiReleases(json, site) {
return json.map((scene) => {
const release = {
entryId: scene.clip_id,
title: scene.title,
description: scene.description,
duration: scene.length,
likes: scene.ratings_up,
dislikes: scene.ratings_down,
};
release.url = `${site.url}/en/video/${scene.url_title}/${release.entryId}`;
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.actors = scene.actors.map(({ name }) => name);
release.director = scene.directors[0].name;
release.tags = scene.master_categories.concat(scene.categories?.map(category => category.name));
const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
if (posterPath) {
release.poster = [
`https://images-evilangel.gammacdn.com/movies${posterPath}`,
`https://transform.gammacdn.com/movies${posterPath}`,
];
}
release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
return release;
});
}
function scrapeAll(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const scenesElements = $('li[data-itemtype=scene]').toArray();
return scenesElements.map((element) => {
const sceneLinkElement = $(element).find('.sceneTitle a');
const url = `${site.url}${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title');
const entryId = $(element).attr('data-itemid');
const date = moment
.utc($(element).find('.sceneDate').text(), 'MM-DD-YYYY')
.toDate();
const actors = $(element).find('.sceneActors a')
.map((actorIndex, actorElement) => $(actorElement).attr('title'))
.toArray();
const [likes, dislikes] = $(element).find('.value')
.toArray()
.map(value => Number($(value).text()));
const poster = $(element).find('.imgLink img').attr('data-original');
const trailer = `https://videothumb.gammacdn.com/307x224/${entryId}.mp4`;
return {
url,
entryId,
title,
actors,
director: 'Mason',
date,
poster,
trailer: {
src: trailer,
quality: 224,
},
rating: {
likes,
dislikes,
},
site,
};
});
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const release = { $ };
const json = $('script[type="application/ld+json"]').html();
const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
const [data, data2] = JSON.parse(json);
const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
[release.entryId] = new URL(url).pathname.split('/').slice(-1);
release.title = data.name;
release.description = data.description;
// date in data object is not the release date of the scene, but the date the entry was added
const dateString = $('.updatedDate').first().text().trim();
const dateMatch = dateString.match(/\d{2,4}-\d{2}-\d{2,4}/)?.[0];
release.date = moment.utc(dateMatch, ['MM-DD-YYYY', 'YYYY-MM-DD']).toDate();
release.director = data.director?.[0].name || data2?.director?.[0].name;
release.actors = data.actor.map(actor => actor.name);
const hasTrans = data.actor.some(actor => actor.gender === 'shemale');
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
if (stars) release.rating = { stars };
release.duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
const rawTags = data.keywords?.split(', ');
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
release.poster = videoData.picPreview;
release.photos = await getPhotos($('.picturesItem a').attr('href'), site);
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
release.trailer = [
{
src: trailer.replace('hd', 'sm'),
quality: 240,
},
{
src: trailer.replace('hd', 'med'),
quality: 360,
},
{
src: trailer.replace('hd', 'big'),
quality: 480,
},
{
// probably 540p
src: trailer,
quality: parseInt(videoData.sizeOnLoad, 10),
},
{
src: trailer.replace('hd', '720p'),
quality: 720,
},
{
src: trailer.replace('hd', '1080p'),
quality: 1080,
},
{
src: trailer.replace('hd', '4k'),
quality: 2160,
},
];
return release;
}
function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
@@ -112,6 +271,113 @@ function scrapeProfile(html, url, actorName, siteSlug) {
return profile;
}
function scrapeApiProfile(data, releases, siteSlug) {
const profile = {};
if (data.male === 1) profile.gender = 'male';
if (data.female === 1) profile.gender = 'female';
if (data.shemale === 1 || data.trans === 1) profile.gender = 'transsexual';
if (data.description) profile.description = data.description.trim();
if (data.attributes.ethnicity) profile.ethnicity = data.attributes.ethnicity;
if (data.attributes.eye_color) profile.eyes = data.attributes.eye_color;
if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
const avatarPath = Object.values(data.pictures).reverse()[0];
if (avatarPath) profile.avatar = `https://images01-evilangel.gammacdn.com/actors${avatarPath}`;
profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
return profile;
}
async function fetchApiCredentials(referer) {
const res = await bhttp.get(referer);
const body = res.body.toString();
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
const apiData = JSON.parse(apiSerial);
const { applicationID: appId, apiKey } = apiData.api.algolia;
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
return {
appId,
apiKey,
userAgent,
apiUrl,
};
}
async function fetchApiLatest(site, page = 1, upcoming = false) {
const referer = `${site.url}/en/videos`;
const { apiUrl } = await fetchApiCredentials(referer);
const res = await bhttp.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]`,
},
],
}, {
headers: {
Referer: referer,
},
encodeJSON: true,
});
return scrapeApiReleases(res.body.results[0].hits, site);
}
async function fetchApiUpcoming(site) {
return fetchApiLatest(site, 1, true);
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
return scrapeAll(res.body.toString(), site);
}
async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/1/upcoming`);
return scrapeAll(res.body.toString(), site);
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site);
}
async function fetchActorScenes(actorName, apiUrl, siteSlug) {
const res = await bhttp.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=0&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["actors.name:${actorName}"]]`,
},
],
}, {
headers: {
Referer: `https://www.${siteSlug}.com/en/videos`,
},
encodeJSON: true,
});
if (res.statusCode === 200 && res.body.results[0].hits.length > 0) {
return res.body.results[0].hits;
}
return [];
}
async function fetchProfile(actorName, siteSlug, altSearchUrl) {
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
const searchUrl = altSearchUrl
@@ -139,38 +405,17 @@ async function fetchProfile(actorName, siteSlug, altSearchUrl) {
return null;
}
async function fetchApiCredentials(referer) {
const res = await bhttp.get(referer);
const body = res.body.toString();
async function fetchApiProfile(actorName, siteSlug) {
const actorSlug = encodeURI(actorName);
const referer = `https://www.${siteSlug}.com/en/search?query=${actorSlug}&tab=actors`;
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
const apiData = JSON.parse(apiSerial);
const { applicationID: appId, apiKey } = apiData.api.algolia;
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
return {
appId,
apiKey,
userAgent,
apiUrl,
};
}
async function fetchLatest(site, page = 1, upcoming = false) {
const referer = `${site.url}/en/videos`;
const { apiUrl } = await fetchApiCredentials(referer);
console.log(referer);
const res = await bhttp.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]`,
indexName: 'all_actors',
params: `query=${actorSlug}`,
},
],
}, {
@@ -180,14 +425,31 @@ async function fetchLatest(site, page = 1, upcoming = false) {
encodeJSON: true,
});
console.log(res.body.results);
if (res.statusCode === 200 && res.body.results[0].hits.length > 0) {
const actorData = res.body.results[0].hits.find(actor => actor.name === actorName);
// return scrape(res.body.results[0].hits, site);
if (actorData) {
const actorScenes = await fetchActorScenes(actorName, apiUrl, siteSlug);
return scrapeApiProfile(actorData, actorScenes, siteSlug);
}
}
return null;
}
module.exports = {
getPhotos,
fetchProfile,
scrapeProfile,
fetchApiLatest,
fetchApiProfile,
fetchApiUpcoming,
fetchLatest,
fetchProfile,
fetchScene,
fetchUpcoming,
getPhotos,
scrapeApiProfile,
scrapeApiReleases,
scrapeProfile,
scrapeAll,
scrapeScene,
};