forked from DebaucheryLibrarian/traxxx
Refactored Gamma scraper, only using API.
This commit is contained in:
DebaucheryLibrarian
@@ -1,107 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const {
|
||||
fetchLatest,
|
||||
fetchApiLatest,
|
||||
fetchUpcoming,
|
||||
fetchApiUpcoming,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
fetchApiProfile,
|
||||
scrapeAll,
|
||||
} = require('./gamma');
|
||||
|
||||
const { get } = require('../utils/qu');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function extractLowArtActors(release) {
|
||||
const actors = release.title
|
||||
.replace(/solo/i, '')
|
||||
.split(/,|\band\b/ig)
|
||||
.map((actor) => actor.trim());
|
||||
|
||||
return {
|
||||
...release,
|
||||
actors,
|
||||
};
|
||||
}
|
||||
|
||||
async function networkFetchLatest(site, page = 1) {
|
||||
if (site.parameters?.api) return fetchApiLatest(site, page, false);
|
||||
|
||||
const releases = await fetchLatest(site, page);
|
||||
|
||||
if (site.slug === 'lowartfilms') {
|
||||
return releases.map((release) => extractLowArtActors(release));
|
||||
}
|
||||
|
||||
return releases;
|
||||
}
|
||||
|
||||
async function networkFetchScene(url, site) {
|
||||
const release = await fetchScene(url, site);
|
||||
|
||||
if (site.slug === 'lowartfilms') {
|
||||
return extractLowArtActors(release);
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function networkFetchUpcoming(site, page = 1) {
|
||||
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
|
||||
|
||||
return fetchUpcoming(site, page);
|
||||
}
|
||||
|
||||
function getActorReleasesUrl(actorPath, page = 1) {
|
||||
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
|
||||
}
|
||||
|
||||
function scrapeClassicProfile({ qu, html }, site) {
|
||||
const profile = {};
|
||||
|
||||
profile.avatar = qu.img('.actorPicture');
|
||||
profile.releases = scrapeAll(html, null, site.url, false);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchClassicProfile(actorName, { site }) {
|
||||
const actorSlug = slugify(actorName);
|
||||
|
||||
const url = `${site.url}/en/pornstars`;
|
||||
const pornstarsRes = await get(url);
|
||||
|
||||
if (!pornstarsRes.ok) return null;
|
||||
|
||||
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
|
||||
.find((el) => slugify(el.textContent) === actorSlug)
|
||||
?.value;
|
||||
|
||||
if (actorPath) {
|
||||
const actorUrl = `${site.url}${actorPath}`;
|
||||
const res = await get(actorUrl);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeClassicProfile(res.item, site);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function networkFetchProfile({ name: actorName }, context, include) {
|
||||
const profile = await ((context.site.parameters?.api && fetchApiProfile(actorName, context, include))
|
||||
|| (context.site.parameters?.classic && include.scenes && fetchClassicProfile(actorName, context, include)) // classic profiles only have scenes, no bio
|
||||
|| fetchProfile({ name: actorName }, context, true, getActorReleasesUrl, include));
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest: networkFetchLatest,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchScene: networkFetchScene,
|
||||
fetchUpcoming: networkFetchUpcoming,
|
||||
};
|
||||
@@ -1,217 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
const util = require('util');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const moment = require('moment');
|
||||
const format = require('template-format');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const logger = require('../logger')(__filename);
|
||||
const qu = require('../utils/qu');
|
||||
const http = require('../utils/http');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function getApiUrl(appId, apiKey) {
|
||||
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
|
||||
|
||||
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
|
||||
|
||||
return {
|
||||
appId,
|
||||
apiKey,
|
||||
userAgent,
|
||||
apiUrl,
|
||||
};
|
||||
}
|
||||
|
||||
function getAvatarFallbacks(avatar) {
|
||||
if (!avatar) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return [
|
||||
avatar.replace(/\d+x\d+/, '500x750'),
|
||||
avatar.replace(/\d+x\d+/, '240x360'),
|
||||
avatar.replace(/\d+x\d+/, '200x300'),
|
||||
avatar,
|
||||
];
|
||||
}
|
||||
|
||||
async function fetchApiCredentials(referer, site) {
|
||||
if (site?.parameters?.appId && site?.parameters?.apiKey) {
|
||||
return getApiUrl(site.parameters.appId, site.parameters.apiKey);
|
||||
}
|
||||
|
||||
const res = await http.get(referer);
|
||||
const body = res.body.toString();
|
||||
|
||||
const apiLine = body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
|
||||
|
||||
if (!apiLine) {
|
||||
throw new Error(`No Gamma API key found for ${referer}`);
|
||||
}
|
||||
|
||||
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
|
||||
const apiData = JSON.parse(apiSerial);
|
||||
|
||||
const { applicationID: appId, apiKey } = apiData.api.algolia;
|
||||
|
||||
return getApiUrl(appId, apiKey);
|
||||
}
|
||||
|
||||
function getAlbumUrl(albumPath, site) {
|
||||
if (site.parameters?.photos) {
|
||||
return /^http/.test(site.parameters.photos)
|
||||
? `${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`
|
||||
: `${site.url}${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`;
|
||||
}
|
||||
|
||||
if (site.url && site.parameters?.photos !== false) {
|
||||
return `${site.url}${albumPath}`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchPhotos(url) {
|
||||
const res = await qu.get(url);
|
||||
|
||||
return res.item;
|
||||
}
|
||||
|
||||
function scrapePhotos({ query }, includeThumbnails = true) {
|
||||
return query.all('.preview .imgLink, .pgFooterThumb a').map((linkEl) => {
|
||||
const url = linkEl.href;
|
||||
|
||||
if (/\/join|\/createaccount/.test(url)) {
|
||||
// URL links to join page instead of full photo, extract thumbnail
|
||||
// /createaccount is used by e.g. Tricky Spa native site
|
||||
const src = query.img(linkEl);
|
||||
|
||||
if (/previews\//.test(src)) {
|
||||
// resource often serves full photo at a modifier URL anyway, add as primary source
|
||||
const highRes = src
|
||||
.replace('previews/', '')
|
||||
.replace('_tb.jpg', '.jpg');
|
||||
|
||||
// keep original thumbnail as fallback in case full photo is not available
|
||||
return [highRes, src];
|
||||
}
|
||||
|
||||
if (!includeThumbnails) return null;
|
||||
|
||||
return src;
|
||||
}
|
||||
|
||||
// URL links to full photo
|
||||
return url;
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
async function getPhotos(albumPath, site, includeThumbnails = true) {
|
||||
const albumUrl = getAlbumUrl(albumPath, site);
|
||||
|
||||
if (!albumUrl) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const item = await fetchPhotos(albumUrl);
|
||||
const photos = scrapePhotos(item, includeThumbnails);
|
||||
|
||||
const lastPage = item.query.url('.Gamma_Paginator a.last')?.match(/\d+$/)[0];
|
||||
|
||||
if (lastPage) {
|
||||
const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
|
||||
|
||||
const otherPhotos = await Promise.map(otherPages, async (page) => {
|
||||
const pageItem = await fetchPhotos(`${albumUrl}/${page}`);
|
||||
|
||||
return scrapePhotos(pageItem, includeThumbnails);
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
|
||||
return photos.concat(otherPhotos.flat());
|
||||
}
|
||||
|
||||
return photos;
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to fetch ${site.name} photos from ${albumUrl}: ${error.message}`);
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function getFullPhotos(entryId, site, parameters) {
|
||||
const res = await http.get(`${parameters.album || site.url}/media/signPhotoset/${entryId}`, {
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && typeof res.body === 'object') { // gives 200 OK even when redirected to signup page
|
||||
return Object.values(res.body);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function getThumbs(entryId, site, parameters) {
|
||||
const referer = parameters?.referer || `${parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
|
||||
const { apiUrl } = await fetchApiCredentials(referer, site);
|
||||
|
||||
const res = await http.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_photosets',
|
||||
params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["set_id:${entryId}"]]`,
|
||||
},
|
||||
],
|
||||
}, {
|
||||
headers: {
|
||||
Referer: referer,
|
||||
},
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) {
|
||||
return res.body.results[0].hits[0].set_pictures.map((img) => img.thumb_path && ([
|
||||
`https://images-fame.gammacdn.com/photo_set${img.thumb_path}`,
|
||||
`https://images01-fame.gammacdn.com/photo_set${img.thumb_path}`,
|
||||
`https://images02-fame.gammacdn.com/photo_set${img.thumb_path}`,
|
||||
`https://images03-fame.gammacdn.com/photo_set${img.thumb_path}`,
|
||||
`https://images04-fame.gammacdn.com/photo_set${img.thumb_path}`,
|
||||
`https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`,
|
||||
`https://transform.gammacdn.com/photo_set${img.thumb_path}`,
|
||||
])).filter(Boolean);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function getPhotosApi(entryId, site, parameters) {
|
||||
const [photos, thumbs] = await Promise.all([
|
||||
getFullPhotos(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
|
||||
getThumbs(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
|
||||
]);
|
||||
|
||||
return photos.concat(thumbs.slice(photos.length));
|
||||
}
|
||||
|
||||
function getImageSources(source) {
|
||||
return [
|
||||
`https://images-fame.gammacdn.com/movies${source}`,
|
||||
`https://images01-fame.gammacdn.com/movies${source}`,
|
||||
`https://images02-fame.gammacdn.com/movies${source}`,
|
||||
`https://images03-fame.gammacdn.com/movies${source}`,
|
||||
`https://images04-fame.gammacdn.com/movies${source}`,
|
||||
`https://images-evilangel.gammacdn.com/movies${source}`,
|
||||
`https://transform.gammacdn.com/movies${source}`,
|
||||
];
|
||||
}
|
||||
|
||||
function curateTitle(title, channel) {
|
||||
// some videos are redundantly prefixed with the name of the site, i.e. Bubblegum Dungeon, Forbidden Seductions and Lady Gonzo
|
||||
return title.replace(new RegExp(`^\\s*${channel.name}\\s*[:|-]\\s`, 'i'), '');
|
||||
@@ -244,7 +38,7 @@ async function scrapeApiReleases(json, site, options) {
|
||||
release.url = `${site.url}/en/video${release.path}`;
|
||||
}
|
||||
|
||||
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
|
||||
release.date = unprint.extractDate(scene.release_date, 'YYYY-MM-DD');
|
||||
release.director = scene.directors[0]?.name || null;
|
||||
|
||||
release.actors = scene.actors.map((actor) => ({
|
||||
@@ -289,6 +83,8 @@ async function scrapeApiReleases(json, site, options) {
|
||||
release.channel = slugify(scene.mainChannel?.id || scene.sitename, ''); // remove -
|
||||
// release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
|
||||
|
||||
console.log(release);
|
||||
|
||||
return {
|
||||
...acc,
|
||||
scenes: acc.scenes.concat(release),
|
||||
@@ -299,7 +95,40 @@ async function scrapeApiReleases(json, site, options) {
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchLatestApi(site, page = 1, options, _preData, upcoming = false) {
|
||||
function getApiUrl(appId, apiKey) {
|
||||
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
|
||||
|
||||
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
|
||||
|
||||
return {
|
||||
appId,
|
||||
apiKey,
|
||||
userAgent,
|
||||
apiUrl,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchApiCredentials(referer, site) {
|
||||
if (site?.parameters?.appId && site?.parameters?.apiKey) {
|
||||
return getApiUrl(site.parameters.appId, site.parameters.apiKey);
|
||||
}
|
||||
|
||||
const res = await unprint.get(referer);
|
||||
const apiLine = res.body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
|
||||
|
||||
if (!apiLine) {
|
||||
throw new Error(`No Gamma API key found for ${referer}`);
|
||||
}
|
||||
|
||||
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
|
||||
const apiData = JSON.parse(apiSerial);
|
||||
|
||||
const { applicationID: appId, apiKey } = apiData.api.algolia;
|
||||
|
||||
return getApiUrl(appId, apiKey);
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1, options, _preData, upcoming = false) {
|
||||
const referer = options.parameters?.referer || `${options.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
|
||||
const { apiUrl } = await fetchApiCredentials(referer, site);
|
||||
const slug = options.parameters.querySlug || site.slug;
|
||||
@@ -308,7 +137,7 @@ async function fetchLatestApi(site, page = 1, options, _preData, upcoming = fals
|
||||
? `&filters=channels.id:${options.parameters.queryChannel === true ? slug : options.parameters.queryChannel}`
|
||||
: `&filters=availableOnSite:${slug}`}`;
|
||||
|
||||
const res = await http.post(apiUrl, {
|
||||
const res = await unprint.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_scenes',
|
||||
@@ -319,213 +148,97 @@ async function fetchLatestApi(site, page = 1, options, _preData, upcoming = fals
|
||||
headers: {
|
||||
Referer: referer,
|
||||
},
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.status === 200 && res.body.results?.[0]?.hits) {
|
||||
return scrapeApiReleases(res.body.results[0].hits, site, options);
|
||||
if (res.ok && res.data.results?.[0]?.hits) {
|
||||
return scrapeApiReleases(res.data.results[0].hits, site, options);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function scrapeAll(scenes, site, networkUrl, hasTeaser = true) {
|
||||
return scenes.map(({ query, el }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('.sceneTitle a, .tlcTitle a', 'href', { origin: networkUrl ? site.parent.url : site.url });
|
||||
|
||||
release.title = query.cnt('.sceneTitle a', 'tlcTitle a', 'title');
|
||||
release.entryId = el.dataset.itemid;
|
||||
|
||||
release.date = query.date('.sceneDate, .tlcSpecsDate .tlcDetailsValue', ['MM-DD-YYYY', 'YYYY-MM-DD']);
|
||||
release.actors = query.cnts('.sceneActors a, .tlcActors a', ' title');
|
||||
|
||||
[release.likes, release.dislikes] = query.all('.value').map((likeEl) => query.number(likeEl));
|
||||
|
||||
release.poster = query.img('.imgLink img, .tlcImageItem', 'data-original') || query.img('.imgLink img, .tlcImageItem');
|
||||
|
||||
if (hasTeaser) {
|
||||
release.teaser = [
|
||||
{ src: `https://videothumb.gammacdn.com/600x339/${release.entryId}.mp4` },
|
||||
{ src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4` },
|
||||
];
|
||||
}
|
||||
|
||||
release.channel = query.el('.fromSite a', 'title')?.replace('.com', '');
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function getLatestUrl(site, page) {
|
||||
if (site.parameters?.latest) {
|
||||
if (/^http/.test(site.parameters.latest)) {
|
||||
return /%d/.test(site.parameters.latest)
|
||||
? util.format(site.parameters.latest, page)
|
||||
: `${site.parameters.latest}${page}`;
|
||||
}
|
||||
|
||||
return /%d/.test(site.parameters.latest)
|
||||
? util.format(`${site.url}${site.parameters.latest}`, page)
|
||||
: `${site.url}${site.parameters.latest}${page}`;
|
||||
}
|
||||
|
||||
return `${site.url}/en/videos/AllCategories/0/${page}`;
|
||||
}
|
||||
|
||||
function getUpcomingUrl(site) {
|
||||
if (site.parameters?.upcoming) {
|
||||
return /^http/.test(site.parameters.upcoming)
|
||||
? `${site.parameters.upcoming}`
|
||||
: `${site.url}${site.parameters.upcoming}`;
|
||||
}
|
||||
|
||||
return `${site.url}/en/videos/AllCategories/0/1/upcoming`;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = getLatestUrl(site, page);
|
||||
const res = await qu.getAll(url, 'li[data-itemtype=scene], div[data-itemtype*=scene]');
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.items, site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchUpcoming(site) {
|
||||
const url = getUpcomingUrl(site);
|
||||
const res = await qu.getAll(url, 'li[data-itemtype=scene], div[data-itemtype*=scene]');
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.items, site, null, false);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function scrapeScene({ query }, url, channel, baseRelease, mobileItem, options) {
|
||||
const release = { query }; // used by XEmpire scraper to resolve channel-specific details
|
||||
|
||||
const json = query.html('script[type="application/ld+json"]');
|
||||
const videoJson = query.htmls('script').find((script) => /ScenePlayerOptions/i.test(script));
|
||||
|
||||
const [data, data2] = json ? JSON.parse(json) : [];
|
||||
const videoData = videoJson && JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
|
||||
|
||||
release.entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
|
||||
release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name;
|
||||
release.description = data?.description;
|
||||
|
||||
release.date = query.date('.updatedDate', ['MM-DD-YYYY', 'YYYY-MM-DD'])
|
||||
|| qu.extractDate(data?.dateCreated, 'YYYY-MM-DD')
|
||||
|| videoData?.playerOptions?.sceneInfos.sceneReleaseDate;
|
||||
|
||||
release.actors = (data?.actor || data2?.actor)?.map((actor) => ({
|
||||
name: actor.name,
|
||||
gender: actor.gender,
|
||||
})) || [];
|
||||
|
||||
release.duration = qu.durationToSeconds(data.duration);
|
||||
release.director = data?.director?.[0]?.name || data2?.director?.[0]?.name;
|
||||
|
||||
release.tags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
|
||||
release.stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5 || null;
|
||||
|
||||
release.channel = slugify(data?.productionCompany?.name
|
||||
|| query.el('.studioLink a, .siteLink a', 'title')
|
||||
|| query.cnt('.siteNameSpan')?.toLowerCase().replace('.com', '')
|
||||
|| query.meta('meta[name="twitter:domain"]')?.replace('.com', ''), '');
|
||||
|
||||
if (videoData?.picPreview && new URL(videoData.picPreview).pathname.length > 1) {
|
||||
// sometimes links to just https://images02-fame.gammacdn.com/
|
||||
const poster = new URL(videoData.picPreview);
|
||||
|
||||
release.poster = [
|
||||
videoData.picPreview, // prefer original URL with width and height parameters, without may give a square crop on e.g. XEmpire
|
||||
`${poster.origin}${poster.pathname}`,
|
||||
];
|
||||
}
|
||||
|
||||
const photoLink = query.url('.picturesItem a');
|
||||
const mobilePhotos = mobileItem?.query.imgs('.preview-displayer a img') || [];
|
||||
|
||||
if (photoLink && options.includePhotos) {
|
||||
const photos = await getPhotos(photoLink, channel, mobilePhotos.length < 3); // only get thumbnails when less than 3 mobile photos are available
|
||||
|
||||
if (photos.length < 7) {
|
||||
release.photos = [...photos, ...mobilePhotos]; // probably only teaser photos available, supplement with mobile album
|
||||
} else {
|
||||
release.photos = photos;
|
||||
}
|
||||
} else {
|
||||
release.photos = mobilePhotos;
|
||||
}
|
||||
|
||||
const trailer = videoData && `${videoData.playerOptions.host}${videoData.url}`;
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = [
|
||||
{
|
||||
src: trailer.replace('hd', 'sm'),
|
||||
quality: 240,
|
||||
},
|
||||
{
|
||||
src: trailer.replace('hd', 'med'),
|
||||
quality: 360,
|
||||
},
|
||||
{
|
||||
src: trailer.replace('hd', 'big'),
|
||||
quality: 480,
|
||||
},
|
||||
{
|
||||
// probably 540p
|
||||
src: trailer,
|
||||
quality: parseInt(videoData.sizeOnLoad, 10),
|
||||
},
|
||||
{
|
||||
src: trailer.replace('hd', '720p'),
|
||||
quality: 720,
|
||||
},
|
||||
{
|
||||
src: trailer.replace('hd', '1080p'),
|
||||
quality: 1080,
|
||||
},
|
||||
{
|
||||
src: trailer.replace('hd', '4k'),
|
||||
quality: 2160,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const movieUrl = query.url('.dvdLink', 'href', { origin: channel.url });
|
||||
|
||||
if (movieUrl) {
|
||||
release.movie = {
|
||||
url: movieUrl,
|
||||
title: query.el('.dvdLink', 'title'),
|
||||
entryId: movieUrl.match(/\/(\d+)(\/|$)/)?.[1],
|
||||
covers: [qu.imgs('.dvdLink img')],
|
||||
};
|
||||
}
|
||||
|
||||
return release;
|
||||
async function fetchUpcoming(site, page = 1, options, preData) {
|
||||
return fetchLatest(site, page, options, preData, true);
|
||||
}
|
||||
|
||||
const qualityMap = {
|
||||
'4k': 2160,
|
||||
};
|
||||
|
||||
async function scrapeReleaseApi(data, site, options, movieScenes) {
|
||||
async function getFullPhotos(entryId, site, parameters) {
|
||||
const res = await unprint.get(`${parameters.album || site.url}/media/signPhotoset/${entryId}`, {
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && typeof res.data === 'object') { // gives 200 OK even when redirected to signup page
|
||||
return Object.values(res.data);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function getThumbs(entryId, site, parameters) {
|
||||
const referer = parameters?.referer || `${parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
|
||||
const { apiUrl } = await fetchApiCredentials(referer, site);
|
||||
|
||||
const res = await unprint.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_photosets',
|
||||
params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["set_id:${entryId}"]]`,
|
||||
},
|
||||
],
|
||||
}, {
|
||||
headers: {
|
||||
Referer: referer,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.data.results?.[0]?.hits[0]?.set_pictures) {
|
||||
return res.data.results[0].hits[0].set_pictures.map((img) => img.thumb_path && [
|
||||
'https://images-fame.gammacdn.com',
|
||||
'https://images01-fame.gammacdn.com',
|
||||
'https://images02-fame.gammacdn.com',
|
||||
'https://images03-fame.gammacdn.com',
|
||||
'https://images04-fame.gammacdn.com',
|
||||
'https://images-evilangel.gammacdn.com',
|
||||
'https://transform.gammacdn.com',
|
||||
].map((origin) => unprint.prefixUrl(`/photo_set${img.thumb_path}`, origin))).filter(Boolean);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function getImageSources(source) {
|
||||
return [
|
||||
'https://images-fame.gammacdn.com',
|
||||
'https://images01-fame.gammacdn.com',
|
||||
'https://images02-fame.gammacdn.com',
|
||||
'https://images03-fame.gammacdn.com',
|
||||
'https://images04-fame.gammacdn.com',
|
||||
'https://images-evilangel.gammacdn.com',
|
||||
'https://transform.gammacdn.com',
|
||||
].map((origin) => unprint.prefixUrl(`/movies${source}`, origin));
|
||||
}
|
||||
|
||||
async function getPhotosApi(entryId, site, parameters) {
|
||||
const [photos, thumbs] = await Promise.all([
|
||||
getFullPhotos(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
|
||||
getThumbs(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
|
||||
]);
|
||||
|
||||
return photos.concat(thumbs.slice(photos.length));
|
||||
}
|
||||
|
||||
async function scrapeScene(data, site, options, movieScenes) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = data.clip_id || data.movie_id;
|
||||
release.title = curateTitle(data.title, site);
|
||||
release.duration = data.length;
|
||||
release.date = (data.date && new Date(data.date * 1000)) || qu.parseDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
|
||||
release.date = (data.date && new Date(data.date * 1000)) || unprint.parseDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
|
||||
release.director = data.directors[0]?.name || null;
|
||||
|
||||
release.actors = data.actors.map((actor) => ({
|
||||
@@ -534,7 +247,7 @@ async function scrapeReleaseApi(data, site, options, movieScenes) {
|
||||
gender: actor.gender,
|
||||
url: options.parameters?.actors
|
||||
? format(options.parameters.actors, { id: actor.actor_id, slug: actor.url_name })
|
||||
: qu.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
|
||||
: unprint.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
|
||||
}));
|
||||
|
||||
release.tags = data.categories.map((category) => category.name);
|
||||
@@ -571,12 +284,12 @@ async function scrapeReleaseApi(data, site, options, movieScenes) {
|
||||
release.movie = {
|
||||
entryId: data.movie_id,
|
||||
title: data.movie_title,
|
||||
url: qu.prefixUrl(`${data.url_movie_title}/${data.movie_id}`, options.parameters.movie ? options.parameters.movie : `${site.url}/en/movie`),
|
||||
url: unprint.prefixUrl(`${data.url_movie_title}/${data.movie_id}`, options.parameters.movie ? options.parameters.movie : `${site.url}/en/movie`),
|
||||
};
|
||||
}
|
||||
|
||||
if (movieScenes?.length > 0) {
|
||||
release.scenes = await Promise.all(movieScenes.map((movieScene) => scrapeReleaseApi(movieScene, site, options)));
|
||||
release.scenes = await Promise.all(movieScenes.map((movieScene) => scrapeScene(movieScene, site, options)));
|
||||
}
|
||||
|
||||
release.channel = slugify(data.mainChannel?.id || data.sitename, ''); // remove -
|
||||
@@ -585,148 +298,13 @@ async function scrapeReleaseApi(data, site, options, movieScenes) {
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchMovieTrailer(release) {
|
||||
if (!release.entryId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = `https://www.evilangel.com/en/dvdtrailer/${release.entryId}`;
|
||||
const res = await qu.get(url);
|
||||
|
||||
if (!res.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trailerHost = res.html.match(/"host":\s*"(.*\.com)"/)?.[1].replace(/\\\//g, '/');
|
||||
const trailerPath = res.html.match(/"url":\s*"(.*\.mp4)"/)?.[1].replace(/\\\//g, '/');
|
||||
|
||||
if (trailerHost && trailerPath) {
|
||||
return qu.prefixUrl(trailerPath, trailerHost);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrapeMovie({ query, el }, url, entity, baseRelease, options) {
|
||||
const release = {};
|
||||
|
||||
const { dataLayer } = query.exec('//script[contains(text(), "dataLayer")]', ['dataLayer']);
|
||||
const rawData = dataLayer?.[0]?.dvdDetails;
|
||||
const data = rawData?.dvdId && rawData; // dvdDetails is mostly empty in some cache states
|
||||
|
||||
if (query.exists('.NotFound-Title')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/\/(\d+)(\/|$)/)?.[1];
|
||||
|
||||
release.covers = [
|
||||
query.img('.frontCoverImg', 'href'),
|
||||
query.img('.backCoverImg', 'href'),
|
||||
];
|
||||
|
||||
release.description = query.cnt('.descriptionText');
|
||||
release.date = qu.extractDate(data?.dvdReleaseDate) || query.date('.updatedOn', 'YYYY-MM-DD');
|
||||
release.title = data?.dvdName || query.cnt('.dvdTitle');
|
||||
release.director = query.el('.directedBy a', 'title');
|
||||
|
||||
release.actors = data?.dvdActors.map((actor) => ({ name: actor.actorName, entryId: actor.actorId }))
|
||||
|| query.all('.actorCarousel a[href*="/pornstar"]').map((actorEl) => ({
|
||||
entryId: query.url(actorEl, null).match(/\/(\d+)/)?.[1],
|
||||
name: query.cnt(actorEl, 'span'),
|
||||
href: query.url(actorEl, null, 'href', { origin: entity.url }),
|
||||
avatar: getAvatarFallbacks(query.img(actorEl)),
|
||||
}));
|
||||
|
||||
release.tags = query.cnts('.dvdCol a');
|
||||
release.scenes = scrapeAll(qu.initAll(el, 'div[data-itemtype*=scene], li[data-itemtype*=scene]'), entity, entity.url);
|
||||
|
||||
if (options.includeTrailers) {
|
||||
release.trailer = await fetchMovieTrailer(release);
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeActorSearch(html, url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
|
||||
|
||||
return actorLink ? actorLink.href : null;
|
||||
}
|
||||
|
||||
async function fetchActorReleases(profileUrl, getActorReleasesUrl, page = 1, accReleases = [], context) {
|
||||
const { origin, pathname } = new URL(profileUrl);
|
||||
const profilePath = `/${pathname.split('/').slice(-2).join('/')}`;
|
||||
|
||||
const url = (context.parameters.actorScenes && format(context.parameters.actorScenes, { path: profilePath, page }))
|
||||
|| getActorReleasesUrl?.(profilePath, page);
|
||||
|
||||
if (!url) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const res = await qu.get(url);
|
||||
|
||||
if (!res.ok) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const releases = scrapeAll(res.item.html, null, origin);
|
||||
const nextPage = res.item.query.url('.Gamma_Paginator a.next');
|
||||
|
||||
if (nextPage) {
|
||||
return fetchActorReleases(profileUrl, getActorReleasesUrl, page + 1, accReleases.concat(releases), context);
|
||||
}
|
||||
|
||||
return accReleases.concat(releases);
|
||||
}
|
||||
|
||||
async function scrapeProfile({ query }, url, actorName, _siteSlug, getActorReleasesUrl, withReleases, context) {
|
||||
const avatar = query.img('img.actorPicture');
|
||||
const hair = query.cnt('.actorProfile .attribute_hair_color');
|
||||
const height = query.cnt('.actorProfile .attribute_height');
|
||||
const weight = query.cnt('.actorProfile .attribute_weight');
|
||||
const alias = query.cnt('.actorProfile .attribute_alternate_names');
|
||||
const nationality = query.cnt('.actorProfile .attribute_home');
|
||||
|
||||
const profile = {
|
||||
name: actorName,
|
||||
};
|
||||
|
||||
if (avatar) {
|
||||
// larger sizes usually available, provide fallbacks
|
||||
const avatars = getAvatarFallbacks(avatar);
|
||||
|
||||
profile.avatar = avatars;
|
||||
}
|
||||
|
||||
profile.description = query.cnt('.actorBio p:not(.bioTitle)');
|
||||
|
||||
if (hair) profile.hairColor = hair.split(':')[1].trim();
|
||||
if (height) profile.height = Number(height.match(/\d+/)[0]);
|
||||
if (weight) profile.weight = Number(weight.match(/\d+/)[0]);
|
||||
if (alias) profile.aliases = alias.split(':')[1].trim().split(', ');
|
||||
if (nationality) profile.nationality = nationality.split(':')[1].trim();
|
||||
|
||||
if ((getActorReleasesUrl || context.parameters.actorScenes) && withReleases) {
|
||||
profile.releases = await fetchActorReleases(url, getActorReleasesUrl, 1, [], context);
|
||||
}
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchUpcomingApi(site, page = 1, options, preData) {
|
||||
return fetchLatestApi(site, page, options, preData, true);
|
||||
}
|
||||
|
||||
async function fetchSceneApi(url, site, baseRelease, options) {
|
||||
async function fetchScene(url, site, baseRelease, options) {
|
||||
const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
|
||||
const { apiUrl } = await fetchApiCredentials(referer, site);
|
||||
|
||||
const entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
|
||||
|
||||
const res = await http.post(apiUrl, {
|
||||
const res = await unprint.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_scenes',
|
||||
@@ -741,28 +319,26 @@ async function fetchSceneApi(url, site, baseRelease, options) {
|
||||
headers: {
|
||||
Referer: referer,
|
||||
},
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.status === 200 && res.body.results?.[0]?.hits.length > 0) {
|
||||
return scrapeReleaseApi(res.body.results[0].hits[0], site, options);
|
||||
if (res.ok && res.data.results?.[0]?.hits.length > 0) {
|
||||
return scrapeScene(res.data.results[0].hits[0], site, options);
|
||||
}
|
||||
|
||||
if (res.status === 200) {
|
||||
if (res.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchMovieApi(url, site, baseRelease, options) {
|
||||
async function fetchMovie(url, site, baseRelease, options) {
|
||||
const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/movies`;
|
||||
const { apiUrl } = await fetchApiCredentials(referer, site);
|
||||
|
||||
const entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
|
||||
|
||||
const res = await http.post(apiUrl, {
|
||||
const res = await unprint.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_movies',
|
||||
@@ -783,12 +359,10 @@ async function fetchMovieApi(url, site, baseRelease, options) {
|
||||
headers: {
|
||||
Referer: referer,
|
||||
},
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.status === 200 && res.body.results?.[0]?.hits.length > 0) {
|
||||
return scrapeReleaseApi(res.body.results[0].hits[0], site, options, res.body.results[1]?.hits);
|
||||
if (res.ok && res.data.results?.[0]?.hits.length > 0) {
|
||||
return scrapeScene(res.data.results[0].hits[0], site, options, res.data.results[1]?.hits);
|
||||
}
|
||||
|
||||
if (res.status === 200) {
|
||||
@@ -798,62 +372,8 @@ async function fetchMovieApi(url, site, baseRelease, options) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function getDeepUrl(url, site, baseRelease, mobile) {
|
||||
const filter = new Set(['en', 'video', 'scene', site.slug, site.parent.slug]);
|
||||
const pathname = baseRelease?.path || new URL(url).pathname
|
||||
.split('/')
|
||||
.filter((component) => !filter.has(component))
|
||||
.join('/'); // reduce to scene ID and title slug
|
||||
|
||||
const sceneId = baseRelease?.entryId || pathname.match(/\/(\d+)\//)?.[1];
|
||||
|
||||
if (mobile && /%d/.test(mobile)) {
|
||||
return util.format(mobile, sceneId);
|
||||
}
|
||||
|
||||
if (mobile && sceneId) {
|
||||
return `${mobile}${pathname}`;
|
||||
}
|
||||
|
||||
if (site.parameters?.deep) {
|
||||
return `${site.parameters.deep}${pathname}`;
|
||||
}
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
async function fetchScene(url, site, baseRelease, options) {
|
||||
if (site.parameters?.deep === false) {
|
||||
return baseRelease;
|
||||
}
|
||||
|
||||
const deepUrl = getDeepUrl(url, site, baseRelease);
|
||||
const mobileUrl = options.includePhotos && getDeepUrl(url, site, baseRelease, site.parameters?.mobile || site.parent?.parameters?.mobile);
|
||||
|
||||
if (deepUrl) {
|
||||
const [res, mobileRes] = await Promise.all([
|
||||
qu.get(deepUrl),
|
||||
mobileUrl && qu.get(mobileUrl, null, {
|
||||
headers: {
|
||||
// don't redirect to main site
|
||||
'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Mobile Safari/537.36',
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
if (res.status === 200) {
|
||||
const mobileItem = mobileRes?.status === 200 ? mobileRes.item : null;
|
||||
const scene = await scrapeScene(res.item, url, site, baseRelease, mobileItem, options);
|
||||
|
||||
return { ...scene, deepUrl };
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchActorScenes(actorName, apiUrl, siteSlug) {
|
||||
const res = await http.post(apiUrl, {
|
||||
const res = await unprint.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_scenes',
|
||||
@@ -864,47 +384,16 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
|
||||
headers: {
|
||||
Referer: `https://www.${siteSlug}.com/en/videos`,
|
||||
},
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.status === 200 && res.body.results[0].hits.length > 0) {
|
||||
return res.body.results[0].hits;
|
||||
if (res.ok && res.data.results[0].hits.length > 0) {
|
||||
return res.data.results[0].hits;
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }, context, include, altSearchUrl, getActorReleasesUrl) {
|
||||
const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug;
|
||||
|
||||
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
|
||||
const searchUrl = altSearchUrl
|
||||
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
|
||||
: `https://www.${siteSlug}.com/en/search/${siteSlug}/actor/${actorSlug}`;
|
||||
const searchRes = await http.get(searchUrl);
|
||||
|
||||
if (searchRes.status !== 200) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
|
||||
|
||||
if (actorUrl) {
|
||||
const url = `https://${siteSlug}.com${actorUrl}`;
|
||||
const actorRes = await qu.get(url);
|
||||
|
||||
if (actorRes.status !== 200) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return scrapeProfile(actorRes.item, url, actorName, siteSlug, getActorReleasesUrl, include.scenes, context);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function scrapeApiProfile(data, releases, siteSlug) {
|
||||
function scrapeProfile(data, releases, siteSlug) {
|
||||
const profile = {};
|
||||
|
||||
if (data.male === 1) profile.gender = 'male';
|
||||
@@ -925,7 +414,7 @@ function scrapeApiProfile(data, releases, siteSlug) {
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchApiProfile({ name: actorName }, context, include) {
|
||||
async function fetchProfile({ name: actorName }, context, include) {
|
||||
const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug;
|
||||
const actorSlug = encodeURI(actorName);
|
||||
const referer = context.parameters.profileReferer || `${context.entity.origin}/en/search`;
|
||||
@@ -940,7 +429,7 @@ async function fetchApiProfile({ name: actorName }, context, include) {
|
||||
: []),
|
||||
]).map((site) => `"availableOnSite:${site}"`).join(',');
|
||||
|
||||
const res = await http.post(apiUrl, {
|
||||
const res = await unprint.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_actors',
|
||||
@@ -952,17 +441,15 @@ async function fetchApiProfile({ name: actorName }, context, include) {
|
||||
headers: {
|
||||
Referer: referer,
|
||||
},
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.status === 200 && res.body.results[0].hits.length > 0) {
|
||||
const actorData = res.body.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
|
||||
if (res.status === 200 && res.data.results[0].hits.length > 0) {
|
||||
const actorData = res.data.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
|
||||
|
||||
if (actorData) {
|
||||
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);
|
||||
|
||||
return scrapeApiProfile(actorData, actorScenes, siteSlug);
|
||||
return scrapeProfile(actorData, actorScenes, siteSlug);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -970,31 +457,9 @@ async function fetchApiProfile({ name: actorName }, context, include) {
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchApiLatest: fetchLatestApi,
|
||||
fetchApiProfile,
|
||||
fetchApiUpcoming: fetchUpcomingApi,
|
||||
fetchLatest,
|
||||
fetchLatestApi,
|
||||
fetchUpcoming,
|
||||
fetchProfile,
|
||||
fetchScene,
|
||||
fetchSceneApi,
|
||||
fetchUpcoming,
|
||||
fetchUpcomingApi,
|
||||
api: {
|
||||
fetchLatest: fetchLatestApi,
|
||||
fetchUpcoming: fetchUpcomingApi,
|
||||
fetchProfile: fetchApiProfile,
|
||||
// fetchScene,
|
||||
fetchScene: fetchSceneApi,
|
||||
// scrapeMovie,
|
||||
fetchMovie: fetchMovieApi,
|
||||
},
|
||||
getPhotos,
|
||||
scrapeApiProfile,
|
||||
scrapeApiReleases,
|
||||
scrapeProfile,
|
||||
scrapeAll,
|
||||
scrapeMovie,
|
||||
scrapeScene,
|
||||
deprecated: true,
|
||||
fetchMovie,
|
||||
};
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
|
||||
const logger = require('../logger');
|
||||
const { fetchApiLatest } = require('./gamma');
|
||||
const qu = require('../utils/qu');
|
||||
const http = require('../utils/http');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
async function fetchActors(entryId, channel, { token, time }) {
|
||||
const url = `${channel.url}/sapi/${token}/${time}/model.getModelContent?_method=model.getModelContent&tz=1&fields[0]=modelId.stageName&fields[1]=_last&fields[2]=modelId.upsellLink&fields[3]=modelId.upsellText&limit=25&transitParameters[contentId]=${entryId}`;
|
||||
const res = await http.get(url);
|
||||
|
||||
if (res.statusCode === 200 && res.body.status === true) {
|
||||
return Object.values(res.body.response.collection).map((actor) => Object.values(actor.modelId.collection)[0].stageName);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function fetchTrailerLocation(entryId, channel) {
|
||||
const url = `${channel.url}/api/download/${entryId}/hd1080/stream`;
|
||||
|
||||
try {
|
||||
const res = await http.get(url, {
|
||||
followRedirects: false,
|
||||
});
|
||||
|
||||
if (res.statusCode === 302) {
|
||||
return res.headers.location;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`${channel.name}: Unable to fetch trailer at '${url}': ${error.message}`);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function scrapeLatest(items, channel) {
|
||||
return items.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('h5 a', null, { origin: channel.url });
|
||||
release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
|
||||
|
||||
release.title = query.cnt('h5 a');
|
||||
|
||||
[release.poster, ...release.photos] = query.imgs('.screenshot').map((src) => [
|
||||
// unnecessarily large
|
||||
// src.replace(/\/\d+/, 3840),
|
||||
// src.replace(/\/\d+/, '/2000'),
|
||||
src.replace(/\/\d+/, '/1500'),
|
||||
src.replace(/\/\d+/, '/1000'),
|
||||
src,
|
||||
]);
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ query, html }, url, channel) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1];
|
||||
|
||||
release.title = query.cnt('h1.description');
|
||||
release.actors = query
|
||||
.all('.video-performer')
|
||||
.map((actorEl) => {
|
||||
const actorUrl = query.url(actorEl, 'a', 'href', { origin: channel.url });
|
||||
const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
|
||||
const avatar = query.img(actorEl, 'img:not([data-bgsrc*="not-available"])', 'data-bgsrc');
|
||||
|
||||
return {
|
||||
name: query.cnt(actorEl, '.video-performer-name'),
|
||||
gender: 'female',
|
||||
avatar: avatar && [
|
||||
avatar.replace(/\/actor\/(\d+)/, '/actor/500'),
|
||||
avatar,
|
||||
],
|
||||
url: actorUrl,
|
||||
entryId,
|
||||
};
|
||||
})
|
||||
.concat({ name: 'Jay Rock', gender: 'male' });
|
||||
|
||||
release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
release.duration = query.number('.release-date:last-child') * 60;
|
||||
|
||||
release.studio = query.cnt('.studio span:nth-child(2)');
|
||||
release.director = query.text('.director');
|
||||
|
||||
release.tags = query.cnts('.tags a');
|
||||
|
||||
const poster = html.match(/url\((https.+\.jpg)\)/)?.[1];
|
||||
const photos = query.imgs('#moreScreenshots img');
|
||||
|
||||
[release.poster, ...release.photos] = [poster]
|
||||
.concat(photos)
|
||||
.filter(Boolean)
|
||||
.map((src) => [
|
||||
src.replace(/\/(\d+)\/\d+/, '/$1/1500'),
|
||||
src.replace(/\/(\d+)\/\d+/, '/$1/1000'),
|
||||
src,
|
||||
]);
|
||||
|
||||
const videoId = html.match(/item: (\d+)/)?.[1];
|
||||
|
||||
if (videoId) {
|
||||
release.trailer = { stream: `https://trailer.adultempire.com/hls/trailer/${videoId}/master.m3u8` };
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function scrapeSceneApi(scene, channel, tokens, deep) {
|
||||
const release = {
|
||||
entryId: scene.id,
|
||||
title: scene.title,
|
||||
duration: scene.length,
|
||||
meta: {
|
||||
tokens, // attach tokens to reduce number of requests required for deep fetching
|
||||
},
|
||||
};
|
||||
|
||||
release.url = `${channel.url}/scene/${release.entryId}/${slugify(release.title, { encode: true })}`;
|
||||
release.date = new Date(scene.sites.collection[scene.id].publishDate);
|
||||
release.poster = scene._resources.primary[0].url;
|
||||
|
||||
if (scene.tags) release.tags = Object.values(scene.tags.collection).map((tag) => tag.alias);
|
||||
if (scene._resources.base) release.photos = scene._resources.base.map((resource) => resource.url);
|
||||
|
||||
if (deep) {
|
||||
// don't make external requests during update scraping, as this would happen for every scene on the page
|
||||
const [actors, trailer] = await Promise.all([
|
||||
fetchActors(release.entryId, channel, tokens),
|
||||
fetchTrailerLocation(release.entryId, channel),
|
||||
]);
|
||||
|
||||
release.actors = actors;
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = { src: trailer, quality: 1080 };
|
||||
}
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeLatestApi(scenes, site, tokens) {
|
||||
return Promise.map(scenes, async (scene) => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
|
||||
}
|
||||
|
||||
async function fetchToken(channel) {
|
||||
const res = await http.get(channel.url);
|
||||
const html = res.body.toString();
|
||||
|
||||
const time = html.match(/"aet":\d+/)[0].split(':')[1];
|
||||
const ah = html.match(/"ah":"[\w-]+"/)[0].split(':')[1].slice(1, -1);
|
||||
const token = ah.split('').reverse().join('');
|
||||
|
||||
return { time, token };
|
||||
}
|
||||
|
||||
async function fetchLatestApi(channel, page = 1) {
|
||||
const { time, token } = await fetchToken(channel);
|
||||
|
||||
// transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory
|
||||
const url = `${channel.url}/sapi/${token}/${time}/content.load?limit=50&offset=${(page - 1) * 50}&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`;
|
||||
const res = await http.get(url);
|
||||
|
||||
if (res.ok && res.body.status) {
|
||||
return scrapeLatestApi(res.body.response.collection, channel, { time, token });
|
||||
}
|
||||
|
||||
return res.ok ? res.body.status : res.status;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1, options, preData) {
|
||||
if (channel.parameters?.useGamma) {
|
||||
return fetchApiLatest(channel, page, preData, options, false);
|
||||
}
|
||||
|
||||
const res = await qu.getAll(`https://jayspov.net/jays-pov-updates.html?view=list&page=${page}`, '.item-grid-list-view > .grid-item');
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatest(res.items, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchSceneApi(url, channel, baseRelease) {
|
||||
const { time, token } = baseRelease?.meta.tokens || await fetchToken(channel); // use attached tokens when deep fetching
|
||||
const { pathname } = new URL(url);
|
||||
const entryId = pathname.split('/')[2];
|
||||
|
||||
const apiUrl = `${channel.url}/sapi/${token}/${time}/content.load?filter[id][fields][0]=id&filter[id][values][0]=${entryId}&transitParameters[v1]=ykYa8ALmUD&transitParameters[preset]=scene`;
|
||||
const res = await http.get(apiUrl);
|
||||
|
||||
if (res.ok && res.body.status) {
|
||||
return scrapeSceneApi(res.body.response.collection[0], channel, { time, token }, true);
|
||||
}
|
||||
|
||||
return res.ok ? res.body.status : res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel) {
|
||||
const res = await qu.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, url, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
api: {
|
||||
fetchLatest: fetchLatestApi,
|
||||
fetchScene: fetchSceneApi,
|
||||
},
|
||||
};
|
||||
@@ -19,7 +19,6 @@ const czechav = require('./czechav');
|
||||
const modelmedia = require('./modelmedia');
|
||||
const dorcel = require('./dorcel');
|
||||
const fabulouscash = require('./fabulouscash');
|
||||
// const famedigital = require('./famedigital');
|
||||
const firstanalquest = require('./firstanalquest');
|
||||
const elevatedx = require('./elevatedx');
|
||||
const exploitedx = require('./exploitedx');
|
||||
@@ -31,7 +30,6 @@ const hush = require('./hush');
|
||||
const innofsin = require('./innofsin');
|
||||
const insex = require('./insex');
|
||||
const inthecrack = require('./inthecrack');
|
||||
const jayrock = require('./jayrock');
|
||||
const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials');
|
||||
const julesjordan = require('./julesjordan');
|
||||
const karups = require('./karups');
|
||||
@@ -137,7 +135,6 @@ module.exports = {
|
||||
insex,
|
||||
interracialpass: hush,
|
||||
inthecrack,
|
||||
jayrock,
|
||||
jerkaoke: modelmedia,
|
||||
jesseloadsmonsterfacials,
|
||||
julesjordan,
|
||||
|
||||
Reference in New Issue
Block a user