469 lines
15 KiB
JavaScript
Executable File
469 lines
15 KiB
JavaScript
Executable File
'use strict';
|
|
|
|
const format = require('template-format');
|
|
const unprint = require('unprint');
|
|
|
|
const logger = require('../logger')(__filename);
|
|
const slugify = require('../utils/slugify');
|
|
|
|
function curateTitle(title, channel) {
|
|
// some videos are redundantly prefixed with the name of the site, i.e. Bubblegum Dungeon, Forbidden Seductions and Lady Gonzo
|
|
return title.replace(new RegExp(`^\\s*${channel.name}\\s*[:|-]\\s`, 'i'), '');
|
|
}
|
|
|
|
async function scrapeApiReleases(json, site, options) {
|
|
return json.reduce((acc, scene) => {
|
|
if (options.parameters?.extract && scene.sitename !== options.parameters.extract) {
|
|
return acc;
|
|
}
|
|
|
|
if (options.parameters?.filterExclusive && scene.availableOnSite.length > 1) {
|
|
return acc;
|
|
}
|
|
|
|
if (options.parameters?.filterChannel && scene.mainChannel?.id !== options.parameters.filterChannel) {
|
|
return acc;
|
|
}
|
|
|
|
const release = {
|
|
entryId: scene.clip_id,
|
|
description: scene.description,
|
|
duration: scene.length,
|
|
likes: scene.ratings_up,
|
|
dislikes: scene.ratings_down,
|
|
};
|
|
|
|
release.title = curateTitle(scene.title, site);
|
|
release.path = `/${scene.url_title}/${release.entryId}`;
|
|
|
|
if (typeof options.parameters?.scene === 'string') {
|
|
release.url = `${options.parameters.scene}${release.path}`;
|
|
} else if (site.url && options.parameters?.scene !== false) {
|
|
const siteSlug = new URL(site.url).pathname.match(/\/sites\/(\w+)/)?.[1];
|
|
release.url = unprint.prefixUrl(`/en/video${siteSlug ? `/${siteSlug}` : ''}${release.path}`, site.origin);
|
|
}
|
|
|
|
release.date = unprint.extractDate(scene.release_date, 'YYYY-MM-DD');
|
|
release.director = scene.directors[0]?.name || null;
|
|
|
|
release.actors = scene.actors.map((actor) => ({
|
|
entryId: actor.actor_id,
|
|
name: actor.name,
|
|
gender: actor.gender,
|
|
avatar: [
|
|
`https://images03-openlife.gammacdn.com/actors/${actor.actor_id}/${actor.actor_id}_500x750.jpg`,
|
|
`https://images03-openlife.gammacdn.com/actors/${actor.actor_id}/${actor.actor_id}_240x360.jpg`,
|
|
`https://images03-openlife.gammacdn.com/actors/${actor.actor_id}/${actor.actor_id}_200x300.jpg`,
|
|
],
|
|
}));
|
|
|
|
/* master categories include e.g. 'transgender' for non-trans Wicked scenes
|
|
release.tags = scene.master_categories
|
|
.concat(scene.categories?.map((category) => category.name))
|
|
.filter(Boolean); // some categories don't have a name
|
|
*/
|
|
|
|
release.tags = scene.categories?.map((category) => category.name).filter(Boolean); // some categories don't have a name
|
|
|
|
if (scene.availableOnSite.length > 1) {
|
|
release.comment = `Also available on ${scene.availableOnSite.filter((sisterSite) => sisterSite !== site.slug).join(', ')}`;
|
|
}
|
|
|
|
const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
|
|
|
|
if (posterPath) {
|
|
release.poster = [
|
|
`https://images-evilangel.gammacdn.com/movies${posterPath}`,
|
|
`https://transform.gammacdn.com/movies${posterPath}`,
|
|
];
|
|
}
|
|
|
|
if (options.parameters.filterNetwork && scene.mainChannel) {
|
|
return {
|
|
...acc,
|
|
unextracted: acc.unextracted.concat(release),
|
|
};
|
|
}
|
|
|
|
release.channel = slugify(scene.mainChannel?.id || scene.sitename, ''); // remove -
|
|
// release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
|
|
|
|
return {
|
|
...acc,
|
|
scenes: acc.scenes.concat(release),
|
|
};
|
|
}, {
|
|
scenes: [],
|
|
unextracted: [],
|
|
});
|
|
}
|
|
|
|
function getApiUrl(appId, apiKey) {
|
|
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
|
|
|
|
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
|
|
|
|
return {
|
|
appId,
|
|
apiKey,
|
|
userAgent,
|
|
apiUrl,
|
|
};
|
|
}
|
|
|
|
async function fetchApiCredentials(referer, site) {
|
|
if (site?.parameters?.appId && site?.parameters?.apiKey) {
|
|
return getApiUrl(site.parameters.appId, site.parameters.apiKey);
|
|
}
|
|
|
|
const res = await unprint.get(referer);
|
|
const apiLine = res.body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
|
|
|
|
if (!apiLine) {
|
|
throw new Error(`No Gamma API key found for ${referer}`);
|
|
}
|
|
|
|
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
|
|
const apiData = JSON.parse(apiSerial);
|
|
|
|
const { applicationID: appId, apiKey } = apiData.api.algolia;
|
|
|
|
return getApiUrl(appId, apiKey);
|
|
}
|
|
|
|
async function fetchLatest(site, page = 1, options, _preData, upcoming = false) {
|
|
const referer = options.parameters?.referer || `${options.parameters?.networkReferer ? site.parent.origin : site.origin}/en/videos`;
|
|
const { apiUrl } = await fetchApiCredentials(referer, site);
|
|
const slug = options.parameters.querySlug || site.slug;
|
|
|
|
const params = `query=&hitsPerPage=60&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"],["compilation:0"]]${options.parameters.queryChannel
|
|
? `&filters=channels.id:${options.parameters.queryChannel === true ? slug : options.parameters.queryChannel}`
|
|
: `&filters=availableOnSite:${slug}`}`;
|
|
|
|
const res = await unprint.post(apiUrl, {
|
|
requests: [
|
|
{
|
|
indexName: 'all_scenes',
|
|
params,
|
|
},
|
|
],
|
|
}, {
|
|
headers: {
|
|
Referer: referer,
|
|
},
|
|
});
|
|
|
|
if (res.ok && res.data.results?.[0]?.hits) {
|
|
return scrapeApiReleases(res.data.results[0].hits, site, options);
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
async function fetchUpcoming(site, page = 1, options, preData) {
|
|
return fetchLatest(site, page, options, preData, true);
|
|
}
|
|
|
|
const qualityMap = {
|
|
'4k': 2160,
|
|
};
|
|
|
|
async function getFullPhotos(entryId, site, parameters) {
|
|
const res = await unprint.get(`${parameters.album || site.url}/media/signPhotoset/${entryId}`, {
|
|
headers: {
|
|
'X-Requested-With': 'XMLHttpRequest',
|
|
},
|
|
});
|
|
|
|
if (res.ok && typeof res.data === 'object') { // gives 200 OK even when redirected to signup page
|
|
return Object.values(res.data);
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
async function getThumbs(entryId, site, parameters) {
|
|
const referer = parameters?.referer || `${parameters?.networkReferer ? site.parent.origin : site.origin}/en/videos`;
|
|
const { apiUrl } = await fetchApiCredentials(referer, site);
|
|
|
|
const res = await unprint.post(apiUrl, {
|
|
requests: [
|
|
{
|
|
indexName: 'all_photosets',
|
|
params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["set_id:${entryId}"]]`,
|
|
},
|
|
],
|
|
}, {
|
|
headers: {
|
|
Referer: referer,
|
|
},
|
|
});
|
|
|
|
if (res.ok && res.data.results?.[0]?.hits[0]?.set_pictures) {
|
|
return res.data.results[0].hits[0].set_pictures.map((img) => img.thumb_path && [
|
|
'https://images-fame.gammacdn.com',
|
|
'https://images01-fame.gammacdn.com',
|
|
'https://images02-fame.gammacdn.com',
|
|
'https://images03-fame.gammacdn.com',
|
|
'https://images04-fame.gammacdn.com',
|
|
'https://images-evilangel.gammacdn.com',
|
|
'https://transform.gammacdn.com',
|
|
].map((origin) => unprint.prefixUrl(`/photo_set${img.thumb_path}`, origin))).filter(Boolean);
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
function getImageSources(source) {
|
|
return [
|
|
'https://images-fame.gammacdn.com',
|
|
'https://images01-fame.gammacdn.com',
|
|
'https://images02-fame.gammacdn.com',
|
|
'https://images03-fame.gammacdn.com',
|
|
'https://images04-fame.gammacdn.com',
|
|
'https://images-evilangel.gammacdn.com',
|
|
'https://transform.gammacdn.com',
|
|
].map((origin) => unprint.prefixUrl(`/movies${source}`, origin));
|
|
}
|
|
|
|
async function getPhotosApi(entryId, site, parameters) {
|
|
const [photos, thumbs] = await Promise.all([
|
|
getFullPhotos(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
|
|
getThumbs(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
|
|
]);
|
|
|
|
return photos.concat(thumbs.slice(photos.length));
|
|
}
|
|
|
|
async function scrapeScene(data, site, options, movieScenes) {
|
|
const release = {};
|
|
|
|
release.entryId = data.clip_id || data.movie_id;
|
|
release.title = curateTitle(data.title, site);
|
|
release.duration = data.length;
|
|
release.date = (data.date && new Date(data.date * 1000)) || unprint.extractDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
|
|
release.director = data.directors[0]?.name || null;
|
|
|
|
release.actors = data.actors.map((actor) => ({
|
|
entryId: actor.actor_id,
|
|
name: actor.name,
|
|
gender: actor.gender,
|
|
url: options.parameters?.actors
|
|
? format(options.parameters.actors, { id: actor.actor_id, slug: actor.url_name })
|
|
: unprint.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
|
|
}));
|
|
|
|
release.tags = data.categories.map((category) => category.name);
|
|
|
|
if (data.pictures) {
|
|
release.poster = [
|
|
...(data.pictures['1920x1080'] ? getImageSources(data.pictures['1920x1080']) : []),
|
|
...(data.pictures.resized ? getImageSources(data.pictures.resized) : []),
|
|
...(data.pictures['960x544'] ? getImageSources(data.pictures['960x544']) : []),
|
|
];
|
|
}
|
|
|
|
if (data.photoset_id && options.includePhotos && options.parameters?.includePhotos !== false) {
|
|
release.photos = await getPhotosApi(data.photoset_id, site, options.parameters);
|
|
}
|
|
|
|
if (data.cover_path) {
|
|
release.covers = [
|
|
getImageSources(`${data.cover_path}_front_400x625.jpg?width=450&height=636&format=webp`),
|
|
getImageSources(`${data.cover_path}_back_400x625.jpg?width=450&height=636&format=webp`),
|
|
];
|
|
}
|
|
|
|
release.teaser = `https://videothumb.gammacdn.com/500x281/${release.entryId}.mp4`;
|
|
|
|
if (data.trailers) {
|
|
release.trailer = Object.entries(data.trailers).map(([quality, source]) => ({
|
|
src: source,
|
|
quality: qualityMap[quality] || parseInt(quality, 10) || quality,
|
|
}));
|
|
}
|
|
|
|
if (data.movie_id && !data.movie_path && options.parameters.sceneMovies !== false) {
|
|
release.movie = {
|
|
entryId: data.movie_id,
|
|
title: data.movie_title,
|
|
url: unprint.prefixUrl(`${data.url_movie_title}/${data.movie_id}`, options.parameters.movie ? options.parameters.movie : `${site.url}/en/movie`),
|
|
};
|
|
}
|
|
|
|
if (movieScenes?.length > 0) {
|
|
release.scenes = await Promise.all(movieScenes.map((movieScene) => scrapeScene(movieScene, site, options)));
|
|
}
|
|
|
|
release.channel = slugify(data.mainChannel?.id || data.sitename, ''); // remove -
|
|
release.qualities = data.download_sizes?.map((quality) => qualityMap[quality] || parseInt(quality, 10) || quality);
|
|
|
|
return release;
|
|
}
|
|
|
|
async function fetchScene(url, site, baseRelease, options) {
|
|
const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.origin : site.origin}/en/videos`;
|
|
const { apiUrl } = await fetchApiCredentials(referer, site);
|
|
|
|
const entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
|
|
|
|
const res = await unprint.post(apiUrl, {
|
|
requests: [
|
|
{
|
|
indexName: 'all_scenes',
|
|
params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["clip_id:${entryId}"]]`,
|
|
},
|
|
{
|
|
indexName: 'all_scenes',
|
|
params: 'query=&page=0&hitsPerPage=1&attributesToRetrieve=[]&attributesToHighlight=[]&attributesToSnippet=[]&tagFilters=&analytics=false&clickAnalytics=false&facets=clip_id',
|
|
},
|
|
],
|
|
}, {
|
|
headers: {
|
|
Referer: referer,
|
|
},
|
|
});
|
|
|
|
if (res.ok && res.data.results?.[0]?.hits.length > 0) {
|
|
return scrapeScene(res.data.results[0].hits[0], site, options);
|
|
}
|
|
|
|
if (res.ok) {
|
|
return null;
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
async function fetchMovie(url, site, baseRelease, options) {
|
|
const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/movies`;
|
|
const { apiUrl } = await fetchApiCredentials(referer, site);
|
|
|
|
const entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
|
|
|
|
const res = await unprint.post(apiUrl, {
|
|
requests: [
|
|
{
|
|
indexName: 'all_movies',
|
|
params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["movie_id:${entryId}"]]`,
|
|
},
|
|
{
|
|
indexName: 'all_scenes_latest_desc',
|
|
params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["movie_id:${entryId}"]]`,
|
|
},
|
|
/*
|
|
{
|
|
indexName: 'all_movies',
|
|
params: 'query=&page=0&hitsPerPage=1&attributesToRetrieve=[]&attributesToHighlight=[]&attributesToSnippet=[]&tagFilters=&analytics=false&clickAnalytics=false&facets=clip_id',
|
|
},
|
|
*/
|
|
],
|
|
}, {
|
|
headers: {
|
|
Referer: referer,
|
|
},
|
|
});
|
|
|
|
if (res.ok && res.data.results?.[0]?.hits.length > 0) {
|
|
return scrapeScene(res.data.results[0].hits[0], site, options, res.data.results[1]?.hits);
|
|
}
|
|
|
|
if (res.status === 200) {
|
|
return null;
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
async function fetchActorScenes(actorName, apiUrl, siteSlug) {
|
|
const res = await unprint.post(apiUrl, {
|
|
requests: [
|
|
{
|
|
indexName: 'all_scenes',
|
|
params: `query=&filters=sitename:${siteSlug}&hitsPerPage=36&maxValuesPerFacet=100&page=0&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["actors.name:${actorName}"]]`,
|
|
},
|
|
],
|
|
}, {
|
|
headers: {
|
|
Referer: `https://www.${siteSlug}.com/en/videos`,
|
|
},
|
|
});
|
|
|
|
if (res.ok && res.data.results[0].hits.length > 0) {
|
|
return res.data.results[0].hits;
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
function scrapeProfile(data, releases, siteSlug) {
|
|
const profile = {};
|
|
|
|
if (data.male === 1) profile.gender = 'male';
|
|
if (data.female === 1) profile.gender = 'female';
|
|
if (data.shemale === 1 || data.trans === 1) profile.gender = 'transsexual';
|
|
|
|
if (data.description) profile.description = data.description.trim();
|
|
|
|
if (data.attributes.ethnicity) profile.ethnicity = data.attributes.ethnicity;
|
|
if (data.attributes.eye_color) profile.eyes = data.attributes.eye_color;
|
|
if (data.attributes.hair_color) profile.hairColor = data.attributes.hair_color;
|
|
|
|
const avatarPaths = Object.values(data.pictures).reverse();
|
|
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map((avatarPath) => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
|
|
|
|
if (releases) profile.releases = releases.map((release) => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
|
|
|
|
return profile;
|
|
}
|
|
|
|
async function fetchProfile({ name: actorName }, context, include) {
|
|
const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug;
|
|
const actorSlug = encodeURI(actorName);
|
|
const referer = context.parameters.profileReferer || `${context.entity.origin}/en/search`;
|
|
|
|
const { apiUrl } = await fetchApiCredentials(referer);
|
|
|
|
const availableOnSites = (context.parameters.actorAvailableOnSites
|
|
|| [
|
|
siteSlug,
|
|
...(context.entity.type === 'network'
|
|
? context.entity.children.map((networkChannel) => networkChannel.slug)
|
|
: []),
|
|
]).map((site) => `"availableOnSite:${site}"`).join(',');
|
|
|
|
const res = await unprint.post(apiUrl, {
|
|
requests: [
|
|
{
|
|
indexName: 'all_actors',
|
|
params: `query=${actorSlug}&facetFilters=[[${availableOnSites}]]`,
|
|
// params: `query=${actorSlug}`,
|
|
},
|
|
],
|
|
}, {
|
|
headers: {
|
|
Referer: referer,
|
|
},
|
|
});
|
|
|
|
if (res.status === 200 && res.data.results[0].hits.length > 0) {
|
|
const actorData = res.data.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
|
|
|
|
if (actorData) {
|
|
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);
|
|
|
|
return scrapeProfile(actorData, actorScenes, siteSlug);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
module.exports = {
|
|
fetchLatest,
|
|
fetchUpcoming,
|
|
fetchProfile,
|
|
fetchScene,
|
|
fetchMovie,
|
|
};
|