DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Refactored Gamma scraper, only using API.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-03-01 23:52:41 +01:00
parent 3696b81e69
commit af57f412c9
7 changed files with 135 additions and 1068 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
seeds/01_networks.js
View File

@@ -59,7 +59,6 @@ const parentNetworks = [
url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
parameters: {
layout: 'api',
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: 'gamma',
@@ -108,7 +107,6 @@ const parentNetworks = [
url: 'https://www.asgmax.com',
parent: 'gamma',
parameters: {
layout: 'api',
scene: 'https://www.asgmax.com/en/video/asgmax',
},
},
@@ -190,7 +188,6 @@ const networks = [
url: 'https://www.21sextreme.com',
description: 'Welcome to 21Sextreme.com, your portal to fisting porn, old and young lesbians, horny grannies & extreme BDSM featuring the best Euro & American Pornstars',
parameters: {
layout: 'api',
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: '21sextury',
@@ -201,7 +198,6 @@ const networks = [
url: 'https://www.21naturals.com',
description: 'Welcome to 21Naturals.com, the porn network featuring the hottest pornstars from all over the world in all natural porn and erotic sex videos. Watch thousands of girls with natural tits',
parameters: {
layout: 'api',
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: '21sextury',
@@ -231,7 +227,6 @@ const networks = [
description: 'Adult Time is a premium streaming service for adults! Watch adult movies, series, and channels from the top names in the industry.',
parent: 'gamma',
parameters: {
layout: 'api',
referer: 'https://freetour.adulttime.com/en/join',
// scene: false,
},
@@ -323,7 +318,6 @@ const networks = [
url: 'https://www.blowpass.com',
description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
parameters: {
layout: 'api',
referer: 'https://www.blowpass.com',
},
parent: 'gamma',
@@ -376,9 +370,6 @@ const networks = [
url: 'https://www.evilangel.com',
description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
parent: 'gamma',
parameters: {
layout: 'api',
},
},
{
slug: 'exploitedx',
@@ -422,7 +413,6 @@ const networks = [
description: 'The world famous Dogfart Interracial series. Online since 1996, we have the largest collection of Interracial videos, pictures and content on the web.',
parent: 'dfxtra',
parameters: {
layout: 'api',
profileReferer: 'https://www.dogfartnetwork.com',
},
},
@@ -436,9 +426,6 @@ const networks = [
slug: 'fantasymassage',
name: 'Fantasy Massage',
url: 'https://www.fantasymassage.com',
parameters: {
layout: 'api',
},
parent: 'gamma',
},
{
@@ -448,7 +435,6 @@ const networks = [
description: 'Watch and download thousands of the best porn videos at FameDigital.com, the largest porn network on the web! The hottest teens, MILFs and more pornstars are all here!',
parameters: {
mobile: 'https://m.dpfanatics.com/en/video',
layout: 'api',
},
parent: 'gamma',
},
@@ -468,7 +454,6 @@ const networks = [
url: 'https://www.filthykings.com',
parent: 'gamma',
parameters: {
layout: 'api',
queryChannel: true,
scene: 'https://www.filthykings.com/en/video/filthykings',
referer: 'https://www.filthykings.com',
@@ -509,9 +494,6 @@ const networks = [
name: 'Girlsway',
url: 'https://www.girlsway.com',
description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!',
parameters: {
layout: 'api',
},
parent: 'gamma',
},
{
@@ -762,7 +744,6 @@ const networks = [
description: 'PureTaboo.com is the ultimate site for family taboo porn, featuring submissive teens & virgins in rough sex videos in ultra 4k HD.',
parent: 'gamma',
parameters: {
layout: 'api',
scene: 'https://www.puretaboo.com/en/video',
referer: 'https://www.puretaboo.com',
},
@@ -859,9 +840,6 @@ const networks = [
url: 'https://www.vivid.com',
description: 'Home of the Kim Kardashian Sex Tape, Porn Parodies, and over 30,000 XXX Movies from The World Leader In Adult Entertainment.',
parent: 'gamma',
parameters: {
layout: 'api',
},
},
{
slug: 'vixen',
@@ -886,7 +864,6 @@ const networks = [
description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
parent: 'gamma',
parameters: {
layout: 'api',
sceneMovies: false,
actorScenes: 'https://www.xempire.com/en/videos/xempire/latest/{page}/All-Categories/0{actorPath}',
},
@@ -896,9 +873,6 @@ const networks = [
name: 'Zero Tolerance',
alias: ['ztod'],
url: 'https://www.zerotolerancefilms.com',
parameters: {
layout: 'api',
},
parent: 'gamma',
},
// ASG MAX

34
seeds/02_sites.js
View File

@@ -579,7 +579,6 @@ const sites = [
tags: ['gay'],
independent: true,
parameters: {
layout: 'api',
scene: 'https://www.chaosmen.com/en/video/chaosmen',
},
},
@@ -601,7 +600,6 @@ const sites = [
parent: 'gamma',
independent: true,
parameters: {
layout: 'api',
scene: 'https://www.tabooheat.com/en/video/tabooheat',
},
},
@@ -1426,7 +1424,6 @@ const sites = [
independent: true,
parameters: {
scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie',
layout: 'api',
},
},
{
@@ -1438,7 +1435,6 @@ const sites = [
independent: true,
parameters: {
scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets',
layout: 'api',
},
},
/* different layout
@@ -1451,7 +1447,6 @@ const sites = [
independent: true,
parameters: {
scene: 'https://www.portagloryhole.com/scenes',
layout: 'api',
},
},
*/
@@ -2685,9 +2680,6 @@ const sites = [
url: 'https://www.biphoria.com',
independent: true,
tags: ['bisexual'],
parameters: {
layout: 'api',
},
parent: 'gamma',
},
// BLOWPASS
@@ -3180,9 +3172,6 @@ const sites = [
alias: ['burna'],
url: 'https://www.burningangel.com',
independent: true,
parameters: {
layout: 'api',
},
parent: 'gamma',
},
// CHERRY PIMPS
@@ -3828,7 +3817,6 @@ const sites = [
parent: 'gamma',
independent: true,
parameters: {
layout: 'api',
deep: 'https://www.diabolic.com/en/video/diabolic',
actors: 'https://www.diabolic.com/en/pornstar/view/{slug}/{id}',
},
@@ -3900,9 +3888,6 @@ const sites = [
name: 'DFXtra',
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
parameters: {
layout: 'api',
},
},
{
slug: 'dfxtraoriginals',
@@ -3910,9 +3895,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
{
slug: 'dfxtracompilations',
@@ -3920,9 +3902,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
{
slug: 'dfxbigbangz',
@@ -3930,9 +3909,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
{
slug: 'dfxsolemates',
@@ -3940,9 +3916,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
{
slug: 'cheatingwithmyex',
@@ -3950,9 +3923,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
// DFXTRA DOGFART
{
@@ -6079,7 +6049,6 @@ const sites = [
url: 'https://cospimps.com',
parent: 'jayrock',
parameters: {
layout: 'api',
/* Gamma scenes are out of date
referer: 'https://www.21sextury.com',
scene: false,
@@ -15427,9 +15396,6 @@ const sites = [
url: 'https://www.wicked.com',
description: 'Welcome to the new Wicked.com! Watch over 25 years of Wicked Pictures\' brand of award-winning porn for couples and women in 4k HD movies & xxx videos',
independent: true,
parameters: {
layout: 'api',
},
parent: 'gamma',
},
// XEMPIRE

107
src/scrapers/famedigital.js
View File

@@ -1,107 +0,0 @@
'use strict';
const {
fetchLatest,
fetchApiLatest,
fetchUpcoming,
fetchApiUpcoming,
fetchScene,
fetchProfile,
fetchApiProfile,
scrapeAll,
} = require('./gamma');
const { get } = require('../utils/qu');
const slugify = require('../utils/slugify');
function extractLowArtActors(release) {
const actors = release.title
.replace(/solo/i, '')
.split(/,|\band\b/ig)
.map((actor) => actor.trim());
return {
...release,
actors,
};
}
async function networkFetchLatest(site, page = 1) {
if (site.parameters?.api) return fetchApiLatest(site, page, false);
const releases = await fetchLatest(site, page);
if (site.slug === 'lowartfilms') {
return releases.map((release) => extractLowArtActors(release));
}
return releases;
}
async function networkFetchScene(url, site) {
const release = await fetchScene(url, site);
if (site.slug === 'lowartfilms') {
return extractLowArtActors(release);
}
return release;
}
async function networkFetchUpcoming(site, page = 1) {
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
return fetchUpcoming(site, page);
}
function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
}
function scrapeClassicProfile({ qu, html }, site) {
const profile = {};
profile.avatar = qu.img('.actorPicture');
profile.releases = scrapeAll(html, null, site.url, false);
return profile;
}
async function fetchClassicProfile(actorName, { site }) {
const actorSlug = slugify(actorName);
const url = `${site.url}/en/pornstars`;
const pornstarsRes = await get(url);
if (!pornstarsRes.ok) return null;
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find((el) => slugify(el.textContent) === actorSlug)
?.value;
if (actorPath) {
const actorUrl = `${site.url}${actorPath}`;
const res = await get(actorUrl);
if (res.ok) {
return scrapeClassicProfile(res.item, site);
}
}
return null;
}
async function networkFetchProfile({ name: actorName }, context, include) {
const profile = await ((context.site.parameters?.api && fetchApiProfile(actorName, context, include))
|| (context.site.parameters?.classic && include.scenes && fetchClassicProfile(actorName, context, include)) // classic profiles only have scenes, no bio
|| fetchProfile({ name: actorName }, context, true, getActorReleasesUrl, include));
return profile;
}
module.exports = {
fetchLatest: networkFetchLatest,
fetchProfile: networkFetchProfile,
fetchScene: networkFetchScene,
fetchUpcoming: networkFetchUpcoming,
};

805
src/scrapers/gamma.js
View File

@@ -1,217 +1,11 @@
'use strict';
const Promise = require('bluebird');
const util = require('util');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const format = require('template-format');
const unprint = require('unprint');
const logger = require('../logger')(__filename);
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
function getApiUrl(appId, apiKey) {
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
return {
appId,
apiKey,
userAgent,
apiUrl,
};
}
function getAvatarFallbacks(avatar) {
if (!avatar) {
return null;
}
return [
avatar.replace(/\d+x\d+/, '500x750'),
avatar.replace(/\d+x\d+/, '240x360'),
avatar.replace(/\d+x\d+/, '200x300'),
avatar,
];
}
async function fetchApiCredentials(referer, site) {
if (site?.parameters?.appId && site?.parameters?.apiKey) {
return getApiUrl(site.parameters.appId, site.parameters.apiKey);
}
const res = await http.get(referer);
const body = res.body.toString();
const apiLine = body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
if (!apiLine) {
throw new Error(`No Gamma API key found for ${referer}`);
}
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
const apiData = JSON.parse(apiSerial);
const { applicationID: appId, apiKey } = apiData.api.algolia;
return getApiUrl(appId, apiKey);
}
function getAlbumUrl(albumPath, site) {
if (site.parameters?.photos) {
return /^http/.test(site.parameters.photos)
? `${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`;
}
if (site.url && site.parameters?.photos !== false) {
return `${site.url}${albumPath}`;
}
return null;
}
async function fetchPhotos(url) {
const res = await qu.get(url);
return res.item;
}
function scrapePhotos({ query }, includeThumbnails = true) {
return query.all('.preview .imgLink, .pgFooterThumb a').map((linkEl) => {
const url = linkEl.href;
if (/\/join|\/createaccount/.test(url)) {
// URL links to join page instead of full photo, extract thumbnail
// /createaccount is used by e.g. Tricky Spa native site
const src = query.img(linkEl);
if (/previews\//.test(src)) {
// resource often serves full photo at a modifier URL anyway, add as primary source
const highRes = src
.replace('previews/', '')
.replace('_tb.jpg', '.jpg');
// keep original thumbnail as fallback in case full photo is not available
return [highRes, src];
}
if (!includeThumbnails) return null;
return src;
}
// URL links to full photo
return url;
}).filter(Boolean);
}
async function getPhotos(albumPath, site, includeThumbnails = true) {
const albumUrl = getAlbumUrl(albumPath, site);
if (!albumUrl) {
return [];
}
try {
const item = await fetchPhotos(albumUrl);
const photos = scrapePhotos(item, includeThumbnails);
const lastPage = item.query.url('.Gamma_Paginator a.last')?.match(/\d+$/)[0];
if (lastPage) {
const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
const otherPhotos = await Promise.map(otherPages, async (page) => {
const pageItem = await fetchPhotos(`${albumUrl}/${page}`);
return scrapePhotos(pageItem, includeThumbnails);
}, {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
}
return photos;
} catch (error) {
logger.warn(`Failed to fetch ${site.name} photos from ${albumUrl}: ${error.message}`);
return [];
}
}
async function getFullPhotos(entryId, site, parameters) {
const res = await http.get(`${parameters.album || site.url}/media/signPhotoset/${entryId}`, {
headers: {
'X-Requested-With': 'XMLHttpRequest',
},
});
if (res.ok && typeof res.body === 'object') { // gives 200 OK even when redirected to signup page
return Object.values(res.body);
}
return [];
}
async function getThumbs(entryId, site, parameters) {
const referer = parameters?.referer || `${parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
const { apiUrl } = await fetchApiCredentials(referer, site);
const res = await http.post(apiUrl, {
requests: [
{
indexName: 'all_photosets',
params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["set_id:${entryId}"]]`,
},
],
}, {
headers: {
Referer: referer,
},
}, {
encodeJSON: true,
});
if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) {
return res.body.results[0].hits[0].set_pictures.map((img) => img.thumb_path && ([
`https://images-fame.gammacdn.com/photo_set${img.thumb_path}`,
`https://images01-fame.gammacdn.com/photo_set${img.thumb_path}`,
`https://images02-fame.gammacdn.com/photo_set${img.thumb_path}`,
`https://images03-fame.gammacdn.com/photo_set${img.thumb_path}`,
`https://images04-fame.gammacdn.com/photo_set${img.thumb_path}`,
`https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`,
`https://transform.gammacdn.com/photo_set${img.thumb_path}`,
])).filter(Boolean);
}
return [];
}
async function getPhotosApi(entryId, site, parameters) {
const [photos, thumbs] = await Promise.all([
getFullPhotos(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
getThumbs(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
]);
return photos.concat(thumbs.slice(photos.length));
}
function getImageSources(source) {
return [
`https://images-fame.gammacdn.com/movies${source}`,
`https://images01-fame.gammacdn.com/movies${source}`,
`https://images02-fame.gammacdn.com/movies${source}`,
`https://images03-fame.gammacdn.com/movies${source}`,
`https://images04-fame.gammacdn.com/movies${source}`,
`https://images-evilangel.gammacdn.com/movies${source}`,
`https://transform.gammacdn.com/movies${source}`,
];
}
function curateTitle(title, channel) {
// some videos are redundantly prefixed with the name of the site, i.e. Bubblegum Dungeon, Forbidden Seductions and Lady Gonzo
return title.replace(new RegExp(`^\\s*${channel.name}\\s*[:|-]\\s`, 'i'), '');
@@ -244,7 +38,7 @@ async function scrapeApiReleases(json, site, options) {
release.url = `${site.url}/en/video${release.path}`;
}
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.date = unprint.extractDate(scene.release_date, 'YYYY-MM-DD');
release.director = scene.directors[0]?.name || null;
release.actors = scene.actors.map((actor) => ({
@@ -289,6 +83,8 @@ async function scrapeApiReleases(json, site, options) {
release.channel = slugify(scene.mainChannel?.id || scene.sitename, ''); // remove -
// release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`;
console.log(release);
return {
...acc,
scenes: acc.scenes.concat(release),
@@ -299,7 +95,40 @@ async function scrapeApiReleases(json, site, options) {
});
}
async function fetchLatestApi(site, page = 1, options, _preData, upcoming = false) {
function getApiUrl(appId, apiKey) {
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
return {
appId,
apiKey,
userAgent,
apiUrl,
};
}
async function fetchApiCredentials(referer, site) {
if (site?.parameters?.appId && site?.parameters?.apiKey) {
return getApiUrl(site.parameters.appId, site.parameters.apiKey);
}
const res = await unprint.get(referer);
const apiLine = res.body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
if (!apiLine) {
throw new Error(`No Gamma API key found for ${referer}`);
}
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
const apiData = JSON.parse(apiSerial);
const { applicationID: appId, apiKey } = apiData.api.algolia;
return getApiUrl(appId, apiKey);
}
async function fetchLatest(site, page = 1, options, _preData, upcoming = false) {
const referer = options.parameters?.referer || `${options.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
const { apiUrl } = await fetchApiCredentials(referer, site);
const slug = options.parameters.querySlug || site.slug;
@@ -308,7 +137,7 @@ async function fetchLatestApi(site, page = 1, options, _preData, upcoming = fals
? `&filters=channels.id:${options.parameters.queryChannel === true ? slug : options.parameters.queryChannel}`
: `&filters=availableOnSite:${slug}`}`;
const res = await http.post(apiUrl, {
const res = await unprint.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
@@ -319,213 +148,97 @@ async function fetchLatestApi(site, page = 1, options, _preData, upcoming = fals
headers: {
Referer: referer,
},
}, {
encodeJSON: true,
});
if (res.status === 200 && res.body.results?.[0]?.hits) {
return scrapeApiReleases(res.body.results[0].hits, site, options);
if (res.ok && res.data.results?.[0]?.hits) {
return scrapeApiReleases(res.data.results[0].hits, site, options);
}
return res.status;
}
function scrapeAll(scenes, site, networkUrl, hasTeaser = true) {
return scenes.map(({ query, el }) => {
const release = {};
release.url = query.url('.sceneTitle a, .tlcTitle a', 'href', { origin: networkUrl ? site.parent.url : site.url });
release.title = query.cnt('.sceneTitle a', 'tlcTitle a', 'title');
release.entryId = el.dataset.itemid;
release.date = query.date('.sceneDate, .tlcSpecsDate .tlcDetailsValue', ['MM-DD-YYYY', 'YYYY-MM-DD']);
release.actors = query.cnts('.sceneActors a, .tlcActors a', ' title');
[release.likes, release.dislikes] = query.all('.value').map((likeEl) => query.number(likeEl));
release.poster = query.img('.imgLink img, .tlcImageItem', 'data-original') || query.img('.imgLink img, .tlcImageItem');
if (hasTeaser) {
release.teaser = [
{ src: `https://videothumb.gammacdn.com/600x339/${release.entryId}.mp4` },
{ src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4` },
];
}
release.channel = query.el('.fromSite a', 'title')?.replace('.com', '');
return release;
});
}
function getLatestUrl(site, page) {
if (site.parameters?.latest) {
if (/^http/.test(site.parameters.latest)) {
return /%d/.test(site.parameters.latest)
? util.format(site.parameters.latest, page)
: `${site.parameters.latest}${page}`;
}
return /%d/.test(site.parameters.latest)
? util.format(`${site.url}${site.parameters.latest}`, page)
: `${site.url}${site.parameters.latest}${page}`;
}
return `${site.url}/en/videos/AllCategories/0/${page}`;
}
function getUpcomingUrl(site) {
if (site.parameters?.upcoming) {
return /^http/.test(site.parameters.upcoming)
? `${site.parameters.upcoming}`
: `${site.url}${site.parameters.upcoming}`;
}
return `${site.url}/en/videos/AllCategories/0/1/upcoming`;
}
async function fetchLatest(site, page = 1) {
const url = getLatestUrl(site, page);
const res = await qu.getAll(url, 'li[data-itemtype=scene], div[data-itemtype*=scene]');
if (res.ok) {
return scrapeAll(res.items, site);
}
return res.status;
}
async function fetchUpcoming(site) {
const url = getUpcomingUrl(site);
const res = await qu.getAll(url, 'li[data-itemtype=scene], div[data-itemtype*=scene]');
if (res.ok) {
return scrapeAll(res.items, site, null, false);
}
return res.status;
}
async function scrapeScene({ query }, url, channel, baseRelease, mobileItem, options) {
const release = { query }; // used by XEmpire scraper to resolve channel-specific details
const json = query.html('script[type="application/ld+json"]');
const videoJson = query.htmls('script').find((script) => /ScenePlayerOptions/i.test(script));
const [data, data2] = json ? JSON.parse(json) : [];
const videoData = videoJson && JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
release.entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name;
release.description = data?.description;
release.date = query.date('.updatedDate', ['MM-DD-YYYY', 'YYYY-MM-DD'])
|| qu.extractDate(data?.dateCreated, 'YYYY-MM-DD')
|| videoData?.playerOptions?.sceneInfos.sceneReleaseDate;
release.actors = (data?.actor || data2?.actor)?.map((actor) => ({
name: actor.name,
gender: actor.gender,
})) || [];
release.duration = qu.durationToSeconds(data.duration);
release.director = data?.director?.[0]?.name || data2?.director?.[0]?.name;
release.tags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
release.stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5 || null;
release.channel = slugify(data?.productionCompany?.name
|| query.el('.studioLink a, .siteLink a', 'title')
|| query.cnt('.siteNameSpan')?.toLowerCase().replace('.com', '')
|| query.meta('meta[name="twitter:domain"]')?.replace('.com', ''), '');
if (videoData?.picPreview && new URL(videoData.picPreview).pathname.length > 1) {
// sometimes links to just https://images02-fame.gammacdn.com/
const poster = new URL(videoData.picPreview);
release.poster = [
videoData.picPreview, // prefer original URL with width and height parameters, without may give a square crop on e.g. XEmpire
`${poster.origin}${poster.pathname}`,
];
}
const photoLink = query.url('.picturesItem a');
const mobilePhotos = mobileItem?.query.imgs('.preview-displayer a img') || [];
if (photoLink && options.includePhotos) {
const photos = await getPhotos(photoLink, channel, mobilePhotos.length < 3); // only get thumbnails when less than 3 mobile photos are available
if (photos.length < 7) {
release.photos = [...photos, ...mobilePhotos]; // probably only teaser photos available, supplement with mobile album
} else {
release.photos = photos;
}
} else {
release.photos = mobilePhotos;
}
const trailer = videoData && `${videoData.playerOptions.host}${videoData.url}`;
if (trailer) {
release.trailer = [
{
src: trailer.replace('hd', 'sm'),
quality: 240,
},
{
src: trailer.replace('hd', 'med'),
quality: 360,
},
{
src: trailer.replace('hd', 'big'),
quality: 480,
},
{
// probably 540p
src: trailer,
quality: parseInt(videoData.sizeOnLoad, 10),
},
{
src: trailer.replace('hd', '720p'),
quality: 720,
},
{
src: trailer.replace('hd', '1080p'),
quality: 1080,
},
{
src: trailer.replace('hd', '4k'),
quality: 2160,
},
];
}
const movieUrl = query.url('.dvdLink', 'href', { origin: channel.url });
if (movieUrl) {
release.movie = {
url: movieUrl,
title: query.el('.dvdLink', 'title'),
entryId: movieUrl.match(/\/(\d+)(\/|$)/)?.[1],
covers: [qu.imgs('.dvdLink img')],
};
}
return release;
async function fetchUpcoming(site, page = 1, options, preData) {
return fetchLatest(site, page, options, preData, true);
}
const qualityMap = {
'4k': 2160,
};
async function scrapeReleaseApi(data, site, options, movieScenes) {
async function getFullPhotos(entryId, site, parameters) {
const res = await unprint.get(`${parameters.album || site.url}/media/signPhotoset/${entryId}`, {
headers: {
'X-Requested-With': 'XMLHttpRequest',
},
});
if (res.ok && typeof res.data === 'object') { // gives 200 OK even when redirected to signup page
return Object.values(res.data);
}
return [];
}
async function getThumbs(entryId, site, parameters) {
const referer = parameters?.referer || `${parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
const { apiUrl } = await fetchApiCredentials(referer, site);
const res = await unprint.post(apiUrl, {
requests: [
{
indexName: 'all_photosets',
params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["set_id:${entryId}"]]`,
},
],
}, {
headers: {
Referer: referer,
},
});
if (res.ok && res.data.results?.[0]?.hits[0]?.set_pictures) {
return res.data.results[0].hits[0].set_pictures.map((img) => img.thumb_path && [
'https://images-fame.gammacdn.com',
'https://images01-fame.gammacdn.com',
'https://images02-fame.gammacdn.com',
'https://images03-fame.gammacdn.com',
'https://images04-fame.gammacdn.com',
'https://images-evilangel.gammacdn.com',
'https://transform.gammacdn.com',
].map((origin) => unprint.prefixUrl(`/photo_set${img.thumb_path}`, origin))).filter(Boolean);
}
return [];
}
function getImageSources(source) {
return [
'https://images-fame.gammacdn.com',
'https://images01-fame.gammacdn.com',
'https://images02-fame.gammacdn.com',
'https://images03-fame.gammacdn.com',
'https://images04-fame.gammacdn.com',
'https://images-evilangel.gammacdn.com',
'https://transform.gammacdn.com',
].map((origin) => unprint.prefixUrl(`/movies${source}`, origin));
}
async function getPhotosApi(entryId, site, parameters) {
const [photos, thumbs] = await Promise.all([
getFullPhotos(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
getThumbs(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }),
]);
return photos.concat(thumbs.slice(photos.length));
}
async function scrapeScene(data, site, options, movieScenes) {
const release = {};
release.entryId = data.clip_id || data.movie_id;
release.title = curateTitle(data.title, site);
release.duration = data.length;
release.date = (data.date && new Date(data.date * 1000)) || qu.parseDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
release.date = (data.date && new Date(data.date * 1000)) || unprint.parseDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
release.director = data.directors[0]?.name || null;
release.actors = data.actors.map((actor) => ({
@@ -534,7 +247,7 @@ async function scrapeReleaseApi(data, site, options, movieScenes) {
gender: actor.gender,
url: options.parameters?.actors
? format(options.parameters.actors, { id: actor.actor_id, slug: actor.url_name })
: qu.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
: unprint.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
}));
release.tags = data.categories.map((category) => category.name);
@@ -571,12 +284,12 @@ async function scrapeReleaseApi(data, site, options, movieScenes) {
release.movie = {
entryId: data.movie_id,
title: data.movie_title,
url: qu.prefixUrl(`${data.url_movie_title}/${data.movie_id}`, options.parameters.movie ? options.parameters.movie : `${site.url}/en/movie`),
url: unprint.prefixUrl(`${data.url_movie_title}/${data.movie_id}`, options.parameters.movie ? options.parameters.movie : `${site.url}/en/movie`),
};
}
if (movieScenes?.length > 0) {
release.scenes = await Promise.all(movieScenes.map((movieScene) => scrapeReleaseApi(movieScene, site, options)));
release.scenes = await Promise.all(movieScenes.map((movieScene) => scrapeScene(movieScene, site, options)));
}
release.channel = slugify(data.mainChannel?.id || data.sitename, ''); // remove -
@@ -585,148 +298,13 @@ async function scrapeReleaseApi(data, site, options, movieScenes) {
return release;
}
async function fetchMovieTrailer(release) {
if (!release.entryId) {
return null;
}
const url = `https://www.evilangel.com/en/dvdtrailer/${release.entryId}`;
const res = await qu.get(url);
if (!res.ok) {
return null;
}
const trailerHost = res.html.match(/"host":\s*"(.*\.com)"/)?.[1].replace(/\\\//g, '/');
const trailerPath = res.html.match(/"url":\s*"(.*\.mp4)"/)?.[1].replace(/\\\//g, '/');
if (trailerHost && trailerPath) {
return qu.prefixUrl(trailerPath, trailerHost);
}
return null;
}
async function scrapeMovie({ query, el }, url, entity, baseRelease, options) {
const release = {};
const { dataLayer } = query.exec('//script[contains(text(), "dataLayer")]', ['dataLayer']);
const rawData = dataLayer?.[0]?.dvdDetails;
const data = rawData?.dvdId && rawData; // dvdDetails is mostly empty in some cache states
if (query.exists('.NotFound-Title')) {
return null;
}
release.entryId = new URL(url).pathname.match(/\/(\d+)(\/|$)/)?.[1];
release.covers = [
query.img('.frontCoverImg', 'href'),
query.img('.backCoverImg', 'href'),
];
release.description = query.cnt('.descriptionText');
release.date = qu.extractDate(data?.dvdReleaseDate) || query.date('.updatedOn', 'YYYY-MM-DD');
release.title = data?.dvdName || query.cnt('.dvdTitle');
release.director = query.el('.directedBy a', 'title');
release.actors = data?.dvdActors.map((actor) => ({ name: actor.actorName, entryId: actor.actorId }))
|| query.all('.actorCarousel a[href*="/pornstar"]').map((actorEl) => ({
entryId: query.url(actorEl, null).match(/\/(\d+)/)?.[1],
name: query.cnt(actorEl, 'span'),
href: query.url(actorEl, null, 'href', { origin: entity.url }),
avatar: getAvatarFallbacks(query.img(actorEl)),
}));
release.tags = query.cnts('.dvdCol a');
release.scenes = scrapeAll(qu.initAll(el, 'div[data-itemtype*=scene], li[data-itemtype*=scene]'), entity, entity.url);
if (options.includeTrailers) {
release.trailer = await fetchMovieTrailer(release);
}
return release;
}
function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
return actorLink ? actorLink.href : null;
}
async function fetchActorReleases(profileUrl, getActorReleasesUrl, page = 1, accReleases = [], context) {
const { origin, pathname } = new URL(profileUrl);
const profilePath = `/${pathname.split('/').slice(-2).join('/')}`;
const url = (context.parameters.actorScenes && format(context.parameters.actorScenes, { path: profilePath, page }))
|| getActorReleasesUrl?.(profilePath, page);
if (!url) {
return [];
}
const res = await qu.get(url);
if (!res.ok) {
return [];
}
const releases = scrapeAll(res.item.html, null, origin);
const nextPage = res.item.query.url('.Gamma_Paginator a.next');
if (nextPage) {
return fetchActorReleases(profileUrl, getActorReleasesUrl, page + 1, accReleases.concat(releases), context);
}
return accReleases.concat(releases);
}
async function scrapeProfile({ query }, url, actorName, _siteSlug, getActorReleasesUrl, withReleases, context) {
const avatar = query.img('img.actorPicture');
const hair = query.cnt('.actorProfile .attribute_hair_color');
const height = query.cnt('.actorProfile .attribute_height');
const weight = query.cnt('.actorProfile .attribute_weight');
const alias = query.cnt('.actorProfile .attribute_alternate_names');
const nationality = query.cnt('.actorProfile .attribute_home');
const profile = {
name: actorName,
};
if (avatar) {
// larger sizes usually available, provide fallbacks
const avatars = getAvatarFallbacks(avatar);
profile.avatar = avatars;
}
profile.description = query.cnt('.actorBio p:not(.bioTitle)');
if (hair) profile.hairColor = hair.split(':')[1].trim();
if (height) profile.height = Number(height.match(/\d+/)[0]);
if (weight) profile.weight = Number(weight.match(/\d+/)[0]);
if (alias) profile.aliases = alias.split(':')[1].trim().split(', ');
if (nationality) profile.nationality = nationality.split(':')[1].trim();
if ((getActorReleasesUrl || context.parameters.actorScenes) && withReleases) {
profile.releases = await fetchActorReleases(url, getActorReleasesUrl, 1, [], context);
}
return profile;
}
async function fetchUpcomingApi(site, page = 1, options, preData) {
return fetchLatestApi(site, page, options, preData, true);
}
async function fetchSceneApi(url, site, baseRelease, options) {
async function fetchScene(url, site, baseRelease, options) {
const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
const { apiUrl } = await fetchApiCredentials(referer, site);
const entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
const res = await http.post(apiUrl, {
const res = await unprint.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
@@ -741,28 +319,26 @@ async function fetchSceneApi(url, site, baseRelease, options) {
headers: {
Referer: referer,
},
}, {
encodeJSON: true,
});
if (res.status === 200 && res.body.results?.[0]?.hits.length > 0) {
return scrapeReleaseApi(res.body.results[0].hits[0], site, options);
if (res.ok && res.data.results?.[0]?.hits.length > 0) {
return scrapeScene(res.data.results[0].hits[0], site, options);
}
if (res.status === 200) {
if (res.ok) {
return null;
}
return res.status;
}
async function fetchMovieApi(url, site, baseRelease, options) {
async function fetchMovie(url, site, baseRelease, options) {
const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/movies`;
const { apiUrl } = await fetchApiCredentials(referer, site);
const entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];
const res = await http.post(apiUrl, {
const res = await unprint.post(apiUrl, {
requests: [
{
indexName: 'all_movies',
@@ -783,12 +359,10 @@ async function fetchMovieApi(url, site, baseRelease, options) {
headers: {
Referer: referer,
},
}, {
encodeJSON: true,
});
if (res.status === 200 && res.body.results?.[0]?.hits.length > 0) {
return scrapeReleaseApi(res.body.results[0].hits[0], site, options, res.body.results[1]?.hits);
if (res.ok && res.data.results?.[0]?.hits.length > 0) {
return scrapeScene(res.data.results[0].hits[0], site, options, res.data.results[1]?.hits);
}
if (res.status === 200) {
@@ -798,62 +372,8 @@ async function fetchMovieApi(url, site, baseRelease, options) {
return res.status;
}
function getDeepUrl(url, site, baseRelease, mobile) {
const filter = new Set(['en', 'video', 'scene', site.slug, site.parent.slug]);
const pathname = baseRelease?.path || new URL(url).pathname
.split('/')
.filter((component) => !filter.has(component))
.join('/'); // reduce to scene ID and title slug
const sceneId = baseRelease?.entryId || pathname.match(/\/(\d+)\//)?.[1];
if (mobile && /%d/.test(mobile)) {
return util.format(mobile, sceneId);
}
if (mobile && sceneId) {
return `${mobile}${pathname}`;
}
if (site.parameters?.deep) {
return `${site.parameters.deep}${pathname}`;
}
return url;
}
async function fetchScene(url, site, baseRelease, options) {
if (site.parameters?.deep === false) {
return baseRelease;
}
const deepUrl = getDeepUrl(url, site, baseRelease);
const mobileUrl = options.includePhotos && getDeepUrl(url, site, baseRelease, site.parameters?.mobile || site.parent?.parameters?.mobile);
if (deepUrl) {
const [res, mobileRes] = await Promise.all([
qu.get(deepUrl),
mobileUrl && qu.get(mobileUrl, null, {
headers: {
// don't redirect to main site
'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Mobile Safari/537.36',
},
}),
]);
if (res.status === 200) {
const mobileItem = mobileRes?.status === 200 ? mobileRes.item : null;
const scene = await scrapeScene(res.item, url, site, baseRelease, mobileItem, options);
return { ...scene, deepUrl };
}
}
return null;
}
async function fetchActorScenes(actorName, apiUrl, siteSlug) {
const res = await http.post(apiUrl, {
const res = await unprint.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
@@ -864,47 +384,16 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
headers: {
Referer: `https://www.${siteSlug}.com/en/videos`,
},
}, {
encodeJSON: true,
});
if (res.status === 200 && res.body.results[0].hits.length > 0) {
return res.body.results[0].hits;
if (res.ok && res.data.results[0].hits.length > 0) {
return res.data.results[0].hits;
}
return [];
}
async function fetchProfile({ name: actorName }, context, include, altSearchUrl, getActorReleasesUrl) {
const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug;
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
const searchUrl = altSearchUrl
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
: `https://www.${siteSlug}.com/en/search/${siteSlug}/actor/${actorSlug}`;
const searchRes = await http.get(searchUrl);
if (searchRes.status !== 200) {
return null;
}
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
if (actorUrl) {
const url = `https://${siteSlug}.com${actorUrl}`;
const actorRes = await qu.get(url);
if (actorRes.status !== 200) {
return null;
}
return scrapeProfile(actorRes.item, url, actorName, siteSlug, getActorReleasesUrl, include.scenes, context);
}
return null;
}
function scrapeApiProfile(data, releases, siteSlug) {
function scrapeProfile(data, releases, siteSlug) {
const profile = {};
if (data.male === 1) profile.gender = 'male';
@@ -925,7 +414,7 @@ function scrapeApiProfile(data, releases, siteSlug) {
return profile;
}
async function fetchApiProfile({ name: actorName }, context, include) {
async function fetchProfile({ name: actorName }, context, include) {
const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug;
const actorSlug = encodeURI(actorName);
const referer = context.parameters.profileReferer || `${context.entity.origin}/en/search`;
@@ -940,7 +429,7 @@ async function fetchApiProfile({ name: actorName }, context, include) {
: []),
]).map((site) => `"availableOnSite:${site}"`).join(',');
const res = await http.post(apiUrl, {
const res = await unprint.post(apiUrl, {
requests: [
{
indexName: 'all_actors',
@@ -952,17 +441,15 @@ async function fetchApiProfile({ name: actorName }, context, include) {
headers: {
Referer: referer,
},
}, {
encodeJSON: true,
});
if (res.status === 200 && res.body.results[0].hits.length > 0) {
const actorData = res.body.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
if (res.status === 200 && res.data.results[0].hits.length > 0) {
const actorData = res.data.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
if (actorData) {
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);
return scrapeApiProfile(actorData, actorScenes, siteSlug);
return scrapeProfile(actorData, actorScenes, siteSlug);
}
}
@@ -970,31 +457,9 @@ async function fetchApiProfile({ name: actorName }, context, include) {
}
module.exports = {
fetchApiLatest: fetchLatestApi,
fetchApiProfile,
fetchApiUpcoming: fetchUpcomingApi,
fetchLatest,
fetchLatestApi,
fetchUpcoming,
fetchProfile,
fetchScene,
fetchSceneApi,
fetchUpcoming,
fetchUpcomingApi,
api: {
fetchLatest: fetchLatestApi,
fetchUpcoming: fetchUpcomingApi,
fetchProfile: fetchApiProfile,
// fetchScene,
fetchScene: fetchSceneApi,
// scrapeMovie,
fetchMovie: fetchMovieApi,
},
getPhotos,
scrapeApiProfile,
scrapeApiReleases,
scrapeProfile,
scrapeAll,
scrapeMovie,
scrapeScene,
deprecated: true,
fetchMovie,
};

226
src/scrapers/jayrock.js
View File

@@ -1,226 +0,0 @@
'use strict';
const Promise = require('bluebird');
const logger = require('../logger');
const { fetchApiLatest } = require('./gamma');
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
async function fetchActors(entryId, channel, { token, time }) {
const url = `${channel.url}/sapi/${token}/${time}/model.getModelContent?_method=model.getModelContent&tz=1&fields[0]=modelId.stageName&fields[1]=_last&fields[2]=modelId.upsellLink&fields[3]=modelId.upsellText&limit=25&transitParameters[contentId]=${entryId}`;
const res = await http.get(url);
if (res.statusCode === 200 && res.body.status === true) {
return Object.values(res.body.response.collection).map((actor) => Object.values(actor.modelId.collection)[0].stageName);
}
return [];
}
async function fetchTrailerLocation(entryId, channel) {
const url = `${channel.url}/api/download/${entryId}/hd1080/stream`;
try {
const res = await http.get(url, {
followRedirects: false,
});
if (res.statusCode === 302) {
return res.headers.location;
}
} catch (error) {
logger.warn(`${channel.name}: Unable to fetch trailer at '${url}': ${error.message}`);
}
return null;
}
function scrapeLatest(items, channel) {
return items.map(({ query }) => {
const release = {};
release.url = query.url('h5 a', null, { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
release.title = query.cnt('h5 a');
[release.poster, ...release.photos] = query.imgs('.screenshot').map((src) => [
// unnecessarily large
// src.replace(/\/\d+/, 3840),
// src.replace(/\/\d+/, '/2000'),
src.replace(/\/\d+/, '/1500'),
src.replace(/\/\d+/, '/1000'),
src,
]);
return release;
});
}
function scrapeScene({ query, html }, url, channel) {
const release = {};
release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1];
release.title = query.cnt('h1.description');
release.actors = query
.all('.video-performer')
.map((actorEl) => {
const actorUrl = query.url(actorEl, 'a', 'href', { origin: channel.url });
const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
const avatar = query.img(actorEl, 'img:not([data-bgsrc*="not-available"])', 'data-bgsrc');
return {
name: query.cnt(actorEl, '.video-performer-name'),
gender: 'female',
avatar: avatar && [
avatar.replace(/\/actor\/(\d+)/, '/actor/500'),
avatar,
],
url: actorUrl,
entryId,
};
})
.concat({ name: 'Jay Rock', gender: 'male' });
release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.number('.release-date:last-child') * 60;
release.studio = query.cnt('.studio span:nth-child(2)');
release.director = query.text('.director');
release.tags = query.cnts('.tags a');
const poster = html.match(/url\((https.+\.jpg)\)/)?.[1];
const photos = query.imgs('#moreScreenshots img');
[release.poster, ...release.photos] = [poster]
.concat(photos)
.filter(Boolean)
.map((src) => [
src.replace(/\/(\d+)\/\d+/, '/$1/1500'),
src.replace(/\/(\d+)\/\d+/, '/$1/1000'),
src,
]);
const videoId = html.match(/item: (\d+)/)?.[1];
if (videoId) {
release.trailer = { stream: `https://trailer.adultempire.com/hls/trailer/${videoId}/master.m3u8` };
}
return release;
}
async function scrapeSceneApi(scene, channel, tokens, deep) {
const release = {
entryId: scene.id,
title: scene.title,
duration: scene.length,
meta: {
tokens, // attach tokens to reduce number of requests required for deep fetching
},
};
release.url = `${channel.url}/scene/${release.entryId}/${slugify(release.title, { encode: true })}`;
release.date = new Date(scene.sites.collection[scene.id].publishDate);
release.poster = scene._resources.primary[0].url;
if (scene.tags) release.tags = Object.values(scene.tags.collection).map((tag) => tag.alias);
if (scene._resources.base) release.photos = scene._resources.base.map((resource) => resource.url);
if (deep) {
// don't make external requests during update scraping, as this would happen for every scene on the page
const [actors, trailer] = await Promise.all([
fetchActors(release.entryId, channel, tokens),
fetchTrailerLocation(release.entryId, channel),
]);
release.actors = actors;
if (trailer) {
release.trailer = { src: trailer, quality: 1080 };
}
}
return release;
}
function scrapeLatestApi(scenes, site, tokens) {
return Promise.map(scenes, async (scene) => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
}
async function fetchToken(channel) {
const res = await http.get(channel.url);
const html = res.body.toString();
const time = html.match(/"aet":\d+/)[0].split(':')[1];
const ah = html.match(/"ah":"[\w-]+"/)[0].split(':')[1].slice(1, -1);
const token = ah.split('').reverse().join('');
return { time, token };
}
async function fetchLatestApi(channel, page = 1) {
const { time, token } = await fetchToken(channel);
// transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory
const url = `${channel.url}/sapi/${token}/${time}/content.load?limit=50&offset=${(page - 1) * 50}&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`;
const res = await http.get(url);
if (res.ok && res.body.status) {
return scrapeLatestApi(res.body.response.collection, channel, { time, token });
}
return res.ok ? res.body.status : res.status;
}
async function fetchLatest(channel, page = 1, options, preData) {
if (channel.parameters?.useGamma) {
return fetchApiLatest(channel, page, preData, options, false);
}
const res = await qu.getAll(`https://jayspov.net/jays-pov-updates.html?view=list&page=${page}`, '.item-grid-list-view > .grid-item');
if (res.ok) {
return scrapeLatest(res.items, channel);
}
return res.status;
}
async function fetchSceneApi(url, channel, baseRelease) {
const { time, token } = baseRelease?.meta.tokens || await fetchToken(channel); // use attached tokens when deep fetching
const { pathname } = new URL(url);
const entryId = pathname.split('/')[2];
const apiUrl = `${channel.url}/sapi/${token}/${time}/content.load?filter[id][fields][0]=id&filter[id][values][0]=${entryId}&transitParameters[v1]=ykYa8ALmUD&transitParameters[preset]=scene`;
const res = await http.get(apiUrl);
if (res.ok && res.body.status) {
return scrapeSceneApi(res.body.response.collection[0], channel, { time, token }, true);
}
return res.ok ? res.body.status : res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
if (res.ok) {
return scrapeScene(res.item, url, channel);
}
return res.status;
}
module.exports = {
fetchLatest,
fetchScene,
api: {
fetchLatest: fetchLatestApi,
fetchScene: fetchSceneApi,
},
};

3
src/scrapers/releases.js
View File

@@ -19,7 +19,6 @@ const czechav = require('./czechav');
const modelmedia = require('./modelmedia');
const dorcel = require('./dorcel');
const fabulouscash = require('./fabulouscash');
// const famedigital = require('./famedigital');
const firstanalquest = require('./firstanalquest');
const elevatedx = require('./elevatedx');
const exploitedx = require('./exploitedx');
@@ -31,7 +30,6 @@ const hush = require('./hush');
const innofsin = require('./innofsin');
const insex = require('./insex');
const inthecrack = require('./inthecrack');
const jayrock = require('./jayrock');
const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials');
const julesjordan = require('./julesjordan');
const karups = require('./karups');
@@ -137,7 +135,6 @@ module.exports = {
insex,
interracialpass: hush,
inthecrack,
jayrock,
jerkaoke: modelmedia,
jesseloadsmonsterfacials,
julesjordan,

2
src/updates.js
View File

@@ -314,8 +314,6 @@ async function scrapeNetworkParallel(networkEntity) {
async function fetchUpdates() {
const includedNetworks = await fetchIncludedEntities();
// console.log(includedNetworks[0]);
const scrapedNetworks = await Promise.map(
includedNetworks,
async (networkEntity) => (networkEntity.parameters?.sequential