Refactored 21Sextury module to use Gamma API.
This commit is contained in:
parent
ee8994582c
commit
65f98c6387
|
|
@ -16,6 +16,7 @@ const sites = [
|
|||
url: 'https://www.assholefever.com',
|
||||
description: 'Welcome to AssholeFever, the most hardcore anal site on the net. Watch your favorite pornstars and anal sluts from all over the world in big booty hardcore porn, anal gape, beads, anal creampie and more! Look inside if you dare!',
|
||||
network: '21sextury',
|
||||
parameters: { networkReferer: true },
|
||||
},
|
||||
{
|
||||
slug: 'buttplays',
|
||||
|
|
|
|||
|
|
@ -82,6 +82,11 @@ async function scrapeReleases(sources, release = null, type = 'scene') {
|
|||
|
||||
const curatedReleases = scrapedReleases.map(scrapedRelease => ({ ...scrapedRelease, type }));
|
||||
|
||||
if (argv.scene && argv.inspect) {
|
||||
// only show when fetching from URL
|
||||
console.log(curatedReleases);
|
||||
}
|
||||
|
||||
if (argv.save) {
|
||||
/*
|
||||
const movie = scrapedRelease.movie
|
||||
|
|
|
|||
|
|
@ -158,8 +158,14 @@ async function scrapeSites() {
|
|||
concurrency: 5,
|
||||
});
|
||||
|
||||
const releases = scrapedNetworks.flat(2);
|
||||
|
||||
if (argv.inspect) {
|
||||
console.log(releases);
|
||||
}
|
||||
|
||||
if (argv.save) {
|
||||
await storeReleases(scrapedNetworks.flat(2));
|
||||
await storeReleases(releases);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,151 +1,10 @@
|
|||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
|
||||
const { getPhotos, fetchProfile } = require('./gamma');
|
||||
|
||||
function scrape(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const scenesElements = $('li[data-itemtype=scene]').toArray();
|
||||
|
||||
return scenesElements.reduce((accReleases, element) => {
|
||||
const siteName = $(element).find('.studioName a').attr('title');
|
||||
|
||||
if (!site.url && siteName.toLowerCase() !== site.name.toLowerCase()) {
|
||||
// using generic overview as fallback, scene from different site
|
||||
return accReleases;
|
||||
}
|
||||
|
||||
const sceneLinkElement = $(element).find('.sceneTitle a');
|
||||
const url = `${site.url || 'https://www.21sextury.com'}${sceneLinkElement.attr('href')}`;
|
||||
const title = sceneLinkElement.attr('title').trim();
|
||||
|
||||
const entryId = $(element).attr('data-itemid');
|
||||
|
||||
const date = moment
|
||||
.utc($(element).find('.sceneDate').text(), 'MM-DD-YYYY')
|
||||
.toDate();
|
||||
|
||||
const actors = $(element).find('.sceneActors a')
|
||||
.map((actorIndex, actorElement) => $(actorElement).attr('title'))
|
||||
.toArray();
|
||||
|
||||
const poster = $(element).find('.imgLink img').attr('data-original');
|
||||
const trailer = `https://videothumb.gammacdn.com/307x224/${entryId}.mp4`;
|
||||
|
||||
const [likes, dislikes] = $(element).find('.value')
|
||||
.toArray()
|
||||
.map(value => Number($(value).text()));
|
||||
|
||||
return [
|
||||
...accReleases,
|
||||
{
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
actors,
|
||||
date,
|
||||
poster,
|
||||
trailer: {
|
||||
src: trailer,
|
||||
},
|
||||
rating: {
|
||||
likes,
|
||||
dislikes,
|
||||
},
|
||||
site,
|
||||
},
|
||||
];
|
||||
}, []);
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const sceneElement = $('#videoWrapper');
|
||||
const json = $('script[type="application/ld+json"]').html();
|
||||
const videoJson = $('script:contains("ScenePlayerOptions")').html();
|
||||
const videoDataString = videoJson.slice(videoJson.indexOf('= {') + 2, videoJson.indexOf('};') + 1);
|
||||
|
||||
const data = JSON.parse(json)[0];
|
||||
const videoData = JSON.parse(videoDataString);
|
||||
const entryId = new URL(url).pathname.split('/').slice(-1)[0];
|
||||
|
||||
const title = videoData?.playerOptions?.sceneInfos?.sceneTitle || (data.isPartOf && data.isPartOf !== 'TBD' ? data.isPartOf.name : data.name);
|
||||
const dataDate = moment.utc(videoData?.playerOptions?.sceneInfos?.sceneReleaseDate, 'YYYY-MM-DD');
|
||||
|
||||
const date = dataDate.isValid()
|
||||
? dataDate.toDate()
|
||||
: moment.utc(sceneElement.find('.updatedDate').text().trim(), 'MM-DD-YYYY').toDate();
|
||||
|
||||
const actors = data.actor.map(actor => actor.name);
|
||||
|
||||
const description = data.description || null; // prevent empty string
|
||||
const likes = Number(sceneElement.find('.rating .state_1 .value').text());
|
||||
const dislikes = Number(sceneElement.find('#infoWrapper .rating .state_2 .value').text());
|
||||
|
||||
const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
|
||||
|
||||
const poster = videoData.picPreview;
|
||||
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
||||
|
||||
const photos = await getPhotos($('.picturesItem a').attr('href'), site);
|
||||
|
||||
const tags = data.keywords.split(', ');
|
||||
const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
|
||||
const channel = siteName && siteName.replace(/\s+/g, '').toLowerCase();
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
date,
|
||||
actors,
|
||||
description,
|
||||
duration,
|
||||
tags,
|
||||
poster,
|
||||
photos,
|
||||
trailer: {
|
||||
src: trailer,
|
||||
},
|
||||
rating: {
|
||||
likes,
|
||||
dislikes,
|
||||
},
|
||||
site,
|
||||
channel,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.url || 'https://21sextury.com'}/en/videos/All-Categories/0/All-Pornstars/0/latest/${page}`;
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return scrape(res.body.toString(), site);
|
||||
}
|
||||
|
||||
async function fetchUpcoming(site) {
|
||||
const url = `${site.url || 'https://21sextury.com'}/en/videos/All-Categories/0/All-Pornstars/0/upcoming`;
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return scrape(res.body.toString(), site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return scrapeScene(res.body.toString(), url, site);
|
||||
}
|
||||
|
||||
async function networkFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, '21sextury', true);
|
||||
}
|
||||
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchUpcoming,
|
||||
fetchLatest: fetchApiLatest,
|
||||
fetchProfile: fetchApiProfile,
|
||||
fetchUpcoming: fetchApiUpcoming,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ const { JSDOM } = require('jsdom');
|
|||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
|
||||
const { ex } = require('../utils/q');
|
||||
|
||||
async function fetchPhotos(url) {
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
|
|
@ -237,39 +239,45 @@ function scrapeActorSearch(html, url, actorName) {
|
|||
|
||||
function scrapeProfile(html, url, actorName, siteSlug) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const { q, qu } = ex(html);
|
||||
|
||||
const avatarEl = document.querySelector('img.actorPicture');
|
||||
const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
|
||||
const hairEl = document.querySelector('.actorProfile .attribute_hair_color');
|
||||
const heightEl = document.querySelector('.actorProfile .attribute_height');
|
||||
const weightEl = document.querySelector('.actorProfile .attribute_weight');
|
||||
const aliasEl = document.querySelector('.actorProfile .attribute_alternate_names');
|
||||
const nationalityEl = document.querySelector('.actorProfile .attribute_home');
|
||||
const avatar = q('img.actorPicture');
|
||||
const hair = q('.actorProfile .attribute_hair_color', true);
|
||||
const height = q('.actorProfile .attribute_height', true);
|
||||
const weight = q('.actorProfile .attribute_weight', true);
|
||||
const alias = q('.actorProfile .attribute_alternate_names', true);
|
||||
const nationality = q('.actorProfile .attribute_home', true);
|
||||
|
||||
const profile = {
|
||||
name: actorName,
|
||||
};
|
||||
|
||||
if (avatarEl) {
|
||||
if (avatar) {
|
||||
// larger sizes usually available, provide fallbacks
|
||||
const avatars = [
|
||||
avatarEl.src.replace(/\d+x\d+/, '500x750'),
|
||||
avatarEl.src.replace(/\d+x\d+/, '240x360'),
|
||||
avatarEl.src.replace(/\d+x\d+/, '200x300'),
|
||||
avatarEl.src,
|
||||
avatar.src.replace(/\d+x\d+/, '500x750'),
|
||||
avatar.src.replace(/\d+x\d+/, '240x360'),
|
||||
avatar.src.replace(/\d+x\d+/, '200x300'),
|
||||
avatar.src,
|
||||
];
|
||||
|
||||
profile.avatar = avatars;
|
||||
}
|
||||
|
||||
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
|
||||
if (hairEl) profile.hair = hairEl.textContent.split(':')[1].trim();
|
||||
if (heightEl) profile.height = Number(heightEl.textContent.match(/\d+/)[0]);
|
||||
if (weightEl) profile.weight = Number(weightEl.textContent.match(/\d+/)[0]);
|
||||
if (aliasEl) profile.aliases = aliasEl.textContent.split(':')[1].trim().split(', ');
|
||||
if (nationalityEl) profile.nationality = nationalityEl.textContent.split(':')[1].trim();
|
||||
profile.description = q('.actorBio p:not(.bioTitle)', true);
|
||||
|
||||
if (hair) profile.hair = hair.split(':')[1].trim();
|
||||
if (height) profile.height = Number(height.match(/\d+/)[0]);
|
||||
if (weight) profile.weight = Number(weight.match(/\d+/)[0]);
|
||||
if (alias) profile.aliases = alias.split(':')[1].trim().split(', ');
|
||||
if (nationality) profile.nationality = nationality.split(':')[1].trim();
|
||||
|
||||
/* not fetching all releases
|
||||
profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://${siteSlug}.com${el.href}`);
|
||||
const moreReleases = qu('.seeAllTop a');
|
||||
|
||||
console.log(moreReleases);
|
||||
*/
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
|
@ -322,7 +330,7 @@ async function fetchApiCredentials(referer) {
|
|||
}
|
||||
|
||||
async function fetchApiLatest(site, page = 1, upcoming = false) {
|
||||
const referer = `${site.url}/en/videos`;
|
||||
const referer = `${site.parameters?.networkReferer ? site.network.url : site.url}/en/videos`;
|
||||
const { apiUrl } = await fetchApiCredentials(referer);
|
||||
|
||||
const res = await bhttp.post(apiUrl, {
|
||||
|
|
@ -339,7 +347,7 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
|
|||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.statuscode === 200 && res.body.results?.[0]?.hits) {
|
||||
if (res.statusCode === 200 && res.body.results?.[0]?.hits) {
|
||||
return scrapeApiReleases(res.body.results[0].hits, site);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue