boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Added proper Gamma parameters to 21Sextury sites.

Browse Source
This commit is contained in:
ThePendulum
2020-02-13 23:05:28 +01:00
parent 14b8993d10
commit 5e461f1f1a
4 changed files with 128 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/scrapers/gamma.js
View File

@@ -14,11 +14,15 @@ const slugify = require('../utils/slugify');
function getAlbumUrl(albumPath, site) {
if (site.parameters?.photos) {
return /^http/.test(site.parameters.photos)
? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`;
? `${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`;
}
return `${site.url}${albumPath}`;
if (site.url) {
return `${site.url}${albumPath}`;
}
return null;
}
async function fetchPhotos(url) {
@@ -59,6 +63,10 @@ function scrapePhotos(html) {
async function getPhotos(albumPath, site) {
const albumUrl = getAlbumUrl(albumPath, site);
if (!albumUrl) {
return [];
}
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -70,7 +78,7 @@ async function getPhotos(albumPath, site) {
const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
const otherPhotos = await Promise.map(otherPages, async (page) => {
const pageUrl = `${site.url}/${albumPath}/${page}`;
const pageUrl = `${albumUrl}/${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml, site);
@@ -107,7 +115,7 @@ async function scrapeApiReleases(json, site) {
release.path = `/${scene.url_title}/${release.entryId}`;
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
@@ -469,12 +477,15 @@ async function fetchScene(url, site, release) {
}
const deepUrl = getDeepUrl(url, site, release);
const res = await bhttp.get(deepUrl);
if (res.statusCode === 200) {
const scene = await scrapeScene(res.body.toString(), url, site, release);
if (deepUrl) {
const res = await bhttp.get(deepUrl);
return { ...scene, deepUrl };
if (res.statusCode === 200) {
const scene = await scrapeScene(res.body.toString(), url, site, release);
return { ...scene, deepUrl };
}
}
return null;