DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added proper Gamma parameters to 21Sextury sites.

Browse Source
This commit is contained in:
ThePendulum
2020-02-13 23:05:28 +01:00
parent 14b8993d10
commit 5e461f1f1a
4 changed files with 128 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/scrape-releases.js
View File

@@ -35,11 +35,11 @@ async function findSite(url, release) {
async function scrapeRelease(source, basicRelease = null, type = 'scene', preflight) {
// profile scraper may return either URLs or pre-scraped scenes
const sourceIsUrl = typeof source === 'string';
const url = sourceIsUrl ? source : source?.url;
const release = sourceIsUrl ? basicRelease : source;
const sourceIsUrlOrEmpty = typeof source === 'string' || source === undefined;
const url = sourceIsUrlOrEmpty ? source : source?.url;
const release = sourceIsUrlOrEmpty ? basicRelease : source;
const site = await findSite(url, release);
const site = basicRelease?.site || await findSite(url, release);
if (!site) {
throw new Error(`Could not find site ${url} in database`);

2
src/scrape-sites.js
View File

@@ -98,7 +98,7 @@ async function deepFetchReleases(baseReleases, preflight) {
return Promise.map(baseReleases, async (release) => {
if (release.url || (release.path && release.site)) {
try {
const fullRelease = await scrapeRelease(release.url || release.path, release, 'scene', preflight);
const fullRelease = await scrapeRelease(release.url, release, 'scene', preflight);
if (fullRelease) {
return {

29
src/scrapers/gamma.js
View File

@@ -14,11 +14,15 @@ const slugify = require('../utils/slugify');
function getAlbumUrl(albumPath, site) {
if (site.parameters?.photos) {
return /^http/.test(site.parameters.photos)
? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`;
? `${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`;
}
return `${site.url}${albumPath}`;
if (site.url) {
return `${site.url}${albumPath}`;
}
return null;
}
async function fetchPhotos(url) {
@@ -59,6 +63,10 @@ function scrapePhotos(html) {
async function getPhotos(albumPath, site) {
const albumUrl = getAlbumUrl(albumPath, site);
if (!albumUrl) {
return [];
}
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -70,7 +78,7 @@ async function getPhotos(albumPath, site) {
const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
const otherPhotos = await Promise.map(otherPages, async (page) => {
const pageUrl = `${site.url}/${albumPath}/${page}`;
const pageUrl = `${albumUrl}/${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml, site);
@@ -107,7 +115,7 @@ async function scrapeApiReleases(json, site) {
release.path = `/${scene.url_title}/${release.entryId}`;
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
@@ -469,12 +477,15 @@ async function fetchScene(url, site, release) {
}
const deepUrl = getDeepUrl(url, site, release);
const res = await bhttp.get(deepUrl);
if (res.statusCode === 200) {
const scene = await scrapeScene(res.body.toString(), url, site, release);
if (deepUrl) {
const res = await bhttp.get(deepUrl);
return { ...scene, deepUrl };
if (res.statusCode === 200) {
const scene = await scrapeScene(res.body.toString(), url, site, release);
return { ...scene, deepUrl };
}
}
return null;