DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added Fantasy Massage sites. Improved Private scraper, added movie link.

Browse Source
This commit is contained in:
ThePendulum
2020-02-08 04:52:32 +01:00
parent ff8ab2fe09
commit d2cb74a252
5 changed files with 143 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/scrapers/gamma.js
View File

@@ -11,6 +11,16 @@ const logger = require('../logger')(__filename);
const { ex, get } = require('../utils/q');
const slugify = require('../utils/slugify');
function getAlbumUrl(albumPath, site) {
if (site.parameters?.photos) {
return /^http/.test(site.parameters.photos)
? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`;
}
return `${site.url}${albumPath}`;
}
async function fetchPhotos(url) {
const res = await bhttp.get(url);
@@ -25,7 +35,7 @@ function scrapePhotos(html) {
if (/\/join|\/createaccount/.test(url)) {
// URL links to join page instead of full photo, extract thumbnail
// /createaccount is used by e.g. Tricky Spa
// /createaccount is used by e.g. Tricky Spa native site
const src = $(linkEl).find('img').attr('src');
if (src.match('previews/')) {
@@ -47,9 +57,9 @@ function scrapePhotos(html) {
}
async function getPhotos(albumPath, site) {
const albumUrl = site.parameters?.photos
? `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${albumPath}`;
const albumUrl = getAlbumUrl(albumPath, site);
console.log(albumUrl);
try {
const html = await fetchPhotos(albumUrl);
@@ -397,28 +407,63 @@ async function fetchApiUpcoming(site) {
return fetchApiLatest(site, 1, true);
}
function getLatestUrl(site, page) {
if (site.parameters?.latest) {
return /^http/.test(site.parameters.latest)
? `${site.parameters.latest}${page}`
: `${site.url}${site.parameters.latest}${page}`;
}
return `${site.url}/en/videos/AllCategories/0/${page}`;
}
function getUpcomingUrl(site) {
if (site.parameters?.upcoming) {
return /^http/.test(site.parameters.upcoming)
? `${site.parameters.upcoming}`
: `${site.url}${site.parameters.upcoming}`;
}
return `${site.url}/en/videos/AllCategories/0/1/upcoming`;
}
async function fetchLatest(site, page = 1) {
const url = `${site.url}${site.parameters?.latest || '/en/videos/AllCategories/0/'}${page}`;
const url = getLatestUrl(site, page);
const res = await bhttp.get(url);
return scrapeAll(res.body.toString(), site);
}
async function fetchUpcoming(site) {
const url = `${site.url}${site.parameters?.upcoming || '/en/videos/AllCategories/0/1/upcoming'}`;
const url = getUpcomingUrl(site);
const res = await bhttp.get(url);
return scrapeAll(res.body.toString(), site);
}
function getDeepUrl(url, site) {
const { pathname } = new URL(url);
if (site.parameters?.deep === 'network') {
return `${site.network.url}${pathname}`;
}
if (site.parameters?.deep) {
return `${site.parameters.deep}${pathname}`;
}
return url;
}
async function fetchScene(url, site, release) {
if (site.parameters?.deep === false) {
return release;
}
const res = await bhttp.get(url);
const deepUrl = getDeepUrl(url, site);
const res = await bhttp.get(deepUrl);
return scrapeScene(res.body.toString(), url, site);
return scrapeScene(res.body.toString(), url, site, deepUrl);
}
async function fetchActorScenes(actorName, apiUrl, siteSlug) {