boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Added profile scraper with scenes to BAM Visions. Passing 'includes' object instead of withReleases boolean to Gamma.

Browse Source
This commit is contained in:
ThePendulum
2020-03-12 02:19:45 +01:00
parent 152813730e
commit 370605554b
9 changed files with 87 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/scrapers/gamma.js
View File

@@ -7,7 +7,6 @@ const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
const argv = require('../argv');
const logger = require('../logger')(__filename);
const { ex, get } = require('../utils/q');
const slugify = require('../utils/slugify');
@@ -312,7 +311,7 @@ async function fetchActorReleases(profileUrl, getActorReleasesUrl, page = 1, acc
return accReleases.concat(releases);
}
async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUrl) {
async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUrl, withReleases) {
const { q } = ex(html);
const avatar = q('img.actorPicture');
@@ -346,7 +345,7 @@ async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUr
if (alias) profile.aliases = alias.split(':')[1].trim().split(', ');
if (nationality) profile.nationality = nationality.split(':')[1].trim();
if (getActorReleasesUrl && argv.withReleases) {
if (getActorReleasesUrl && withReleases) {
profile.releases = await fetchActorReleases(url, getActorReleasesUrl);
}
@@ -553,7 +552,7 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
return [];
}
async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesUrl) {
async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesUrl, include) {
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
const searchUrl = altSearchUrl
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
@@ -574,7 +573,7 @@ async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesU
return null;
}
return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug, getActorReleasesUrl);
return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug, getActorReleasesUrl, include.scenes);
}
return null;