DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added profile scraper with scenes to BAM Visions. Passing 'includes' object instead of withReleases boolean to Gamma.

Browse Source
This commit is contained in:
ThePendulum
2020-03-12 02:19:45 +01:00
parent 152813730e
commit 370605554b
9 changed files with 87 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/scrapers/gamma.js
View File

@@ -7,7 +7,6 @@ const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
const argv = require('../argv');
const logger = require('../logger')(__filename);
const { ex, get } = require('../utils/q');
const slugify = require('../utils/slugify');
@@ -312,7 +311,7 @@ async function fetchActorReleases(profileUrl, getActorReleasesUrl, page = 1, acc
return accReleases.concat(releases);
}
async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUrl) {
async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUrl, withReleases) {
const { q } = ex(html);
const avatar = q('img.actorPicture');
@@ -346,7 +345,7 @@ async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUr
if (alias) profile.aliases = alias.split(':')[1].trim().split(', ');
if (nationality) profile.nationality = nationality.split(':')[1].trim();
if (getActorReleasesUrl && argv.withReleases) {
if (getActorReleasesUrl && withReleases) {
profile.releases = await fetchActorReleases(url, getActorReleasesUrl);
}
@@ -553,7 +552,7 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
return [];
}
async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesUrl) {
async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesUrl, include) {
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
const searchUrl = altSearchUrl
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
@@ -574,7 +573,7 @@ async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesU
return null;
}
return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug, getActorReleasesUrl);
return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug, getActorReleasesUrl, include.scenes);
}
return null;