boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Switched to tabs. Adding missing actor entries when scraping actors, with batch ID.

Browse Source
This commit is contained in:
ThePendulum
2020-05-14 04:26:05 +02:00
parent f1eb29c713
commit 11eb66f834
178 changed files with 16594 additions and 16929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
src/scrapers/famedigital.js
View File

@@ -1,115 +1,115 @@
'use strict';
const {
fetchLatest,
fetchApiLatest,
fetchUpcoming,
fetchApiUpcoming,
fetchScene,
fetchProfile,
fetchApiProfile,
scrapeAll,
fetchLatest,
fetchApiLatest,
fetchUpcoming,
fetchApiUpcoming,
fetchScene,
fetchProfile,
fetchApiProfile,
scrapeAll,
} = require('./gamma');
const { get } = require('../utils/q');
const slugify = require('../utils/slugify');
function extractLowArtActors(release) {
const actors = release.title
.replace(/solo/i, '')
.split(/,|\band\b/ig)
.map(actor => actor.trim());
const actors = release.title
.replace(/solo/i, '')
.split(/,|\band\b/ig)
.map(actor => actor.trim());
return {
...release,
actors,
};
return {
...release,
actors,
};
}
async function networkFetchLatest(site, page = 1) {
if (site.parameters?.api) return fetchApiLatest(site, page, false);
if (site.parameters?.api) return fetchApiLatest(site, page, false);
const releases = await fetchLatest(site, page);
const releases = await fetchLatest(site, page);
if (site.slug === 'lowartfilms') {
return releases.map(release => extractLowArtActors(release));
}
if (site.slug === 'lowartfilms') {
return releases.map(release => extractLowArtActors(release));
}
return releases;
return releases;
}
async function networkFetchScene(url, site) {
const release = await fetchScene(url, site);
const release = await fetchScene(url, site);
if (site.slug === 'lowartfilms') {
return extractLowArtActors(release);
}
if (site.slug === 'lowartfilms') {
return extractLowArtActors(release);
}
return release;
return release;
}
async function networkFetchUpcoming(site, page = 1) {
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
return fetchUpcoming(site, page);
return fetchUpcoming(site, page);
}
function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
}
async function fetchClassicProfile(actorName, siteSlug) {
const actorSlug = slugify(actorName);
const actorSlug = slugify(actorName);
const url = `https://${siteSlug}.com/en/pornstars`;
const pornstarsRes = await get(url);
const url = `https://${siteSlug}.com/en/pornstars`;
const pornstarsRes = await get(url);
if (!pornstarsRes.ok) return null;
if (!pornstarsRes.ok) return null;
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find(el => slugify(el.textContent) === actorSlug)
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find(el => slugify(el.textContent) === actorSlug)
?.value;
if (actorPath) {
const actorUrl = `https://${siteSlug}.com${actorPath}`;
const res = await get(actorUrl);
if (actorPath) {
const actorUrl = `https://${siteSlug}.com${actorPath}`;
const res = await get(actorUrl);
if (res.ok) {
const releases = scrapeAll(res.item, null, `https://www.${siteSlug}.com`, false);
if (res.ok) {
const releases = scrapeAll(res.item, null, `https://www.${siteSlug}.com`, false);
return { releases };
}
}
return { releases };
}
}
return null;
return null;
}
async function networkFetchProfile(actorName, scraperSlug, site, include) {
// not all Fame Digital sites offer Gamma actors
const [devils, rocco, peter, silvia] = await Promise.all([
fetchApiProfile(actorName, 'devilsfilm', true),
fetchApiProfile(actorName, 'roccosiffredi'),
include.scenes ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl, include) : [],
include.scenes ? fetchClassicProfile(actorName, 'silviasaint') : [],
include.scenes ? fetchClassicProfile(actorName, 'silverstonedvd') : [],
]);
// not all Fame Digital sites offer Gamma actors
const [devils, rocco, peter, silvia] = await Promise.all([
fetchApiProfile(actorName, 'devilsfilm', true),
fetchApiProfile(actorName, 'roccosiffredi'),
include.scenes ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl, include) : [],
include.scenes ? fetchClassicProfile(actorName, 'silviasaint') : [],
include.scenes ? fetchClassicProfile(actorName, 'silverstonedvd') : [],
]);
if (devils || rocco || peter) {
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []);
if (devils || rocco || peter) {
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []);
return {
...peter,
...rocco,
...devils,
releases,
};
}
return {
...peter,
...rocco,
...devils,
releases,
};
}
return null;
return null;
}
module.exports = {
fetchLatest: networkFetchLatest,
fetchProfile: networkFetchProfile,
fetchScene: networkFetchScene,
fetchUpcoming: networkFetchUpcoming,
fetchLatest: networkFetchLatest,
fetchProfile: networkFetchProfile,
fetchScene: networkFetchScene,
fetchUpcoming: networkFetchUpcoming,
};