DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Switched to tabs. Adding missing actor entries when scraping actors, with batch ID.

Browse Source
This commit is contained in:
ThePendulum
2020-05-14 04:26:05 +02:00
parent f1eb29c713
commit 11eb66f834
178 changed files with 16594 additions and 16929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/scrapers/xempire.js
View File

@@ -5,31 +5,31 @@ const bhttp = require('bhttp');
const { fetchLatest, fetchUpcoming, scrapeScene, fetchProfile } = require('./gamma');
async function fetchScene(url, site) {
const res = await bhttp.get(url);
const res = await bhttp.get(url);
const release = await scrapeScene(res.body.toString(), url, site);
const release = await scrapeScene(res.body.toString(), url, site);
const siteDomain = release.$('meta[name="twitter:domain"]').attr('content') || 'allblackx.com'; // only AllBlackX has no twitter domain, no other useful hints available
const siteSlug = siteDomain && siteDomain.split('.')[0].toLowerCase();
// const siteUrl = siteDomain && `https://www.${siteDomain}`;
const siteDomain = release.$('meta[name="twitter:domain"]').attr('content') || 'allblackx.com'; // only AllBlackX has no twitter domain, no other useful hints available
const siteSlug = siteDomain && siteDomain.split('.')[0].toLowerCase();
// const siteUrl = siteDomain && `https://www.${siteDomain}`;
release.channel = siteSlug;
release.director = 'Mason';
release.channel = siteSlug;
release.director = 'Mason';
return release;
return release;
}
function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.xempire.com/en/videos/xempire/latest/${page}/All-Categories/0${actorPath}`;
return `https://www.xempire.com/en/videos/xempire/latest/${page}/All-Categories/0${actorPath}`;
}
async function networkFetchProfile(actorName, scraperSlug, site, include) {
return fetchProfile(actorName, scraperSlug, null, getActorReleasesUrl, include);
return fetchProfile(actorName, scraperSlug, null, getActorReleasesUrl, include);
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchUpcoming,
fetchScene,
fetchLatest,
fetchProfile: networkFetchProfile,
fetchUpcoming,
fetchScene,
};