DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added scene count to actor inspect. Preferring network slug over data brand for scene URLs in MindGeek scraper, since milehighmedia.com's brand is milehigh, resulting in milehigh.com.

Browse Source
This commit is contained in:
ThePendulum
2020-02-09 03:09:06 +01:00
parent 2068202ca6
commit 9d9eda29be
4 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/scrapers/mindgeek.js
View File

@@ -73,7 +73,7 @@ async function scrapeLatest(items, site) {
return latestReleases.filter(Boolean);
}
function scrapeScene(data, url, _site) {
function scrapeScene(data, url, _site, networkName) {
const release = {};
const { id: entryId, title, description } = data;
@@ -100,7 +100,7 @@ function scrapeScene(data, url, _site) {
const siteName = data.collections[0]?.name || data.brand;
release.channel = siteName.replace(/\s+/g, '').toLowerCase();
release.url = url || `https://www.${data.brand}.com/scene/${entryId}/`;
release.url = url || `https://www.${networkName || data.brand}.com/scene/${entryId}/`;
return release;
}
@@ -139,7 +139,7 @@ async function getSession(url) {
return { session, instanceToken };
}
function scrapeProfile(data, html, releases = []) {
function scrapeProfile(data, html, releases = [], networkName) {
const { qa, qd } = ex(html);
const profile = {
@@ -170,7 +170,7 @@ function scrapeProfile(data, html, releases = []) {
const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent));
if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY');
profile.releases = releases.map(release => scrapeScene(release));
profile.releases = releases.map(release => scrapeScene(release, null, null, networkName));
return profile;
}
@@ -247,11 +247,11 @@ async function fetchProfile(actorName, networkName, actorPath = 'model') {
]);
if (actorRes.statusCode === 200 && actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result);
return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, networkName);
}
if (actorRes.statusCode === 200) {
return scrapeProfile(actorData, actorRes.body.toString());
return scrapeProfile(actorData, actorRes.body.toString(), null, networkName);
}
}
}