boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Added Black for Wife to JayRock. Switched parameters field to JSON type.

Browse Source
This commit is contained in:
ThePendulum
2020-02-09 19:41:39 +01:00
parent f7f4da24ec
commit 0f513266a0
18 changed files with 173 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/scrapers/gamma.js
View File

@@ -100,13 +100,14 @@ async function scrapeApiReleases(json, site) {
dislikes: scene.ratings_down,
};
release.url = site.parameters?.scene
? `${site.parameters.scene}/${scene.url_title}/${release.entryId}`
: `${site.url}/en/video/${scene.url_title}/${release.entryId}`;
release.path = `/${scene.url_title}/${release.entryId}`;
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.actors = scene.actors.map(({ name }) => name);
release.director = scene.directors[0].name;
release.director = scene.directors[0]?.name || null;
release.tags = scene.master_categories
.concat(scene.categories?.map(category => category.name))
@@ -171,9 +172,9 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
});
}
async function scrapeScene(html, url, site) {
async function scrapeScene(html, url, site, scrapedRelease) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const release = { $, url };
const release = { $ };
const json = $('script[type="application/ld+json"]').html();
const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
@@ -181,7 +182,7 @@ async function scrapeScene(html, url, site) {
const [data, data2] = json ? JSON.parse(json) : [];
const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
[release.entryId] = new URL(url).pathname.split('/').slice(-1);
[release.entryId] = (scrapedRelease?.path || new URL(url).pathname).split('/').slice(-1);
release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name;
// date in data object is not the release date of the scene, but the date the entry was added; only use as fallback
@@ -194,7 +195,8 @@ async function scrapeScene(html, url, site) {
if (data) {
release.description = data.description;
release.director = data.director?.[0].name || data2?.director?.[0].name;
if (data.director?.[0]?.name) release.director = data.director[0].name;
else if (data2?.director?.[0]?.name) release.director = data2.director[0].name;
const actors = data?.actor || data2?.actor || [];
release.actors = actors.map(actor => actor.name);
@@ -437,8 +439,8 @@ async function fetchUpcoming(site) {
return scrapeAll(res.body.toString(), site);
}
function getDeepUrl(url, site) {
const { pathname } = new URL(url);
function getDeepUrl(url, site, release) {
const pathname = release.path || new URL(url).pathname;
if (site.parameters?.deep === 'network') {
return `${site.network.url}${pathname}`;
@@ -456,11 +458,11 @@ async function fetchScene(url, site, release) {
return release;
}
const deepUrl = getDeepUrl(url, site);
const deepUrl = getDeepUrl(url, site, release);
const res = await bhttp.get(deepUrl);
if (res.statusCode === 200) {
return scrapeScene(res.body.toString(), url, site, deepUrl);
return scrapeScene(res.body.toString(), url, site, release);
}
return null;