DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updated dependencies. Added periodic memory logger.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-11-20 23:59:15 +01:00
parent a867817dc1
commit 26539b74a5
109 changed files with 10238 additions and 10833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/scrapers/mindgeek.js
View File

@@ -14,14 +14,14 @@ const { inchesToCm, lbsToKg } = require('../utils/convert');
function getThumbs(scene) {
if (scene.images.poster) {
return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
.filter(img => typeof img === 'object') // remove alternateText property
.map(image => image.xl.url);
.filter((img) => typeof img === 'object') // remove alternateText property
.map((image) => image.xl.url);
}
if (scene.images.card_main_rect) {
return scene.images.card_main_rect
.concat(scene.images.card_secondary_rect || [])
.map(image => image.xl.url.replace('.thumb', ''));
.map((image) => image.xl.url.replace('.thumb', ''));
}
return [];
@@ -29,14 +29,14 @@ function getThumbs(scene) {
function getVideos(data) {
const teaserSources = data.videos.mediabook?.files;
const trailerSources = data.children.find(child => child.type === 'trailer')?.videos.full?.files;
const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
const teaser = teaserSources && Object.values(teaserSources).map(source => ({
const teaser = teaserSources && Object.values(teaserSources).map((source) => ({
src: source.urls.view,
quality: parseInt(source.format, 10),
}));
const trailer = trailerSources && Object.values(trailerSources).map(source => ({
const trailer = trailerSources && Object.values(trailerSources).map((source) => ({
src: source.urls.view,
quality: parseInt(source.format, 10),
}));
@@ -59,8 +59,8 @@ function scrapeLatestX(data, site, filterChannel) {
release.date = new Date(data.dateReleased);
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map(tag => tag.name);
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map((tag) => tag.name);
[release.poster, ...release.photos] = getThumbs(data);
@@ -69,15 +69,15 @@ function scrapeLatestX(data, site, filterChannel) {
if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({
release.chapters = data.timeTags?.map((chapter) => ({
time: chapter.startTime,
duration: chapter.endTime - chapter.startTime,
tags: [chapter.name],
}));
if ((site.parameters?.extract === true && data.collections.length > 0) // release should not belong to any channel
|| (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) // release should belong to specific channel
|| (filterChannel && !data.collections?.some(collection => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes
|| (typeof site.parameters?.extract === 'string' && !data.collections.some((collection) => collection.shortName === site.parameters.extract)) // release should belong to specific channel
|| (filterChannel && !data.collections?.some((collection) => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes
return {
...release,
exclude: true,
@@ -88,11 +88,11 @@ function scrapeLatestX(data, site, filterChannel) {
}
async function scrapeLatest(items, site, filterChannel) {
const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel));
const latestReleases = items.map((data) => scrapeLatestX(data, site, filterChannel));
return {
scenes: latestReleases.filter(scene => !scene.exclude),
unextracted: latestReleases.filter(scene => scene.exclude),
scenes: latestReleases.filter((scene) => !scene.exclude),
unextracted: latestReleases.filter((scene) => scene.exclude),
};
}
@@ -108,8 +108,8 @@ function scrapeScene(data, url, _site, networkName) {
release.date = new Date(data.dateReleased);
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map(tag => tag.name);
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map((tag) => tag.name);
[release.poster, ...release.photos] = getThumbs(data);
@@ -118,7 +118,7 @@ function scrapeScene(data, url, _site, networkName) {
if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({
release.chapters = data.timeTags?.map((chapter) => ({
time: chapter.startTime,
duration: chapter.endTime - chapter.startTime,
tags: [chapter.name],
@@ -213,18 +213,18 @@ function scrapeProfile(data, html, releases = [], networkName) {
|| data.images.card_main_rect[0].xs?.url;
}
const birthdate = query.all('li').find(el => /Date of Birth/.test(el.textContent));
const birthdate = query.all('li').find((el) => /Date of Birth/.test(el.textContent));
if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');
if (data.tags.some(tag => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
if (data.tags.some((tag) => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
profile.naturalBoobs = true;
}
if (data.tags.some(tag => /boob type/i.test(tag.category) && /enhanced/i.test(tag.name))) {
if (data.tags.some((tag) => /boob type/i.test(tag.category) && /enhanced/i.test(tag.name))) {
profile.naturalBoobs = false;
}
profile.releases = releases.map(release => scrapeScene(release, null, null, networkName));
profile.releases = releases.map((release) => scrapeScene(release, null, null, networkName));
return profile;
}
@@ -325,7 +325,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
});
if (res.statusCode === 200) {
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
const actorData = res.body.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
if (actorData) {
const actorUrl = `https://www.${entity.slug}.com/${entity.parameters?.actorPath || 'model'}/${actorData.id}/${actorSlug}`;