DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updated dependencies. Added periodic memory logger.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-11-20 23:59:15 +01:00
parent a867817dc1
commit 26539b74a5
109 changed files with 10238 additions and 10833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/scrapers/gamma.js
View File

@@ -33,7 +33,7 @@ async function fetchApiCredentials(referer, site) {
const res = await http.get(referer);
const body = res.body.toString();
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
const apiLine = body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
if (!apiLine) {
throw new Error(`No Gamma API key found for ${referer}`);
@@ -169,7 +169,7 @@ async function getThumbs(entryId, site, parameters) {
});
if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) {
return res.body.results[0].hits[0].set_pictures.map(img => ([
return res.body.results[0].hits[0].set_pictures.map((img) => ([
`https://transform.gammacdn.com/photo_set${img.thumb_path}`,
`https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`,
]));
@@ -214,7 +214,7 @@ async function scrapeApiReleases(json, site) {
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.director = scene.directors[0]?.name || null;
release.actors = scene.actors.map(actor => ({
release.actors = scene.actors.map((actor) => ({
entryId: actor.actor_id,
name: actor.name,
gender: actor.gender,
@@ -226,7 +226,7 @@ async function scrapeApiReleases(json, site) {
}));
release.tags = scene.master_categories
.concat(scene.categories?.map(category => category.name))
.concat(scene.categories?.map((category) => category.name))
.filter(Boolean); // some categories don't have a name
const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
@@ -272,7 +272,7 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
[release.likes, release.dislikes] = $(element).find('.value')
.toArray()
.map(value => Number($(value).text()));
.map((value) => Number($(value).text()));
const posterEl = $(element).find('.imgLink img, .tlcImageItem');
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
@@ -327,13 +327,13 @@ async function scrapeScene(html, url, site, baseRelease, mobileHtml, options) {
const actors = data?.actor || data2?.actor;
if (actors) {
release.actors = actors.map(actor => ({
release.actors = actors.map((actor) => ({
name: actor.name,
gender: actor.gender,
}));
}
const hasTrans = release.actors?.some(actor => actor.gender === 'shemale');
const hasTrans = release.actors?.some((actor) => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
@@ -420,7 +420,7 @@ async function scrapeSceneApi(data, site, options) {
release.duration = data.length;
release.date = new Date(data.date * 1000) || qu.parseDate(data.release_date, 'YYYY-MM-DD');
release.actors = data.actors.map(actor => ({
release.actors = data.actors.map((actor) => ({
entryId: actor.actor_id,
name: actor.name,
gender: actor.gender,
@@ -429,7 +429,7 @@ async function scrapeSceneApi(data, site, options) {
: qu.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
}));
release.tags = data.categories.map(category => category.name);
release.tags = data.categories.map((category) => category.name);
if (data.pictures) {
release.poster = [
@@ -501,7 +501,7 @@ async function scrapeMovie({ query, html }, window, url, entity, options) {
release.date = qu.extractDate(data.dvdReleaseDate);
release.title = data.dvdName;
release.actors = data.dvdActors.map(actor => ({ name: actor.actorName, entryId: actor.actorId }));
release.actors = data.dvdActors.map((actor) => ({ name: actor.actorName, entryId: actor.actorId }));
release.tags = query.cnts('.dvdCol a');
release.scenes = scrapeAll(html, entity, entity.url);
@@ -602,9 +602,9 @@ function scrapeApiProfile(data, releases, siteSlug) {
if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
const avatarPaths = Object.values(data.pictures).reverse();
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map(avatarPath => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map((avatarPath) => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
if (releases) profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
if (releases) profile.releases = releases.map((release) => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
return profile;
}
@@ -723,7 +723,7 @@ function getDeepUrl(url, site, baseRelease, mobile) {
const filter = new Set(['en', 'video', 'scene', site.slug, site.parent.slug]);
const pathname = baseRelease?.path || new URL(url).pathname
.split('/')
.filter(component => !filter.has(component))
.filter((component) => !filter.has(component))
.join('/'); // reduce to scene ID and title slug
const sceneId = baseRelease?.entryId || pathname.match(/\/(\d+)\//)?.[1];
@@ -863,7 +863,7 @@ async function fetchApiProfile({ name: actorName }, context, include) {
});
if (res.status === 200 && res.body.results[0].hits.length > 0) {
const actorData = res.body.results[0].hits.find(actor => slugify(actor.name) === slugify(actorName));
const actorData = res.body.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
if (actorData) {
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);