DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Showing all unique descriptions on profile with network logo. Fixed Fame Digital scraper.

Browse Source
This commit is contained in:
ThePendulum
2020-05-19 04:46:49 +02:00
parent 9883c3d9c2
commit c0898b84d6
23 changed files with 131 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/scrapers/bang.js
View File

@@ -39,9 +39,8 @@ function decodeId(id) {
.toString('hex');
}
function scrapeScene(scene, site) {
function scrapeScene(scene) {
const release = {
site,
entryId: scene.id,
title: scene.name,
description: scene.description,
@@ -80,11 +79,11 @@ function scrapeScene(scene, site) {
return release;
}
function scrapeAll(scenes, site) {
return scenes.map(({ _source: scene }) => scrapeScene(scene, site));
function scrapeAll(scenes) {
return scenes.map(({ _source: scene }) => scrapeScene(scene));
}
async function fetchActorReleases(actor, site) {
async function fetchActorReleases(actor) {
const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50,
query: {
@@ -138,11 +137,10 @@ async function fetchActorReleases(actor, site) {
},
});
return scrapeAll(res.body.hits.hits, site);
return scrapeAll(res.body.hits.hits);
}
async function scrapeProfile(actor, site, include) {
async function scrapeProfile(actor, include) {
const profile = {};
profile.aliases = actor.aliases;
@@ -174,7 +172,7 @@ async function scrapeProfile(actor, site, include) {
if (actor.image) profile.avatar = `https://i.bang.com/pornstars/${actor.identifier}.jpg`;
if (include.releases) {
profile.releases = await fetchActorReleases(actor, site);
profile.releases = await fetchActorReleases(actor);
}
return profile;
@@ -267,7 +265,7 @@ async function fetchLatest(site, page = 1) {
return scrapeAll(res.body.hits.hits, site);
}
async function fetchScene(url, site) {
async function fetchScene(url) {
const encodedId = new URL(url).pathname.split('/')[2];
const entryId = decodeId(encodedId);
@@ -277,10 +275,10 @@ async function fetchScene(url, site) {
},
});
return scrapeScene(res.body._source, site); // eslint-disable-line no-underscore-dangle
return scrapeScene(res.body._source); // eslint-disable-line no-underscore-dangle
}
async function fetchProfile(actorName, actorSlug, site, include) {
async function fetchProfile(actorName, context, include) {
const res = await post(`https://${clusterId}.us-east-1.aws.found.io/actors/actor/_search`, {
size: 5,
sort: [{
@@ -315,7 +313,7 @@ async function fetchProfile(actorName, actorSlug, site, include) {
const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase());
if (actor) {
return scrapeProfile(actor._source, site, include);
return scrapeProfile(actor._source, include);
}
return null;