Passing context object with site or network instead of scraper slug and 'site or network' to all profile scrapers.

This commit is contained in:
2020-05-18 03:22:03 +02:00
parent 8733fdc657
commit 885aa4f627
22 changed files with 161 additions and 79 deletions

View File

@@ -80,11 +80,69 @@ function scrapeScene(scene, site) {
return release;
}
function scrapeLatest(scenes, site) {
function scrapeAll(scenes, site) {
return scenes.map(({ _source: scene }) => scrapeScene(scene, site));
}
function scrapeProfile(actor) {
async function fetchActorReleases(actor, site) {
const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50,
query: {
bool: {
must: [
{
match: {
status: 'ok',
},
},
{
nested: {
path: 'actors',
query: {
bool: {
must: [
{
match: {
'actors.mongoId': {
operator: 'AND',
query: actor.id,
},
},
},
],
},
},
},
},
],
must_not: [
{
match: {
type: 'trailer',
},
},
],
},
},
sort: [
{
releaseDate: {
order: 'desc',
},
},
],
}, {
encodeJSON: true,
headers: {
Authorization: `Basic ${authKey}`,
},
});
return scrapeAll(res.body.hits.hits, site);
}
async function scrapeProfile(actor, site, include) {
const profile = {};
profile.aliases = actor.aliases;
@@ -115,7 +173,9 @@ function scrapeProfile(actor) {
if (actor.twitter) profile.social = [`https://www.twitter.com/${actor.twitter}`];
if (actor.image) profile.avatar = `https://i.bang.com/pornstars/${actor.identifier}.jpg`;
// TODO: get releases
if (include.releases) {
profile.releases = await fetchActorReleases(actor, site);
}
return profile;
}
@@ -204,7 +264,7 @@ async function fetchLatest(site, page = 1) {
},
});
return scrapeLatest(res.body.hits.hits, site);
return scrapeAll(res.body.hits.hits, site);
}
async function fetchScene(url, site) {
@@ -220,7 +280,7 @@ async function fetchScene(url, site) {
return scrapeScene(res.body._source, site); // eslint-disable-line no-underscore-dangle
}
async function fetchProfile(actorName) {
async function fetchProfile(actorName, actorSlug, site, include) {
const res = await post(`https://${clusterId}.us-east-1.aws.found.io/actors/actor/_search`, {
size: 5,
sort: [{
@@ -255,7 +315,7 @@ async function fetchProfile(actorName) {
const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase());
if (actor) {
return scrapeProfile(actor._source);
return scrapeProfile(actor._source, site, include);
}
return null;