From 87e2d6bbfd0cc7862c44d779cd97ad2a1a3994d6 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Sat, 1 Feb 2020 04:42:35 +0100 Subject: [PATCH] Added actor releases to MindGeek module. --- src/scrapers/mindgeek.js | 67 ++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index cebd1eac..a6649947 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -61,36 +61,36 @@ async function scrapeLatest(items, site) { return Promise.all(items.map(async data => scrapeLatestX(data, site))); } -async function scrapeScene(data, url, site) { +function scrapeScene(data, url, _site) { + const release = {}; + const { id: entryId, title, description } = data; - const date = new Date(data.dateReleased); - const actors = data.actors.map(actor => actor.name); - const tags = data.tags.map(tag => tag.name); + release.entryId = data.id; + release.title = title; + release.description = description; + + release.date = new Date(data.dateReleased); + release.actors = data.actors.map(actor => actor.name); + + release.tags = data.tags.map(tag => tag.name); + + [release.poster, ...release.photos] = getThumbs(data); - const [poster, ...photos] = getThumbs(data); const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']); - - const siteName = data.collections[0].name; - const channel = siteName.replace(/\s+/g, '').toLowerCase(); - - return { - url, - entryId, - title, - description, - actors, - tags, - poster, - photos, - trailer: trailer && { + if (trailer) { + release.trailer = { src: trailer.urls.view, quality: parseInt(trailer.format, 10), - }, - date, - site, - channel, - }; + }; + } + + const siteName = data.collections[0].name; + release.channel = siteName.replace(/\s+/g, '').toLowerCase(); + + release.url = url || `https://www.realitykings.com/scene/${entryId}/`; + + return release; } function getUrl(site) { @@ -124,7 +124,7 @@ async function getSession(url) { return { session, instanceToken }; } -function scrapeProfile(data, html) { +function scrapeProfile(data, html, releases = []) { const { qa, qd } = ex(html); const profile = { @@ -155,6 +155,8 @@ function scrapeProfile(data, html) { const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent)); if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY'); + profile.releases = releases.map(release => scrapeScene(release)); + return profile; } @@ -218,7 +220,20 @@ async function fetchProfile(actorName, networkName, actorPath = 'model') { if (actorData) { const actorUrl = `https://www.${networkName}.com/${actorPath}/${actorData.id}/`; - const actorRes = await bhttp.get(actorUrl); + const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`; + + const [actorRes, actorReleasesRes] = await Promise.all([ + bhttp.get(actorUrl), + session.get(actorReleasesUrl, { + headers: { + Instance: instanceToken, + }, + }), + ]); + + if (actorRes.statusCode === 200 && actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) { + return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result); + } if (actorRes.statusCode === 200) { return scrapeProfile(actorData, actorRes.body.toString());