Fixed countries seed file. Updated MOFOS scraper. Improved Reality Kings scraper. Limiting photos for XEmpire scraper.

src/scrapers/realitykings.js

@@ -4,8 +4,10 @@
 const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
-const { JSDOM } = require('jsdom');
+const moment = require('moment');
 
+const { fetchSites } = require('../sites');
+const { cookieToData } = require('../utils/cookies');
 const { matchTags } = require('../tags');
 
 function getThumbs(scene) {
@@ -22,40 +24,18 @@ function getThumbs(scene) {
     return [];
 }
 
-async function scrapeLatest(html, site) {
-    const { document } = new JSDOM(html).window;
-
-    const scriptString = document.querySelector('script').textContent;
-    const prefix = 'window.__JUAN.initialState = {';
-
-    const dataStart = scriptString.slice(scriptString.indexOf(prefix) + prefix.length - 1);
-    const dataString = dataStart.slice(0, dataStart.indexOf('};') + 1);
-    const data = JSON.parse(dataString);
-
-    const actorsMap = data.entities.actors;
-    const tagsMap = data.entities.tags;
-    const scenes = Object.values(data.entities.releases);
-
-    return Promise.all(scenes.map(async (scene) => {
-        const {
-            id: entryId,
-            title,
-            description,
-        } = scene;
-
+async function scrapeLatest(items, site) {
+    return Promise.all(items.map(async (data) => {
+        const { id: entryId, title, description } = data;
         const url = `https://www.realitykings.com/scene/${entryId}/`;
-        const date = new Date(scene.dateReleased);
-        const actors = scene.actors.map(actorId => actorsMap[actorId].name);
-        const duration = scene.videos.mediabook && scene.videos.mediabook.length;
+        const date = new Date(data.dateReleased);
+        const actors = data.actors.map(actor => actor.name);
 
-        const rawTags = scene.tags.map(tagId => tagsMap[tagId].name);
+        const rawTags = data.tags.map(tag => tag.name);
         const tags = await matchTags(rawTags);
 
-        const [poster, ...photos] = getThumbs(scene);
-        const trailer720p = scene.videos.mediabook && scene.videos.mediabook.files['720p'] && scene.videos.mediabook.files['720p'].urls.view;
-        const trailer360p = scene.videos.mediabook && scene.videos.mediabook.files['360p'] && scene.videos.mediabook.files['360p'].urls.view;
-
-        const { likes, dislikes } = scene.stats;
+        const [poster, ...photos] = getThumbs(data);
+        const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
         return {
             url,
@@ -63,41 +43,44 @@ async function scrapeLatest(html, site) {
             title,
             description,
             actors,
-            date,
             tags,
-            duration,
             poster,
             photos,
-            trailer: {
-                src: trailer720p || trailer360p,
-                quality: trailer720p ? 720 : 360,
+            trailer: trailer && {
+                src: trailer.urls.view,
+                quality: parseInt(trailer.format, 10),
             },
-            rating: { likes, dislikes },
+            date,
             site,
         };
     }));
 }
 
 async function scrapeScene(data, url, site) {
-    const {
-        id: entryId,
-        title,
-        description,
-    } = data;
-
+    const { id: entryId, title, description } = data;
     const date = new Date(data.dateReleased);
     const actors = data.actors.map(actor => actor.name);
 
-    const { likes, dislikes } = data.stats;
-
     const rawTags = data.tags.map(tag => tag.name);
-    const tags = await matchTags(rawTags);
 
     const [poster, ...photos] = getThumbs(data);
+    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
-    const duration = data.videos.mediabook && data.videos.mediabook.length;
-    const trailer720p = data.videos.mediabook && data.videos.mediabook.files['720p'] && data.videos.mediabook.files['720p'].urls.view;
-    const trailer360p = data.videos.mediabook && data.videos.mediabook.files['360p'] && data.videos.mediabook.files['360p'].urls.view;
+    const siteName = data.collections[0].name;
+    const siteId = data.collections[0].id;
+    const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
+    const siteUrl = `https://www.realitykings.com/scenes?site=${siteId}`;
+
+    const [[channelSite], tags] = await Promise.all([
+        site.isFallback
+            ? fetchSites({
+                slug: siteSlug,
+                name: siteName,
+                url: siteUrl,
+            })
+            : [site],
+        matchTags(rawTags),
+    ]);
 
     return {
         url,
@@ -105,41 +88,56 @@ async function scrapeScene(data, url, site) {
         title,
         description,
         actors,
-        date,
-        duration,
         tags,
         poster,
         photos,
-        trailer: {
-            src: trailer720p || trailer360p,
-            quality: trailer720p ? 720 : 360,
+        trailer: trailer && {
+            src: trailer.urls.view,
+            quality: parseInt(trailer.format, 10),
         },
-        rating: {
-            likes,
-            dislikes,
-        },
-        site,
+        date,
+        site: channelSite,
     };
 }
 
-async function fetchLatest(site, page = 1) {
+function getUrl(site) {
     const { hostname, search } = new URL(site.url);
 
     if (hostname.match(/(www\.)?realitykings\.com/) && search.match(/\?site=\d+/)) {
-        const res = await bhttp.get(`${site.url}&page=${page}`);
-
-        return scrapeLatest(res.body.toString(), site);
+        return site.url;
     }
 
     if (site.parameters && site.parameters.siteId) {
-        const res = await bhttp.get(`https://www.realitykings.com/scenes?site=${site.parameters.siteId}&page=${page}`);
-
-        return scrapeLatest(res.body.toString(), site);
+        return `https://www.realitykings.com/scenes?site=${site.parameters.siteId}`;
     }
 
     throw new Error(`Reality Kings site '${site.name}' (${site.url}) not supported`);
 }
 
+async function fetchLatest(site, page = 1) {
+    const url = getUrl(site);
+    const { search } = new URL(url);
+    const siteId = new URLSearchParams(search).get('site');
+
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
+
+    await session.get(url);
+
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
+    const limit = 10;
+    const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeLatest(res.body.result, site);
+}
+
 async function fetchScene(url, site) {
     const entryId = url.match(/\d+/)[0];
 
@@ -148,8 +146,8 @@ async function fetchScene(url, site) {
 
     await session.get(url);
 
-    const cookies = await cookieJar.getCookieStringAsync(url);
-    const instanceToken = cookies.split(';')[0].split('=')[1];
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
 
     const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
         headers: {
@@ -157,7 +155,7 @@ async function fetchScene(url, site) {
         },
     });
 
-    return scrapeScene(res.body.result.parent || res.body.result, url, site);
+    return scrapeScene(res.body.result, url, site);
 }
 
 module.exports = {