traxxx/src/scrapers/realitykings.js

165 lines
4.8 KiB
JavaScript
Raw Normal View History

'use strict';
/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const moment = require('moment');
const { fetchSites } = require('../sites');
const { cookieToData } = require('../utils/cookies');
const { matchTags } = require('../tags');
function getThumbs(scene) {
if (scene.images.poster) {
return scene.images.poster.map(image => image.xl.url);
}
if (scene.images.card_main_rect) {
return scene.images.card_main_rect
.concat(scene.images.card_secondary_rect || [])
.map(image => image.xl.url.replace('.thumb', ''));
}
return [];
}
async function scrapeLatest(items, site) {
return Promise.all(items.map(async (data) => {
const { id: entryId, title, description } = data;
const url = `https://www.realitykings.com/scene/${entryId}/`;
const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name);
const rawTags = data.tags.map(tag => tag.name);
const tags = await matchTags(rawTags);
const [poster, ...photos] = getThumbs(data);
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
return {
url,
entryId,
title,
description,
actors,
tags,
poster,
photos,
trailer: trailer && {
src: trailer.urls.view,
quality: parseInt(trailer.format, 10),
},
date,
site,
};
}));
}
async function scrapeScene(data, url, site) {
const { id: entryId, title, description } = data;
const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name);
const rawTags = data.tags.map(tag => tag.name);
const [poster, ...photos] = getThumbs(data);
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
const siteName = data.collections[0].name;
const siteId = data.collections[0].id;
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
const siteUrl = `https://www.realitykings.com/scenes?site=${siteId}`;
const [[channelSite], tags] = await Promise.all([
site.isFallback
? fetchSites({
slug: siteSlug,
name: siteName,
url: siteUrl,
})
: [site],
matchTags(rawTags),
]);
return {
url,
entryId,
title,
description,
actors,
tags,
poster,
photos,
trailer: trailer && {
src: trailer.urls.view,
quality: parseInt(trailer.format, 10),
},
date,
site: channelSite,
};
}
function getUrl(site) {
const { hostname, search } = new URL(site.url);
if (hostname.match(/(www\.)?realitykings\.com/) && search.match(/\?site=\d+/)) {
return site.url;
}
if (site.parameters && site.parameters.siteId) {
return `https://www.realitykings.com/scenes?site=${site.parameters.siteId}`;
}
throw new Error(`Reality Kings site '${site.name}' (${site.url}) not supported`);
}
async function fetchLatest(site, page = 1) {
const url = getUrl(site);
const { search } = new URL(url);
const siteId = new URLSearchParams(search).get('site');
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
await session.get(url);
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 10;
const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
headers: {
Instance: instanceToken,
},
});
return scrapeLatest(res.body.result, site);
}
async function fetchScene(url, site) {
const entryId = url.match(/\d+/)[0];
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
await session.get(url);
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
headers: {
Instance: instanceToken,
},
});
return scrapeScene(res.body.result, url, site);
}
module.exports = {
fetchLatest,
fetchScene,
};