traxxx/src/scrapers/jayrock.js

116 lines
3.9 KiB
JavaScript
Raw Normal View History

2020-01-10 21:10:11 +00:00
'use strict';
/* example for other ModelCentro scrapers */
2020-01-10 21:10:11 +00:00
const Promise = require('bluebird');
const bhttp = require('bhttp');
2020-01-12 04:30:46 +00:00
const logger = require('../logger');
2020-01-10 21:10:11 +00:00
const slugify = require('../utils/slugify');
async function fetchToken(site) {
const res = await bhttp.get(site.url);
2020-01-10 21:10:11 +00:00
const html = res.body.toString();
const time = html.match(/"aet":\d+/)[0].split(':')[1];
const ah = html.match(/"ah":"[\w-]+"/)[0].split(':')[1].slice(1, -1);
const token = ah.split('').reverse().join('');
return { time, token };
}
async function fetchActors(entryId, site, { token, time }) {
const url = `${site.url}/sapi/${token}/${time}/model.getModelContent?_method=model.getModelContent&tz=1&fields[0]=modelId.stageName&fields[1]=_last&fields[2]=modelId.upsellLink&fields[3]=modelId.upsellText&limit=25&transitParameters[contentId]=${entryId}`;
2020-01-10 21:10:11 +00:00
const res = await bhttp.get(url);
if (res.statusCode === 200 && res.body.status === true) {
return Object.values(res.body.response.collection).map(actor => Object.values(actor.modelId.collection)[0].stageName);
}
return [];
}
async function fetchTrailerLocation(entryId, site) {
const url = `${site.url}/api/download/${entryId}/hd1080/stream`;
2020-01-10 21:10:11 +00:00
2020-01-12 04:30:46 +00:00
try {
const res = await bhttp.get(url, {
followRedirects: false,
});
if (res.statusCode === 302) {
return res.headers.location;
}
} catch (error) {
logger.warn(`${site.name}: Unable to fetch trailer at '${url}': ${error.message}`);
2020-01-10 21:10:11 +00:00
}
return null;
}
async function scrapeScene(scene, site, tokens) {
const release = {
entryId: scene.id,
title: scene.title,
duration: scene.length,
site,
meta: {
tokens, // attach tokens to reduce number of requests required for deep fetching
},
2020-01-10 21:10:11 +00:00
};
release.url = `${site.url}/scene/${release.entryId}/${slugify(release.title, { encode: true })}`;
2020-01-10 21:10:11 +00:00
release.date = new Date(scene.sites.collection[scene.id].publishDate);
release.poster = scene._resources.primary[0].url;
if (scene.tags) release.tags = Object.values(scene.tags.collection).map(tag => tag.alias);
if (scene._resources.base) release.photos = scene._resources.base.map(resource => resource.url);
const [actors, trailer] = await Promise.all([
fetchActors(release.entryId, site, tokens),
fetchTrailerLocation(release.entryId, site),
2020-01-10 21:10:11 +00:00
]);
release.actors = actors;
if (trailer) release.trailer = { src: trailer, quality: 1080 };
return release;
}
function scrapeLatest(scenes, site, tokens) {
return Promise.map(scenes, async scene => scrapeScene(scene, site, tokens), { concurrency: 10 });
}
2020-01-12 04:30:46 +00:00
async function fetchLatest(site, page = 1) {
const { time, token } = await fetchToken(site);
2020-01-10 21:10:11 +00:00
// transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory
2020-01-12 04:30:46 +00:00
const url = `${site.url}/sapi/${token}/${time}/content.load?limit=50&offset=${(page - 1) * 50}&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`;
2020-01-10 21:10:11 +00:00
const res = await bhttp.get(url);
if (res.statusCode === 200 && res.body.status) {
return scrapeLatest(res.body.response.collection, site, { time, token });
}
return null;
}
async function fetchScene(url, site, release) {
const { time, token } = release?.meta.tokens || await fetchToken(site); // use attached tokens when deep fetching
2020-01-10 21:10:11 +00:00
const { pathname } = new URL(url);
const entryId = pathname.split('/')[2];
const apiUrl = `${site.url}/sapi/${token}/${time}/content.load?filter[id][fields][0]=id&filter[id][values][0]=${entryId}&transitParameters[v1]=ykYa8ALmUD&transitParameters[preset]=scene`;
const res = await bhttp.get(apiUrl);
2020-01-10 21:10:11 +00:00
if (res.statusCode === 200 && res.body.status) {
return scrapeScene(res.body.response.collection[0], site, { time, token });
}
return null;
}
module.exports = {
fetchLatest,
fetchScene,
};