Fixed countries seed file. Updated MOFOS scraper. Improved Reality Kings scraper. Limiting photos for XEmpire scraper.
This commit is contained in:
@@ -1,67 +1,84 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
|
||||
const moment = require('moment');
|
||||
|
||||
const knex = require('../knex');
|
||||
const { fetchSites } = require('../sites');
|
||||
const { cookieToData } = require('../utils/cookies');
|
||||
const { matchTags } = require('../tags');
|
||||
|
||||
function getThumbs(scene) {
|
||||
if (scene.images.poster) {
|
||||
return scene.images.poster.map(image => image.xl.url);
|
||||
}
|
||||
|
||||
if (scene.images.card_main_rect) {
|
||||
return scene.images.card_main_rect
|
||||
.concat(scene.images.card_secondary_rect || [])
|
||||
.map(image => image.xl.url.replace('.thumb', ''));
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/* eslint-disable newline-per-chained-call */
|
||||
function scrape(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const sceneElements = $('.widget-release-card').toArray();
|
||||
async function scrapeLatest(items, site) {
|
||||
return Promise.all(items.map(async (data) => {
|
||||
const { id: entryId, title, description } = data;
|
||||
const url = `https://www.mofos.com/scene/${entryId}/`;
|
||||
const date = new Date(data.dateReleased);
|
||||
const actors = data.actors.map(actor => actor.name);
|
||||
|
||||
return sceneElements.map((element) => {
|
||||
const sceneLinkElement = $(element).find('.title a');
|
||||
const rawTags = data.tags.map(tag => tag.name);
|
||||
const tags = await matchTags(rawTags);
|
||||
|
||||
const title = sceneLinkElement.text().trim();
|
||||
const url = `https://www.mofos.com${sceneLinkElement.attr('href')}`;
|
||||
const entryId = url.split('/').slice(-2, -1)[0];
|
||||
|
||||
const date = moment.utc($(element).find('.date-added').text(), 'MMM DD, YYYY').toDate();
|
||||
const actors = $(element).find('.girls-name a').map((actorIndex, actorElement) => $(actorElement).attr('title').replace(/\s+/g, ' ')).toArray();
|
||||
|
||||
const stars = Number($(element).find('.rating').text().slice(0, -1).trim()) / 20;
|
||||
const [poster, ...photos] = getThumbs(data);
|
||||
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
description,
|
||||
actors,
|
||||
date,
|
||||
rating: {
|
||||
stars,
|
||||
tags,
|
||||
poster,
|
||||
photos,
|
||||
trailer: {
|
||||
src: trailer.urls.view,
|
||||
quality: parseInt(trailer.format, 10),
|
||||
},
|
||||
date,
|
||||
site,
|
||||
};
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const sceneElement = $('.video-info');
|
||||
async function scrapeScene(data, url, site) {
|
||||
const { id: entryId, title, description } = data;
|
||||
const date = new Date(data.dateReleased);
|
||||
const actors = data.actors.map(actor => actor.name);
|
||||
|
||||
const entryId = url.split('/').slice(-2, -1)[0];
|
||||
const title = sceneElement.find('.title').text();
|
||||
const description = sceneElement.find('.desc').text();
|
||||
const actors = sceneElement.find('.girls-site-box a.model-name').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
|
||||
const rawTags = data.tags.map(tag => tag.name);
|
||||
|
||||
const siteElement = sceneElement.find('.site-name');
|
||||
const sitename = siteElement.text().trim();
|
||||
const siteId = sitename.replace(/\s+/g, '').toLowerCase();
|
||||
const siteUrl = siteElement.attr('href').split('/').slice(0, 4).join('/');
|
||||
const [poster, ...photos] = getThumbs(data);
|
||||
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
|
||||
|
||||
const stars = Number(sceneElement.find('.rating-box .rating').text().slice(0, -1).trim()) / 20;
|
||||
const siteName = data.collections[0].name;
|
||||
const siteId = data.collections[0].id;
|
||||
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
|
||||
const siteUrl = `https://www.mofos.com/scenes?site=${siteId}`;
|
||||
|
||||
const rawTags = sceneElement.find('.categories a').map((tagIndex, tagElement) => $(tagElement).text().trim()).toArray();
|
||||
|
||||
const [channelSite, tags] = await Promise.all([
|
||||
knex('sites')
|
||||
.where({ slug: siteId })
|
||||
.orWhere({ url: `https://www.mofos.com${siteUrl}` })
|
||||
.orWhere({ name: sitename })
|
||||
.first(),
|
||||
const [[channelSite], tags] = await Promise.all([
|
||||
site.isFallback
|
||||
? fetchSites({
|
||||
slug: siteSlug,
|
||||
name: siteName,
|
||||
url: siteUrl,
|
||||
})
|
||||
: [site],
|
||||
matchTags(rawTags),
|
||||
]);
|
||||
|
||||
@@ -72,35 +89,61 @@ async function scrapeScene(html, url, site) {
|
||||
description,
|
||||
actors,
|
||||
tags,
|
||||
rating: {
|
||||
stars,
|
||||
poster,
|
||||
photos,
|
||||
trailer: {
|
||||
src: trailer.urls.view,
|
||||
quality: parseInt(trailer.format, 10),
|
||||
},
|
||||
site: channelSite || site,
|
||||
date,
|
||||
site: channelSite,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const res = page > 1
|
||||
? await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/${page}/`)
|
||||
: await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/`); // explicit page 1 redirects to homepage
|
||||
const { search } = new URL(site.url);
|
||||
const siteId = new URLSearchParams(search).get('site');
|
||||
|
||||
return scrape(res.body.toString(), site);
|
||||
}
|
||||
const cookieJar = new CookieJar();
|
||||
const session = bhttp.session({ cookieJar });
|
||||
|
||||
async function fetchUpcoming(site) {
|
||||
const res = await bhttp.get(`${site.url}/all-models/all-categories/upcoming/bydate/`);
|
||||
await session.get(site.url);
|
||||
|
||||
return scrape(res.body.toString(), site);
|
||||
const cookieString = await cookieJar.getCookieStringAsync(site.url);
|
||||
const { instance_token: instanceToken } = cookieToData(cookieString);
|
||||
|
||||
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
|
||||
const limit = 10;
|
||||
const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
|
||||
headers: {
|
||||
Instance: instanceToken,
|
||||
},
|
||||
});
|
||||
|
||||
return scrapeLatest(res.body.result, site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const res = await bhttp.get(url);
|
||||
const entryId = url.match(/\d+/)[0];
|
||||
|
||||
return scrapeScene(res.body.toString(), url, site);
|
||||
const cookieJar = new CookieJar();
|
||||
const session = bhttp.session({ cookieJar });
|
||||
|
||||
await session.get(url);
|
||||
|
||||
const cookieString = await cookieJar.getCookieStringAsync(url);
|
||||
const { instance_token: instanceToken } = cookieToData(cookieString);
|
||||
|
||||
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
|
||||
headers: {
|
||||
Instance: instanceToken,
|
||||
},
|
||||
});
|
||||
|
||||
return scrapeScene(res.body.result, url, site);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchUpcoming,
|
||||
fetchScene,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user