DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Fixed countries seed file. Updated MOFOS scraper. Improved Reality Kings scraper. Limiting photos for XEmpire scraper.

Browse Source
This commit is contained in:
ThePendulum
2019-11-27 04:58:38 +01:00
parent de36ed97e4
commit d113123778
61 changed files with 2182 additions and 2005 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/scrapers/xempire.js
View File

@@ -7,6 +7,7 @@ const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');
const defaultTags = {
hardx: [],
@@ -36,7 +37,7 @@ function scrapePhotos(html) {
return unlockedPhotos.concat(lockedThumbnails);
}
async function getPhotos(albumPath, siteDomain) {
async function getPhotos(albumPath, siteDomain, site) {
const albumUrl = `https://${siteDomain}${albumPath}`;
const html = await fetchPhotos(albumUrl);
@@ -54,7 +55,14 @@ async function getPhotos(albumPath, siteDomain) {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
const allPhotos = photos.concat(otherPhotos.flat());
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
return pluckedPhotos;
}
function scrape(html, site) {
@@ -140,7 +148,7 @@ async function scrapeScene(html, url, site) {
const poster = videoData.picPreview;
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain);
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
const rawTags = data.keywords.split(', ');