DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added basic filename copy. Added HTTP helper to q. Fetching all actor release pages from Naughty America. Added various high res network logos.

Browse Source
This commit is contained in:
ThePendulum
2020-02-04 00:18:53 +01:00
parent bffa6d2c9e
commit ef602a3a15
42 changed files with 1483 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/scrapers/naughtyamerica.js
View File

@@ -6,7 +6,7 @@ const cheerio = require('cheerio');
const moment = require('moment');
const slugify = require('../utils/slugify');
const { ex } = require('../utils/q');
const { ex, get } = require('../utils/q');
function titleExtractor(pathname) {
const components = pathname.split('/')[2].split('-');
@@ -100,7 +100,13 @@ function scrapeScene(html, url, site) {
};
}
function scrapeProfile(html) {
async function fetchActorReleases(url) {
const { qus } = await get(url);
return qus('.contain-block:not(.live-scenes) .scene-item > a:first-child'); // live scenes repeat on all pages
}
async function scrapeProfile(html) {
const { q, qus } = ex(html);
const profile = {};
@@ -109,7 +115,11 @@ function scrapeProfile(html) {
const avatar = q('img.performer-pic', 'src');
if (avatar) profile.avatar = `https:${avatar}`;
profile.releases = qus('.scene-item > a:first-child');
const releases = qus('.scene-item > a:first-child');
const otherPages = qus('.pagination a:not([rel=next]):not([rel=prev])');
const olderReleases = await Promise.all(otherPages.map(async page => fetchActorReleases(page)));
profile.releases = releases.concat(olderReleases.flat());
return profile;
}