DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Refactored and fixed Naughty America scraper.

This commit is contained in:
DebaucheryLibrarian 2024-07-06 06:08:35 +02:00
parent 7fb24cb31b
commit 56163f2b3d
4 changed files with 92 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
package-lock.json generated
View File

@ -88,7 +88,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.37", "ua-parser-js": "^1.0.37",
"undici": "^5.28.1", "undici": "^5.28.1",
"unprint": "^0.11.2", "unprint": "^0.11.5",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3", "v-tooltip": "^2.1.3",
"video.js": "^8.6.1", "video.js": "^8.6.1",
@ -18293,9 +18293,9 @@
} }
}, },
"node_modules/unprint": { "node_modules/unprint": {
"version": "0.11.2", "version": "0.11.5",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.11.2.tgz", "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.11.5.tgz",
"integrity": "sha512-i4WJxmEnd6LKYbcnKAjX8bkaPRdyDlhAAqpxej0qIX0pjK5d17hp51x/RGDMfEe63dlcJtGCn9bhZrGcMY4PXQ==", "integrity": "sha512-tLhiFGeSU40GN12625+9oqmNGDFSToMPME60pB+DSGT9wd9fJM0L/lyZMQeNFmWMSThwa/id/FHAOnN7cE1aOw==",
"dependencies": { "dependencies": {
"axios": "^0.27.2", "axios": "^0.27.2",
"bottleneck": "^2.19.5", "bottleneck": "^2.19.5",

2
package.json
View File

@ -147,7 +147,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.37", "ua-parser-js": "^1.0.37",
"undici": "^5.28.1", "undici": "^5.28.1",
"unprint": "^0.11.2", "unprint": "^0.11.5",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3", "v-tooltip": "^2.1.3",
"video.js": "^8.6.1", "video.js": "^8.6.1",

12
seeds/02_sites.js
View File

@ -7120,7 +7120,8 @@ const sites = [
{ {
slug: 'myfriendshotmom', slug: 'myfriendshotmom',
name: 'My Friend\'s Hot Mom', name: 'My Friend\'s Hot Mom',
alias: ['mfhm'], alias: ['mfhm', 'watchyourmom', 'watch your mom'],
comment: 'Merged with Watch Your Mom.',
url: 'https://www.naughtyamerica.com/site/my-friend-s-hot-mom', url: 'https://www.naughtyamerica.com/site/my-friend-s-hot-mom',
parent: 'naughtyamerica', parent: 'naughtyamerica',
}, },
@ -7145,6 +7146,7 @@ const sites = [
{ {
slug: 'watchyourmom', slug: 'watchyourmom',
name: 'Watch Your Mom', name: 'Watch Your Mom',
comment: 'Merged into My Friend\'s Hot Mom.',
url: 'https://www.naughtyamerica.com/site/watch-your-mom', url: 'https://www.naughtyamerica.com/site/watch-your-mom',
parent: 'naughtyamerica', parent: 'naughtyamerica',
}, },
@ -7506,7 +7508,7 @@ const sites = [
{ {
slug: 'afterschool', slug: 'afterschool',
name: 'After School', name: 'After School',
url: 'https://www.naughtyamerica.com/site/after-school', url: 'https://www.naughtyamerica.com/site/after-schooll', // sic
parent: 'naughtyamerica', parent: 'naughtyamerica',
}, },
{ {
@ -7617,6 +7619,12 @@ const sites = [
url: 'https://www.naughtyamerica.com/site/tonight-s-fuck', url: 'https://www.naughtyamerica.com/site/tonight-s-fuck',
parent: 'naughtyamerica', parent: 'naughtyamerica',
}, },
{
slug: 'fans',
name: 'Fans',
url: 'https://www.naughtyamerica.com/site/fans',
parent: 'naughtyamerica',
},
// NEBRASKA COEDS // NEBRASKA COEDS
{ {
name: 'Nebraska Coeds', name: 'Nebraska Coeds',

187
src/scrapers/naughtyamerica.js
View File

@ -1,158 +1,123 @@
'use strict'; 'use strict';
/* eslint-disable newline-per-chained-call */ const unprint = require('unprint');
const cheerio = require('cheerio');
const moment = require('moment');
const http = require('../utils/http');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
const qu = require('../utils/q');
function titleExtractor(pathname) { function scrapeLatest(scenes) {
const components = pathname.split('/')[2].split('-'); return scenes.map(({ query }) => {
const entryId = components.slice(-1)[0]; const release = {};
const url = query.url('a');
const title = components.slice(0, -1).reduce((accTitle, word, index) => `${accTitle}${index > 0 ? ' ' : ''}${word.slice(0, 1).toUpperCase()}${word.slice(1)}`, ''); release.url = url;
release.entryId = query.attribute('a', 'data-scene-id') || (url && new URL(url).pathname.match(/-(\d+)$/)?.[1]) || null;
return { title, entryId }; release.date = query.date('.entry-date', 'MMM D, YYYY');
} release.duration = query.duration('.scene-runtime');
function scrapeLatest(html, site) { release.actors = query.all('.contain-actors a').map((actorEl) => ({
const $ = cheerio.load(html, { normalizeWhitespace: true }); name: unprint.query.content(actorEl),
const sceneElements = $('.site-list .scene-item').toArray(); url: unprint.query.url(actorEl, null),
}));
return sceneElements.map((item) => { release.poster = [
const element = $(item); ...(query.sourceSet('source[data-srcset*="scenes/"][type="image/jpeg"]', 'data-srcset') || []),
query.img('.main-scene-img', { attribute: 'data-srcset' }),
];
const sceneLinkElement = element.find('a').first(); release.tags = query.contents('.flag-bg');
const { protocol, hostname, pathname } = new URL(sceneLinkElement.attr('href'));
const url = `${protocol}//${hostname}${pathname}`;
const { title, entryId } = titleExtractor(pathname);
const date = moment.utc(element.find('.entry-date').text(), 'MMM D, YYYY').toDate(); release.qualities = [
const actors = element.find('.contain-actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); query.exists('//a[contains(@class, "label-four-k") and contains(text(), "4K")]') && 2160, // label-four-k is also used for non-4K tags
query.exists('//a[contains(@class, "label-hd") and contains(text(), "HD")]') && 720,
].filter(Boolean);
const duration = Number(element.find('.scene-runtime').text().slice(0, -4)) * 60; release.channel = slugify(query.content('.site-title'), '');
const posterString = sceneLinkElement.find('img[data-srcset]').attr('data-srcset') || sceneLinkElement.find('img[data-src]').attr('data-src'); return release;
const poster = `https:${posterString.match(/[\w/.]+$/)[0]}`;
return {
url,
entryId,
title,
actors,
date,
duration,
poster,
rating: null,
site,
};
}); });
} }
function scrapeScene(html, url, site) { function scrapeScene({ query }, { url }) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const release = {};
const sceneElement = $('.scene-info'); release.entryId = new URL(url).pathname.match(/-(\d+)$/)?.[1];
const { protocol, hostname, pathname } = new URL(url); release.title = query.content('.scene-title');
const originalUrl = `${protocol}//${hostname}${pathname}`; release.description = query.text('.synopsis');
const entryId = originalUrl.split('-').slice(-1)[0]; release.date = query.date('.entry-date', 'MMM D, YYYY');
const title = sceneElement.find('h1.scene-title').text(); release.duration = query.duration('.duration');
const description = sceneElement.find('.synopsis').contents().slice(2).text().replace(/[\s\n]+/g, ' ').trim();
const date = moment.utc(sceneElement.find('span.entry-date').text()?.match(/\w+ \d{1,2}, \d{4}/), 'MMM D, YYYY').toDate(); release.actors = query.all('.performer-list a').map((actorEl) => ({
const actors = $('.performer-list a, h1 a.scene-title').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null),
}));
const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60; release.poster = [
...(query.sourceSet('source[data-srcset*="scenes/"][type="image/jpeg"]', 'data-srcset') || []),
query.img('.play-trailer img[data-srcset*="scenes/"]', { attribute: 'data-srcset' }),
];
const posterPath = $('video, dl8-video').attr('poster') || $('img.start-card').attr('src'); release.photos = query.els('.contain-scene-images.desktop-only .scene-image').map((imgEl) => [
const poster = posterPath && `https:${posterPath}`; unprint.query.url(imgEl, null),
const photos = $('.contain-scene-images.desktop-only a').map((index, el) => $(el).attr('href')).toArray().filter(Boolean).map((photo) => `https:${photo}`); unprint.query.img(imgEl, 'img', { attribute: 'srcset' }),
]);
const trailerEl = $('source'); const trailer = query.video('video source');
const trailerSrc = trailerEl.attr('src');
const trailerType = trailerEl.attr('type');
const siteName = sceneElement.find('a.site-title').text(); if (trailer) {
const channel = siteName.replace(/[\s']+/g, '').toLowerCase(); release.trailer = [
{
const tags = $('.categories a.cat-tag').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); source: trailer.replace(/_\d+\.mp4/, '_1080.mp4'),
quality: 1080,
return { },
url, trailer,
entryId, ];
title,
description,
actors,
date,
duration,
tags,
photos,
poster,
trailer: trailerSrc ? {
src: trailerSrc,
type: trailerType,
} : null,
rating: null,
site,
channel,
};
} }
async function fetchActorReleases(url) { release.channel = slugify(query.content('.site-title'), '');
const res = await qu.get(url);
return res.ok release.tags = query.contents('.categories a');
? res.item.query.urls('.contain-block:not(.live-scenes) .scene-item > a:first-child') // live scenes repeat on all pages
: []; release.qualities = [
query.exists('//a[contains(@class, "label-four-k") and contains(text(), "4K")]') && 2160, // label-four-k is also used for non-4K tags
query.exists('img.icon-1080') && 1080,
query.exists('//a[contains(@class, "label-hd") and contains(text(), "HD")]') && 720,
].filter(Boolean);
return release;
} }
async function scrapeProfile(html) { async function scrapeProfile({ query }) {
const { query } = qu.extract(html);
const profile = {}; const profile = {};
profile.description = query.q('.bio_about_text', true); profile.description = query.content('.bio_about_text');
profile.avatar = query.img('img.performer-pic');
const avatar = query.q('img.performer-pic', 'src');
if (avatar) profile.avatar = `https:${avatar}`;
const releases = query.urls('.scene-item > a:first-child');
const otherPages = query.urls('.pagination a:not([rel=next]):not([rel=prev])');
const olderReleases = await Promise.all(otherPages.map(async (page) => fetchActorReleases(page)));
profile.releases = releases.concat(olderReleases.flat());
return profile; return profile;
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const res = await http.get(`${site.url}?page=${page}`); const res = await unprint.get(`${site.url}?page=${page}`, { selectAll: '.site-list .scene-item' });
return scrapeLatest(res.body.toString(), site); return scrapeLatest(res.context, site);
} }
async function fetchScene(url, site) { async function fetchProfile({ slug }) {
const res = await http.get(url); const res = await unprint.get(`https://www.naughtyamerica.com/pornstar/${slug}`, { select: '.bio-info' });
return scrapeScene(res.body.toString(), url, site); if (res.ok) {
return scrapeProfile(res.context);
} }
async function fetchProfile({ name: actorName }) { return res.status;
const actorSlug = slugify(actorName);
const res = await http.get(`https://www.naughtyamerica.com/pornstar/${actorSlug}`);
if (res.statusCode === 200) {
return scrapeProfile(res.body.toString());
}
return null;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene,
fetchProfile, fetchProfile,
scrapeScene: {
scraper: scrapeScene,
unprint: true,
},
}; };