boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added sparse date mode. Fixed profile fetch error catching. Updated Kelly Madison scraper, using site IDs and fixed profile scraper.

This commit is contained in:
DebaucheryLibrarian 2024-12-30 01:57:26 +01:00
parent ffea90b5e8
commit 23f76fd1be
9 changed files with 73 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config/default.js
View File

@ -273,6 +273,7 @@ module.exports = {
'topwebmodels',
'pascalssubsluts',
'kellymadison',
'5kporn',
'private',
'bangbros',
'hitzefrei',

20
seeds/02_sites.js
View File

@ -5281,23 +5281,30 @@ const sites = [
name: 'Teen Fidelity',
alias: ['tf'],
url: 'https://www.teenfidelity.com',
description: 'Home of Kelly Madison and Ryan Madison',
parent: 'kellymadison',
parameters: {
siteId: 3,
},
},
{
slug: 'pornfidelity',
name: 'Porn Fidelity',
alias: ['pf'],
url: 'https://www.pornfidelity.com',
description: 'Home of Kelly Madison and Ryan Madison',
parent: 'kellymadison',
parameters: {
siteId: 2,
},
},
{
slug: 'kellymadison',
name: 'Kelly Madison',
url: 'https://www.pornfidelity.com',
url: 'https://www.kellymadison.com',
description: 'Home of Kelly Madison and Ryan Madison',
parent: 'kellymadison',
parameters: {
siteId: 1,
},
},
{
slug: '5kporn',
@ -5305,6 +5312,10 @@ const sites = [
url: 'https://www.5kporn.com',
tags: ['5k'],
parent: 'kellymadison',
parameters: {
// IDs overlap with Fidelity sites
siteId: 1,
},
},
{
slug: '5kteens',
@ -5312,6 +5323,9 @@ const sites = [
url: 'https://www.5kteens.com',
tags: ['5k'],
parent: 'kellymadison',
parameters: {
siteId: 2,
},
},
// KILLERGRAM
{

2
src/actors.js
View File

@ -614,7 +614,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
const profiles = Promise.map(validSources, async (source) => {
try {
// config may group sources to try until success
return [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
try {
const entity = entitiesBySlug[scraperSlug] || null;

2
src/app.js
View File

@ -108,6 +108,8 @@ async function startMemorySample(snapshotTriggers = []) {
async function init() {
try {
await redis.connect();
if (argv.server) {
await initServer();
return;

6
src/argv.js
View File

@ -174,6 +174,12 @@ const { argv } = yargs
default: config.upcomingMissingDateLimit,
alias: ['upcoming-null-date-limit'],
})
.option('filter-sparse-dates', {
describe: 'If some but not all scenes have dates, filter out scenes without dates, instead of using missing date limit.',
type: 'boolean',
default: false,
alias: ['sparse'],
})
.option('page', {
describe: 'Page to start scraping at',
type: 'number',

2
src/redis.js
View File

@ -10,6 +10,6 @@ const redisClient = redis.createClient({
socket: config.redis,
});
redisClient.connect();
// redisClient.connect();
module.exports = redisClient;

63
src/scrapers/kellymadison.js
View File

@ -1,9 +1,11 @@
'use strict';
const unprint = require('unprint');
const slugify = require('../utils/slugify');
const qu = require('../utils/qu');
const http = require('../utils/http');
const { feetInchesToCm } = require('../utils/convert');
const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
const siteMapByKey = {
PF: 'pornfidelity',
@ -16,14 +18,11 @@ const siteMapByKey = {
const siteMapBySlug = Object.entries(siteMapByKey).reduce((acc, [key, value]) => ({ ...acc, [value]: key }), {});
function scrapeLatest(scenes, site) {
return scenes.reduce((acc, { query }) => {
return scenes.map(({ query }) => {
const release = {};
release.shootId = query.q('.card-meta .text-right, .row .text-right, .card-footer-item:last-child', true);
const siteId = release.shootId.match(/\d?\w{2}/)[0];
const siteSlug = siteMapByKey[siteId];
const { pathname } = new URL(query.url('h5 a, .ep-title a, .title a'));
[release.entryId] = pathname.match(/\d+$/);
release.url = `${site.url}${pathname}`;
@ -47,15 +46,19 @@ function scrapeLatest(scenes, site) {
};
}
/* using site ID, filter no longer needed
const siteId = release.shootId.match(/\d?\w{2}/)[0];
const siteSlug = siteMapByKey[siteId];
if (site.slug !== siteSlug) {
// using generic network overview, scene is not from the site we want
return { ...acc, unextracted: [...acc.unextracted, release] };
}
return { ...acc, scenes: [...acc.scenes, release] };
}, {
scenes: [],
unextracted: [],
*/
return release;
});
}
@ -114,34 +117,47 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session)
}));
}
console.log(release);
return release;
}
function scrapeProfile({ query }) {
const profile = {};
const bioKeys = query.all('table.table td:nth-child(1)', true);
const bioValues = query.all('table.table td:nth-child(2)', true);
const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key.slice(0, -1)]: bioValues[index] }), {});
const bioKeys = query.contents('table.table td:nth-child(1), table.table th');
const bioValues = query.contents('table.table td:nth-child(2)');
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Measurements) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio.Birthplace) profile.birthPlace = bio.Birthplace;
const bio = bioKeys.reduce((acc, key, index) => ({
...acc,
[slugify(key, '_')]: bioValues[index],
}), {});
if (bio.Height) {
const [feet, inches] = bio.Height.match(/\d+/g);
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
if (bio.measurements) profile.measurements = bio.measurements;
if (bio.birthplace) profile.birthPlace = bio.birthplace;
if (bio.shoe_size) profile.foot = femaleFeetUsToEu(bio.shoe_size);
if (bio.height) {
const [feet, inches] = bio.height.match(/\d+/g);
profile.height = feetInchesToCm(feet, inches);
}
profile.avatar = query.img('img[src*="model"]');
if (bio.birthday) {
const [month, day] = bio.birthday.split('/');
const birthday = new Date(Date.UTC(0, Number(month) - 1, Number(day)));
birthday.setUTCFullYear(0); // indicate birth year is unknown
profile.dateOfBirth = new Date(birthday);
}
profile.avatar = query.img('img[src*="model"][src*="headshot"]');
profile.photos = query.imgs('img[src*="model"][src*="thumb_image"], img[src*="model"][src*="bg_image"]');
return profile;
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/episodes/search?page=${page}`; // TLS issues with teenfidelity.com, same overview on all sites
const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
const res = await http.get(url, {
headers: {
'X-Requested-With': 'XMLHttpRequest',
@ -165,16 +181,17 @@ async function fetchScene(url, channel, baseRelease) {
return res.ok ? scrapeScene(res.item, url, baseRelease, channel, session) : res.status;
}
async function fetchProfile({ name: actorName }) {
async function fetchProfile({ name: actorName }, { entity }) {
const actorSlug = slugify(actorName);
const res = await qu.get(`https://www.kellymadison.com/models/${actorSlug}`, null, {
const res = await unprint.get(`${entity.url}/models/${actorSlug}`, {
headers: {
'X-Requested-With': 'XMLHttpRequest',
},
});
if (res.ok) {
return scrapeProfile(res.item);
return scrapeProfile(res.context);
}
return res.status;

1
src/scrapers/scrapers.js
View File

@ -265,6 +265,7 @@ const scrapers = {
julesjordan,
karups,
kellymadison,
'5kporn': kellymadison,
killergram,
kink,
kinkmen: kink,

6
src/updates.js
View File

@ -147,10 +147,12 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const releases = await scrapeReleasesPage(argv.page || 1, []);
const hasDates = releases.every((release) => !!release.date);
const hasDates = argv.filterSparseDates
? releases.some((release) => !!release.date)
: releases.every((release) => !!release.date);
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|| (hasDates && releases.filter((release) => release.date && moment(release.date).isAfter(argv.after)))
|| releases.slice(0, Math.max(isUpcoming ? argv.upcomingMissingDateLimit : argv.missingDateLimit, 0));
const { uniqueReleases, duplicateReleases } = argv.force