forked from DebaucheryLibrarian/traxxx
Added Sex Like Real. Added separate missing date limit for upcoming updates.
This commit is contained in:
parent
6cc872821c
commit
51bd1bff14
|
@ -380,6 +380,7 @@ module.exports = {
|
||||||
},
|
},
|
||||||
fetchAfter: [1, 'week'],
|
fetchAfter: [1, 'week'],
|
||||||
missingDateLimit: 3,
|
missingDateLimit: 3,
|
||||||
|
upcomingMissingDateLimit: 20,
|
||||||
memorySampling: {
|
memorySampling: {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
sampleDuration: 300000, // 5 minutes
|
sampleDuration: 300000, // 5 minutes
|
||||||
|
|
|
@ -1269,6 +1269,26 @@ const tags = [
|
||||||
implies: ['anal'],
|
implies: ['anal'],
|
||||||
description: 'Ass is the new pussy! Zero vaginal sex, anal only, guaranteed.',
|
description: 'Ass is the new pussy! Zero vaginal sex, anal only, guaranteed.',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'fisheye',
|
||||||
|
slug: 'fisheye',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'passthrough',
|
||||||
|
slug: 'passthrough',
|
||||||
|
description: 'Passthrough is a feature on VR headsets that allows you to see your real environment behind the VR content.',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'AI passthrough',
|
||||||
|
slug: 'ai-passthrough',
|
||||||
|
description: '[Passthrough](/tag/passthrough) VR that is filmed with a regular background that is removed by AI, rather than traditional chroma keying (green screen).',
|
||||||
|
implies: ['passthrough'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'scripts',
|
||||||
|
slug: 'scripts',
|
||||||
|
description: 'Scripts for haptic sex toys.',
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const aliases = [
|
const aliases = [
|
||||||
|
@ -1411,6 +1431,14 @@ const aliases = [
|
||||||
name: 'boob fucking',
|
name: 'boob fucking',
|
||||||
for: 'titty-fucking',
|
for: 'titty-fucking',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'tits fucking',
|
||||||
|
for: 'titty-fucking',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'titfucking',
|
||||||
|
for: 'titty-fucking',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: 'bts',
|
name: 'bts',
|
||||||
for: 'bts',
|
for: 'bts',
|
||||||
|
@ -2603,6 +2631,14 @@ const aliases = [
|
||||||
name: 'anal only',
|
name: 'anal only',
|
||||||
for: 'only-anal',
|
for: 'only-anal',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: '200°',
|
||||||
|
for: 'fisheye',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'sex toy scripts',
|
||||||
|
for: 'scripts',
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const priorities = [ // higher index is higher priority
|
const priorities = [ // higher index is higher priority
|
||||||
|
|
|
@ -10651,6 +10651,13 @@ const sites = [
|
||||||
url: 'https://www.milfbundle.com/yourwifemymeat',
|
url: 'https://www.milfbundle.com/yourwifemymeat',
|
||||||
parent: 'score',
|
parent: 'score',
|
||||||
},
|
},
|
||||||
|
// SEX LIKE REAL
|
||||||
|
{
|
||||||
|
name: 'Sex Like Real',
|
||||||
|
slug: 'sexlikereal',
|
||||||
|
url: 'https://www.sexlikereal.com',
|
||||||
|
tags: ['vr'],
|
||||||
|
},
|
||||||
// SEXY HUB
|
// SEXY HUB
|
||||||
{
|
{
|
||||||
slug: 'danejones',
|
slug: 'danejones',
|
||||||
|
|
|
@ -168,6 +168,12 @@ const { argv } = yargs
|
||||||
default: config.missingDateLimit,
|
default: config.missingDateLimit,
|
||||||
alias: ['null-date-limit'],
|
alias: ['null-date-limit'],
|
||||||
})
|
})
|
||||||
|
.option('upcoming-missing-date-limit', {
|
||||||
|
describe: 'Limit amount of scenes when dates are missing.',
|
||||||
|
type: 'number',
|
||||||
|
default: config.upcomingMissingDateLimit,
|
||||||
|
alias: ['upcoming-null-date-limit'],
|
||||||
|
})
|
||||||
.option('page', {
|
.option('page', {
|
||||||
describe: 'Page to start scraping at',
|
describe: 'Page to start scraping at',
|
||||||
type: 'number',
|
type: 'number',
|
||||||
|
|
|
@ -60,6 +60,7 @@ const privateNetwork = require('./private'); // reserved keyword
|
||||||
const purgatoryx = require('./purgatoryx');
|
const purgatoryx = require('./purgatoryx');
|
||||||
const radical = require('./radical');
|
const radical = require('./radical');
|
||||||
const rickysroom = require('./rickysroom');
|
const rickysroom = require('./rickysroom');
|
||||||
|
const sexlikereal = require('./sexlikereal');
|
||||||
const score = require('./score');
|
const score = require('./score');
|
||||||
const spizoo = require('./spizoo');
|
const spizoo = require('./spizoo');
|
||||||
const teamskeet = require('./teamskeet');
|
const teamskeet = require('./teamskeet');
|
||||||
|
@ -159,6 +160,7 @@ const scrapers = {
|
||||||
rickysroom,
|
rickysroom,
|
||||||
sayuncle: teamskeet,
|
sayuncle: teamskeet,
|
||||||
score,
|
score,
|
||||||
|
sexlikereal,
|
||||||
sexyhub: aylo,
|
sexyhub: aylo,
|
||||||
spizoo,
|
spizoo,
|
||||||
swallowsalon: julesjordan,
|
swallowsalon: julesjordan,
|
||||||
|
@ -304,6 +306,7 @@ const scrapers = {
|
||||||
sayuncle: teamskeet,
|
sayuncle: teamskeet,
|
||||||
score,
|
score,
|
||||||
seehimfuck: hush,
|
seehimfuck: hush,
|
||||||
|
sexlikereal,
|
||||||
sexyhub: aylo,
|
sexyhub: aylo,
|
||||||
silverstonedvd: famedigital,
|
silverstonedvd: famedigital,
|
||||||
silviasaint: famedigital,
|
silviasaint: famedigital,
|
||||||
|
|
|
@ -0,0 +1,170 @@
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const unprint = require('unprint');
|
||||||
|
|
||||||
|
function scrapeAll(scenes, channel) {
|
||||||
|
return scenes.map(({ query }) => {
|
||||||
|
const release = {};
|
||||||
|
const data = query.json('script[type="application/ld+json"]');
|
||||||
|
|
||||||
|
release.url = unprint.prefixUrl(data?.url, channel.url) || query.url('article a[href*="/scenes"]');
|
||||||
|
release.entryId = query.attribute(null, 'data-scene-id');
|
||||||
|
|
||||||
|
release.title = data?.name || query.content('.c-grid-item-footer-title');
|
||||||
|
release.description = data?.description;
|
||||||
|
|
||||||
|
release.date = unprint.extractDate(data?.datePublished, 'YYYY-MM-DD');
|
||||||
|
release.duration = query.duration('.c-grid-ratio-bottom');
|
||||||
|
|
||||||
|
release.tags = [
|
||||||
|
query.exists('.c-grid-badge--fisheye-bg') && 'fisheye',
|
||||||
|
query.exists('.c-grid-badge--fleshlight-badge-multi') && 'scripts',
|
||||||
|
query.exists('.c-grid-badge--passthrough') && 'passthrough',
|
||||||
|
query.exists('.c-grid-badge--passthrough-ai') && 'ai-passthrough',
|
||||||
|
].filter(Boolean);
|
||||||
|
|
||||||
|
const poster = query.img('img[data-qa="grid-item-photo-img"]', { attribute: 'data-srcset' });
|
||||||
|
|
||||||
|
if (poster) {
|
||||||
|
release.poster = [
|
||||||
|
poster.replace('-app.', '-desktop.'),
|
||||||
|
poster,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
release.teaser = query.video('img[data-qa="grid-item-photo-img"]', { attribute: 'data-videosrc' });
|
||||||
|
|
||||||
|
return release;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeScene({ query, window }, { url, entity }) {
|
||||||
|
const release = {};
|
||||||
|
const data = query.json('//script[contains(@type, "application/ld+json") and contains(text(), "VideoObject")]');
|
||||||
|
const videoData = window.vrPlayerSettings?.videoData;
|
||||||
|
|
||||||
|
release.entryId = videoData?.id || (data?.url || new URL(url).pathname).split('-').at(-1);
|
||||||
|
|
||||||
|
release.title = videoData?.title || data?.name || query.content('h1[data-qa="scene-title"]');
|
||||||
|
release.description = query.content('div[data-qa="scene-about-tab-text"]'); // data text is cut off
|
||||||
|
|
||||||
|
release.date = unprint.extractDate(data?.datePublished, 'YYYY-MM-DD') || query.date('time[data-qa="page-scene-studio-date"]', 'YYYY-MM-DD', { attribute: 'datetime' });
|
||||||
|
// release.duration = unprint.extractTimestamp(data?.duration); // video duration data seems to be missing hours, regularly leading to wrong numbers; rely on front page duration
|
||||||
|
|
||||||
|
release.actors = query.all('div[data-qa="scene-model-list-item"]').map((actorEl) => {
|
||||||
|
const avatar = unprint.query.content(actorEl, 'img[data-qa="scene-model-list-item-photo-img"]', { attribute: 'data-src' });
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: unprint.query.content(actorEl, 'a[data-qa="scene-model-list-item-name"]'),
|
||||||
|
url: unprint.query.url(actorEl, 'a[data-qa="scene-model-list-item-photo-link-to-profile"], a[data-qa="scene-model-list-item-name"]', { origin: entity.url }),
|
||||||
|
avatar: [
|
||||||
|
avatar?.replace('-small.', '.'),
|
||||||
|
avatar,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}) || data?.actor.map((actor) => actor.name); // prefer html actors for url and avatar
|
||||||
|
|
||||||
|
release.tags = query.contents('a[data-qa="scene-tags-list-item-link"]');
|
||||||
|
|
||||||
|
const fallbackPoster = data?.thumbnail || query.img(`link[rel="preload"][as="image"][href*="images/${release.entryId}"]`);
|
||||||
|
|
||||||
|
release.poster = [
|
||||||
|
videoData?.posterURL,
|
||||||
|
fallbackPoster?.replace('-app.', '-desktop.'),
|
||||||
|
fallbackPoster,
|
||||||
|
];
|
||||||
|
|
||||||
|
release.photos = query.imgs('.mediabox-img', { attribute: 'data-srcset' });
|
||||||
|
|
||||||
|
release.trailer = videoData?.src
|
||||||
|
.filter((src) => src.encoding === 'h264')
|
||||||
|
?.map((src) => ({
|
||||||
|
src: src.url,
|
||||||
|
type: src.mimeType,
|
||||||
|
quality: parseInt(src.quality, 10),
|
||||||
|
expectType: {
|
||||||
|
'binary/octet-stream': 'video/mp4',
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
release.chapters = videoData?.timeStamps?.map((chapter) => ({
|
||||||
|
time: chapter.ts,
|
||||||
|
tags: [chapter.name],
|
||||||
|
}));
|
||||||
|
|
||||||
|
release.qualities = release.trailer?.map((trailer) => trailer.quality);
|
||||||
|
|
||||||
|
return release;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeProfile({ query }, entity) {
|
||||||
|
const profile = {};
|
||||||
|
const data = query.json('//script[contains(@type, "application/ld+json") and contains(text(), "Person")]');
|
||||||
|
|
||||||
|
if (!data) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
profile.url = unprint.prefixUrl(data.url, entity.url);
|
||||||
|
|
||||||
|
profile.dateOfBirth = unprint.extractDate(data.birthDate, 'MMMM DD, YYYY');
|
||||||
|
profile.birthPlace = data.nationality; // origin country rather than nationality
|
||||||
|
|
||||||
|
// height and weight are provided in both cm and lbs, but this seems to be a manual conversion; the format isn't always the same
|
||||||
|
profile.height = unprint.extractNumber(data.height, { match: /(\d+)\s*cm/, matchIndex: 1 });
|
||||||
|
profile.weight = unprint.extractNumber(data.weight, { match: /(\d+)\s*kg/, matchIndex: 1 });
|
||||||
|
|
||||||
|
profile.description = data.description;
|
||||||
|
|
||||||
|
profile.avatar = [
|
||||||
|
data.image?.replace('-small.', '.'),
|
||||||
|
data.image,
|
||||||
|
];
|
||||||
|
|
||||||
|
return profile;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchLatest(channel, page = 1) {
|
||||||
|
const url = `https://www.sexlikereal.com/studios/slr-originals?sort=most_recent&page=${page}`;
|
||||||
|
const res = await unprint.get(url, { selectAll: '.c-grid-item--scene' });
|
||||||
|
|
||||||
|
if (res.ok) {
|
||||||
|
return scrapeAll(res.context, channel);
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchUpcoming(channel) {
|
||||||
|
const url = 'https://www.sexlikereal.com/studios/slr-originals?type=upcoming';
|
||||||
|
const res = await unprint.get(url, { selectAll: '.c-grid-item--scene' });
|
||||||
|
|
||||||
|
if (res.ok) {
|
||||||
|
return scrapeAll(res.context, channel);
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchProfile({ slug }, entity) {
|
||||||
|
const url = `${entity.url}/pornstars/${slug}`;
|
||||||
|
const res = await unprint.get(url);
|
||||||
|
|
||||||
|
if (res.ok) {
|
||||||
|
return scrapeProfile(res.context, entity);
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
fetchLatest,
|
||||||
|
fetchUpcoming,
|
||||||
|
fetchProfile,
|
||||||
|
scrapeScene: {
|
||||||
|
scraper: scrapeScene,
|
||||||
|
parser: {
|
||||||
|
runScripts: 'dangerously',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
|
@ -151,7 +151,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
|
||||||
|
|
||||||
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|
||||||
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|
||||||
|| releases.slice(0, Math.max(argv.missingDateLimit, 0));
|
|| releases.slice(0, Math.max(isUpcoming ? argv.upcomingMissingDateLimit : argv.missingDateLimit, 0));
|
||||||
|
|
||||||
const { uniqueReleases, duplicateReleases } = argv.force
|
const { uniqueReleases, duplicateReleases } = argv.force
|
||||||
? { uniqueReleases: limitedReleases, duplicateReleases: [] }
|
? { uniqueReleases: limitedReleases, duplicateReleases: [] }
|
||||||
|
|
Loading…
Reference in New Issue