Compare commits
2 Commits
6cb990dbdb
...
c85915bf97
| Author | SHA1 | Date |
|---|---|---|
DebaucheryLibrarian
|
c85915bf97 | |
|
DebaucheryLibrarian
|
6338e8fb8d |
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.245.12",
|
||||
"version": "1.245.13",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "traxxx",
|
||||
"version": "1.245.12",
|
||||
"version": "1.245.13",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.458.0",
|
||||
|
|
@ -93,7 +93,7 @@
|
|||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.18.6",
|
||||
"unprint": "^0.18.7",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
|
|
@ -20340,9 +20340,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/unprint": {
|
||||
"version": "0.18.6",
|
||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.6.tgz",
|
||||
"integrity": "sha512-kcDpsaTaMrxY0AkoHq1bGPuVz6Cv1umC0kA1U58Th+UhFarmwPB5racAY514eWEpjC9AXGEhOvIa+n2hErQmRg==",
|
||||
"version": "0.18.7",
|
||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.7.tgz",
|
||||
"integrity": "sha512-Su7SQ7UxvM5SHuvkLmSGMiWi+nS6Qh3489qqbPYaqvH9DFBDc5C4544mGyysU/7ZKO+5RtAi59prVTNPo8rlSw==",
|
||||
"dependencies": {
|
||||
"bottleneck": "^2.19.5",
|
||||
"cookie": "^1.1.1",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.245.12",
|
||||
"version": "1.245.13",
|
||||
"description": "All the latest porn releases in one place",
|
||||
"main": "src/app.js",
|
||||
"scripts": {
|
||||
|
|
@ -152,7 +152,7 @@
|
|||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.18.6",
|
||||
"unprint": "^0.18.7",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
|
|
|
|||
|
|
@ -818,9 +818,6 @@ const sites = [
|
|||
slug: 'archangel',
|
||||
name: 'ArchAngel',
|
||||
url: 'https://www.archangelvideo.com',
|
||||
parameters: {
|
||||
path: '/tour',
|
||||
},
|
||||
},
|
||||
// ASSYLUM
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,171 +1,35 @@
|
|||
'use strict';
|
||||
|
||||
// ALSO USED BY THE FLOURISH
|
||||
|
||||
const unprint = require('unprint');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const { feetInchesToCm } = require('../utils/convert');
|
||||
const { convert } = require('../utils/convert');
|
||||
const tryUrls = require('../utils/try-urls');
|
||||
|
||||
const placeholder = /images\/p\d+\.jpe?g/i;
|
||||
|
||||
function getEntryId(release) {
|
||||
return slugify(new URL(release.url).pathname.match(/\/([\w-]+)\.html/)?.[1]
|
||||
|| [unprint.formatDate(release.date, 'YYYY-MM-DD'), release.title, ...release.actors]);
|
||||
}
|
||||
|
||||
function scrapeAll(scenes) {
|
||||
function scrapeAll(scenes, channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('a');
|
||||
release.entryId = slugify(new URL(release.url).pathname.match(/trailers\/(.*)/)[1]);
|
||||
|
||||
release.title = query.content('a span');
|
||||
|
||||
release.date = query.date('.timeDate', 'YYYY-MM-DD');
|
||||
release.duration = query.duration('.timeDate');
|
||||
release.title = query.content('h2 a');
|
||||
|
||||
release.actors = query.all('a[href*="models/"], a[href*="sets.php"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
url: unprint.query.url(actorEl, null, { origin: channel.url }),
|
||||
}));
|
||||
|
||||
const poster = query.img('img.mainThumb');
|
||||
const previewCount = query.number('img.mainThumb', { attribute: 'cnt' });
|
||||
|
||||
if (poster && !placeholder.test(poster)) {
|
||||
const posterFallbacks = [
|
||||
poster.replace('-1x', '-3x'),
|
||||
poster.replace('-1x', '-2x'),
|
||||
poster.replace('-1x', '-4x'),
|
||||
poster,
|
||||
];
|
||||
|
||||
release.poster = posterFallbacks;
|
||||
}
|
||||
|
||||
if (previewCount) {
|
||||
release.photos = Array.from(
|
||||
{ length: previewCount - 1 },
|
||||
(value, index) => [3, 2, 4, 1].map((scale) => unprint.prefixUrl(query.img('img.mainThumb', { attribute: `src${index + 1}_${scale}x` }))).filter(Boolean), // 4x is unnecessarily big and possibly upscaled
|
||||
).filter(Boolean);
|
||||
}
|
||||
|
||||
release.photoCount = query.number('.timeDate', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
|
||||
release.entryId = getEntryId(release);
|
||||
release.poster = query.img('.thumbnail img');
|
||||
release.teaser = query.video('.thumbnail img', { attribute: 'data-vid' }); // not a mistake, video source is on img tag
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ query, html }, { url, entity, baseRelease }) {
|
||||
const release = { url };
|
||||
|
||||
release.title = query.content('.title h2');
|
||||
release.description = query.content('.description p');
|
||||
|
||||
release.date = query.date('.info p', 'MMMM D, YYYY');
|
||||
release.duration = query.duration('.info p');
|
||||
|
||||
release.actors = query.all('.info a[href*="models/"], .info a[href*="sets.php"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
const poster = unprint.prefixUrl(query.img('.update_thumb') || html.match(/poster="(.*\.jpg)"/)?.[1], entity.url);
|
||||
|
||||
if (poster && !placeholder.test(poster)) {
|
||||
const posterFallbacks = [
|
||||
poster.replace('-1x', '-3x'),
|
||||
poster.replace('-1x', '-2x'),
|
||||
poster.replace('-1x', '-4x'),
|
||||
poster,
|
||||
];
|
||||
|
||||
// scene page poster usually different from overview page, don't replace
|
||||
if (baseRelease?.poster && baseRelease.poster !== poster) {
|
||||
release.photos = baseRelease.photos
|
||||
? [posterFallbacks, ...baseRelease.photos]
|
||||
: [posterFallbacks];
|
||||
} else {
|
||||
release.poster = posterFallbacks;
|
||||
}
|
||||
}
|
||||
|
||||
const trailer = html.match(/src="(.*\.mp4)"/)?.[1];
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = unprint.prefixUrl(encodeURI(trailer), entity.url);
|
||||
}
|
||||
|
||||
release.tags = query.contents('.info .tags a');
|
||||
|
||||
release.photoCount = query.number('.info', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
release.entryId = getEntryId(release);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeMovie({ query, element }, { entity, url }) {
|
||||
const release = { url };
|
||||
|
||||
release.title = query.content('.title h2');
|
||||
release.description = query.content('.aboutArea p');
|
||||
|
||||
release.covers = [[
|
||||
query.img('.update_thumb', { attribute: 'src0_2x', origin: entity.url }),
|
||||
query.img('.update_thumb', { attribute: 'src0_1x', origin: entity.url }),
|
||||
query.img('.update_thumb', { attribute: 'src0', origin: entity.url }),
|
||||
// usually upscaled
|
||||
query.img('.update_thumb', { attribute: 'src0_4x', origin: entity.url }),
|
||||
query.img('.update_thumb', { attribute: 'src0_3x', origin: entity.url }),
|
||||
].filter(Boolean)];
|
||||
|
||||
release.entryId = getEntryId(release);
|
||||
|
||||
release.scenes = scrapeAll(unprint.initAll(element, '.item-video'));
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query, element }, { url, entity }) {
|
||||
const profile = { url };
|
||||
|
||||
const bio = Object.fromEntries(query.all('.stats li')
|
||||
.map((row) => [
|
||||
slugify(unprint.query.content(row, '.data-name, span'), '_'),
|
||||
unprint.query.text(row),
|
||||
])
|
||||
.filter(([key, value]) => key && value));
|
||||
|
||||
profile.description = query.content('.aboutArea p');
|
||||
|
||||
profile.birthPlace = bio.place_of_birth;
|
||||
profile.dateOfBirth = unprint.extractDate(bio.age, 'MMMM D, YYYY');
|
||||
|
||||
profile.height = Number(bio.height?.match(/(\d+)\s*cm/)?.[1]) || (/\d fe*t \d+ inch/i.test(bio.height) && feetInchesToCm(bio.height)) || null;
|
||||
profile.measurements = bio.measurements;
|
||||
|
||||
profile.hairColor = bio.hair_color;
|
||||
profile.eyes = bio.eye_color;
|
||||
|
||||
profile.avatar = [
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_4x', origin: entity.url }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_3x', origin: entity.url }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_2x', origin: entity.url }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_1x', origin: entity.url }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0', origin: entity.url }),
|
||||
].filter((avatar) => avatar && !placeholder.test(avatar));
|
||||
|
||||
profile.scenes = scrapeAll(unprint.initAll(element, '.item-video'));
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1, context) {
|
||||
const url = `${channel.url}${context.parameters.path || ''}/categories/movies_${page}_d.html`;
|
||||
const res = await unprint.get(url, { selectAll: '.item-video' });
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/porn-categories/movies/?sort=most-recent&page=${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '.content div[data-setid]' });
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.context, channel);
|
||||
|
|
@ -174,32 +38,63 @@ async function fetchLatest(channel, page = 1, context) {
|
|||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName, url: actorUrl }, { entity, include, parameters }) {
|
||||
const res = await [
|
||||
function scrapeScene({ query }, { url }) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = slugify(new URL(url).pathname.match(/trailers\/(.*)/)[1]);
|
||||
|
||||
release.title = query.content('h1.title_bar');
|
||||
release.description = query.content('.description-text');
|
||||
|
||||
release.date = query.date('//label[contains(text(), \'Date\')]/following-sibling::p[1]', 'YYYY-MM-DD');
|
||||
|
||||
release.actors = query.all('.text a[href*="/models"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
release.tags = query.contents('.text a[href*="categories/"]');
|
||||
|
||||
release.poster = query.poster('#preview video');
|
||||
release.trailer = query.video('#preview video source');
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query }, { url }) {
|
||||
const profile = { url };
|
||||
|
||||
const bio = Object.fromEntries(query.all('.model-details > div').map((bioEl) => [
|
||||
slugify(unprint.query.content(bioEl, 'h2'), '_'),
|
||||
unprint.query.text(bioEl),
|
||||
]));
|
||||
|
||||
profile.avatar = [
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_3x' }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_2x' }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_1x' }),
|
||||
];
|
||||
|
||||
profile.description = [query.content('.model-bio-text'), bio.funfact].filter(Boolean).join(' ');
|
||||
profile.aliases = bio.alias?.split(/[,\n]/).map((alias) => alias.trim());
|
||||
|
||||
profile.age = parseInt(bio.age, 10) || null;
|
||||
profile.dateOfBirth = unprint.extractDate(bio.age, 'MM/DD/YYYY');
|
||||
profile.measurements = bio.measurements;
|
||||
profile.height = Number(bio.height.match(/(\d+)\s*cm/)?.[1]) || convert(bio.height, 'cm');
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName, url: actorUrl }, { entity, include }) {
|
||||
const { res, url } = await tryUrls([
|
||||
actorUrl,
|
||||
`${entity.url}${parameters.path || ''}/models/${slugify(actorName, '-')}.html`,
|
||||
`${entity.url}${parameters.path || ''}/models/${slugify(actorName, '')}.html`,
|
||||
].reduce(async (chain, url) => {
|
||||
const prevRes = await chain;
|
||||
|
||||
if (prevRes.ok || !url) {
|
||||
return prevRes;
|
||||
}
|
||||
|
||||
const actorRes = await unprint.get(url);
|
||||
|
||||
if (actorRes.ok) {
|
||||
return {
|
||||
...actorRes,
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
return prevRes;
|
||||
}, Promise.resolve({ ok: false, status: null }));
|
||||
`${entity.url}/models/${slugify(actorName, '')}.html`,
|
||||
`${entity.url}/models/${slugify(actorName, '-')}.html`,
|
||||
]);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.context, { entity, include, url: res.url });
|
||||
return scrapeProfile(res.context, { entity, include, url });
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
|
@ -209,5 +104,4 @@ module.exports = {
|
|||
fetchLatest,
|
||||
fetchProfile,
|
||||
scrapeScene,
|
||||
scrapeMovie,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ const unprint = require('unprint');
|
|||
|
||||
const slugify = require('../utils/slugify');
|
||||
const { convert } = require('../utils/convert');
|
||||
const tryUrls = require('../utils/try-urls');
|
||||
|
||||
const channelCodes = {
|
||||
ao: 'analoverdose',
|
||||
|
|
@ -151,36 +152,17 @@ function scrapeProfile({ query }, url) {
|
|||
}
|
||||
|
||||
async function fetchProfile({ name: actorName, url: actorUrl }) {
|
||||
if (actorUrl) {
|
||||
const res = await unprint.get(actorUrl);
|
||||
const { res, url } = await tryUrls([
|
||||
actorUrl,
|
||||
`https://pervcity.com/models/${slugify(actorName)}.html`,
|
||||
`https://pervcity.com/models/${slugify(actorName, '')}.html`,
|
||||
]);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.context);
|
||||
}
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.context, url);
|
||||
}
|
||||
|
||||
const url = `https://pervcity.com/models/${slugify(actorName)}.html`;
|
||||
const url2 = `https://pervcity.com/models/${slugify(actorName, '')}.html`;
|
||||
|
||||
if (url !== actorUrl) {
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.context, url);
|
||||
}
|
||||
}
|
||||
|
||||
if (url2 !== actorUrl) {
|
||||
const res = await unprint.get(url2);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.context, url);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
return null;
|
||||
return res.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ const teamskeet = require('./teamskeet');
|
|||
const teencoreclub = require('./teencoreclub');
|
||||
const teenmegaworld = require('./teenmegaworld');
|
||||
const testedefudelidade = require('./testedefudelidade');
|
||||
const theflourish = require('./theflourish');
|
||||
const tokyohot = require('./tokyohot');
|
||||
// const topwebmodels = require('./topwebmodels');
|
||||
const traxxx = require('./traxxx');
|
||||
|
|
@ -92,13 +93,14 @@ const scrapers = {
|
|||
// gamma
|
||||
gamma,
|
||||
// daringsex,
|
||||
// arch angel
|
||||
archangel,
|
||||
// etc
|
||||
amateurallure,
|
||||
americanpornstar,
|
||||
amateureuro: porndoe,
|
||||
amnesiac,
|
||||
angelogodshackoriginal,
|
||||
archangel,
|
||||
asiam: modelmedia,
|
||||
assylum,
|
||||
aziani,
|
||||
|
|
@ -177,11 +179,11 @@ const scrapers = {
|
|||
snowvalley,
|
||||
spizoo,
|
||||
swallowsalon: julesjordan,
|
||||
theflourish: archangel,
|
||||
teencoreclub,
|
||||
teenmegaworld,
|
||||
teamskeet,
|
||||
testedefudelidade,
|
||||
theflourish,
|
||||
tokyohot,
|
||||
transbella: porndoe,
|
||||
traxxx,
|
||||
|
|
@ -254,20 +256,27 @@ const scrapers = {
|
|||
'3rddegreefilms': gamma,
|
||||
addicted2girls: gamma,
|
||||
genderxfilms: gamma,
|
||||
// mike adriano
|
||||
trueanal: mikeadriano,
|
||||
swallowed: mikeadriano,
|
||||
nympho: mikeadriano,
|
||||
dirtyauditions: mikeadriano,
|
||||
analonly: mikeadriano,
|
||||
allanal: mikeadriano,
|
||||
// the flourish
|
||||
theflourishxxx: theflourish,
|
||||
// etc
|
||||
'18vr': badoink,
|
||||
adultempire,
|
||||
allanal: mikeadriano,
|
||||
archangel,
|
||||
allherluv: missax,
|
||||
amateureuro: porndoe,
|
||||
americanpornstar,
|
||||
analbbc: fullpornnetwork,
|
||||
analized: fullpornnetwork,
|
||||
analonly: mikeadriano,
|
||||
analviolation: fullpornnetwork,
|
||||
angelogodshackoriginal,
|
||||
anilos: nubiles,
|
||||
archangel,
|
||||
asiam: modelmedia,
|
||||
aziani,
|
||||
'2poles1hole': aziani,
|
||||
|
|
@ -288,7 +297,6 @@ const scrapers = {
|
|||
cherrypimps,
|
||||
cumlouder,
|
||||
deeplush: nubiles,
|
||||
dirtyauditions: mikeadriano,
|
||||
dorcelclub: dorcel,
|
||||
doubleviewcasting: firstanalquest,
|
||||
dtfsluts: fullpornnetwork,
|
||||
|
|
@ -338,7 +346,6 @@ const scrapers = {
|
|||
nfbusty: nubiles,
|
||||
nubilefilms: nubiles,
|
||||
nubiles,
|
||||
nympho: mikeadriano,
|
||||
onlyprince: fullpornnetwork,
|
||||
pascalssubsluts,
|
||||
pervcity,
|
||||
|
|
@ -370,12 +377,6 @@ const scrapers = {
|
|||
uralesbian: snowvalley,
|
||||
rawattack: spizoo,
|
||||
spizoo,
|
||||
swallowed: mikeadriano,
|
||||
milfcandy: archangel,
|
||||
theflourishamateurs: archangel,
|
||||
theflourishpov: archangel,
|
||||
theflourishfetish: archangel,
|
||||
theflourishxxx: archangel,
|
||||
teamskeet,
|
||||
teencoreclub,
|
||||
teenmegaworld,
|
||||
|
|
@ -384,7 +385,6 @@ const scrapers = {
|
|||
tokyohot,
|
||||
transbella: porndoe,
|
||||
tranzvr: wankzvr,
|
||||
trueanal: mikeadriano,
|
||||
vipsexvault: porndoe,
|
||||
virtualtaboo,
|
||||
darkroomvr: virtualtaboo,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,213 @@
|
|||
'use strict';
|
||||
|
||||
// ALSO USED BY THE FLOURISH
|
||||
|
||||
const unprint = require('unprint');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const { feetInchesToCm } = require('../utils/convert');
|
||||
|
||||
const placeholder = /images\/p\d+\.jpe?g/i;
|
||||
|
||||
function getEntryId(release) {
|
||||
return slugify(new URL(release.url).pathname.match(/\/([\w-]+)\.html/)?.[1]
|
||||
|| [unprint.formatDate(release.date, 'YYYY-MM-DD'), release.title, ...release.actors]);
|
||||
}
|
||||
|
||||
function scrapeAll(scenes) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('a');
|
||||
|
||||
release.title = query.content('a span');
|
||||
|
||||
release.date = query.date('.timeDate', 'YYYY-MM-DD');
|
||||
release.duration = query.duration('.timeDate');
|
||||
|
||||
release.actors = query.all('a[href*="models/"], a[href*="sets.php"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
const poster = query.img('img.mainThumb');
|
||||
const previewCount = query.number('img.mainThumb', { attribute: 'cnt' });
|
||||
|
||||
if (poster && !placeholder.test(poster)) {
|
||||
const posterFallbacks = [
|
||||
poster.replace('-1x', '-3x'),
|
||||
poster.replace('-1x', '-2x'),
|
||||
poster.replace('-1x', '-4x'),
|
||||
poster,
|
||||
];
|
||||
|
||||
release.poster = posterFallbacks;
|
||||
}
|
||||
|
||||
if (previewCount) {
|
||||
release.photos = Array.from(
|
||||
{ length: previewCount - 1 },
|
||||
(value, index) => [3, 2, 4, 1].map((scale) => unprint.prefixUrl(query.img('img.mainThumb', { attribute: `src${index + 1}_${scale}x` }))).filter(Boolean), // 4x is unnecessarily big and possibly upscaled
|
||||
).filter(Boolean);
|
||||
}
|
||||
|
||||
release.photoCount = query.number('.timeDate', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
|
||||
release.entryId = getEntryId(release);
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/categories/movies_${page}_d.html`;
|
||||
const res = await unprint.get(url, { selectAll: '.item-video' });
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.context, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function scrapeScene({ query, html }, { url, entity, baseRelease }) {
|
||||
const release = { url };
|
||||
|
||||
release.title = query.content('.title h2');
|
||||
release.description = query.content('.description p');
|
||||
|
||||
release.date = query.date('.info p', 'MMMM D, YYYY');
|
||||
release.duration = query.duration('.info p');
|
||||
|
||||
release.actors = query.all('.info a[href*="models/"], .info a[href*="sets.php"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
const poster = unprint.prefixUrl(query.img('.update_thumb') || html.match(/poster="(.*\.jpg)"/)?.[1], entity.url);
|
||||
|
||||
if (poster && !placeholder.test(poster)) {
|
||||
const posterFallbacks = [
|
||||
poster.replace('-1x', '-3x'),
|
||||
poster.replace('-1x', '-2x'),
|
||||
poster.replace('-1x', '-4x'),
|
||||
poster,
|
||||
];
|
||||
|
||||
// scene page poster usually different from overview page, don't replace
|
||||
if (baseRelease?.poster && baseRelease.poster !== poster) {
|
||||
release.photos = baseRelease.photos
|
||||
? [posterFallbacks, ...baseRelease.photos]
|
||||
: [posterFallbacks];
|
||||
} else {
|
||||
release.poster = posterFallbacks;
|
||||
}
|
||||
}
|
||||
|
||||
const trailer = html.match(/src="(.*\.mp4)"/)?.[1];
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = unprint.prefixUrl(encodeURI(trailer), entity.url);
|
||||
}
|
||||
|
||||
release.tags = query.contents('.info .tags a');
|
||||
|
||||
release.photoCount = query.number('.info', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
release.entryId = getEntryId(release);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeMovie({ query, element }, { entity, url }) {
|
||||
const release = { url };
|
||||
|
||||
release.title = query.content('.title h2');
|
||||
release.description = query.content('.aboutArea p');
|
||||
|
||||
release.covers = [[
|
||||
query.img('.update_thumb', { attribute: 'src0_2x', origin: entity.url }),
|
||||
query.img('.update_thumb', { attribute: 'src0_1x', origin: entity.url }),
|
||||
query.img('.update_thumb', { attribute: 'src0', origin: entity.url }),
|
||||
// usually upscaled
|
||||
query.img('.update_thumb', { attribute: 'src0_4x', origin: entity.url }),
|
||||
query.img('.update_thumb', { attribute: 'src0_3x', origin: entity.url }),
|
||||
].filter(Boolean)];
|
||||
|
||||
release.entryId = getEntryId(release);
|
||||
|
||||
release.scenes = scrapeAll(unprint.initAll(element, '.item-video'));
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query, element }, { url, entity }) {
|
||||
const profile = { url };
|
||||
|
||||
const bio = Object.fromEntries(query.all('.stats li')
|
||||
.map((row) => [
|
||||
slugify(unprint.query.content(row, '.data-name, span'), '_'),
|
||||
unprint.query.text(row),
|
||||
])
|
||||
.filter(([key, value]) => key && value));
|
||||
|
||||
profile.description = query.content('.aboutArea p');
|
||||
|
||||
profile.birthPlace = bio.place_of_birth;
|
||||
profile.dateOfBirth = unprint.extractDate(bio.age, 'MMMM D, YYYY');
|
||||
|
||||
profile.height = Number(bio.height?.match(/(\d+)\s*cm/)?.[1]) || (/\d fe*t \d+ inch/i.test(bio.height) && feetInchesToCm(bio.height)) || null;
|
||||
profile.measurements = bio.measurements;
|
||||
|
||||
profile.hairColor = bio.hair_color;
|
||||
profile.eyes = bio.eye_color;
|
||||
|
||||
profile.avatar = [
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_4x', origin: entity.url }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_3x', origin: entity.url }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_2x', origin: entity.url }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0_1x', origin: entity.url }),
|
||||
query.img('.model_bio_thumb', { attribute: 'src0', origin: entity.url }),
|
||||
].filter((avatar) => avatar && !placeholder.test(avatar));
|
||||
|
||||
profile.scenes = scrapeAll(unprint.initAll(element, '.item-video'));
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName, url: actorUrl }, { entity, include, parameters }) {
|
||||
const res = await [
|
||||
actorUrl,
|
||||
`${entity.url}${parameters.path || ''}/models/${slugify(actorName, '-')}.html`,
|
||||
`${entity.url}${parameters.path || ''}/models/${slugify(actorName, '')}.html`,
|
||||
].reduce(async (chain, url) => {
|
||||
const prevRes = await chain;
|
||||
|
||||
if (prevRes.ok || !url) {
|
||||
return prevRes;
|
||||
}
|
||||
|
||||
const actorRes = await unprint.get(url);
|
||||
|
||||
if (actorRes.ok) {
|
||||
return {
|
||||
...actorRes,
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
return prevRes;
|
||||
}, Promise.resolve({ ok: false, status: null }));
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.context, { entity, include, url: res.url });
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile,
|
||||
scrapeScene,
|
||||
scrapeMovie,
|
||||
};
|
||||
|
|
@ -60,7 +60,7 @@ function kgToLbs(kgs) {
|
|||
}
|
||||
|
||||
function curateConvertInput(string) {
|
||||
if (/['’]|(fe*t)/.test(string)) {
|
||||
if (/['’]|(fe*o*t)/.test(string)) {
|
||||
const result = string.match(/(\d+).*(\d+)/);
|
||||
|
||||
if (result) {
|
||||
|
|
@ -74,9 +74,6 @@ function curateConvertInput(string) {
|
|||
function convertManyApi(input, to) {
|
||||
const curatedInput = curateConvertInput(input);
|
||||
|
||||
console.log('CONVERT', input);
|
||||
console.log('RESULT', curatedInput);
|
||||
|
||||
return Math.round(convertMany(curatedInput).to(to)) || null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
const unprint = require('unprint');
|
||||
|
||||
async function tryUrls(urls, options) {
|
||||
return urls.filter(Boolean).reduce(async (chain, url) => {
|
||||
const acc = await chain;
|
||||
|
||||
if (acc?.res.ok) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
const res = await unprint.get(url, options);
|
||||
|
||||
return {
|
||||
res,
|
||||
url,
|
||||
};
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
module.exports = tryUrls;
|
||||
|
|
@ -113,6 +113,10 @@ const actors = [
|
|||
// perv city
|
||||
{ entity: 'pervcity', name: 'Brooklyn Gray', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'dpdiva', name: 'Liz Jordan', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
// arch angel
|
||||
{ entity: 'archangel', name: 'Summer Brielle', fields: ['avatar', 'description', 'dateOfBirth', 'age', 'measurements', 'height', 'aliases'] },
|
||||
// the flourish
|
||||
{ entity: 'theflourishxxx', name: 'XWifeKaren', fields: ['avatar', 'description'] },
|
||||
];
|
||||
|
||||
const actorScrapers = scrapers.actors;
|
||||
|
|
@ -194,7 +198,7 @@ async function init() {
|
|||
}
|
||||
|
||||
if (source && source !== entitySlug) {
|
||||
console.log('____', entitySlug);
|
||||
// console.log('____', entitySlug);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue