DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Migrated remaining scrapers from qu/http to unprint, likely fixing TeamSkeet and PinkyXXX (verification needed).

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-10-19 07:35:03 +02:00
parent 03a5ce5bb1
commit 4ede0724a7
24 changed files with 301 additions and 1732 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scrapers/actors.js
-3
View File
@@ -29,7 +29,6 @@ const karups = require('./karups');
const kellymadison = require('./kellymadison');
const kink = require('./kink');
const mariskax = require('./mariskax');
// const analvids = require('./analvids');
const pornbox = require('./pornbox');
const littlecapricedreams = require('./littlecapricedreams');
const loveherfilms = require('./loveherfilms');
@@ -59,7 +58,6 @@ const teenmegaworld = require('./teenmegaworld');
const testedefudelidade = require('./testedefudelidade');
const theflourish = require('./theflourish');
const tokyohot = require('./tokyohot');
// const topwebmodels = require('./topwebmodels');
const virtualtaboo = require('./virtualtaboo');
const vixen = require('./vixen');
const wankzvr = require('./wankzvr');
@@ -229,7 +227,6 @@ module.exports = {
boyfun: karups,
kellymadison,
'8kmembers': kellymadison,
// analvids,
analvids: pornbox,
littlecapricedreams,
mariskax,
src/scrapers/americanpornstar.js
+14 -14
View File
@@ -1,45 +1,45 @@
'use strict';
const { fetchLatest, fetchProfile } = require('./julesjordan');
const unprint = require('unprint');
const qu = require('../utils/q');
const { fetchLatest, fetchProfile } = require('./julesjordan');
const slugify = require('../utils/slugify');
function scrapeScene({ query }, channel) {
const release = {};
const title = query.cnt('.update_title', false);
const title = query.content('.update_title', { trim: false });
release.title = title.trim();
release.description = query.cnt('.latest_update_description');
release.description = query.content('.latest_update_description');
release.date = query.date('.update_date', 'MM/DD/YYYY');
release.actors = query.all('.tour_update_models a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null),
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null),
}));
release.entryId = slugify(release.title);
release.duration = Number(query.cnt('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
release.duration = Number(query.content('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
const poster = query.img('.large_update_thumb', 'src', { origin: channel.url });
const trailer = query.q('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
const poster = query.img('.large_update_thumb', { attribute: 'src', origin: channel.url });
const trailer = query.attribute('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
release.poster = [
poster.replace(/.jpg$/, '-full.jpg'),
poster,
];
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
release.photos = query.imgs('.small_update_thumb', { attribute: 'src', origin: channel.url }).map((img) => [
img.replace(/.jpg$/, '-full.jpg'),
img,
]);
if (trailer) {
release.trailer = qu.prefixUrl(encodeURI(trailer), channel.url);
release.trailer = unprint.prefixUrl(encodeURI(trailer), channel.url);
}
release.tags = query.cnts('.tour_update_tags a');
release.tags = query.contents('.tour_update_tags a');
return release;
}
@@ -49,10 +49,10 @@ async function fetchLatestLocal(channel, page, include, preData) {
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, channel);
return scrapeScene(res.context, channel);
}
return res.status;
src/scrapers/analvids.js
-174
View File
@@ -1,174 +0,0 @@
'use strict';
const unprint = require('unprint');
const slugify = require('../utils/slugify');
function extractTitle(originalTitle) {
const titleComponents = originalTitle.split(' ');
// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
return { shootId, title };
}
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('.card-scene__view > a');
release.entryId = query.dataset(null, 'content') || new URL(release.url).pathname.match(/watch\/(\d+)/)?.[1];
release.title = query.content('.card-scene__text');
release.shootId = extractTitle(release.title).shootId;
release.date = query.date('.label--time:nth-child(2)', 'YYYY-MM-DD'); // only available on front-page, not on studio page
release.duration = query.duration('.label--time:first-child');
release.poster = query.img('.card-scene__view img', { attribute: 'data-src' });
const caps = query.json('.card-scene__view > a', { attribute: 'data-casting' })?.map((timestamp) => `${channel.url}/casting/${release.entryId}/${timestamp}`);
if (caps && release.poster) {
release.caps = caps;
} else if (caps) {
[release.poster, ...release.caps] = caps;
}
release.teaser = query.video('.card-scene__view > a', { attribute: 'data-preview' });
return release;
});
}
async function fetchLatest(channel, page) {
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
// studios as channels
const url = `${channel.url}/latest/${page}`;
const res = await unprint.get(url, {
selectAll: '.card-scene',
headers: {
Referer: url,
},
});
if (res.ok) {
return scrapeAll(res.context, channel);
}
return res.status;
}
function scrapeScene({ query }, url) {
const release = {};
release.entryId = new URL(url).pathname.match(/watch\/(\d+)/)?.[1];
const featuring = query.content('.watch__title .watch__featuring_models');
release.title = query.content('.watch__title').replace(featuring, '');
release.description = query.content('.text-mob-more');
release.shootId = extractTitle(release.title).shootId;
release.date = query.date('.bi-calendar3', 'YYYY-MM-DD');
release.duration = query.duration('.bi-clock');
release.actors = query.all('.watch__title a[href*="/model"]').map((el) => ({
name: unprint.query.content(el),
url: unprint.query.url(el, null),
}));
release.tags = query.contents('.genres-list a[href*="/genre"]');
// release.studio = slugify(query.content('.genres-list a[href*="/studios"]'), '', { removePunctuation: true });
release.poster = query.img('.watch__video video', { attribute: 'data-poster' });
release.trailer = query.all('.watch__video source').map((el) => ({
src: unprint.query.video(el, null, { attribute: 'src' }),
quality: unprint.query.number(el, null, { attribute: 'size' }),
}));
return release;
}
async function fetchScene(url) {
const res = await unprint.get(url, {
headers: {
Referer: url,
},
});
if (res.ok) {
return scrapeScene(res.context, url);
}
return res.status;
}
function scrapeProfile({ query }, url, channel) {
const profile = { url };
profile.nationality = query.content('.model__info a[href*="/nationality"]');
profile.age = query.number('//td[contains(text(), "Age")]/following-sibling::td');
profile.avatar = query.img('.model__left img');
profile.scenes = scrapeAll(unprint.initAll(query.all('.card-scene')), channel);
return profile;
}
async function getActorUrl(actor, channel) {
if (actor.url) {
return actor.url;
}
const searchUrl = `${channel.url}/api/autocomplete/search?q=${slugify(actor.name, '+')}`;
const searchRes = await unprint.get(searchUrl, {
headers: {
Referer: actor.url,
},
});
if (!searchRes.ok) {
return searchRes.status;
}
const result = searchRes.body.terms.find((item) => item.type === 'model');
if (result) {
return result.url;
}
return null;
}
async function fetchProfile(actor, { channel }) {
const actorUrl = await getActorUrl(actor, channel);
if (typeof actorUrl !== 'string') {
return actorUrl;
}
const bioRes = await unprint.get(actorUrl, {
headers: {
Referer: actor.url,
},
});
if (bioRes.ok) {
return scrapeProfile(bioRes.context, actorUrl, channel);
}
return bioRes.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
};
src/scrapers/analvids_legacy.js
-203
View File
@@ -1,203 +0,0 @@
'use strict';
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
function extractTitle(originalTitle) {
const titleComponents = originalTitle.split(' ');
// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
return { shootId, title };
}
function getPoster(posterElement, sceneId) {
const posterStyle = posterElement.attr('style');
if (posterStyle) {
return posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
}
const posterRange = posterElement.attr('data-casting');
const posterRangeData = posterRange ? JSON.parse(posterRange) : null;
const posterTimeRange = posterRangeData[Math.floor(Math.random() * posterRangeData.length)];
if (!posterTimeRange) {
return null;
}
if (typeof posterTimeRange === 'number') {
// poster time is already a single time value
return `https://analvids.com/casting/${sceneId}/${posterTimeRange}`;
}
const [max, min] = posterTimeRange.split('-');
const posterTime = Math.floor(Math.random() * (Number(max) - Number(min) + 1) + Number(min));
return `https://analvids.com/casting/${sceneId}/${posterTime}`;
}
function scrapeAll(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const scenesElements = $('.thumbnails > div').toArray();
return scenesElements.map((element) => {
const sceneLinkElement = $(element).find('.thumbnail-title a');
const url = sceneLinkElement.attr('href');
const originalTitle = sceneLinkElement.text().trim(); // title attribute breaks when they use \\ escaping
const { shootId, title } = extractTitle(originalTitle);
const entryId = new URL(url).pathname.split('/')[2];
const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
const sceneId = $(element).attr('data-content');
const posterElement = $(element).find('.thumbnail-avatar');
const poster = getPoster(posterElement, sceneId);
return {
url,
shootId,
entryId,
title,
date,
poster,
};
});
}
async function scrapeScene(html, url, site, useGallery) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const playerObject = $('script:contains("new WatchPage")').html();
const playerData = playerObject && playerObject.slice(playerObject.indexOf('{"swf":'), playerObject.lastIndexOf('},') + 1);
const data = playerData && JSON.parse(playerData);
const release = { url };
const originalTitle = $('h1.watchpage-title').text().trim();
const { shootId, title } = extractTitle(originalTitle);
release.shootId = shootId;
release.entryId = new URL(url).pathname.split('/')[2];
release.title = title;
release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
release.description = $('meta[name="description"]')?.attr('content')?.trim()
|| (descriptionElement && $(descriptionElement).find('dd').text().trim());
release.actors = $(actorsElement)
.find('a[href*="com/model"]')
.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const photos = useGallery
? $('.gallery a img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray()
: $('.screenshots img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray();
release.photos = photos.map((source) => {
// source without parameters sometimes serves larger preview photo
const { origin, pathname } = new URL(source);
return `${origin}${pathname}`;
});
const posterStyle = $('#player').attr('style');
const poster = posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
release.poster = poster || release.photos.slice(Math.floor(release.photos.length / 3) * -1); // poster unavailable, try last 1/3rd of high res photos as fallback
if (data) {
const qualityMap = {
web: 240,
vga: 480,
hd: 720,
'1080p': 1080,
};
release.trailer = data.clip.qualities.map((trailer) => ({
src: trailer.src,
type: trailer.type,
quality: qualityMap[trailer.quality] || trailer.quality,
}));
}
const studioName = $('.watchpage-studioname').first().text().trim();
release.studio = slugify(studioName, '', { removePunctuation: true });
return release;
}
async function scrapeProfile(html, _url, actorName) {
const { document } = new JSDOM(html).window;
const profile = {
name: actorName,
};
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
const bio = entries
.filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
profile.birthPlace = bio.Nationality;
if (bio.Age) profile.age = bio.Age;
if (avatarEl) profile.avatar = avatarEl.src;
profile.releases = scrapeAll(html);
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await http.get(`${site.url}/new-videos/${page}`);
return scrapeAll(res.body.toString(), site);
}
async function fetchScene(url, site) {
const useGallery = true;
// TODO: fall back on screenshots when gallery is not available
const res = useGallery
? await http.get(`${url}/gallery#gallery`)
: await http.get(`${url}/screenshots#screenshots`);
return scrapeScene(res.body.toString(), url, site, useGallery);
}
async function fetchProfile({ name: actorName }) {
const res = await http.get(`https://www.analvids.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
const data = res.body;
const result = data.terms.find((item) => item.type === 'model');
if (result) {
const bioRes = await http.get(result.url);
const html = bioRes.body.toString();
return scrapeProfile(html, result.url, actorName);
}
return null;
}
module.exports = {
fetchLatest,
fetchProfile,
fetchScene,
};
src/scrapers/assylum.js
+30 -27
View File
@@ -1,6 +1,9 @@
'use strict';
const { get, geta, ctxa } = require('../utils/q');
const unprint = require('unprint');
// paths are relative, read raw attributes instead of resolved URLs
const raw = { forceGetAttribute: true };
function extractActors(actorString) {
return actorString
@@ -20,48 +23,48 @@ function matchActors(actorString, models) {
}
function scrapeLatest(scenes, site, models) {
return scenes.map(({ qu }) => {
return scenes.map(({ query }) => {
const release = {};
const pathname = qu.url('a.itemimg').slice(1);
const pathname = query.attribute('a.itemimg', 'href', raw).slice(1);
[release.entryId] = pathname.split('/').slice(-1);
release.url = `${site.url}${pathname}`;
release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
release.title = query.attribute('.itemimg img', 'alt') || query.content('h4 a');
release.description = query.content('.mas_longdescription');
release.date = query.date('.movie_info2', 'MM/DD/YY', { match: /\d{2}\/\d{2}\/\d{2}/ });
const actorString = qu.q('.mas_description', true);
const actorString = query.content('.mas_description');
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
const posterPath = qu.img('.itemimg img');
const posterPath = query.attribute('.itemimg img', 'src', raw);
release.poster = `${site.url}/${posterPath}`;
return release;
});
}
function scrapeScene({ html, qu }, url, site, include, models) {
function scrapeScene({ html, query }, url, site, include, models) {
const release = { url };
[release.entryId] = url.split('/').slice(-1);
release.title = qu.q('.mas_title', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.title = query.content('.mas_title');
release.description = query.content('.mas_longdescription');
release.date = query.date('.mas_description', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
if (models) {
const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
const actorString = query.content('.mas_description').replace(/\w+ \d{1,2}, \d{4}/, '');
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
}
release.tags = qu.all('.tags a', true);
release.tags = query.contents('.tags a');
release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
release.photos = query.attributes('.stills img', 'src', raw).map((photoPath) => `${site.url}/${photoPath}`);
const posterIndex = 'splash:';
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
@@ -74,20 +77,20 @@ function scrapeScene({ html, qu }, url, site, include, models) {
return release;
}
function extractModels({ el }, site) {
const models = ctxa(el, '.item');
function extractModels({ element }, site) {
const models = unprint.initAll(element, '.item');
return models.map(({ qu }) => {
return models.map(({ query }) => {
const actor = { gender: 'female' };
const avatar = qu.q('.itemimg img');
const avatar = query.element('.itemimg img');
actor.avatar = `${site.url}/${avatar.src}`;
actor.name = avatar.alt
.split(':').slice(-1)[0]
.replace(/xtreme girl|nurse/ig, '')
.trim();
const actorPath = qu.url('.itemimg');
const actorPath = query.attribute('.itemimg', 'href', raw);
actor.url = `${site.url}${actorPath.slice(1)}`;
return actor;
@@ -96,11 +99,11 @@ function extractModels({ el }, site) {
async function fetchModels(site, page = 1, accModels = []) {
const url = `${site.url}/?models/${page}`;
const res = await get(url);
const res = await unprint.get(url);
if (res.ok) {
const models = extractModels(res.item, site);
const nextPage = res.item.qa('.pagenumbers', true)
const models = extractModels(res.context, site);
const nextPage = res.context.query.contents('.pagenumbers')
.map((pageX) => Number(pageX))
.filter(Boolean) // remove << and >>
.includes(page + 1);
@@ -117,16 +120,16 @@ async function fetchModels(site, page = 1, accModels = []) {
async function fetchLatest(site, page = 1, include, { beforeFetchLatest }) {
const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
const res = await geta(url, '.item');
const res = await unprint.get(url, { selectAll: '.item' });
return res.ok ? scrapeLatest(res.items, site, beforeFetchLatest) : res.status;
return res.ok ? scrapeLatest(res.context, site, beforeFetchLatest) : res.status;
}
async function fetchScene(url, site, release, include, beforeFetchLatest) {
const models = beforeFetchLatest || await fetchModels(site);
const res = await get(url);
const res = await unprint.get(url);
return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
return res.ok ? scrapeScene(res.context, url, site, include, models) : res.status;
}
module.exports = {
src/scrapers/badoink.js
+35 -31
View File
@@ -1,6 +1,7 @@
'use strict';
const qu = require('../utils/qu');
const unprint = require('unprint');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -21,23 +22,23 @@ function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.entryId = query.q(null, 'data-video-card-scene-id') || query.q('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.url = query.url('a', { origin: channel.url });
release.entryId = query.attribute(null, 'data-video-card-scene-id') || query.attribute('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.video-card-title span') || query.q('.video-card-title', 'title');
release.description = query.cnt('.video-card-description');
release.title = query.content('.video-card-title span') || query.attribute('.video-card-title', 'title');
release.description = query.content('.video-card-description');
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
release.duration = query.duration('.video-card-duration', { attribute: 'content' }) || query.number('.video-card-duration') * 60;
release.actors = query.all('.video-card-details--cast a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
name: unprint.query.content(el),
url: unprint.query.url(el, null, { origin: channel.url }),
}));
release.tags = query.cnts('.video-card-tags a');
release.tags = query.contents('.video-card-tags a');
const posterSources = query.srcset('picture source', 'data-srcset') || [query.img('.video-card-image', 'data-src')];
const posterSources = query.sourceSet('picture source', 'data-srcset') || [query.img('.video-card-image', { attribute: 'data-src' })];
release.poster = getPoster(posterSources);
release.teaser = query.video('video') || (channel.parameters?.teasers !== false ? `https://p.badoinkvr.com/d/scene_files/${release.entryId}/preview/preview.mp4` : null);
@@ -46,29 +47,29 @@ function scrapeAll(scenes, channel) {
});
}
function scrapeScene({ query }, url, channel) {
function scrapeScene({ query }, { url, entity: channel }) {
const release = {};
release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.video-title', '.breadcrumbs__item:last-child [itemprop=name]');
release.description = query.cnt('.video-description');
release.title = query.content(['.video-title', '.breadcrumbs__item:last-child [itemprop=name]']);
release.description = query.content('.video-description');
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-upload-date', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.duration = query.duration('.video-duration', { attribute: 'content' }) || query.number('.video-duration') * 60;
release.actors = query.all('.video-actors a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
name: unprint.query.content(el),
url: unprint.query.url(el, null, { origin: channel.url }),
}));
release.tags = query.cnts('.video-tags a');
release.tags = query.contents('.video-tags a');
const posterSources = query.srcset('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
const posterSources = query.sourceSet('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
release.poster = getPoster(posterSources);
const photos = query.imgs('.gallery .gallery-item', 'data-big-image');
const extraPhotoCount = query.number('.gallery .gallery-item:last-child', /\d+/, 'data-desktop-text');
const photos = query.imgs('.gallery .gallery-item', { attribute: 'data-big-image' });
const extraPhotoCount = query.number('.gallery .gallery-item:last-child', { match: /\d+/, attribute: 'data-desktop-text' });
const lastPhoto = photos.slice(-1)[0];
const photosHaveIndex = /_\d+\./.test(lastPhoto);
@@ -87,10 +88,10 @@ function scrapeProfile({ query }, url, entity) {
const bio = query.all('.girl-details-stats-item').reduce((acc, el) => ({
...acc,
[slugify(qu.query.cnt(el, '.girl-details-stat'))]: qu.query.cnt(el, '.girl-details-stat-value'),
[slugify(unprint.query.content(el, '.girl-details-stat'))]: unprint.query.content(el, '.girl-details-stat-value'),
}), {});
profile.description = query.cnt('.girl-details-bio');
profile.description = query.content('.girl-details-bio');
profile.age = bio.age;
profile.birthPlace = bio.country;
@@ -104,23 +105,26 @@ function scrapeProfile({ query }, url, entity) {
profile.hairColor = bio.hair;
profile.eyes = bio.eyes;
const avatarSources = query.srcset('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
const avatarSources = query.sourceSet('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
profile.avatar = getPoster(avatarSources).slice(0, 3); // returns a metric ton of links, if first few don't work the rest presumably won't either
profile.social = query.urls('.girl-details-social-media-list a');
profile.scenes = scrapeAll(qu.initAll(query.all('.video-card')), entity);
profile.scenes = scrapeAll(unprint.initAll(query.all('.video-card')), entity);
return profile;
}
async function fetchLatest(channel, page) {
const res = await qu.getAll(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, '.video-card', {
Cookie: 'affsubid=12345-;', // required to show teaser video, exact number doesn't seem to matter
const res = await unprint.get(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, {
selectAll: '.video-card',
cookies: {
affsubid: '12345-', // required to show teaser video, exact number doesn't seem to matter
},
});
if (res.ok) {
return scrapeAll(res.items, channel);
return scrapeAll(res.context, channel);
}
return res.status;
@@ -128,10 +132,10 @@ async function fetchLatest(channel, page) {
async function fetchProfile(baseActor, { entity }) {
const url = `${entity.url}/${entity.parameters?.actor || 'pornstar'}/${slugify(baseActor.name, '-')}/`;
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeProfile(res.item, url, entity);
return scrapeProfile(res.context, url, entity);
}
return res.status;
src/scrapers/czechav.js
+17 -17
View File
@@ -1,29 +1,29 @@
'use strict';
const qu = require('../utils/q');
const unprint = require('unprint');
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.url = query.url('a', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/tour\/preview\/(.+)/)[1];
release.title = query.cnt('.nice-title');
release.title = query.content('.nice-title');
release.poster = query.img('img.poster');
release.photos = query.q('img.poster', 'data-urls')?.split(' ');
release.teaser = query.video('img.poster', 'data-thumbnail');
release.photos = query.attribute('img.poster', 'data-urls')?.split(' ');
release.teaser = query.video('img.poster', { attribute: 'data-thumbnail' });
if (query.exists('.info')) {
release.date = query.date('.updated', 'MM/DD/YYYY');
release.actors = [{
name: query.cnt('.name'),
age: query.cnt('.age'),
name: query.content('.name'),
age: query.content('.age'),
}];
release.tags = query.cnts('.tags a');
release.tags = query.contents('.tags a');
}
return release;
@@ -35,38 +35,38 @@ function scrapeScene({ query }, url) {
release.entryId = new URL(url).pathname.match(/\/tour\/preview\/(.+)/)[1];
release.title = query.cnt('.nice-title');
release.description = query.cnt('.desc-text p');
release.title = query.content('.nice-title');
release.description = query.content('.desc-text p');
release.tags = query.cnts('.tags a');
release.tags = query.contents('.tags a');
release.poster = query.poster();
release.photos = query.imgs('.thumbnails img, .thumb img');
// some videos may not have unique trailers, but many do
release.trailer = query.video();
release.trailer = query.video('source');
release.formats = query.cnts('.download-quality option');
release.formats = query.contents('.download-quality option');
return release;
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/tour/videos/page-${page}`;
const res = await qu.getAll(url, '.episode-list .episode, .content .girl');
const res = await unprint.get(url, { selectAll: '.episode-list .episode, .content .girl' });
if (res.ok) {
return scrapeAll(res.items, channel);
return scrapeAll(res.context, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, url, channel);
return scrapeScene(res.context, url, channel);
}
return res.status;
src/scrapers/elevatedx.js
+50 -51
View File
@@ -1,8 +1,8 @@
'use strict';
const format = require('template-format');
const unprint = require('unprint');
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -10,11 +10,11 @@ function deriveEntryId(release) {
if (release.date && release.url) {
const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
}
if (release.date && release.title) {
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
}
return null;
@@ -22,7 +22,7 @@ function deriveEntryId(release) {
function extractPoster(posterPath, channel, baseRelease) {
if (posterPath && !/400.jpg/.test(posterPath)) {
const poster = qu.prefixUrl(posterPath, channel.parameters?.media || channel.url);
const poster = unprint.prefixUrl(posterPath, channel.parameters?.media || channel.url);
const posterSources = [
poster,
// upscaled
@@ -40,20 +40,20 @@ function extractPoster(posterPath, channel, baseRelease) {
return [baseRelease?.poster || null, []];
}
function getImageWithFallbacks(q, selector, site, el) {
function getImageWithFallbacks(query, selector, site, el) {
const sources = el
? [
q(el, selector, 'src0_3x'),
q(el, selector, 'src0_2x'),
q(el, selector, 'src0_1x'),
unprint.query.attribute(el, selector, 'src0_3x'),
unprint.query.attribute(el, selector, 'src0_2x'),
unprint.query.attribute(el, selector, 'src0_1x'),
]
: [
q(selector, 'src0_3x'),
q(selector, 'src0_2x'),
q(selector, 'src0_1x'),
query.attribute(selector, 'src0_3x'),
query.attribute(selector, 'src0_2x'),
query.attribute(selector, 'src0_1x'),
];
return sources.filter(Boolean).map((src) => qu.prefixUrl(src, site.parameters?.media || site.url));
return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url));
}
function scrapeAllClassic(scenes, channel) {
@@ -61,16 +61,16 @@ function scrapeAllClassic(scenes, channel) {
const release = {};
release.url = query.url('.updateInfo h5 a:not([href*="content/"]):not([href*="#coming"])');
release.entryId = query.url('.updateThumb img', 'alt');
release.entryId = query.attribute('.updateThumb img', 'alt');
release.title = query.cnt('.updateInfo h5 a');
release.title = query.content('.updateInfo h5 a');
release.actors = query.cnts('.tour_update_models a');
release.actors = query.contents('.tour_update_models a');
release.date = query.date('.availdate, .updateInfo p span:last-child', 'MM/DD/YYYY');
release.poster = query.img('.updateThumb img');
release.poster = query.img('.updateThumb img', { origin: channel.url }); // relative to site root, not listing page
const trailer = query.q('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
const trailer = query.attribute('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
if (trailer) {
release.trailer = `${channel.url}${trailer}`;
@@ -84,13 +84,14 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
release.title = query.attribute('h4 a', 'title') || query.content('h4 a');
release.url = query.url('h4 a');
release.date = query.date('.more-info-div', 'MMM D, YYYY');
release.duration = query.dur('.more-info-div');
release.duration = query.duration('.more-info-div');
const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
// raw path, prefixed with media URL below
const posterPath = query.attribute('.img-div img', 'src0_1x') || query.attribute('img.video_placeholder', 'data-src') || query.attribute('img.video_placeholder', 'src', { forceGetAttribute: true });
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${channel.parameters?.media || channel.url}${posterPath}`;
@@ -102,7 +103,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
];
}
release.teaser = query.video();
release.teaser = query.video('source');
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
@@ -116,11 +117,11 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
});
}
function scrapeSceneClassic({ query, html }, url, channel) {
function scrapeSceneClassic({ query, html }, { entity: channel }) {
const release = {};
release.title = query.q('.updatesBlock h2', true);
release.poster = query.meta('property="og:image"');
release.title = query.content('.updatesBlock h2');
release.poster = query.attribute('meta[property="og:image"]', 'content');
release.entryId = release.poster.match(/\/content\/(.*)\//)?.[1];
const trailer = html.match(/src="(.+\.mp4)"/)?.[1];
@@ -134,34 +135,34 @@ function scrapeSceneClassic({ query, html }, url, channel) {
return release;
}
function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
function scrapeSceneTubular({ query, html }, { entity, url, baseRelease }) {
const release = {};
release.title = query.q('.video-player .section-title', true) || query.q('.bio-section-head .section-title', true);
release.title = query.content('.video-player .section-title') || query.content('.bio-section-head .section-title');
release.description = query.text('.update-info-block .descriptionFull') || query.text('.update-info-block .description');
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)');
release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
release.date = query.date('.update-info-row', 'MMM D, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.duration = query.duration('.update-info-row:nth-child(2)');
release.photoCount = query.number('.update-info-row:nth-child(2)', { match: /(\d+) photos/i, matchIndex: 1 });
release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.cnt(el, 'span'),
avatar: getImageWithFallbacks(query.q, 'img', entity, el),
url: query.url(el, null),
name: unprint.query.content(el, 'span'),
avatar: getImageWithFallbacks(query, 'img', entity, el),
url: unprint.query.url(el, null),
}));
release.tags = query.all('.tags a', true);
release.tags = query.contents('.tags a');
const posterPath = query.q('.player-thumb img', 'src0_1x');
const posterPath = query.attribute('.player-thumb img', 'src0_1x');
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
[release.poster, release.photos] = extractPoster(posterPath, entity, baseRelease);
if (trailer) {
release.trailer = { src: qu.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
release.trailer = { src: unprint.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
}
const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
const stars = query.content('.update-rating').match(/\d.\d/)?.[0];
if (stars) release.stars = Number(stars);
if (entity.type === 'network') {
@@ -181,11 +182,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
async function scrapeProfile({ query }, entity, parameters) {
const profile = {};
const bio = query.cnt('.model_bio, .detail-div');
const avatarEl = query.q('.model_bio_pic img, .model_bio_thumb');
const bio = query.content('.model_bio, .detail-div');
const avatarEl = query.element('.model_bio_pic img, .model_bio_thumb');
profile.age = Number(bio?.match(/Age:\s*(\d{2})/)?.[1]) || null;
profile.dateOfBirth = qu.parseDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
profile.dateOfBirth = unprint.extractDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
profile.height = convert(bio?.match(/\d+\s*(feet|')\s*\d+\s*(inches|"|$)/)?.[0], 'cm');
profile.measurements = bio?.match(/\w+[-x]\d+[-x]\d+/)?.[0] || null;
@@ -201,17 +202,17 @@ async function scrapeProfile({ query }, entity, parameters) {
avatarEl.getAttribute('src'),
]
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map((avatar) => qu.prefixUrl(avatar, entity.url));
.map((avatar) => unprint.prefixUrl(avatar, entity.url));
if (avatarSources.length) profile.avatar = avatarSources;
}
if (parameters?.layout === 'classic') {
profile.scenes = scrapeAllClassic(qu.initAll(query.all('.bodyArea .updateItem')), entity);
profile.scenes = scrapeAllClassic(unprint.initAll(query.all('.bodyArea .updateItem')), entity);
}
if (parameters?.layout === 'tubular') {
profile.scenes = scrapeAllTubular(qu.initAll(query.all('.modelfeature, .item-video')), entity);
profile.scenes = scrapeAllTubular(unprint.initAll(query.all('.modelfeature, .item-video')), entity);
}
return profile;
@@ -221,20 +222,20 @@ async function fetchLatest(site, page = 1, options, preData, allScraper) {
const url = (site.parameters?.latest && format(site.parameters.latest, { page }))
|| `${site.url}/categories/movies_${page}_d.html`;
const res = await qu.getAll(url, '.modelfeature, .item-video, .bodyArea .updateItem');
const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .bodyArea .updateItem' });
if (!res.ok) {
return res.status;
}
return allScraper(res.items, site, preData?.uniqueReleases);
return allScraper(res.context, site, preData?.uniqueReleases);
}
async function fetchUpcomingClassic(channel) {
const res = await qu.getAll(channel.url, '#owl-upcomingScenes .updateItem');
const res = await unprint.get(channel.url, { selectAll: '#owl-upcomingScenes .updateItem' });
if (res.ok) {
return scrapeAllClassic(res.items, channel);
return scrapeAllClassic(res.context, channel);
}
return res.status;
@@ -273,10 +274,10 @@ async function fetchProfile({ name: actorName, url }, { entity, parameters }) {
return null;
}
const res = await qu.get(profileUrl);
const res = await unprint.get(profileUrl);
if (res.statusCode === 200) {
return scrapeProfile(res.item, entity, parameters);
if (res.status === 200) {
return scrapeProfile(res.context, entity, parameters);
}
return null;
@@ -290,14 +291,12 @@ module.exports = {
fetchProfile,
scrapeAll: scrapeAllClassic,
scrapeScene: scrapeSceneClassic,
deprecated: true,
},
tubular: {
fetchLatest: fetchLatestTubular,
fetchProfile,
scrapeAll: scrapeAllTubular,
scrapeScene: scrapeSceneTubular,
deprecated: true,
},
getImageWithFallbacks,
};
src/scrapers/insex.js
+44 -39
View File
@@ -1,25 +1,25 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const unprint = require('unprint');
const slugify = require('../utils/slugify');
function scrapeLatest(scenes, site) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
release.url = query.url('figure a', { origin: site.parameters.latest });
release.title = query.cnt('.has-text-weight-bold, .is-size-6');
release.title = query.content('.has-text-weight-bold, .is-size-6');
release.date = query.date('span.tag', 'YYYY-MM-DD');
release.actors = query.cnts('a.tag');
release.actors = query.contents('a.tag');
const cover = query.img('.image img');
release.poster = cover.replace('poster_noplay', 'trailer_noplay');
release.covers = [cover];
release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
return release;
});
@@ -30,11 +30,11 @@ function scrapeLatestLegacy(scenes, site) {
// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
const release = {};
const titleEl = query.q('.galleryTitleText, .articleTitleText');
const titleEl = query.element('.galleryTitleText, .articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
const url = query.url(titleEl, 'a');
const url = unprint.query.attribute(titleEl, 'a', 'href');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
@@ -44,20 +44,20 @@ function scrapeLatestLegacy(scenes, site) {
} else {
// title should contain date instead, not applicable in brief mode
release.title = title.slice(title.indexOf(':') + 1).trim();
release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
}
release.actors = actors.map((actor) => actor.trim());
const description = query.q('.articleCopyText', true);
const description = query.content('.articleCopyText');
if (description) release.description = description.slice(0, description.lastIndexOf('('));
const duration = query.dur('.articleCopyText a:nth-child(2)');
const duration = query.duration('.articleCopyText a:nth-child(2)');
if (duration) release.duration = duration;
release.likes = parseInt(query.q('.articlePostDateText td:nth-child(3)', true), 10);
release.likes = query.number('.articlePostDateText td:nth-child(3)');
const cover = query.img('a img');
const cover = query.img('a img', { origin: site.url });
release.covers = [[
cover.replace('_thumbnail', ''),
@@ -68,25 +68,26 @@ function scrapeLatestLegacy(scenes, site) {
});
}
async function scrapeScene({ query }, url, channel, parameters, session) {
async function scrapeScene({ query }, channel, include, cookies) {
const release = {};
release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
release.title = query.content('.columns div.is-size-5.has-text-weight-bold');
release.description = query.content('.has-background-black-ter > div:nth-child(4)');
release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
release.actors = query.contents('.has-text-white-ter a.tag[href*="home.php"]');
release.tags = query.contents('.has-background-black-ter > div:nth-child(6) > span');
release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
release.poster = query.img('#videoPlayer, #iodvideo', { attribute: 'poster' });
release.photos = Array.from((query.html('body > div:nth-child(6)') || '').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
release.trailer = query.video();
release.trailer = query.video('source');
if (!release.trailer && parameters.includeTrailers) {
const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
if (!release.trailer && include.includeTrailers) {
// play API relies on the session cookie set by the scene page
const trailerRes = await unprint.get(`${channel.url}/api/play-api.php`, { cookies, extract: false });
if (trailerRes.ok) {
release.trailer = trailerRes.body;
@@ -96,22 +97,22 @@ async function scrapeScene({ query }, url, channel, parameters, session) {
return release;
}
function scrapeSceneLegacy({ query }, site) {
function scrapeSceneLegacy({ query }, { entity: site }) {
const release = {};
const titleEl = query.q('.articleTitleText');
const titleEl = query.element('.articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const url = query.url(titleEl, 'a');
const url = unprint.query.attribute(titleEl, 'a', 'href');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
release.title = title.trim();
release.description = query.q('.articleCopyText', true);
release.description = query.content('.articleCopyText');
release.actors = actors.map((actor) => actor.trim());
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
release.duration = query.dur('.articlePostDateText a:nth-child(2)');
release.duration = query.duration('.articlePostDateText a:nth-child(2)');
const [cover, ...photos] = query.imgs('img[src*="images"]');
release.covers = [cover];
@@ -119,7 +120,7 @@ function scrapeSceneLegacy({ query }, site) {
release.poster = query.poster();
const trailer = query.trailer();
const trailer = query.video('source');
if (trailer) release.trailer = { src: trailer };
return release;
@@ -127,10 +128,13 @@ function scrapeSceneLegacy({ query }, site) {
async function fetchLatest(site, page = 1) {
const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
const res = await unprint.get(url, {
selectAll: 'body > .columns .column',
cookies: { consent: 'yes' },
});
if (res.ok) {
return scrapeLatest(res.items, site);
return scrapeLatest(res.context, site);
}
return res.status;
@@ -138,21 +142,22 @@ async function fetchLatest(site, page = 1) {
async function fetchLatestLegacy(site, page = 1) {
const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
const res = await qu.get(url); // JSON containing html as a property
const res = await unprint.get(url); // JSON containing html as a property
if (res.ok) {
return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
if (res.ok && res.data?.html) {
return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
}
return res.status;
}
async function fetchScene(url, site, baseRelease, parameters) {
const session = http.session();
const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
async function fetchScene(url, site, baseRelease, include) {
const res = await unprint.get(url, {
cookies: { consent: 'yes' },
});
if (res.ok) {
return scrapeScene(res.item, url, site, parameters, session);
return scrapeScene(res.context, site, include, { consent: 'yes', ...res.cookies });
}
return res.status;
src/scrapers/jesseloadsmonsterfacials.js
+25 -22
View File
@@ -1,11 +1,14 @@
'use strict';
const { get, initAll } = require('../utils/qu');
const unprint = require('unprint');
// paths are relative to /visitors/, read raw attributes instead of resolved URLs
const raw = { forceGetAttribute: true };
function scrapeLatest(scenes, dates, site) {
return scenes.map(({ qu }, index) => {
return scenes.map(({ query }, index) => {
const release = {};
const path = qu.url('a[href*="videos/"]');
const path = query.attribute('a[href*="videos/"]', 'href', raw);
if (path) {
if (/\.wmv$/.test(path)) {
@@ -16,30 +19,30 @@ function scrapeLatest(scenes, dates, site) {
}
if (dates && dates[index]) {
release.date = dates[index].qu.date(null, 'MM/DD/YYYY');
release.date = dates[index].query.date(null, 'MM/DD/YYYY');
}
// release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
|| qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1];
|| query.attribute('img[src*="graphics/fft"]', 'src', raw)?.match(/fft_(\w+).gif/)?.[1];
release.description = qu.q('tbody tr:nth-child(3) font', true);
release.description = query.content('tbody tr:nth-child(3) font');
const infoLine = qu.q('font[color="#663366"]', true);
const infoLine = query.content('font[color="#663366"]');
if (infoLine) {
release.duration = Number(infoLine.match(/(\d+) min/i)?.[1] || infoLine.match(/video: (\d+)/i)?.[1]) * 60 || null;
}
const poster = qu.img('img[src*="photos/"][width="400"]');
const poster = query.attribute('img[src*="photos/"][width="400"]', 'src', raw);
release.poster = `${site.url}/visitors/${poster}`;
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
release.photos = query.attributes('img[src*="photos/"]:not([width="400"])', 'src', raw).map((source) => `${site.url}/visitors/${source}`);
return release;
});
}
function scrapeScene({ qu }, url, site) {
function scrapeScene({ query }, url, site) {
const release = { url };
const { pathname } = new URL(url);
@@ -52,47 +55,47 @@ function scrapeScene({ qu }, url, site) {
return release;
}
const actor = qu.q('font[color="#990033"] strong', true);
const actor = query.content('font[color="#990033"] strong');
release.actors = [actor];
const hdTrailer = qu.url('a[href*="hd_trailer.mp4"]');
const sdTrailer = qu.url('a[href*="hd_trailer_mobile.mp4"]');
const hdTrailer = query.attribute('a[href*="hd_trailer.mp4"]', 'href', raw);
const sdTrailer = query.attribute('a[href*="hd_trailer_mobile.mp4"]', 'href', raw);
release.trailer = [
{
hdTrailer && {
src: `${site.url}/visitors/videos/${hdTrailer}`,
quality: 1080,
},
{
sdTrailer && {
src: `${site.url}/visitors/videos/${sdTrailer}`,
quality: 270,
},
];
].filter(Boolean);
return release;
}
async function fetchLatest(site, page = 1) {
const url = `https://jesseloadsmonsterfacials.com/visitors/tour_${page.toString().padStart(2, '0')}.html`;
const res = await get(url);
const res = await unprint.get(url);
if (!res.ok) {
return res.status;
}
const { el } = res.item;
const { element } = res.context;
const scenes = initAll(el, 'table[width="880"]');
const dates = initAll(el, 'font[color="#000000"] strong:not(:empty)');
const scenes = unprint.initAll(element, 'table[width="880"]');
const dates = unprint.initAll(element, 'font[color="#000000"] strong:not(:empty)');
return scrapeLatest(scenes, dates, site);
}
async function fetchScene(url, site) {
const res = await get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, url, site);
return scrapeScene(res.context, url, site);
}
return res.status;
src/scrapers/perfectgonzo.js
+15 -15
View File
@@ -2,7 +2,7 @@
const blake2 = require('blake2');
const qu = require('../utils/qu');
const unprint = require('unprint');
function getHash(identifier) {
const hash = blake2.createHash('blake2b', { digestLength: 8 });
@@ -45,8 +45,8 @@ async function scrapeLatest(scenes, site) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.q('a', 'title');
release.url = query.url('a', 'href', { origin: site.url });
release.title = query.attribute('a', 'title');
release.url = query.url('a', { origin: site.url });
release.date = query.date('.nm-date', 'MM/DD/YYYY');
const slug = new URL(release.url).pathname.split('/')[2];
@@ -56,7 +56,7 @@ async function scrapeLatest(scenes, site) {
[release.poster, ...release.photos] = query.imgs('.bloc-link img');
release.tags = query.cnts('.dropdown ul a').slice(1);
release.tags = query.contents('.dropdown ul a').slice(1);
release.duration = query.duration('.dropdown p:first-child');
return release;
@@ -66,22 +66,22 @@ async function scrapeLatest(scenes, site) {
async function scrapeScene({ query }, site, url) {
const release = { url, site };
release.title = query.cnt('#movie-header h2');
release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.title = query.content('#movie-header h2');
release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.description = query.cnt('.container .mg-md');
release.description = query.content('.container .mg-md');
release.duration = query.duration('#video-ribbon .container > div > span:nth-child(3)');
release.actors = query.cnts('#video-info a').concat(extractMaleModelsFromTags(query.q('.tag-container')));
release.tags = query.cnts('.tag-container a');
release.actors = query.contents('#video-info a').concat(extractMaleModelsFromTags(query.element('.tag-container')));
release.tags = query.contents('.tag-container a');
const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
const uhd = query.content('#video-ribbon .container > div > span:nth-child(2)');
if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
release.poster = query.poster();
const trailer = query.trailer();
const trailer = query.video('source');
if (trailer) release.trailer = { src: trailer };
if (release.photos.length > 0) release.channel = await extractChannelFromPhoto(release.photos[0], site);
@@ -99,20 +99,20 @@ async function scrapeScene({ query }, site, url) {
async function fetchLatest(site, page = 1) {
const url = `${site.url}/movies/page-${page}`;
const res = await qu.getAll(url, '#content-main [class^="item"]');
const res = await unprint.get(url, { selectAll: '#content-main [class^="item"]' });
if (res.ok) {
return scrapeLatest(res.items, site);
return scrapeLatest(res.context, site);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, channel, url);
return scrapeScene(res.context, channel, url);
}
return res.status;
src/scrapers/pinkyxxx.js
+46 -39
View File
@@ -1,24 +1,42 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const unprint = require('unprint');
function parseJson(res) {
// admin-ajax responds with JSON labeled as text/html
if (typeof res.data === 'object') {
return res.data;
}
try {
return JSON.parse(res.body);
} catch (_error) {
return null;
}
}
async function getTrailer(entryId, channel) {
const videoLinksRes = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
const videoLinksRes = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
action: 'vidplay',
vid: entryId,
type: 'promo',
}, {
encodeJSON: false,
decodeJSON: true,
form: true,
});
if (!videoLinksRes.ok || !videoLinksRes.body.size1) {
const videoLinks = videoLinksRes.ok && parseJson(videoLinksRes);
if (!videoLinks?.size1) {
return null;
}
const scriptUrl = qu.extract(videoLinksRes.body.size1)?.query.url('script', 'src', { origin: channel.url });
const videoScriptRes = await http.get(scriptUrl);
const scriptUrl = unprint.init(videoLinks.size1, null, { origin: channel.url })?.query.url('script', { attribute: 'src' });
if (!scriptUrl) {
return null;
}
const videoScriptRes = await unprint.get(scriptUrl, { extract: false });
if (!videoScriptRes.ok) {
return null;
@@ -28,13 +46,11 @@ async function getTrailer(entryId, channel) {
const videoData = videoDataString && JSON.parse(videoDataString);
const posterUrlString = videoScriptRes.body.match(/src="(.*poster.*?)"/)?.[1];
const posterUrlEl = videoScriptRes.document.createElement('span');
posterUrlEl.innerHTML = posterUrlString;
const poster = posterUrlString && unprint.init(`<span>${posterUrlString}</span>`)?.query.content('span'); // decode HTML entities
return {
trailer: videoData?.url || null,
poster: posterUrlEl.textContent || null,
poster: poster || null,
};
}
@@ -48,7 +64,7 @@ function scrapeAll(scenes) {
release.title = scene.title;
release.description = scene.description;
release.date = qu.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || qu.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
release.date = unprint.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || unprint.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
release.poster = scene.poster[0];
release.photos = Object.entries(scene).reduce((acc, [key, data]) => {
@@ -63,20 +79,20 @@ function scrapeAll(scenes) {
});
}
async function scrapeScene({ query }, channel, baseScene) {
async function scrapeScene({ query }, { entity, baseRelease }) {
const release = {};
release.title = query.cnt('.title');
release.description = query.cnt('.description');
release.title = query.content('.title');
release.description = query.content('.description');
release.entryId = query.q('body', 'class').match(/postid-(\d+)/)?.[1];
release.entryId = query.attribute('body', 'class')?.match(/postid-(\d+)/)?.[1];
const { poster, trailer } = await getTrailer(release.entryId, channel) || {};
const { poster, trailer } = await getTrailer(release.entryId, entity) || {};
release.poster = baseScene?.poster || poster;
release.photos = poster && baseScene?.poster
? [poster, ...baseScene.photos]
: baseScene?.photos;
release.poster = baseRelease?.poster || poster;
release.photos = poster && baseRelease?.poster
? [poster, ...baseRelease.photos]
: baseRelease?.photos;
release.trailer = trailer;
@@ -84,33 +100,24 @@ async function scrapeScene({ query }, channel, baseScene) {
}
async function fetchLatest(channel, page) {
const res = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
const res = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
action: 'vls',
limit: 10,
offset: (page - 1) * 10,
}, {
encodeJSON: false,
decodeJSON: true,
form: true,
});
if (res.ok) {
return scrapeAll(res.body.listings);
const data = res.ok && parseJson(res);
if (data?.listings) {
return scrapeAll(data.listings);
}
return res.status;
}
async function fetchScene(url, channel, baseScene) {
const res = await qu.get(url);
if (res.ok) {
return scrapeScene(res.item, channel, baseScene);
}
return res.status;
return res.ok ? null : res.status;
}
module.exports = {
fetchLatest,
fetchScene,
scrapeScene,
};
src/scrapers/releases.js
-2
View File
@@ -39,7 +39,6 @@ const karups = require('./karups');
const kellymadison = require('./kellymadison');
const kink = require('./kink');
const mariskax = require('./mariskax');
// const analvids = require('./analvids');
const pornbox = require('./pornbox');
const littlecapricedreams = require('./littlecapricedreams');
const loveherfilms = require('./loveherfilms');
@@ -74,7 +73,6 @@ const teenmegaworld = require('./teenmegaworld');
const testedefudelidade = require('./testedefudelidade');
const theflourish = require('./theflourish');
const tokyohot = require('./tokyohot');
// const topwebmodels = require('./topwebmodels');
const traxxx = require('./traxxx');
const virtualtaboo = require('./virtualtaboo');
const vixen = require('./vixen');
src/scrapers/rickysroom.js
+17 -19
View File
@@ -1,22 +1,23 @@
'use strict';
const qu = require('../utils/q');
const unprint = require('unprint');
const { lbsToKg, feetInchesToCm } = require('../utils/convert');
function scrapeScene(data, channel) {
const release = {};
release.entryId = data.id;
release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url);
release.url = unprint.prefixUrl(`/videos/${data.slug}`, channel.url);
release.title = data.title;
release.description = data.description;
release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
release.duration = qu.durationToSeconds(data.videos_duration);
release.date = unprint.extractDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
release.duration = unprint.extractDuration(data.videos_duration);
release.actors = data?.models_thumbs.map((model) => ({
name: model.name,
url: qu.prefixUrl(`/models/${model.slug}`, channel.url),
url: unprint.prefixUrl(`/models/${model.slug}`, channel.url),
avatar: model.thumb,
})) || data.models;
@@ -42,13 +43,12 @@ function scrapeScene(data, channel) {
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.contents?.data) {
if (data?.props?.pageProps?.contents?.data) {
const scenes = data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel));
return scenes;
@@ -66,13 +66,12 @@ async function fetchScene(url, channel, baseRelease) {
return baseRelease;
}
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.content) {
if (data?.props?.pageProps?.content) {
return scrapeScene(data.props.pageProps.content, channel);
}
@@ -86,11 +85,11 @@ function scrapeProfile(data, scenes, entity) {
const profile = {};
profile.entryId = data.id;
profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
profile.url = unprint.prefixUrl(`/models/${data.slug}`, entity.url);
profile.description = data.Bio || data.bio;
profile.birthPlace = data.Born || data.born;
profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
profile.dateOfBirth = unprint.extractDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
profile.measurements = data.Measurements || data.Measurements;
profile.height = feetInchesToCm(data.Height || data.height);
@@ -108,13 +107,12 @@ function scrapeProfile(data, scenes, entity) {
async function fetchProfile({ slug }, entity) {
const url = `${entity.url}/models/${slug}`;
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.model) {
if (data?.props?.pageProps?.model) {
return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity);
}
src/scrapers/teamskeet.js
+3 -4
View File
@@ -2,7 +2,6 @@
const unprint = require('unprint');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -99,10 +98,10 @@ async function fetchLatest(channel, page = 1, { parameters }) {
? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
: `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
const res = await http.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
if (res.ok && res.data?.hits) {
return scrapeAll(res.data.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
}
return res.status;
src/scrapers/topwebmodels.js
-144
View File
@@ -1,144 +0,0 @@
'use strict';
const mime = require('mime');
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
function scrapeSceneX(scene) {
const release = {};
release.entryId = scene.id;
release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title, '-', { removePunctuation: true })}`;
release.title = scene.title;
release.description = scene.description;
release.duration = qu.durationToSeconds(scene.videos_duration);
release.date = new Date(scene.release_date);
release.actors = scene.models
.map((actor) => (/&/.test(actor.name)
? actor.name.split(/\s*&\s*/)
: {
name: actor.name,
gender: actor.gender || null,
avatar: actor.thumb,
url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
}))
.flat();
release.stars = scene.rating;
release.tags = scene.tags.map((tag) => tag.name);
if (mime.getType(scene.thumb) === 'image/gif') {
release.teaser = scene.thumb;
} else {
release.poster = scene.thumb;
}
release.channel = slugify(scene.sites[0]?.name, '');
return release;
}
function scrapeAll(scenes) {
return scenes.map(scrapeSceneX);
}
async function scrapeProfile(actor, options) {
const profile = {};
profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch
profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch
profile.gender = actor.attributes.gender || null;
profile.height = convert(actor.attributes.height.value, 'cm');
profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg');
const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || [];
profile.bust = Number(bust);
profile.cup = cup;
profile.waist = Number(waist);
profile.hip = Number(hip);
profile.ethnicity = actor.attributes.ethnicity.value;
profile.birthPlace = actor.attributes.born.value;
profile.eyes = actor.attributes.eyes.value;
profile.hairColor = actor.attributes.hair.value.split('/')[0];
profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`;
profile.avatar = actor.thumb;
if (options.includeActorScenes) {
const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } });
if (res.ok) {
profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items);
}
}
return profile;
}
async function fetchLatest(channel, page) {
const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
headers: {
Referer: 'https://tour.topwebmodels.com',
'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey,
'x-Requested-With': 'XMLHttpRequest',
},
});
if (res.ok) {
return scrapeAll(res.body.videos.items);
}
return res.status;
}
async function fetchScene(url) {
const res = await http.get(url, { extract: { runScripts: 'dangerously' } });
if (res.ok) {
return {
...scrapeSceneX(res.window.__DATA__.data.video),
...(/\.gif/.test(res.window.__DATA__.data.video.thumb) && { teaser: res.window.__DATA__.data.video.thumb }),
poster: res.window.__DATA__.data.file_poster,
};
}
return res.status;
}
async function fetchProfile(baseActor, entity, options) {
const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, {
headers: {
Referer: 'https://tour.topwebmodels.com',
'api-key': entity.parameters?.apiKey,
'x-Requested-With': 'XMLHttpRequest',
},
});
if (!searchRes.ok) {
return searchRes.status;
}
const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
if (actor) {
return scrapeProfile(actor, options);
}
return null;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
};