DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Migrated remaining scrapers from qu/http to unprint, likely fixing TeamSkeet and PinkyXXX (verification needed).

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-10-19 07:35:03 +02:00
parent 03a5ce5bb1
commit 4ede0724a7
24 changed files with 301 additions and 1732 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/deep.js
+1 -30
View File
@@ -9,7 +9,6 @@ const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const { fetchReleaseEntities, urlToHostname } = require('./entities');
const logger = require('./logger')(__filename);
const qu = require('./utils/qu');
const getRecursiveParameters = require('./utils/get-recursive-parameters');
const windows = require('./utils/http-windows');
@@ -85,35 +84,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
}
if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
/*
if (scraper.useUnprint || (type === 'scene' && scraper.scrapeScene?.unprint) || (type === 'movie' && scraper.scrapeMovie?.unprint)) {
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
}
*/
if (!scraper.deprecated) {
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
}
const session = qu.session();
const res = await qu.get(url, null, null, {
session,
rejectUnauthorized: false,
});
const cookie = await session._sessionOptions.cookieJar.get(url);
if (res.ok) {
return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.item, url, entity, baseRelease, options, {
session,
headers: res.headers,
cookieJar: session._sessionOptions.cookieJar,
cookie,
});
}
return res.status;
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
}
return null;
src/media.js
+3 -3
View File
@@ -15,6 +15,7 @@ const sharp = require('sharp');
const blake2 = require('blake2');
const { format } = require('date-fns');
const taskQueue = require('promise-task-queue');
const unprint = require('unprint');
const { Upload } = require('@aws-sdk/lib-storage');
const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
@@ -25,7 +26,6 @@ const knex = require('./knex');
const http = require('./utils/http');
const batchInsert = require('./utils/batch-insert');
const chunk = require('./utils/chunk');
const { get } = require('./utils/qu');
const { fetchEntityReleaseIds } = require('./entity-releases');
const streamQueue = taskQueue();
@@ -383,10 +383,10 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
};
}
const res = await get(baseSource.url);
const res = await unprint.get(baseSource.url);
if (res.ok) {
const src = await baseSource.extract(res.item);
const src = await baseSource.extract(res.context);
return {
...baseSource,
src/scrapers/actors.js
-3
View File
@@ -29,7 +29,6 @@ const karups = require('./karups');
const kellymadison = require('./kellymadison');
const kink = require('./kink');
const mariskax = require('./mariskax');
// const analvids = require('./analvids');
const pornbox = require('./pornbox');
const littlecapricedreams = require('./littlecapricedreams');
const loveherfilms = require('./loveherfilms');
@@ -59,7 +58,6 @@ const teenmegaworld = require('./teenmegaworld');
const testedefudelidade = require('./testedefudelidade');
const theflourish = require('./theflourish');
const tokyohot = require('./tokyohot');
// const topwebmodels = require('./topwebmodels');
const virtualtaboo = require('./virtualtaboo');
const vixen = require('./vixen');
const wankzvr = require('./wankzvr');
@@ -229,7 +227,6 @@ module.exports = {
boyfun: karups,
kellymadison,
'8kmembers': kellymadison,
// analvids,
analvids: pornbox,
littlecapricedreams,
mariskax,
src/scrapers/americanpornstar.js
+14 -14
View File
@@ -1,45 +1,45 @@
'use strict';
const { fetchLatest, fetchProfile } = require('./julesjordan');
const unprint = require('unprint');
const qu = require('../utils/q');
const { fetchLatest, fetchProfile } = require('./julesjordan');
const slugify = require('../utils/slugify');
function scrapeScene({ query }, channel) {
const release = {};
const title = query.cnt('.update_title', false);
const title = query.content('.update_title', { trim: false });
release.title = title.trim();
release.description = query.cnt('.latest_update_description');
release.description = query.content('.latest_update_description');
release.date = query.date('.update_date', 'MM/DD/YYYY');
release.actors = query.all('.tour_update_models a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null),
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null),
}));
release.entryId = slugify(release.title);
release.duration = Number(query.cnt('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
release.duration = Number(query.content('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
const poster = query.img('.large_update_thumb', 'src', { origin: channel.url });
const trailer = query.q('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
const poster = query.img('.large_update_thumb', { attribute: 'src', origin: channel.url });
const trailer = query.attribute('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
release.poster = [
poster.replace(/.jpg$/, '-full.jpg'),
poster,
];
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
release.photos = query.imgs('.small_update_thumb', { attribute: 'src', origin: channel.url }).map((img) => [
img.replace(/.jpg$/, '-full.jpg'),
img,
]);
if (trailer) {
release.trailer = qu.prefixUrl(encodeURI(trailer), channel.url);
release.trailer = unprint.prefixUrl(encodeURI(trailer), channel.url);
}
release.tags = query.cnts('.tour_update_tags a');
release.tags = query.contents('.tour_update_tags a');
return release;
}
@@ -49,10 +49,10 @@ async function fetchLatestLocal(channel, page, include, preData) {
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, channel);
return scrapeScene(res.context, channel);
}
return res.status;
src/scrapers/analvids.js
-174
View File
@@ -1,174 +0,0 @@
'use strict';
const unprint = require('unprint');
const slugify = require('../utils/slugify');
function extractTitle(originalTitle) {
const titleComponents = originalTitle.split(' ');
// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
return { shootId, title };
}
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('.card-scene__view > a');
release.entryId = query.dataset(null, 'content') || new URL(release.url).pathname.match(/watch\/(\d+)/)?.[1];
release.title = query.content('.card-scene__text');
release.shootId = extractTitle(release.title).shootId;
release.date = query.date('.label--time:nth-child(2)', 'YYYY-MM-DD'); // only available on front-page, not on studio page
release.duration = query.duration('.label--time:first-child');
release.poster = query.img('.card-scene__view img', { attribute: 'data-src' });
const caps = query.json('.card-scene__view > a', { attribute: 'data-casting' })?.map((timestamp) => `${channel.url}/casting/${release.entryId}/${timestamp}`);
if (caps && release.poster) {
release.caps = caps;
} else if (caps) {
[release.poster, ...release.caps] = caps;
}
release.teaser = query.video('.card-scene__view > a', { attribute: 'data-preview' });
return release;
});
}
async function fetchLatest(channel, page) {
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
// studios as channels
const url = `${channel.url}/latest/${page}`;
const res = await unprint.get(url, {
selectAll: '.card-scene',
headers: {
Referer: url,
},
});
if (res.ok) {
return scrapeAll(res.context, channel);
}
return res.status;
}
function scrapeScene({ query }, url) {
const release = {};
release.entryId = new URL(url).pathname.match(/watch\/(\d+)/)?.[1];
const featuring = query.content('.watch__title .watch__featuring_models');
release.title = query.content('.watch__title').replace(featuring, '');
release.description = query.content('.text-mob-more');
release.shootId = extractTitle(release.title).shootId;
release.date = query.date('.bi-calendar3', 'YYYY-MM-DD');
release.duration = query.duration('.bi-clock');
release.actors = query.all('.watch__title a[href*="/model"]').map((el) => ({
name: unprint.query.content(el),
url: unprint.query.url(el, null),
}));
release.tags = query.contents('.genres-list a[href*="/genre"]');
// release.studio = slugify(query.content('.genres-list a[href*="/studios"]'), '', { removePunctuation: true });
release.poster = query.img('.watch__video video', { attribute: 'data-poster' });
release.trailer = query.all('.watch__video source').map((el) => ({
src: unprint.query.video(el, null, { attribute: 'src' }),
quality: unprint.query.number(el, null, { attribute: 'size' }),
}));
return release;
}
async function fetchScene(url) {
const res = await unprint.get(url, {
headers: {
Referer: url,
},
});
if (res.ok) {
return scrapeScene(res.context, url);
}
return res.status;
}
function scrapeProfile({ query }, url, channel) {
const profile = { url };
profile.nationality = query.content('.model__info a[href*="/nationality"]');
profile.age = query.number('//td[contains(text(), "Age")]/following-sibling::td');
profile.avatar = query.img('.model__left img');
profile.scenes = scrapeAll(unprint.initAll(query.all('.card-scene')), channel);
return profile;
}
async function getActorUrl(actor, channel) {
if (actor.url) {
return actor.url;
}
const searchUrl = `${channel.url}/api/autocomplete/search?q=${slugify(actor.name, '+')}`;
const searchRes = await unprint.get(searchUrl, {
headers: {
Referer: actor.url,
},
});
if (!searchRes.ok) {
return searchRes.status;
}
const result = searchRes.body.terms.find((item) => item.type === 'model');
if (result) {
return result.url;
}
return null;
}
async function fetchProfile(actor, { channel }) {
const actorUrl = await getActorUrl(actor, channel);
if (typeof actorUrl !== 'string') {
return actorUrl;
}
const bioRes = await unprint.get(actorUrl, {
headers: {
Referer: actor.url,
},
});
if (bioRes.ok) {
return scrapeProfile(bioRes.context, actorUrl, channel);
}
return bioRes.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
};
src/scrapers/analvids_legacy.js
-203
View File
@@ -1,203 +0,0 @@
'use strict';
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
function extractTitle(originalTitle) {
const titleComponents = originalTitle.split(' ');
// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
return { shootId, title };
}
function getPoster(posterElement, sceneId) {
const posterStyle = posterElement.attr('style');
if (posterStyle) {
return posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
}
const posterRange = posterElement.attr('data-casting');
const posterRangeData = posterRange ? JSON.parse(posterRange) : null;
const posterTimeRange = posterRangeData[Math.floor(Math.random() * posterRangeData.length)];
if (!posterTimeRange) {
return null;
}
if (typeof posterTimeRange === 'number') {
// poster time is already a single time value
return `https://analvids.com/casting/${sceneId}/${posterTimeRange}`;
}
const [max, min] = posterTimeRange.split('-');
const posterTime = Math.floor(Math.random() * (Number(max) - Number(min) + 1) + Number(min));
return `https://analvids.com/casting/${sceneId}/${posterTime}`;
}
function scrapeAll(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const scenesElements = $('.thumbnails > div').toArray();
return scenesElements.map((element) => {
const sceneLinkElement = $(element).find('.thumbnail-title a');
const url = sceneLinkElement.attr('href');
const originalTitle = sceneLinkElement.text().trim(); // title attribute breaks when they use \\ escaping
const { shootId, title } = extractTitle(originalTitle);
const entryId = new URL(url).pathname.split('/')[2];
const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
const sceneId = $(element).attr('data-content');
const posterElement = $(element).find('.thumbnail-avatar');
const poster = getPoster(posterElement, sceneId);
return {
url,
shootId,
entryId,
title,
date,
poster,
};
});
}
async function scrapeScene(html, url, site, useGallery) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const playerObject = $('script:contains("new WatchPage")').html();
const playerData = playerObject && playerObject.slice(playerObject.indexOf('{"swf":'), playerObject.lastIndexOf('},') + 1);
const data = playerData && JSON.parse(playerData);
const release = { url };
const originalTitle = $('h1.watchpage-title').text().trim();
const { shootId, title } = extractTitle(originalTitle);
release.shootId = shootId;
release.entryId = new URL(url).pathname.split('/')[2];
release.title = title;
release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
release.description = $('meta[name="description"]')?.attr('content')?.trim()
|| (descriptionElement && $(descriptionElement).find('dd').text().trim());
release.actors = $(actorsElement)
.find('a[href*="com/model"]')
.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const photos = useGallery
? $('.gallery a img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray()
: $('.screenshots img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray();
release.photos = photos.map((source) => {
// source without parameters sometimes serves larger preview photo
const { origin, pathname } = new URL(source);
return `${origin}${pathname}`;
});
const posterStyle = $('#player').attr('style');
const poster = posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
release.poster = poster || release.photos.slice(Math.floor(release.photos.length / 3) * -1); // poster unavailable, try last 1/3rd of high res photos as fallback
if (data) {
const qualityMap = {
web: 240,
vga: 480,
hd: 720,
'1080p': 1080,
};
release.trailer = data.clip.qualities.map((trailer) => ({
src: trailer.src,
type: trailer.type,
quality: qualityMap[trailer.quality] || trailer.quality,
}));
}
const studioName = $('.watchpage-studioname').first().text().trim();
release.studio = slugify(studioName, '', { removePunctuation: true });
return release;
}
async function scrapeProfile(html, _url, actorName) {
const { document } = new JSDOM(html).window;
const profile = {
name: actorName,
};
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
const bio = entries
.filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
profile.birthPlace = bio.Nationality;
if (bio.Age) profile.age = bio.Age;
if (avatarEl) profile.avatar = avatarEl.src;
profile.releases = scrapeAll(html);
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await http.get(`${site.url}/new-videos/${page}`);
return scrapeAll(res.body.toString(), site);
}
async function fetchScene(url, site) {
const useGallery = true;
// TODO: fall back on screenshots when gallery is not available
const res = useGallery
? await http.get(`${url}/gallery#gallery`)
: await http.get(`${url}/screenshots#screenshots`);
return scrapeScene(res.body.toString(), url, site, useGallery);
}
async function fetchProfile({ name: actorName }) {
const res = await http.get(`https://www.analvids.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
const data = res.body;
const result = data.terms.find((item) => item.type === 'model');
if (result) {
const bioRes = await http.get(result.url);
const html = bioRes.body.toString();
return scrapeProfile(html, result.url, actorName);
}
return null;
}
module.exports = {
fetchLatest,
fetchProfile,
fetchScene,
};
src/scrapers/assylum.js
+30 -27
View File
@@ -1,6 +1,9 @@
'use strict';
const { get, geta, ctxa } = require('../utils/q');
const unprint = require('unprint');
// paths are relative, read raw attributes instead of resolved URLs
const raw = { forceGetAttribute: true };
function extractActors(actorString) {
return actorString
@@ -20,48 +23,48 @@ function matchActors(actorString, models) {
}
function scrapeLatest(scenes, site, models) {
return scenes.map(({ qu }) => {
return scenes.map(({ query }) => {
const release = {};
const pathname = qu.url('a.itemimg').slice(1);
const pathname = query.attribute('a.itemimg', 'href', raw).slice(1);
[release.entryId] = pathname.split('/').slice(-1);
release.url = `${site.url}${pathname}`;
release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
release.title = query.attribute('.itemimg img', 'alt') || query.content('h4 a');
release.description = query.content('.mas_longdescription');
release.date = query.date('.movie_info2', 'MM/DD/YY', { match: /\d{2}\/\d{2}\/\d{2}/ });
const actorString = qu.q('.mas_description', true);
const actorString = query.content('.mas_description');
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
const posterPath = qu.img('.itemimg img');
const posterPath = query.attribute('.itemimg img', 'src', raw);
release.poster = `${site.url}/${posterPath}`;
return release;
});
}
function scrapeScene({ html, qu }, url, site, include, models) {
function scrapeScene({ html, query }, url, site, include, models) {
const release = { url };
[release.entryId] = url.split('/').slice(-1);
release.title = qu.q('.mas_title', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.title = query.content('.mas_title');
release.description = query.content('.mas_longdescription');
release.date = query.date('.mas_description', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
if (models) {
const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
const actorString = query.content('.mas_description').replace(/\w+ \d{1,2}, \d{4}/, '');
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
}
release.tags = qu.all('.tags a', true);
release.tags = query.contents('.tags a');
release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
release.photos = query.attributes('.stills img', 'src', raw).map((photoPath) => `${site.url}/${photoPath}`);
const posterIndex = 'splash:';
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
@@ -74,20 +77,20 @@ function scrapeScene({ html, qu }, url, site, include, models) {
return release;
}
function extractModels({ el }, site) {
const models = ctxa(el, '.item');
function extractModels({ element }, site) {
const models = unprint.initAll(element, '.item');
return models.map(({ qu }) => {
return models.map(({ query }) => {
const actor = { gender: 'female' };
const avatar = qu.q('.itemimg img');
const avatar = query.element('.itemimg img');
actor.avatar = `${site.url}/${avatar.src}`;
actor.name = avatar.alt
.split(':').slice(-1)[0]
.replace(/xtreme girl|nurse/ig, '')
.trim();
const actorPath = qu.url('.itemimg');
const actorPath = query.attribute('.itemimg', 'href', raw);
actor.url = `${site.url}${actorPath.slice(1)}`;
return actor;
@@ -96,11 +99,11 @@ function extractModels({ el }, site) {
async function fetchModels(site, page = 1, accModels = []) {
const url = `${site.url}/?models/${page}`;
const res = await get(url);
const res = await unprint.get(url);
if (res.ok) {
const models = extractModels(res.item, site);
const nextPage = res.item.qa('.pagenumbers', true)
const models = extractModels(res.context, site);
const nextPage = res.context.query.contents('.pagenumbers')
.map((pageX) => Number(pageX))
.filter(Boolean) // remove << and >>
.includes(page + 1);
@@ -117,16 +120,16 @@ async function fetchModels(site, page = 1, accModels = []) {
async function fetchLatest(site, page = 1, include, { beforeFetchLatest }) {
const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
const res = await geta(url, '.item');
const res = await unprint.get(url, { selectAll: '.item' });
return res.ok ? scrapeLatest(res.items, site, beforeFetchLatest) : res.status;
return res.ok ? scrapeLatest(res.context, site, beforeFetchLatest) : res.status;
}
async function fetchScene(url, site, release, include, beforeFetchLatest) {
const models = beforeFetchLatest || await fetchModels(site);
const res = await get(url);
const res = await unprint.get(url);
return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
return res.ok ? scrapeScene(res.context, url, site, include, models) : res.status;
}
module.exports = {
src/scrapers/badoink.js
+35 -31
View File
@@ -1,6 +1,7 @@
'use strict';
const qu = require('../utils/qu');
const unprint = require('unprint');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -21,23 +22,23 @@ function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.entryId = query.q(null, 'data-video-card-scene-id') || query.q('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.url = query.url('a', { origin: channel.url });
release.entryId = query.attribute(null, 'data-video-card-scene-id') || query.attribute('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.video-card-title span') || query.q('.video-card-title', 'title');
release.description = query.cnt('.video-card-description');
release.title = query.content('.video-card-title span') || query.attribute('.video-card-title', 'title');
release.description = query.content('.video-card-description');
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
release.duration = query.duration('.video-card-duration', { attribute: 'content' }) || query.number('.video-card-duration') * 60;
release.actors = query.all('.video-card-details--cast a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
name: unprint.query.content(el),
url: unprint.query.url(el, null, { origin: channel.url }),
}));
release.tags = query.cnts('.video-card-tags a');
release.tags = query.contents('.video-card-tags a');
const posterSources = query.srcset('picture source', 'data-srcset') || [query.img('.video-card-image', 'data-src')];
const posterSources = query.sourceSet('picture source', 'data-srcset') || [query.img('.video-card-image', { attribute: 'data-src' })];
release.poster = getPoster(posterSources);
release.teaser = query.video('video') || (channel.parameters?.teasers !== false ? `https://p.badoinkvr.com/d/scene_files/${release.entryId}/preview/preview.mp4` : null);
@@ -46,29 +47,29 @@ function scrapeAll(scenes, channel) {
});
}
function scrapeScene({ query }, url, channel) {
function scrapeScene({ query }, { url, entity: channel }) {
const release = {};
release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.video-title', '.breadcrumbs__item:last-child [itemprop=name]');
release.description = query.cnt('.video-description');
release.title = query.content(['.video-title', '.breadcrumbs__item:last-child [itemprop=name]']);
release.description = query.content('.video-description');
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-upload-date', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.duration = query.duration('.video-duration', { attribute: 'content' }) || query.number('.video-duration') * 60;
release.actors = query.all('.video-actors a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
name: unprint.query.content(el),
url: unprint.query.url(el, null, { origin: channel.url }),
}));
release.tags = query.cnts('.video-tags a');
release.tags = query.contents('.video-tags a');
const posterSources = query.srcset('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
const posterSources = query.sourceSet('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
release.poster = getPoster(posterSources);
const photos = query.imgs('.gallery .gallery-item', 'data-big-image');
const extraPhotoCount = query.number('.gallery .gallery-item:last-child', /\d+/, 'data-desktop-text');
const photos = query.imgs('.gallery .gallery-item', { attribute: 'data-big-image' });
const extraPhotoCount = query.number('.gallery .gallery-item:last-child', { match: /\d+/, attribute: 'data-desktop-text' });
const lastPhoto = photos.slice(-1)[0];
const photosHaveIndex = /_\d+\./.test(lastPhoto);
@@ -87,10 +88,10 @@ function scrapeProfile({ query }, url, entity) {
const bio = query.all('.girl-details-stats-item').reduce((acc, el) => ({
...acc,
[slugify(qu.query.cnt(el, '.girl-details-stat'))]: qu.query.cnt(el, '.girl-details-stat-value'),
[slugify(unprint.query.content(el, '.girl-details-stat'))]: unprint.query.content(el, '.girl-details-stat-value'),
}), {});
profile.description = query.cnt('.girl-details-bio');
profile.description = query.content('.girl-details-bio');
profile.age = bio.age;
profile.birthPlace = bio.country;
@@ -104,23 +105,26 @@ function scrapeProfile({ query }, url, entity) {
profile.hairColor = bio.hair;
profile.eyes = bio.eyes;
const avatarSources = query.srcset('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
const avatarSources = query.sourceSet('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
profile.avatar = getPoster(avatarSources).slice(0, 3); // returns a metric ton of links, if first few don't work the rest presumably won't either
profile.social = query.urls('.girl-details-social-media-list a');
profile.scenes = scrapeAll(qu.initAll(query.all('.video-card')), entity);
profile.scenes = scrapeAll(unprint.initAll(query.all('.video-card')), entity);
return profile;
}
async function fetchLatest(channel, page) {
const res = await qu.getAll(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, '.video-card', {
Cookie: 'affsubid=12345-;', // required to show teaser video, exact number doesn't seem to matter
const res = await unprint.get(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, {
selectAll: '.video-card',
cookies: {
affsubid: '12345-', // required to show teaser video, exact number doesn't seem to matter
},
});
if (res.ok) {
return scrapeAll(res.items, channel);
return scrapeAll(res.context, channel);
}
return res.status;
@@ -128,10 +132,10 @@ async function fetchLatest(channel, page) {
async function fetchProfile(baseActor, { entity }) {
const url = `${entity.url}/${entity.parameters?.actor || 'pornstar'}/${slugify(baseActor.name, '-')}/`;
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeProfile(res.item, url, entity);
return scrapeProfile(res.context, url, entity);
}
return res.status;
src/scrapers/czechav.js
+17 -17
View File
@@ -1,29 +1,29 @@
'use strict';
const qu = require('../utils/q');
const unprint = require('unprint');
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.url = query.url('a', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/tour\/preview\/(.+)/)[1];
release.title = query.cnt('.nice-title');
release.title = query.content('.nice-title');
release.poster = query.img('img.poster');
release.photos = query.q('img.poster', 'data-urls')?.split(' ');
release.teaser = query.video('img.poster', 'data-thumbnail');
release.photos = query.attribute('img.poster', 'data-urls')?.split(' ');
release.teaser = query.video('img.poster', { attribute: 'data-thumbnail' });
if (query.exists('.info')) {
release.date = query.date('.updated', 'MM/DD/YYYY');
release.actors = [{
name: query.cnt('.name'),
age: query.cnt('.age'),
name: query.content('.name'),
age: query.content('.age'),
}];
release.tags = query.cnts('.tags a');
release.tags = query.contents('.tags a');
}
return release;
@@ -35,38 +35,38 @@ function scrapeScene({ query }, url) {
release.entryId = new URL(url).pathname.match(/\/tour\/preview\/(.+)/)[1];
release.title = query.cnt('.nice-title');
release.description = query.cnt('.desc-text p');
release.title = query.content('.nice-title');
release.description = query.content('.desc-text p');
release.tags = query.cnts('.tags a');
release.tags = query.contents('.tags a');
release.poster = query.poster();
release.photos = query.imgs('.thumbnails img, .thumb img');
// some videos may not have unique trailers, but many do
release.trailer = query.video();
release.trailer = query.video('source');
release.formats = query.cnts('.download-quality option');
release.formats = query.contents('.download-quality option');
return release;
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/tour/videos/page-${page}`;
const res = await qu.getAll(url, '.episode-list .episode, .content .girl');
const res = await unprint.get(url, { selectAll: '.episode-list .episode, .content .girl' });
if (res.ok) {
return scrapeAll(res.items, channel);
return scrapeAll(res.context, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, url, channel);
return scrapeScene(res.context, url, channel);
}
return res.status;
src/scrapers/elevatedx.js
+50 -51
View File
@@ -1,8 +1,8 @@
'use strict';
const format = require('template-format');
const unprint = require('unprint');
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -10,11 +10,11 @@ function deriveEntryId(release) {
if (release.date && release.url) {
const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
}
if (release.date && release.title) {
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
}
return null;
@@ -22,7 +22,7 @@ function deriveEntryId(release) {
function extractPoster(posterPath, channel, baseRelease) {
if (posterPath && !/400.jpg/.test(posterPath)) {
const poster = qu.prefixUrl(posterPath, channel.parameters?.media || channel.url);
const poster = unprint.prefixUrl(posterPath, channel.parameters?.media || channel.url);
const posterSources = [
poster,
// upscaled
@@ -40,20 +40,20 @@ function extractPoster(posterPath, channel, baseRelease) {
return [baseRelease?.poster || null, []];
}
function getImageWithFallbacks(q, selector, site, el) {
function getImageWithFallbacks(query, selector, site, el) {
const sources = el
? [
q(el, selector, 'src0_3x'),
q(el, selector, 'src0_2x'),
q(el, selector, 'src0_1x'),
unprint.query.attribute(el, selector, 'src0_3x'),
unprint.query.attribute(el, selector, 'src0_2x'),
unprint.query.attribute(el, selector, 'src0_1x'),
]
: [
q(selector, 'src0_3x'),
q(selector, 'src0_2x'),
q(selector, 'src0_1x'),
query.attribute(selector, 'src0_3x'),
query.attribute(selector, 'src0_2x'),
query.attribute(selector, 'src0_1x'),
];
return sources.filter(Boolean).map((src) => qu.prefixUrl(src, site.parameters?.media || site.url));
return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url));
}
function scrapeAllClassic(scenes, channel) {
@@ -61,16 +61,16 @@ function scrapeAllClassic(scenes, channel) {
const release = {};
release.url = query.url('.updateInfo h5 a:not([href*="content/"]):not([href*="#coming"])');
release.entryId = query.url('.updateThumb img', 'alt');
release.entryId = query.attribute('.updateThumb img', 'alt');
release.title = query.cnt('.updateInfo h5 a');
release.title = query.content('.updateInfo h5 a');
release.actors = query.cnts('.tour_update_models a');
release.actors = query.contents('.tour_update_models a');
release.date = query.date('.availdate, .updateInfo p span:last-child', 'MM/DD/YYYY');
release.poster = query.img('.updateThumb img');
release.poster = query.img('.updateThumb img', { origin: channel.url }); // relative to site root, not listing page
const trailer = query.q('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
const trailer = query.attribute('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
if (trailer) {
release.trailer = `${channel.url}${trailer}`;
@@ -84,13 +84,14 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
release.title = query.attribute('h4 a', 'title') || query.content('h4 a');
release.url = query.url('h4 a');
release.date = query.date('.more-info-div', 'MMM D, YYYY');
release.duration = query.dur('.more-info-div');
release.duration = query.duration('.more-info-div');
const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
// raw path, prefixed with media URL below
const posterPath = query.attribute('.img-div img', 'src0_1x') || query.attribute('img.video_placeholder', 'data-src') || query.attribute('img.video_placeholder', 'src', { forceGetAttribute: true });
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${channel.parameters?.media || channel.url}${posterPath}`;
@@ -102,7 +103,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
];
}
release.teaser = query.video();
release.teaser = query.video('source');
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
@@ -116,11 +117,11 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
});
}
function scrapeSceneClassic({ query, html }, url, channel) {
function scrapeSceneClassic({ query, html }, { entity: channel }) {
const release = {};
release.title = query.q('.updatesBlock h2', true);
release.poster = query.meta('property="og:image"');
release.title = query.content('.updatesBlock h2');
release.poster = query.attribute('meta[property="og:image"]', 'content');
release.entryId = release.poster.match(/\/content\/(.*)\//)?.[1];
const trailer = html.match(/src="(.+\.mp4)"/)?.[1];
@@ -134,34 +135,34 @@ function scrapeSceneClassic({ query, html }, url, channel) {
return release;
}
function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
function scrapeSceneTubular({ query, html }, { entity, url, baseRelease }) {
const release = {};
release.title = query.q('.video-player .section-title', true) || query.q('.bio-section-head .section-title', true);
release.title = query.content('.video-player .section-title') || query.content('.bio-section-head .section-title');
release.description = query.text('.update-info-block .descriptionFull') || query.text('.update-info-block .description');
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)');
release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
release.date = query.date('.update-info-row', 'MMM D, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.duration = query.duration('.update-info-row:nth-child(2)');
release.photoCount = query.number('.update-info-row:nth-child(2)', { match: /(\d+) photos/i, matchIndex: 1 });
release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.cnt(el, 'span'),
avatar: getImageWithFallbacks(query.q, 'img', entity, el),
url: query.url(el, null),
name: unprint.query.content(el, 'span'),
avatar: getImageWithFallbacks(query, 'img', entity, el),
url: unprint.query.url(el, null),
}));
release.tags = query.all('.tags a', true);
release.tags = query.contents('.tags a');
const posterPath = query.q('.player-thumb img', 'src0_1x');
const posterPath = query.attribute('.player-thumb img', 'src0_1x');
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
[release.poster, release.photos] = extractPoster(posterPath, entity, baseRelease);
if (trailer) {
release.trailer = { src: qu.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
release.trailer = { src: unprint.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
}
const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
const stars = query.content('.update-rating').match(/\d.\d/)?.[0];
if (stars) release.stars = Number(stars);
if (entity.type === 'network') {
@@ -181,11 +182,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
async function scrapeProfile({ query }, entity, parameters) {
const profile = {};
const bio = query.cnt('.model_bio, .detail-div');
const avatarEl = query.q('.model_bio_pic img, .model_bio_thumb');
const bio = query.content('.model_bio, .detail-div');
const avatarEl = query.element('.model_bio_pic img, .model_bio_thumb');
profile.age = Number(bio?.match(/Age:\s*(\d{2})/)?.[1]) || null;
profile.dateOfBirth = qu.parseDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
profile.dateOfBirth = unprint.extractDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
profile.height = convert(bio?.match(/\d+\s*(feet|')\s*\d+\s*(inches|"|$)/)?.[0], 'cm');
profile.measurements = bio?.match(/\w+[-x]\d+[-x]\d+/)?.[0] || null;
@@ -201,17 +202,17 @@ async function scrapeProfile({ query }, entity, parameters) {
avatarEl.getAttribute('src'),
]
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map((avatar) => qu.prefixUrl(avatar, entity.url));
.map((avatar) => unprint.prefixUrl(avatar, entity.url));
if (avatarSources.length) profile.avatar = avatarSources;
}
if (parameters?.layout === 'classic') {
profile.scenes = scrapeAllClassic(qu.initAll(query.all('.bodyArea .updateItem')), entity);
profile.scenes = scrapeAllClassic(unprint.initAll(query.all('.bodyArea .updateItem')), entity);
}
if (parameters?.layout === 'tubular') {
profile.scenes = scrapeAllTubular(qu.initAll(query.all('.modelfeature, .item-video')), entity);
profile.scenes = scrapeAllTubular(unprint.initAll(query.all('.modelfeature, .item-video')), entity);
}
return profile;
@@ -221,20 +222,20 @@ async function fetchLatest(site, page = 1, options, preData, allScraper) {
const url = (site.parameters?.latest && format(site.parameters.latest, { page }))
|| `${site.url}/categories/movies_${page}_d.html`;
const res = await qu.getAll(url, '.modelfeature, .item-video, .bodyArea .updateItem');
const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .bodyArea .updateItem' });
if (!res.ok) {
return res.status;
}
return allScraper(res.items, site, preData?.uniqueReleases);
return allScraper(res.context, site, preData?.uniqueReleases);
}
async function fetchUpcomingClassic(channel) {
const res = await qu.getAll(channel.url, '#owl-upcomingScenes .updateItem');
const res = await unprint.get(channel.url, { selectAll: '#owl-upcomingScenes .updateItem' });
if (res.ok) {
return scrapeAllClassic(res.items, channel);
return scrapeAllClassic(res.context, channel);
}
return res.status;
@@ -273,10 +274,10 @@ async function fetchProfile({ name: actorName, url }, { entity, parameters }) {
return null;
}
const res = await qu.get(profileUrl);
const res = await unprint.get(profileUrl);
if (res.statusCode === 200) {
return scrapeProfile(res.item, entity, parameters);
if (res.status === 200) {
return scrapeProfile(res.context, entity, parameters);
}
return null;
@@ -290,14 +291,12 @@ module.exports = {
fetchProfile,
scrapeAll: scrapeAllClassic,
scrapeScene: scrapeSceneClassic,
deprecated: true,
},
tubular: {
fetchLatest: fetchLatestTubular,
fetchProfile,
scrapeAll: scrapeAllTubular,
scrapeScene: scrapeSceneTubular,
deprecated: true,
},
getImageWithFallbacks,
};
src/scrapers/insex.js
+44 -39
View File
@@ -1,25 +1,25 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const unprint = require('unprint');
const slugify = require('../utils/slugify');
function scrapeLatest(scenes, site) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
release.url = query.url('figure a', { origin: site.parameters.latest });
release.title = query.cnt('.has-text-weight-bold, .is-size-6');
release.title = query.content('.has-text-weight-bold, .is-size-6');
release.date = query.date('span.tag', 'YYYY-MM-DD');
release.actors = query.cnts('a.tag');
release.actors = query.contents('a.tag');
const cover = query.img('.image img');
release.poster = cover.replace('poster_noplay', 'trailer_noplay');
release.covers = [cover];
release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
return release;
});
@@ -30,11 +30,11 @@ function scrapeLatestLegacy(scenes, site) {
// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
const release = {};
const titleEl = query.q('.galleryTitleText, .articleTitleText');
const titleEl = query.element('.galleryTitleText, .articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
const url = query.url(titleEl, 'a');
const url = unprint.query.attribute(titleEl, 'a', 'href');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
@@ -44,20 +44,20 @@ function scrapeLatestLegacy(scenes, site) {
} else {
// title should contain date instead, not applicable in brief mode
release.title = title.slice(title.indexOf(':') + 1).trim();
release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
}
release.actors = actors.map((actor) => actor.trim());
const description = query.q('.articleCopyText', true);
const description = query.content('.articleCopyText');
if (description) release.description = description.slice(0, description.lastIndexOf('('));
const duration = query.dur('.articleCopyText a:nth-child(2)');
const duration = query.duration('.articleCopyText a:nth-child(2)');
if (duration) release.duration = duration;
release.likes = parseInt(query.q('.articlePostDateText td:nth-child(3)', true), 10);
release.likes = query.number('.articlePostDateText td:nth-child(3)');
const cover = query.img('a img');
const cover = query.img('a img', { origin: site.url });
release.covers = [[
cover.replace('_thumbnail', ''),
@@ -68,25 +68,26 @@ function scrapeLatestLegacy(scenes, site) {
});
}
async function scrapeScene({ query }, url, channel, parameters, session) {
async function scrapeScene({ query }, channel, include, cookies) {
const release = {};
release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
release.title = query.content('.columns div.is-size-5.has-text-weight-bold');
release.description = query.content('.has-background-black-ter > div:nth-child(4)');
release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
release.actors = query.contents('.has-text-white-ter a.tag[href*="home.php"]');
release.tags = query.contents('.has-background-black-ter > div:nth-child(6) > span');
release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
release.poster = query.img('#videoPlayer, #iodvideo', { attribute: 'poster' });
release.photos = Array.from((query.html('body > div:nth-child(6)') || '').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
release.trailer = query.video();
release.trailer = query.video('source');
if (!release.trailer && parameters.includeTrailers) {
const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
if (!release.trailer && include.includeTrailers) {
// play API relies on the session cookie set by the scene page
const trailerRes = await unprint.get(`${channel.url}/api/play-api.php`, { cookies, extract: false });
if (trailerRes.ok) {
release.trailer = trailerRes.body;
@@ -96,22 +97,22 @@ async function scrapeScene({ query }, url, channel, parameters, session) {
return release;
}
function scrapeSceneLegacy({ query }, site) {
function scrapeSceneLegacy({ query }, { entity: site }) {
const release = {};
const titleEl = query.q('.articleTitleText');
const titleEl = query.element('.articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const url = query.url(titleEl, 'a');
const url = unprint.query.attribute(titleEl, 'a', 'href');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
release.title = title.trim();
release.description = query.q('.articleCopyText', true);
release.description = query.content('.articleCopyText');
release.actors = actors.map((actor) => actor.trim());
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
release.duration = query.dur('.articlePostDateText a:nth-child(2)');
release.duration = query.duration('.articlePostDateText a:nth-child(2)');
const [cover, ...photos] = query.imgs('img[src*="images"]');
release.covers = [cover];
@@ -119,7 +120,7 @@ function scrapeSceneLegacy({ query }, site) {
release.poster = query.poster();
const trailer = query.trailer();
const trailer = query.video('source');
if (trailer) release.trailer = { src: trailer };
return release;
@@ -127,10 +128,13 @@ function scrapeSceneLegacy({ query }, site) {
async function fetchLatest(site, page = 1) {
const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
const res = await unprint.get(url, {
selectAll: 'body > .columns .column',
cookies: { consent: 'yes' },
});
if (res.ok) {
return scrapeLatest(res.items, site);
return scrapeLatest(res.context, site);
}
return res.status;
@@ -138,21 +142,22 @@ async function fetchLatest(site, page = 1) {
async function fetchLatestLegacy(site, page = 1) {
const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
const res = await qu.get(url); // JSON containing html as a property
const res = await unprint.get(url); // JSON containing html as a property
if (res.ok) {
return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
if (res.ok && res.data?.html) {
return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
}
return res.status;
}
async function fetchScene(url, site, baseRelease, parameters) {
const session = http.session();
const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
async function fetchScene(url, site, baseRelease, include) {
const res = await unprint.get(url, {
cookies: { consent: 'yes' },
});
if (res.ok) {
return scrapeScene(res.item, url, site, parameters, session);
return scrapeScene(res.context, site, include, { consent: 'yes', ...res.cookies });
}
return res.status;
src/scrapers/jesseloadsmonsterfacials.js
+25 -22
View File
@@ -1,11 +1,14 @@
'use strict';
const { get, initAll } = require('../utils/qu');
const unprint = require('unprint');
// paths are relative to /visitors/, read raw attributes instead of resolved URLs
const raw = { forceGetAttribute: true };
function scrapeLatest(scenes, dates, site) {
return scenes.map(({ qu }, index) => {
return scenes.map(({ query }, index) => {
const release = {};
const path = qu.url('a[href*="videos/"]');
const path = query.attribute('a[href*="videos/"]', 'href', raw);
if (path) {
if (/\.wmv$/.test(path)) {
@@ -16,30 +19,30 @@ function scrapeLatest(scenes, dates, site) {
}
if (dates && dates[index]) {
release.date = dates[index].qu.date(null, 'MM/DD/YYYY');
release.date = dates[index].query.date(null, 'MM/DD/YYYY');
}
// release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
|| qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1];
|| query.attribute('img[src*="graphics/fft"]', 'src', raw)?.match(/fft_(\w+).gif/)?.[1];
release.description = qu.q('tbody tr:nth-child(3) font', true);
release.description = query.content('tbody tr:nth-child(3) font');
const infoLine = qu.q('font[color="#663366"]', true);
const infoLine = query.content('font[color="#663366"]');
if (infoLine) {
release.duration = Number(infoLine.match(/(\d+) min/i)?.[1] || infoLine.match(/video: (\d+)/i)?.[1]) * 60 || null;
}
const poster = qu.img('img[src*="photos/"][width="400"]');
const poster = query.attribute('img[src*="photos/"][width="400"]', 'src', raw);
release.poster = `${site.url}/visitors/${poster}`;
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
release.photos = query.attributes('img[src*="photos/"]:not([width="400"])', 'src', raw).map((source) => `${site.url}/visitors/${source}`);
return release;
});
}
function scrapeScene({ qu }, url, site) {
function scrapeScene({ query }, url, site) {
const release = { url };
const { pathname } = new URL(url);
@@ -52,47 +55,47 @@ function scrapeScene({ qu }, url, site) {
return release;
}
const actor = qu.q('font[color="#990033"] strong', true);
const actor = query.content('font[color="#990033"] strong');
release.actors = [actor];
const hdTrailer = qu.url('a[href*="hd_trailer.mp4"]');
const sdTrailer = qu.url('a[href*="hd_trailer_mobile.mp4"]');
const hdTrailer = query.attribute('a[href*="hd_trailer.mp4"]', 'href', raw);
const sdTrailer = query.attribute('a[href*="hd_trailer_mobile.mp4"]', 'href', raw);
release.trailer = [
{
hdTrailer && {
src: `${site.url}/visitors/videos/${hdTrailer}`,
quality: 1080,
},
{
sdTrailer && {
src: `${site.url}/visitors/videos/${sdTrailer}`,
quality: 270,
},
];
].filter(Boolean);
return release;
}
async function fetchLatest(site, page = 1) {
const url = `https://jesseloadsmonsterfacials.com/visitors/tour_${page.toString().padStart(2, '0')}.html`;
const res = await get(url);
const res = await unprint.get(url);
if (!res.ok) {
return res.status;
}
const { el } = res.item;
const { element } = res.context;
const scenes = initAll(el, 'table[width="880"]');
const dates = initAll(el, 'font[color="#000000"] strong:not(:empty)');
const scenes = unprint.initAll(element, 'table[width="880"]');
const dates = unprint.initAll(element, 'font[color="#000000"] strong:not(:empty)');
return scrapeLatest(scenes, dates, site);
}
async function fetchScene(url, site) {
const res = await get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, url, site);
return scrapeScene(res.context, url, site);
}
return res.status;
src/scrapers/perfectgonzo.js
+15 -15
View File
@@ -2,7 +2,7 @@
const blake2 = require('blake2');
const qu = require('../utils/qu');
const unprint = require('unprint');
function getHash(identifier) {
const hash = blake2.createHash('blake2b', { digestLength: 8 });
@@ -45,8 +45,8 @@ async function scrapeLatest(scenes, site) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.q('a', 'title');
release.url = query.url('a', 'href', { origin: site.url });
release.title = query.attribute('a', 'title');
release.url = query.url('a', { origin: site.url });
release.date = query.date('.nm-date', 'MM/DD/YYYY');
const slug = new URL(release.url).pathname.split('/')[2];
@@ -56,7 +56,7 @@ async function scrapeLatest(scenes, site) {
[release.poster, ...release.photos] = query.imgs('.bloc-link img');
release.tags = query.cnts('.dropdown ul a').slice(1);
release.tags = query.contents('.dropdown ul a').slice(1);
release.duration = query.duration('.dropdown p:first-child');
return release;
@@ -66,22 +66,22 @@ async function scrapeLatest(scenes, site) {
async function scrapeScene({ query }, site, url) {
const release = { url, site };
release.title = query.cnt('#movie-header h2');
release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.title = query.content('#movie-header h2');
release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.description = query.cnt('.container .mg-md');
release.description = query.content('.container .mg-md');
release.duration = query.duration('#video-ribbon .container > div > span:nth-child(3)');
release.actors = query.cnts('#video-info a').concat(extractMaleModelsFromTags(query.q('.tag-container')));
release.tags = query.cnts('.tag-container a');
release.actors = query.contents('#video-info a').concat(extractMaleModelsFromTags(query.element('.tag-container')));
release.tags = query.contents('.tag-container a');
const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
const uhd = query.content('#video-ribbon .container > div > span:nth-child(2)');
if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
release.poster = query.poster();
const trailer = query.trailer();
const trailer = query.video('source');
if (trailer) release.trailer = { src: trailer };
if (release.photos.length > 0) release.channel = await extractChannelFromPhoto(release.photos[0], site);
@@ -99,20 +99,20 @@ async function scrapeScene({ query }, site, url) {
async function fetchLatest(site, page = 1) {
const url = `${site.url}/movies/page-${page}`;
const res = await qu.getAll(url, '#content-main [class^="item"]');
const res = await unprint.get(url, { selectAll: '#content-main [class^="item"]' });
if (res.ok) {
return scrapeLatest(res.items, site);
return scrapeLatest(res.context, site);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, channel, url);
return scrapeScene(res.context, channel, url);
}
return res.status;
src/scrapers/pinkyxxx.js
+46 -39
View File
@@ -1,24 +1,42 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const unprint = require('unprint');
function parseJson(res) {
// admin-ajax responds with JSON labeled as text/html
if (typeof res.data === 'object') {
return res.data;
}
try {
return JSON.parse(res.body);
} catch (_error) {
return null;
}
}
async function getTrailer(entryId, channel) {
const videoLinksRes = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
const videoLinksRes = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
action: 'vidplay',
vid: entryId,
type: 'promo',
}, {
encodeJSON: false,
decodeJSON: true,
form: true,
});
if (!videoLinksRes.ok || !videoLinksRes.body.size1) {
const videoLinks = videoLinksRes.ok && parseJson(videoLinksRes);
if (!videoLinks?.size1) {
return null;
}
const scriptUrl = qu.extract(videoLinksRes.body.size1)?.query.url('script', 'src', { origin: channel.url });
const videoScriptRes = await http.get(scriptUrl);
const scriptUrl = unprint.init(videoLinks.size1, null, { origin: channel.url })?.query.url('script', { attribute: 'src' });
if (!scriptUrl) {
return null;
}
const videoScriptRes = await unprint.get(scriptUrl, { extract: false });
if (!videoScriptRes.ok) {
return null;
@@ -28,13 +46,11 @@ async function getTrailer(entryId, channel) {
const videoData = videoDataString && JSON.parse(videoDataString);
const posterUrlString = videoScriptRes.body.match(/src="(.*poster.*?)"/)?.[1];
const posterUrlEl = videoScriptRes.document.createElement('span');
posterUrlEl.innerHTML = posterUrlString;
const poster = posterUrlString && unprint.init(`<span>${posterUrlString}</span>`)?.query.content('span'); // decode HTML entities
return {
trailer: videoData?.url || null,
poster: posterUrlEl.textContent || null,
poster: poster || null,
};
}
@@ -48,7 +64,7 @@ function scrapeAll(scenes) {
release.title = scene.title;
release.description = scene.description;
release.date = qu.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || qu.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
release.date = unprint.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || unprint.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
release.poster = scene.poster[0];
release.photos = Object.entries(scene).reduce((acc, [key, data]) => {
@@ -63,20 +79,20 @@ function scrapeAll(scenes) {
});
}
async function scrapeScene({ query }, channel, baseScene) {
async function scrapeScene({ query }, { entity, baseRelease }) {
const release = {};
release.title = query.cnt('.title');
release.description = query.cnt('.description');
release.title = query.content('.title');
release.description = query.content('.description');
release.entryId = query.q('body', 'class').match(/postid-(\d+)/)?.[1];
release.entryId = query.attribute('body', 'class')?.match(/postid-(\d+)/)?.[1];
const { poster, trailer } = await getTrailer(release.entryId, channel) || {};
const { poster, trailer } = await getTrailer(release.entryId, entity) || {};
release.poster = baseScene?.poster || poster;
release.photos = poster && baseScene?.poster
? [poster, ...baseScene.photos]
: baseScene?.photos;
release.poster = baseRelease?.poster || poster;
release.photos = poster && baseRelease?.poster
? [poster, ...baseRelease.photos]
: baseRelease?.photos;
release.trailer = trailer;
@@ -84,33 +100,24 @@ async function scrapeScene({ query }, channel, baseScene) {
}
async function fetchLatest(channel, page) {
const res = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
const res = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
action: 'vls',
limit: 10,
offset: (page - 1) * 10,
}, {
encodeJSON: false,
decodeJSON: true,
form: true,
});
if (res.ok) {
return scrapeAll(res.body.listings);
const data = res.ok && parseJson(res);
if (data?.listings) {
return scrapeAll(data.listings);
}
return res.status;
}
async function fetchScene(url, channel, baseScene) {
const res = await qu.get(url);
if (res.ok) {
return scrapeScene(res.item, channel, baseScene);
}
return res.status;
return res.ok ? null : res.status;
}
module.exports = {
fetchLatest,
fetchScene,
scrapeScene,
};
src/scrapers/releases.js
-2
View File
@@ -39,7 +39,6 @@ const karups = require('./karups');
const kellymadison = require('./kellymadison');
const kink = require('./kink');
const mariskax = require('./mariskax');
// const analvids = require('./analvids');
const pornbox = require('./pornbox');
const littlecapricedreams = require('./littlecapricedreams');
const loveherfilms = require('./loveherfilms');
@@ -74,7 +73,6 @@ const teenmegaworld = require('./teenmegaworld');
const testedefudelidade = require('./testedefudelidade');
const theflourish = require('./theflourish');
const tokyohot = require('./tokyohot');
// const topwebmodels = require('./topwebmodels');
const traxxx = require('./traxxx');
const virtualtaboo = require('./virtualtaboo');
const vixen = require('./vixen');
src/scrapers/rickysroom.js
+17 -19
View File
@@ -1,22 +1,23 @@
'use strict';
const qu = require('../utils/q');
const unprint = require('unprint');
const { lbsToKg, feetInchesToCm } = require('../utils/convert');
function scrapeScene(data, channel) {
const release = {};
release.entryId = data.id;
release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url);
release.url = unprint.prefixUrl(`/videos/${data.slug}`, channel.url);
release.title = data.title;
release.description = data.description;
release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
release.duration = qu.durationToSeconds(data.videos_duration);
release.date = unprint.extractDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
release.duration = unprint.extractDuration(data.videos_duration);
release.actors = data?.models_thumbs.map((model) => ({
name: model.name,
url: qu.prefixUrl(`/models/${model.slug}`, channel.url),
url: unprint.prefixUrl(`/models/${model.slug}`, channel.url),
avatar: model.thumb,
})) || data.models;
@@ -42,13 +43,12 @@ function scrapeScene(data, channel) {
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.contents?.data) {
if (data?.props?.pageProps?.contents?.data) {
const scenes = data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel));
return scenes;
@@ -66,13 +66,12 @@ async function fetchScene(url, channel, baseRelease) {
return baseRelease;
}
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.content) {
if (data?.props?.pageProps?.content) {
return scrapeScene(data.props.pageProps.content, channel);
}
@@ -86,11 +85,11 @@ function scrapeProfile(data, scenes, entity) {
const profile = {};
profile.entryId = data.id;
profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
profile.url = unprint.prefixUrl(`/models/${data.slug}`, entity.url);
profile.description = data.Bio || data.bio;
profile.birthPlace = data.Born || data.born;
profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
profile.dateOfBirth = unprint.extractDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
profile.measurements = data.Measurements || data.Measurements;
profile.height = feetInchesToCm(data.Height || data.height);
@@ -108,13 +107,12 @@ function scrapeProfile(data, scenes, entity) {
async function fetchProfile({ slug }, entity) {
const url = `${entity.url}/models/${slug}`;
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.model) {
if (data?.props?.pageProps?.model) {
return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity);
}
src/scrapers/teamskeet.js
+3 -4
View File
@@ -2,7 +2,6 @@
const unprint = require('unprint');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -99,10 +98,10 @@ async function fetchLatest(channel, page = 1, { parameters }) {
? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
: `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
const res = await http.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
if (res.ok && res.data?.hits) {
return scrapeAll(res.data.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
}
return res.status;
src/scrapers/topwebmodels.js
-144
View File
@@ -1,144 +0,0 @@
'use strict';
const mime = require('mime');
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
function scrapeSceneX(scene) {
const release = {};
release.entryId = scene.id;
release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title, '-', { removePunctuation: true })}`;
release.title = scene.title;
release.description = scene.description;
release.duration = qu.durationToSeconds(scene.videos_duration);
release.date = new Date(scene.release_date);
release.actors = scene.models
.map((actor) => (/&/.test(actor.name)
? actor.name.split(/\s*&\s*/)
: {
name: actor.name,
gender: actor.gender || null,
avatar: actor.thumb,
url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
}))
.flat();
release.stars = scene.rating;
release.tags = scene.tags.map((tag) => tag.name);
if (mime.getType(scene.thumb) === 'image/gif') {
release.teaser = scene.thumb;
} else {
release.poster = scene.thumb;
}
release.channel = slugify(scene.sites[0]?.name, '');
return release;
}
function scrapeAll(scenes) {
return scenes.map(scrapeSceneX);
}
async function scrapeProfile(actor, options) {
const profile = {};
profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch
profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch
profile.gender = actor.attributes.gender || null;
profile.height = convert(actor.attributes.height.value, 'cm');
profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg');
const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || [];
profile.bust = Number(bust);
profile.cup = cup;
profile.waist = Number(waist);
profile.hip = Number(hip);
profile.ethnicity = actor.attributes.ethnicity.value;
profile.birthPlace = actor.attributes.born.value;
profile.eyes = actor.attributes.eyes.value;
profile.hairColor = actor.attributes.hair.value.split('/')[0];
profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`;
profile.avatar = actor.thumb;
if (options.includeActorScenes) {
const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } });
if (res.ok) {
profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items);
}
}
return profile;
}
async function fetchLatest(channel, page) {
const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
headers: {
Referer: 'https://tour.topwebmodels.com',
'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey,
'x-Requested-With': 'XMLHttpRequest',
},
});
if (res.ok) {
return scrapeAll(res.body.videos.items);
}
return res.status;
}
async function fetchScene(url) {
const res = await http.get(url, { extract: { runScripts: 'dangerously' } });
if (res.ok) {
return {
...scrapeSceneX(res.window.__DATA__.data.video),
...(/\.gif/.test(res.window.__DATA__.data.video.thumb) && { teaser: res.window.__DATA__.data.video.thumb }),
poster: res.window.__DATA__.data.file_poster,
};
}
return res.status;
}
async function fetchProfile(baseActor, entity, options) {
const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, {
headers: {
Referer: 'https://tour.topwebmodels.com',
'api-key': entity.parameters?.apiKey,
'x-Requested-With': 'XMLHttpRequest',
},
});
if (!searchRes.ok) {
return searchRes.status;
}
const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
if (actor) {
return scrapeProfile(actor, options);
}
return null;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
};
src/store-releases.js
+1 -2
View File
@@ -12,7 +12,6 @@ const slugify = require('./utils/slugify');
const bulkInsert = require('./utils/bulk-insert');
const chunk = require('./utils/chunk');
const redis = require('./redis');
const { formatDate } = require('./utils/qu');
const { associateActors, associateDirectors, scrapeActors, toBaseActors } = require('./actors');
const { associateReleaseTags } = require('./tags');
const { curateEntity } = require('./entities');
@@ -32,7 +31,7 @@ const commonContext = {
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
const slugBase = release.title
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
|| (release.date && `${release.entity.slug} ${unprint.formatDate(release.date, 'YYYY MM DD')}`)
|| null;
const slug = slugify(slugBase, '-', {
src/tools/dogfart.js
-76
View File
@@ -1,76 +0,0 @@
'use strict';
const fs = require('fs').promises;
const Promise = require('bluebird');
const qu = require('../utils/qu');
const qualities = {
sm: 360,
med: 480,
big: 720,
};
/*
async function scrape() {
const urlsByPage = await Promise.map(Array.from({ length: 140 }), async (value, index) => {
const res = await qu.get(`https://www.dogfartnetwork.com/tour/scenes/?p=${index + 1}`);
if (res.ok) {
return res.item.query.urls('.recent-updates > a', 'href', { origin: 'https://www.dogfartnetwork.com' });
}
return [];
}, { concurrency: 1 });
const urls = urlsByPage.flat();
await fs.writeFile('./dogfart-links', urls.join('\n'));
console.log(`Saved ${urls.length} URLs to file`);
}
async function compare() {
const newLinksFile = await fs.readFile('./dogfart-links', 'utf8');
const oldLinksFile = await fs.readFile('./dogfart-got', 'utf8');
const newLinks = newLinksFile.split('\n').filter(Boolean);
const oldLinks = new Set(oldLinksFile.split('\n').filter(Boolean));
const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
console.log(getLinks);
}
*/
async function scrapeMembers() {
const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
if (res.ok) {
return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
actors: query.contents('a[href*="model.php"]'),
trailer: query.urls('.trailer-link a').map((url) => ({
src: url,
quality: qualities[url.match(/_([a-z]+).mp4/)[1]],
})),
}));
}
return [];
}, { concurrency: 1 });
const urls = titlesByPage.flat().map((data) => JSON.stringify(data));
console.log(urls);
await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
console.log(`Saved ${urls.length} URLs to file`);
}
scrapeMembers();
src/utils/media.js
-110
View File
@@ -1,110 +0,0 @@
'use strict';
const config = require('config');
const fs = require('fs');
const fsPromises = require('fs').promises;
const Promise = require('bluebird');
const blake2 = require('blake2');
const sharp = require('sharp');
const { nanoid } = require('nanoid');
const { PassThrough } = require('stream');
const http = require('./http');
function getMemoryUsage() {
return process.memoryUsage().rss / (10 ** 6);
}
let peakMemoryUsage = getMemoryUsage();
async function fetchSource(link) {
const id = nanoid();
const hasher = new blake2.Hash('blake2b');
hasher.setEncoding('hex');
const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
const tempFileStream = fs.createWriteStream(tempFilePath);
const hashStream = new PassThrough();
hashStream.on('data', (chunk) => hasher.write(chunk));
try {
const res = await http.get(link, null, {
stream: true,
transforms: [hashStream],
destination: tempFileStream,
timeout: 5000,
});
if (!res.ok) {
throw new Error(res.status);
}
hasher.end();
const hash = hasher.read();
const memoryUsage = getMemoryUsage();
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
return {
id,
path: tempFilePath,
hash,
};
} catch (error) {
await fsPromises.unlink(tempFilePath);
throw error;
}
}
async function init() {
const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
const links = linksFile.split('\n').filter(Boolean);
await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
console.time('thumbs');
const files = await Promise.map(links, async (link) => {
try {
return await fetchSource(link);
} catch (error) {
console.log(`Failed to fetch ${link}: ${error.message}`);
return null;
}
});
await Promise.map(files.filter(Boolean), async (file) => {
const image = sharp(file.path).jpeg();
const [{ width, height }, { size }] = await Promise.all([
image.metadata(),
fsPromises.stat(file.path),
]);
await Promise.all([
image
.toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
image
.resize({
height: config.media.thumbnailSize,
withoutEnlargement: true,
})
.toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
]);
const memoryUsage = getMemoryUsage();
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
}, { concurrency: 10 });
console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
console.timeEnd('thumbs');
}
init();
src/utils/q.js
-5
View File
@@ -1,5 +0,0 @@
'use strict';
const qu = require('./qu');
module.exports = qu;
src/utils/qu.js
-682
View File
@@ -1,682 +0,0 @@
'use strict';
const { JSDOM } = require('jsdom');
const moment = require('moment');
const http = require('./http');
const virtualConsole = require('./virtual-console')(__filename);
const { window: globalWindow } = new JSDOM('', { virtualConsole });
function trim(str) {
if (typeof str !== 'string') {
return str;
}
return str.trim().replace(/\s+/g, ' ');
}
function extractDate(dateString, format, match) {
if (!dateString) {
return null;
}
if (match) {
const dateStamp = trim(dateString).match(match);
if (dateStamp) {
const dateValue = moment.utc(dateStamp[0], format);
return dateValue.isValid() ? dateValue.toDate() : null;
}
return null;
}
const dateValue = moment.utc(trim(dateString), format);
return dateValue.isValid() ? dateValue.toDate() : null;
}
function formatDate(dateValue, format, inputFormat) {
if (inputFormat) {
return moment(dateValue, inputFormat).format(format);
}
return moment(dateValue).format(format);
}
function durationToSeconds(durationString, match) {
const durationMatch = durationString.match(match || /(\d+:)?\d+:\d+/);
if (durationMatch) {
const segments = ['00'].concat(durationMatch[0].split(/[:hm]/)).slice(-3);
return moment.duration(segments.join(':')).asSeconds();
}
return null;
}
function prefixUrl(urlValue, origin, protocol = 'https') {
if (!urlValue) {
return null;
}
if (/^http/.test(urlValue)) {
return urlValue;
}
if (protocol && /^\/\//.test(urlValue)) {
return `${protocol}:${urlValue}`;
}
if (origin && /^\//.test(urlValue)) {
return `${origin}${urlValue}`;
}
if (origin && /^\.\//.test(urlValue)) {
return `${origin}${urlValue.slice(1)}`;
}
if (origin) {
return `${origin}/${urlValue}`;
}
return urlValue;
}
function iterateXPathResult(iterator, results = []) {
const element = iterator.iterateNext();
if (element) {
return iterateXPathResult(iterator, [...results, element]);
}
return results;
}
function getElements(context, selector, first = false) {
if (!selector) {
return context;
}
if (/^\/\//.test(selector)) {
// XPath selector
const iterator = globalWindow.document.evaluate(selector, context, null, globalWindow.XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
if (first) {
return iterator.iterateNext();
}
return iterateXPathResult(iterator);
}
if (first) {
return context.querySelector(selector);
}
return Array.from(context.querySelectorAll(selector));
}
function q(context, selectors, attrArg, applyTrim = true) {
if (!selectors && context.nodeName === '#document') {
return null;
}
const attr = attrArg === true ? 'textContent' : attrArg;
const element = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, true), null);
if (!element) {
return null;
}
if (attr) {
const value = element[attr] || element.getAttribute(attr);
return applyTrim && typeof value === 'string' ? trim(value) : value;
}
return element;
}
function all(context, selectors, attrArg, applyTrim = true) {
const attr = attrArg === true ? 'textContent' : attrArg;
const elements = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector), null);
if (!Array.isArray(elements)) {
return [];
}
if (attr) {
return elements.map((el) => q(el, null, attr, applyTrim));
}
return elements;
}
function exists(context, selector) {
return !!q(context, selector);
}
function count(context, selector) {
return all(context, selector)?.length || 0;
}
function content(context, selector, applyTrim = true) {
return q(context, selector, 'textContent', applyTrim);
}
function contents(context, selector, applyTrim) {
return all(context, selector, 'textContent', applyTrim);
}
function html(context, selector) {
const el = q(context, selector, null, true);
return el && el.innerHTML;
}
function htmls(context, selector) {
const els = all(context, selector, null, true);
return els.map((el) => el.innerHTML);
}
function execute(context, selector = 'script') {
const scripts = htmls(context, selector);
const originalGlobal = Object.fromEntries(Object.entries(global));
const errors = scripts?.reduce((accErrors, script) => {
try {
Function(script)(); /* eslint-disable-line no-new-func */
return accErrors;
} catch (error) {
// the script failed
return [...accErrors, error];
}
}, []);
const data = Object.fromEntries(Object.entries(global).filter(([key, value]) => {
if (originalGlobal[key] !== value) {
delete global[key];
return true;
}
return false;
}));
return {
...data,
errors,
};
}
function json(context, selector) {
const el = q(context, selector, null, true);
try {
return JSON.parse(el?.innerHTML);
} catch (error) {
return null;
}
}
function jsons(context, selector) {
const els = all(context, selector, null, true);
return els.map((el) => {
try {
return JSON.parse(el?.innerHTML);
} catch (error) {
return null;
}
});
}
function texts(context, selector, applyTrim = true, filter = true) {
const el = q(context, selector, null, applyTrim);
if (!el) return null;
const nodes = Array.from(el.childNodes)
.filter((node) => node.nodeName === '#text')
.map((node) => (applyTrim ? trim(node.textContent) : node.textContent));
return filter ? nodes.filter(Boolean) : nodes;
}
function text(context, selector, applyTrim = true) {
const nodes = texts(context, selector, applyTrim, true);
if (!nodes) return null;
const textValue = nodes.join(' ');
return applyTrim ? trim(textValue) : textValue;
}
function removeStyleFunctionSpaces(el) {
// jsdom appears to have a bug where it ignores inline CSS attributes set to a function() containing spaces, e.g. url( image.png )
el.setAttribute('style', el.getAttribute('style').replace(/\(\s+(.*)\s+\)/g, (match, cssArgs) => `(${cssArgs})`));
}
function style(context, selector, styleAttr) {
const el = q(context, selector);
if (el?.hasAttribute('style')) {
removeStyleFunctionSpaces(el);
return styleAttr ? el.style[styleAttr] : el.style;
}
return null;
}
function styles(context, selector, styleAttr) {
const elStyles = Array.from(context.querySelectorAll(selector), (el) => {
removeStyleFunctionSpaces(el);
return styleAttr ? el.style[styleAttr] : el.style;
});
return elStyles;
}
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) {
const value = q(context, selector, attr);
if (value && match) {
return Number(value.match(match)?.[matchIndex]);
}
if (value) {
return Number(value);
}
return null;
}
function meta(context, selector, attrArg = 'content', applyTrim = true) {
if (/meta\[.*\]/.test(selector)) {
return q(context, selector, attrArg, applyTrim);
}
return q(context, `meta[${selector}]`, attrArg, applyTrim);
}
function date(context, selector, format, match, attr = 'textContent') {
const dateString = q(context, selector, attr, true);
if (!dateString) return null;
return extractDate(dateString, format, match);
}
function dateAgo(context, selector, match = /(\d+)\s*(\w+)/, attr = 'textContent') {
const timeString = q(context, selector, attr, 'textContent');
if (!timeString) {
return null;
}
const timeMatch = timeString.match(match);
if (timeMatch) {
const [n, period] = timeMatch.slice(1);
const thenDate = moment.utc().subtract(Number(n), period);
return {
date: thenDate.toDate(),
precision: period.replace(/s$/, ''),
};
}
return null;
}
function image(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
const imageEl = (attr && q(context, selector, attr))
|| q(context, selector, 'data-src')
|| q(context, selector, 'src');
return prefixUrl(imageEl, origin, protocol);
}
function images(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
const attribute = attr
|| (q(context, selector, 'data-src') && 'data-src')
|| (q(context, selector, 'src') && 'src');
const imageEls = all(context, selector, attribute);
return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol));
}
function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
const urlEl = q(context, selector, attr);
const prefixedUrl = prefixUrl(urlEl, origin, protocol);
if (prefixedUrl && object) {
return new URL(prefixedUrl);
}
return prefixedUrl;
}
function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
const urlEls = all(context, selector, attr);
return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls;
}
function sourceSet(context, selector, attr = 'srcset', options = {}) {
const srcset = q(context, selector, attr);
if (!srcset) {
return null;
}
const sources = srcset
.split(/\s*,\s*/)
.map((source) => {
const [link, descriptor] = source.split(' ');
if (link) {
return {
descriptor: descriptor || 'fallback',
url: prefixUrl(link, options.origin, options.protocol),
};
}
return null;
})
.filter(Boolean)
.sort((sourceA, sourceB) => {
if (sourceB.descriptor === 'fallback' || parseInt(sourceA.descriptor, 10) > parseInt(sourceB.descriptor, 10)) {
return -1;
}
if (parseInt(sourceA.descriptor, 10) < parseInt(sourceB.descriptor, 10)) {
return 1;
}
return 0;
});
if (options.includeDescriptor) {
return sources;
}
return sources.map((source) => source.url);
}
function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
const posterEl = q(context, selector, attr);
return attr ? prefixUrl(posterEl, origin, protocol) : posterEl;
}
function video(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
const trailerEl = q(context, selector, attr);
return attr ? prefixUrl(trailerEl, origin, protocol) : trailerEl;
}
function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
const trailerEls = all(context, selector, attr);
return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
}
function duration(context, selector, match, attr = 'textContent') {
const durationString = q(context, selector, attr);
if (!durationString) {
return null;
}
const durationMatch = durationToSeconds(durationString, match);
if (durationMatch) {
return durationMatch;
}
const timestampMatch = durationString.match(/(\d+H)?\s*(\d+M)?\s*\d+S?/i);
if (timestampMatch) {
const hours = timestampMatch[0].match(/(\d+)H/i)?.[1] || 0;
const minutes = timestampMatch[0].match(/(\d+)M/i)?.[1] || 0;
const seconds = timestampMatch[0].match(/(\d+)(S|$)/i)?.[1] || 0;
return (Number(hours) * 3600) + (Number(minutes) * 60) + Number(seconds);
}
return null;
}
const legacyFuncs = {
q,
qa: all,
qall: all,
qd: date,
qdate: date,
qh: html,
qhtml: html,
qi: image,
qimage: image,
qimages: images,
qis: images,
ql: duration,
qlength: duration,
qm: meta,
qmeta: meta,
qp: poster,
qposter: poster,
qs: all,
qt: video,
qtext: text,
qtexts: texts,
qtrailer: video,
qtrailers: videos,
qts: videos,
qtx: text,
qtxs: texts,
qtxt: text,
qtxts: texts,
// qu: url,
qurl: url,
qurls: urls,
qus: urls,
};
const quFuncs = {
all,
cnt: content,
cnts: contents,
content,
contents,
count,
date,
dateAgo,
dur: duration,
duration,
el: q,
element: q,
execute,
exec: execute,
exists,
html,
htmls,
image,
images,
img: image,
imgs: images,
json,
jsons,
length: duration,
meta,
num: number,
number,
poster,
q,
sourceSet,
sources: sourceSet,
srcs: sourceSet,
srcset: sourceSet,
style,
styles,
text,
texts,
trailer: video,
url,
urls,
video,
videos,
};
function init(context, selector, window) {
if (!context) {
return null;
}
const element = selector ? context.querySelector(selector) : context;
if (!element) {
return null;
}
const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context
.reduce((acc, [key, func]) => ({
...acc,
[key]: (...args) => (args[0] instanceof globalWindow.HTMLElement // allow for different context
? func(...args)
: func(element, ...args)),
}), {});
const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
.reduce((acc, [key, func]) => ({
...acc,
[key]: (...args) => (args[0]?.nodeType === undefined // allow for different context
? func(element, ...args)
: func(...args)),
}), {});
return {
element,
el: element,
html: element.outerHTML || element.body.outerHTML,
text: trim(element.textContent),
...(window && {
window,
document: window.document,
}),
...legacyContextFuncs,
qu: quContextFuncs,
query: quContextFuncs,
};
}
function initAll(context, selector, window) {
if (Array.isArray(context)) {
return context.map((element) => init(element, null, window));
}
return Array.from(context.querySelectorAll(selector))
.map((element) => init(element, null, window));
}
function extract(htmlValue, selector, options) {
const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
return init(window.document, selector, window);
}
function extractAll(htmlValue, selector, options) {
const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
return initAll(window.document, selector, window);
}
async function request(method = 'get', urlValue, body, selector, headers, options, queryAll = false) {
const res = await (method === 'post'
? http.post(urlValue, body, { ...options, headers })
: http[method](urlValue, {
...options,
headers,
parse: true,
}));
if (res.ok) {
const item = queryAll
? initAll(res.document, selector, res.window)
: init(res.document, selector, res.window);
return {
item,
items: all ? item : [item],
res,
ok: true,
status: res.statusCode,
...res,
};
}
return {
item: null,
items: [],
res,
ok: false,
status: res.statusCode,
...res,
};
}
async function get(urlValue, selector, headers, options) {
return request('get', urlValue, null, selector, headers, options, false);
}
async function post(urlValue, body, selector, headers, options) {
return request('post', urlValue, body, selector, headers, options, false);
}
async function getAll(urlValue, selector, headers, options) {
return request('get', urlValue, null, selector, headers, options, true);
}
async function postAll(urlValue, body, selector, headers, options) {
return request('post', urlValue, body, selector, headers, options, true);
}
function session(headers, options) {
return http.session(headers, options);
}
module.exports = {
extractDate,
extract,
extractAll,
durationToSeconds,
init,
initAll,
formatDate,
get,
getAll,
http,
fetch: get,
fetchAll: getAll,
context: init,
contextAll: initAll,
ed: extractDate,
ex: extract,
exa: extractAll,
fd: formatDate,
parseDate: extractDate,
ctx: init,
ctxa: initAll,
geta: getAll,
qu: quFuncs,
query: quFuncs,
post,
postAll,
prefixUrl,
session,
...legacyFuncs,
};
src/utils/timeout.js
-20
View File
@@ -1,20 +0,0 @@
'use strict';
const http = require('./http');
const sleep = 5000;
const timeout = 1000;
async function init() {
try {
const res = await http.get(`https://httpstat.us/200?sleep=${sleep}`, {
timeout,
});
console.log(res.statusCode);
} catch (error) {
console.log(error);
}
}
init();