Added WankzVR update, scene and profile scraper.

This commit is contained in:
DebaucheryLibrarian 2021-02-04 01:13:02 +01:00
parent 0b99e72924
commit ff123b99b7
37 changed files with 251 additions and 5 deletions

View File

@ -171,6 +171,9 @@ module.exports = {
'pervertgallery', 'pervertgallery',
'povperverts', 'povperverts',
], ],
'wankzvr',
'milfvr',
'tranzvr',
'topwebmodels', 'topwebmodels',
'pascalssubsluts', 'pascalssubsluts',
'kellymadison', 'kellymadison',
@ -252,7 +255,7 @@ module.exports = {
thumbnailQuality: 100, thumbnailQuality: 100,
lazySize: 90, lazySize: 90,
lazyQuality: 90, lazyQuality: 90,
trailerQuality: [480, 720, 360, 1080, 320, 540, 2160, 270, 240, 180], trailerQuality: [480, 540, 360, 720, 1080, 320, 1440, 1600, 1920, 2160, 270, 240, 180],
limit: 25, // max number of photos per release limit: 25, // max number of photos per release
streamConcurrency: 2, // max number of video streams (m3u8 etc.) to fetch and process at once streamConcurrency: 2, // max number of video streams (m3u8 etc.) to fetch and process at once
}, },

View File

@ -34,6 +34,7 @@ exports.up = knex => Promise.resolve()
table.integer('quality', 6); table.integer('quality', 6);
table.integer('width', 6); table.integer('width', 6);
table.integer('height', 6); table.integer('height', 6);
table.boolean('vr');
table.float('entropy'); table.float('entropy');
table.float('sharpness'); table.float('sharpness');

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 62 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 8.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 62 KiB

View File

@ -0,0 +1 @@
<svg id="logo-tvr" viewBox="0 62.813 73.107 42.218"><g fill="#FFF"><path d="M8.164 66.296v22.647H4.27V66.296H0v-3.45h12.467v3.45H8.164zM23.674 88.943L19.37 76.816V74.46h3.826v-8.266h-4.918v22.75h-3.894V62.847h10.35c1.571 0 2.357.797 2.357 2.391v9.496c0 1.548-.774 2.345-2.323 2.392h-1.332l4.304 11.682v.137h-4.066zM39.501 88.943l-.786-4.953h-4.884l-.786 4.953h-3.86v-.068l5.055-26.063h4.099l5.021 26.063v.068h-3.859zm-3.244-20.085l-1.913 11.784h3.826l-1.913-11.784zM55.318 88.943l-6.251-16.43c.023.228.057.456.103.683.045.182.08.381.103.598s.034.416.034.598v14.551h-3.792V62.846h3.416l6.251 16.088-.137-.683a3.178 3.178 0 0 1-.085-.598 11.907 11.907 0 0 1-.017-.598v-14.21h3.792v26.097h-3.417zM61.013 88.943v-2.084l7.31-20.495h-6.729v-3.518h11.375V65.1l-7.275 20.324h7.31v3.519H61.013z"></path></g><path fill="#E41577" d="M72.22 93.854h-1c.023.06.038.119.061.187.276.962.709 4.104.873 5.835h.067c.485 0 .888-.389.888-.858v-4.305a.886.886 0 0 0-.889-.859M46.637 93.854h-1c-.484 0-.888.388-.888.858v4.305c0 .47.396.858.888.858h.067c.164-1.731.597-4.873.873-5.835.023-.066.038-.126.06-.186M71.325 100.659c-.007-.202-.03-.471-.061-.783-.164-1.693-.581-4.716-.835-5.604a3.124 3.124 0 0 0-.15-.418c-.334-.806-.939-1.313-2.066-1.597-1.246-.321-2.97-.984-9.289-.984s-8.043.663-9.281.977c-1.127.284-1.724.791-2.067 1.597-.052.135-.104.269-.149.419-.254.888-.672 3.908-.836 5.602-.029.313-.052.583-.066.784-.068 1.283.223 1.828 1.074 2.38.552.358 1.059.687 1.97 1.097.902.41 2.007.902 3.215.902.761 0 3.253-.484 4.17-.641.597-.098 1.403-.157 1.97-.157s1.373.052 1.97.157c.917.148 3.41.641 4.171.641 1.208 0 2.312-.492 3.215-.902.903-.41 1.41-.738 1.961-1.097.86-.546 1.151-1.091 1.084-2.373"></path><path fill="#FFF" d="M49.002 93.943h2.76l2.521 4.142 2.566-4.142h.149l.447-.007h7.431c1.097 0 1.984.291 2.626.858.292.23.5.462.627.693.253.433.388.873.388 1.299v.208c0 .664-.328 1.307-.977 1.91l-.015.008-.015.007a3.972 3.972 0 0 1-.5.352c.53.671.806 1.499.806 2.47v.447h-2.671v-.561c0-.641-.313-1.141-.947-1.536-.366-.216-.873-.328-1.507-.328h-3.91V97.45h6.2c.544 0 .746-.246.85-.478.016-.066.023-.112.031-.148 0-.067-.031-.217-.254-.403a.994.994 0 0 0-.529-.156h-.448v-.022h-6.454l-3.902 6.543-5.273-8.843z"></path></svg>

After

Width:  |  Height:  |  Size: 2.2 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 99 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 99 KiB

View File

@ -57,6 +57,14 @@ const groups = [
]; ];
const tags = [ const tags = [
{
name: '180°',
slug: '180',
},
{
name: '60 FPS',
slug: '60fps',
},
{ {
name: '3D', name: '3D',
slug: '3d', slug: '3d',
@ -72,6 +80,16 @@ const tags = [
slug: '5k', slug: '5k',
description: 'Available in very high quality 5K resolution.', description: 'Available in very high quality 5K resolution.',
}, },
{
name: '7K',
slug: '7k',
description: 'Available in super high quality 7K resolution.',
},
{
name: '8K',
slug: '8k',
description: 'Available in extremely high quality 8K resolution.',
},
{ {
name: '69', name: '69',
slug: '69', slug: '69',

View File

@ -543,6 +543,11 @@ const networks = [
url: 'https://www.vixen.com', url: 'https://www.vixen.com',
description: 'Vixen.com features the worlds finest cinematic adult films with 4K quality and high-end erotic photography.', description: 'Vixen.com features the worlds finest cinematic adult films with 4K quality and high-end erotic photography.',
}, },
{
slug: 'wankzvr',
name: 'WankzVR',
url: 'https://www.wankzvr.com',
},
{ {
slug: 'pierrewoodman', slug: 'pierrewoodman',
name: 'Pierre Woodman', name: 'Pierre Woodman',

View File

@ -9365,6 +9365,28 @@ const sites = [
url: 'https://www.vogov.com', url: 'https://www.vogov.com',
description: 'Top rated models. Graceful locations. Best gonzo scenes. 4K UHD 60 FPS. So, in general Vogov is a website that is worth visiting and exploring carefully. It gives a chance to spend a fantastic night with gorgeous girls ready to experiment and to full around with their lovers.', description: 'Top rated models. Graceful locations. Best gonzo scenes. 4K UHD 60 FPS. So, in general Vogov is a website that is worth visiting and exploring carefully. It gives a chance to spend a fantastic night with gorgeous girls ready to experiment and to full around with their lovers.',
}, },
// WANKZ VR
{
name: 'WankzVR',
slug: 'wankzvr',
url: 'https://www.wankzvr.com',
tags: ['vr'],
parent: 'wankzvr',
},
{
name: 'MilfVR',
slug: 'milfvr',
url: 'https://www.milfvr.com',
tags: ['vr', 'milf'],
parent: 'wankzvr',
},
{
name: 'TranzVR',
slug: 'tranzvr',
url: 'https://www.tranzvr.com',
tags: ['vr', 'transsexual'],
parent: 'wankzvr',
},
// WHALE MEMBER // WHALE MEMBER
{ {
name: 'Cum 4K', name: 'Cum 4K',

View File

@ -56,10 +56,17 @@ async function fetchScene(scraper, url, entity, baseRelease, options) {
} }
if (scraper.scrapeScene) { if (scraper.scrapeScene) {
const res = await qu.get(url); const session = qu.session();
const res = await qu.get(url, null, null, { session });
const cookie = await session._sessionOptions.cookieJar.get(url);
if (res.ok) { if (res.ok) {
return scraper.scrapeScene(res.item, url, entity, baseRelease, options); return scraper.scrapeScene(res.item, url, entity, baseRelease, options, {
session,
headers: res.headers,
cookieJar: session._sessionOptions.cookieJar,
cookie,
});
} }
return res.status; return res.status;

View File

@ -108,6 +108,8 @@ function toBaseSource(rawSource) {
if (rawSource.interval) baseSource.interval = rawSource.interval; if (rawSource.interval) baseSource.interval = rawSource.interval;
if (rawSource.concurrency) baseSource.concurrency = rawSource.concurrency; if (rawSource.concurrency) baseSource.concurrency = rawSource.concurrency;
if (rawSource.vr) baseSource.vr = rawSource.vr;
if (rawSource.credit !== undefined) baseSource.credit = rawSource.credit; if (rawSource.credit !== undefined) baseSource.credit = rawSource.credit;
if (rawSource.comment) baseSource.comment = rawSource.comment; if (rawSource.comment) baseSource.comment = rawSource.comment;
if (rawSource.group) baseSource.group = rawSource.group; if (rawSource.group) baseSource.group = rawSource.group;
@ -155,17 +157,19 @@ function baseSourceToBaseMedia(baseSource, role, metadata) {
function sortBaseTrailersByQuality(sources, role) { function sortBaseTrailersByQuality(sources, role) {
if (role === 'trailers') { if (role === 'trailers') {
const sortedSources = sources.sort((sourceA, sourceB) => { const sortedSources = sources.sort((sourceA, sourceB) => {
if (config.media.trailerQuality.indexOf(sourceA.quality) > config.media.trailerQuality.indexOf(sourceB.quality)) { if (config.media.trailerQuality.includes(sourceB.quality) && config.media.trailerQuality.indexOf(sourceA.quality) > config.media.trailerQuality.indexOf(sourceB.quality)) {
return 1; return 1;
} }
if (config.media.trailerQuality.indexOf(sourceA.quality) < config.media.trailerQuality.indexOf(sourceB.quality)) { if (config.media.trailerQuality.includes(sourceA.quality) && config.media.trailerQuality.indexOf(sourceA.quality) < config.media.trailerQuality.indexOf(sourceB.quality)) {
return -1; return -1;
} }
return 0; return 0;
}); });
console.log(sortedSources);
return sortedSources; return sortedSources;
} }
@ -632,6 +636,7 @@ function curateMediaEntry(media, index) {
size: media.meta.size, size: media.meta.size,
width: media.meta.width, width: media.meta.width,
height: media.meta.height, height: media.meta.height,
vr: media.vr,
entropy: media.meta.entropy, entropy: media.meta.entropy,
sharpness: media.meta.sharpness, sharpness: media.meta.sharpness,
source: media.src, source: media.src,

View File

@ -57,6 +57,7 @@ const traxxx = require('./traxxx');
const vivid = require('./vivid'); const vivid = require('./vivid');
const vixen = require('./vixen'); const vixen = require('./vixen');
const vogov = require('./vogov'); const vogov = require('./vogov');
const wankzvr = require('./wankzvr');
const whalemember = require('./whalemember'); const whalemember = require('./whalemember');
const xempire = require('./xempire'); const xempire = require('./xempire');
@ -138,6 +139,7 @@ const scrapers = {
vivid, vivid,
vixen, vixen,
vogov, vogov,
wankzvr,
whalemember, whalemember,
xempire, xempire,
}, },
@ -212,6 +214,7 @@ const scrapers = {
men: mindgeek, men: mindgeek,
metrohd: mindgeek, metrohd: mindgeek,
milehighmedia: mindgeek, milehighmedia: mindgeek,
milfvr: wankzvr,
mofos: mindgeek, mofos: mindgeek,
mugfucked: fullpornnetwork, mugfucked: fullpornnetwork,
naughtyamerica, naughtyamerica,
@ -248,6 +251,7 @@ const scrapers = {
topwebmodels, topwebmodels,
transangels: mindgeek, transangels: mindgeek,
transbella: porndoe, transbella: porndoe,
tranzvr: wankzvr,
trueanal: mikeadriano, trueanal: mikeadriano,
tushy: vixen, tushy: vixen,
tushyraw: vixen, tushyraw: vixen,
@ -255,6 +259,7 @@ const scrapers = {
vipsexvault: porndoe, vipsexvault: porndoe,
vixen, vixen,
vrcosplayx: badoink, vrcosplayx: badoink,
wankzvr,
wicked: gamma, wicked: gamma,
wildoncam: cherrypimps, wildoncam: cherrypimps,
xempire, xempire,

177
src/scrapers/wankzvr.js Normal file
View File

@ -0,0 +1,177 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
async function getTrailerUrl(release, channel, request) {
const csrfToken = request.cookie.match('csrfst=(.*?);')?.[1];
if (!csrfToken) {
return null;
}
const res = await http.post(`${channel.url}/ajax/player-config.json`, {
item_id: release.entryId,
}, {
headers: {
'X-CSRF-Token': csrfToken,
},
session: request.session,
encodeJSON: false,
});
if (res.ok) {
const trailers = res.body.streams.map(trailer => ({
src: trailer.url,
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
vr: true,
}));
return {
trailers,
poster: qu.prefixUrl(res.body.poster, res.body.thumbCDN),
};
}
return null;
}
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.card__h');
release.date = query.date('.card__date', 'D MMMM, YYYY');
release.actors = query.all('.card__links a').map(el => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
}));
const poster = query.srcset('picture source[type="image/jpeg"]', 'data-srcset')
|| query.srcset('picture source[type="image/jpeg"]', 'srcset')
|| query.srcset('.video__cover', 'srcset');
if (poster?.[0]) {
release.poster = [
poster[0].replace(/small|tiny/, 'large'),
...poster,
];
release.teaser = poster[0].replace(/\b(cover|hero|\d+)\/[a-z0-9_]+\.[a-z]+$/i, 'roll.webm'); // actually how site generates teaser URL
}
release.channel = channel.slug; // avoid being assigned to WankzVR network
return release;
});
}
async function scrapeScene({ query }, url, channel, baseRelease, options, request) {
const release = {};
release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.detail__title');
release.description = query.cnt('.detail__txt');
release.date = query.date('.detail__date', 'D MMMM, YYYY');
release.duration = query.number('.time') * 60;
release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map(el => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
}));
release.tags = query.cnts('.tag-list .tag').concat(query.cnts('.detail__specs-list .detail__specs-item'));
release.photos = query.all('.photo-strip__slide').map(el => ([
qu.query.img(el, null, 'data-src'),
qu.query.img(el, 'img', 'src'),
]));
if (options.includePosters || options.includeTrailers) {
const { trailers, poster } = await getTrailerUrl(release, channel, request);
release.trailer = trailers;
release.poster = poster;
}
return release;
}
async function fetchActorScenes({ query }, url, entity, page = 1, accScenes = []) {
const scenes = scrapeAll(qu.initAll(query.all('.cards-list .card')), entity);
const hasNextPage = !query.exists('.pagenav__link.inactive');
if (hasNextPage) {
const { origin, pathname, searchParams } = new URL(url);
searchParams.set('p', page + 1);
const res = await qu.get(`${origin}${pathname}?${searchParams}`);
if (res.ok) {
return fetchActorScenes(res.item, url, entity, page + 1, accScenes.concat(scenes));
}
}
return accScenes.concat(scenes);
}
async function scrapeProfile({ query }, url, entity, options) {
const profile = {};
const bio = query.all('.person__meta__item').reduce((acc, el) => ({
...acc,
[slugify(qu.query.cnt(el, '.person__meta__label'))]: qu.query.text(el),
}), {});
profile.description = query.cnt('.person__content');
profile.gender = entity.slug === 'tranzvr' ? 'transsexual' : 'female';
profile.age = bio.age;
profile.birthPlace = bio.birthplace;
profile.height = parseInt(bio.height, 10);
profile.measurements = bio.measurements;
profile.avatar = query.srcset('.person__avatar img');
if (options.includeActorScenes) {
profile.scenes = await fetchActorScenes({ query }, url, entity);
}
return profile;
}
async function fetchLatest(channel, page) {
const res = await qu.getAll(`${channel.url}/videos?o=d&p=${page}`, '.cards-list .card');
if (res.ok) {
return scrapeAll(res.items, channel);
}
return res.status;
}
async function fetchProfile(baseActor, { entity }, options) {
const url = `${entity.url}/${baseActor.slug}`;
const res = await qu.get(url);
if (res.ok) {
return scrapeProfile(res.item, url, entity, options);
}
return res.status;
}
module.exports = {
fetchLatest,
scrapeScene,
fetchProfile,
};