Added VogoV (no trailer yet). Fixed MindGeek profile scraper.

This commit is contained in:
ThePendulum 2020-01-28 03:05:53 +01:00
parent 601196ddeb
commit 76852daf6d
12 changed files with 128 additions and 5 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

View File

@ -161,6 +161,12 @@ const networks = [
url: 'https://www.vixen.com',
description: 'Vixen.com features the worlds finest cinematic adult films with 4K quality and high-end erotic photography.',
},
{
slug: 'vogov',
name: 'VogoV',
url: 'https://www.vogov.com',
description: 'Fantastic collection of exclusive porn movies with the most beautiful porn models in leading roles saisfies the most picky visitor of the site.',
},
{
slug: 'xempire',
name: 'XEmpire',

View File

@ -3192,6 +3192,14 @@ function getSites(networksMap) {
url: 'https://www.deeper.com',
network_id: networksMap.vixen,
},
// VOGOV
{
slug: 'vogov',
name: 'VogoV',
url: 'https://www.vogov.com',
description: 'Top rated models. Graceful locations. Best gonzo scenes. 4K UHD 60 FPS. So, in general Vogov is a website that is worth visiting and exploring carefully. It gives a chance to spend a fantastic night with gorgeous girls ready to experiment and to full around with their lovers.',
network_id: networksMap.vogov,
},
// XEMPIRE
{
slug: 'hardx',

View File

@ -11,6 +11,7 @@
"no-underscore-dangle": 0,
"indent": "off",
"template-curly-spacing": "off",
"object-curly-newline": "off",
"max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}],
}
}

View File

@ -54,7 +54,10 @@ async function scrapeRelease(url, release, type = 'scene') {
? await scraper.fetchScene(url, site, release)
: await scraper.fetchMovie(url, site, release);
return scrapedRelease;
return {
...scrapedRelease,
site,
};
}
async function scrapeReleases(urls, release, type = 'scene') {

View File

@ -124,7 +124,9 @@ async function scrapeSites() {
}
try {
return await scrapeSiteReleases(scraper, site);
const siteReleases = await scrapeSiteReleases(scraper, site);
return siteReleases.map(release => ({ ...release, site }));
} catch (error) {
logger.error(`${site.name}: Failed to scrape releases: ${error.message}`);

View File

@ -194,12 +194,15 @@ async function fetchProfile(actorName, networkName, actorPath = 'model') {
if (res.statusCode === 200) {
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
if (actorData) {
const actorRes = await bhttp.get(`https://www.${networkName}.com/${actorPath}/${actorData.id}/`);
if (actorData && actorRes.statusCode === 200) {
if (actorRes.statusCode === 200) {
return scrapeProfile(actorData, actorRes.body.toString());
}
}
}
return null;
}

View File

@ -19,6 +19,7 @@ const naughtyamerica = require('./naughtyamerica');
const realitykings = require('./realitykings');
const teamskeet = require('./teamskeet');
const vixen = require('./vixen');
const vogov = require('./vogov');
// releases and profiles
const blowpass = require('./blowpass');
@ -68,6 +69,7 @@ module.exports = {
realitykings,
teamskeet,
vixen,
vogov,
xempire,
},
actors: {

91
src/scrapers/vogov.js Normal file
View File

@ -0,0 +1,91 @@
'use strict';
const bhttp = require('bhttp');
const { ex, ctxa } = require('../utils/q');
// const slugify = require('../utils/slugify');
function scrapeLatest(html) {
const { document } = ex(html);
return ctxa(document, '.video-post').map(({ q, qa, qd }) => {
const release = {};
// release.entryId = slugify(release.title);
release.entryId = q('.ico-fav-0').dataset.favVideoId;
const titleEl = q('.video-title-title');
release.title = titleEl.title;
release.url = titleEl.href;
release.date = qd('.video-data em', 'MMM DD, YYYY');
release.actors = qa('.video-model-list a', true);
const posterData = q('img.thumb').dataset;
release.poster = posterData.src;
release.trailer = posterData.preview;
return release;
});
}
function scrapeScene(html, url) {
const { q, qa, qd, qu, ql, qm } = ex(html);
const release = { url };
// release.entryId = slugify(release.title);
[release.entryId] = q('link[rel="canonical"]').href.match(/\d+/);
release.title = qm('meta[property="og:title"]') || q('.video-page-header h1', true);
release.description = qm('meta[property="og:description"]') || q('.info-video-description', true);
release.date = qd('.info-video-details li:first-child span', 'MMM DD, YYYY');
release.duration = ql('.info-video-details li:nth-child(2) span');
release.actors = qa('.info-video-models a', true);
release.tags = qa('.info-video-category a', true);
release.photos = qu('.swiper-wrapper .swiper-slide a').map(source => source.replace('.jpg/', '.jpg'));
release.poster = qm('meta[property="og:image"');
if (!release.poster) {
const previewStart = html.indexOf('preview_url');
release.poster = html.slice(html.indexOf('http', previewStart), html.indexOf('.jpg', previewStart) + 4);
}
/*
const trailerStart = html.indexOf('video_url');
const trailerUrl = html.slice(html.indexOf('http', trailerStart), html.indexOf('.mp4', trailerStart) + 4);
const rnd = new Date().getTime();
release.trailer = `${trailerUrl}/?rnd=${rnd}`;
*/
// console.log(release);
return release;
}
async function fetchLatest(site, page = 1) {
const url = `https://vogov.com/latest-videos/?sort_by=post_date&from=${page}`;
const res = await bhttp.get(url);
if (res.statusCode === 200) {
return scrapeLatest(res.body.toString(), site);
}
return null;
}
async function fetchScene(url) {
const res = await bhttp.get(url);
if (res.statusCode === 200) {
return scrapeScene(res.body.toString(), url);
}
return null;
}
module.exports = {
fetchLatest,
fetchScene,
};

View File

@ -25,6 +25,10 @@ function qall(context, selector, attrArg, trim = true) {
return Array.from(context.querySelectorAll(selector));
}
function qmeta(context, selector, attrArg = 'content', trim = true) {
return q(context, selector, attrArg, trim);
}
function qdate(context, selector, format, match, attr = 'textContent') {
const dateString = context.querySelector(selector)?.[attr];
@ -76,6 +80,7 @@ const funcs = {
qimages,
qposter,
qlength,
qmeta,
qtrailer,
qurls,
qa: qall,
@ -83,6 +88,7 @@ const funcs = {
qi: qimages,
qp: qposter,
ql: qlength,
qm: qmeta,
qt: qtrailer,
qu: qurls,
};
@ -98,6 +104,7 @@ function ctx(element, window) {
return {
element,
document: element,
...(window && { window }),
...contextFuncs,
};