Completed Evil Angel scraper. Added trans tags.
This commit is contained in:
parent
298eabe56e
commit
2c32ed6549
|
|
@ -4,3 +4,5 @@ public/js/*
|
|||
public/css/*
|
||||
config/*
|
||||
!config/default.js
|
||||
assets/js/config/
|
||||
!assets/js/config/default.js
|
||||
|
|
|
|||
|
|
@ -15,11 +15,11 @@ function init() {
|
|||
watch: {
|
||||
pageTitle(title) {
|
||||
if (title) {
|
||||
document.title = `Porn Radar - ${title}`;
|
||||
document.title = `traxxx - ${title}`;
|
||||
return;
|
||||
}
|
||||
|
||||
document.title = 'Porn Radar';
|
||||
document.title = 'traxxx';
|
||||
},
|
||||
},
|
||||
methods: {
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
'use strict';
|
||||
|
||||
const React = require('react');
|
||||
|
||||
const Header = () => (
|
||||
<header className="header">
|
||||
<h1>Porn Radar</h1>
|
||||
</header>
|
||||
);
|
||||
|
||||
module.exports = Header;
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
'use strict';
|
||||
|
||||
const React = require('react');
|
||||
const moment = require('moment');
|
||||
const PropTypes = require('prop-types');
|
||||
|
||||
const Layout = require('./layout.jsx');
|
||||
|
||||
const Home = ({ releases }) => (
|
||||
<Layout>
|
||||
<ul className="scenes">
|
||||
{releases.map(release => (
|
||||
<li key={release.id} className="scene">
|
||||
<span
|
||||
className="scene-banner"
|
||||
>
|
||||
<span className="scene-details">
|
||||
<a
|
||||
href={`/site/${release.site.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="scene-site site-link"
|
||||
title={release.network.name}
|
||||
>
|
||||
{release.site.name}
|
||||
</a>
|
||||
|
||||
<a
|
||||
href={release.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="scene-date"
|
||||
>
|
||||
{moment(release.date).format('MMM D, YYYY')}
|
||||
</a>
|
||||
</span>
|
||||
|
||||
<a
|
||||
href={`/scene/${release.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="scene-row scene-link"
|
||||
>
|
||||
<img
|
||||
src={`/${release.site.id}/${release.id}/0.jpg`}
|
||||
alt={release.id}
|
||||
className="scene-thumbnail"
|
||||
/>
|
||||
</a>
|
||||
</span>
|
||||
|
||||
<div className="scene-info">
|
||||
<a
|
||||
href={`/scene/${release.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="scene-row scene-link"
|
||||
>
|
||||
<h2 className="scene-title">{release.title}</h2>
|
||||
</a>
|
||||
|
||||
<span className="scene-row">
|
||||
<ul className="scene-actors nolist">
|
||||
{release.actors.map(actor => (
|
||||
<li
|
||||
key={actor.id}
|
||||
className="scene-actor"
|
||||
>
|
||||
<a
|
||||
href={`/actor/${actor.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="actor-link"
|
||||
>
|
||||
{actor.name}
|
||||
</a>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</span>
|
||||
|
||||
<span
|
||||
title={release.tags.map(tag => tag.tag).join(', ')}
|
||||
className="scene-row"
|
||||
>
|
||||
<ul className="scene-tags nolist">
|
||||
{release.tags.map(tag => (
|
||||
<li
|
||||
key={tag.tag}
|
||||
className="scene-tag"
|
||||
>
|
||||
<a
|
||||
href={`/tag/${tag.tag}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="tag-link"
|
||||
>
|
||||
{tag.tag}
|
||||
</a>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</span>
|
||||
</div>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</Layout>
|
||||
);
|
||||
|
||||
Home.propTypes = {
|
||||
releases: PropTypes.arrayOf(PropTypes.object),
|
||||
};
|
||||
|
||||
Home.defaultProps = {
|
||||
releases: [],
|
||||
};
|
||||
|
||||
module.exports = Home;
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
'use strict';
|
||||
|
||||
const React = require('react');
|
||||
const PropTypes = require('prop-types');
|
||||
|
||||
const Header = require('./header.jsx');
|
||||
|
||||
const Layout = ({ children, title }) => (
|
||||
<html lang="en">
|
||||
<head>
|
||||
{title
|
||||
? <title>Porn Radar | {title}</title>
|
||||
: <title>Porn Radar</title>
|
||||
}
|
||||
|
||||
<link href="/css/style.css" rel="stylesheet" />
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<Header />
|
||||
|
||||
<div className="content">
|
||||
{children}
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
|
||||
Layout.propTypes = {
|
||||
children: PropTypes.node.isRequired,
|
||||
title: PropTypes.string,
|
||||
};
|
||||
|
||||
Layout.defaultProps = {
|
||||
title: null,
|
||||
};
|
||||
|
||||
module.exports = Layout;
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
'use strict';
|
||||
|
||||
const React = require('react');
|
||||
const PropTypes = require('prop-types');
|
||||
const moment = require('moment');
|
||||
|
||||
const Layout = require('./layout.jsx');
|
||||
|
||||
const Release = ({ release, thumbnails }) => (
|
||||
<Layout title={release.title}>
|
||||
<div>
|
||||
<h2>{release.title}</h2>
|
||||
<p>{moment(release.date).format('MMMM DD, YYYY')}</p>
|
||||
<p>{release.shootId}</p>
|
||||
|
||||
<h3>
|
||||
<a
|
||||
href={`/site/${release.site.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
title={release.network.name}
|
||||
>
|
||||
{release.site.name}
|
||||
</a>
|
||||
</h3>
|
||||
|
||||
<p>{release.network.name}</p>
|
||||
|
||||
<p>
|
||||
<a
|
||||
href={release.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
View on {new URL(release.site.url).host.replace('www.', '')}
|
||||
</a>
|
||||
</p>
|
||||
|
||||
<ul>
|
||||
{release.actors.map(actor => (
|
||||
<li key={actor.id}>
|
||||
<a
|
||||
href={`/actor/${actor.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="actor-link"
|
||||
>
|
||||
{actor.name}
|
||||
</a>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
|
||||
<p>{release.description}</p>
|
||||
|
||||
<ul>
|
||||
{release.tags.map(tag => (
|
||||
<li key={tag.tag}>
|
||||
<a
|
||||
href={`/tag/${tag.tag}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="actor-link"
|
||||
>
|
||||
{tag.tag}
|
||||
</a>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
|
||||
{thumbnails.map((thumbnail, index) => (
|
||||
<img
|
||||
key={thumbnail}
|
||||
src={`/${release.site.id}/${release.id}/${thumbnail}`}
|
||||
alt={`Thumbnail ${index}`}
|
||||
className="thumbnail"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
|
||||
Release.propTypes = {
|
||||
release: PropTypes.object,
|
||||
thumbnails: PropTypes.arrayOf(PropTypes.string),
|
||||
};
|
||||
|
||||
Release.defaultProps = {
|
||||
release: null,
|
||||
thumbnails: [],
|
||||
};
|
||||
|
||||
module.exports = Release;
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Porn Radar</title>
|
||||
<title>traxxx</title>
|
||||
|
||||
<script src="/js/bundle.js" defer></script>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
'use strict';
|
||||
|
||||
/* eslint-disable max-len */
|
||||
exports.seed = knex => Promise.resolve()
|
||||
.then(() => knex.raw(`${knex('networks').insert([
|
||||
|
|
@ -33,6 +31,12 @@ exports.seed = knex => Promise.resolve()
|
|||
url: 'https://ddfnetwork.com',
|
||||
description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
|
||||
},
|
||||
{
|
||||
slug: 'evilangel',
|
||||
name: 'Evil Angel',
|
||||
url: 'https://evilangel.com',
|
||||
description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
|
||||
},
|
||||
{
|
||||
slug: 'julesjordan',
|
||||
name: 'Jules Jordan',
|
||||
|
|
|
|||
|
|
@ -828,6 +828,14 @@ exports.seed = knex => Promise.resolve()
|
|||
description: 'Fantasy Blowjobs & POV Cock Sucking Videos and Photos Produced in VR, 4K and full HD featuring Sexy European Pornstars',
|
||||
network_id: networksMap['ddfnetwork'],
|
||||
},
|
||||
// EVIL ANGEL
|
||||
{
|
||||
slug: 'evilangel',
|
||||
name: 'Evil Angel',
|
||||
url: 'https://evilangel.com',
|
||||
description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
|
||||
network_id: networksMap['evilangel'],
|
||||
},
|
||||
// JULES JORDAN
|
||||
{
|
||||
slug: 'julesjordan',
|
||||
|
|
|
|||
|
|
@ -721,6 +721,11 @@ exports.seed = knex => Promise.resolve()
|
|||
slug: 'toys',
|
||||
alias_for: null,
|
||||
},
|
||||
{
|
||||
tag: 'transsexual',
|
||||
slug: 'transsexual',
|
||||
alias_for: null,
|
||||
},
|
||||
{
|
||||
tag: 'triple penetration',
|
||||
slug: 'triple-penetration',
|
||||
|
|
@ -1157,6 +1162,10 @@ exports.seed = knex => Promise.resolve()
|
|||
tag: 'mff',
|
||||
alias_for: tagsMap['fmf'],
|
||||
},
|
||||
{
|
||||
tag: 'mature & milf',
|
||||
alias_for: tagsMap['milf'],
|
||||
},
|
||||
{
|
||||
tag: 'natural',
|
||||
alias_for: tagsMap['natural-boobs'],
|
||||
|
|
@ -1321,6 +1330,10 @@ exports.seed = knex => Promise.resolve()
|
|||
tag: 'tittyfuck',
|
||||
alias_for: tagsMap['titty-fuck'],
|
||||
},
|
||||
{
|
||||
tag: 'trans',
|
||||
alias_for: tagsMap['transsexual'],
|
||||
},
|
||||
{
|
||||
tag: 'trimmed pussy',
|
||||
alias_for: tagsMap['trimmed'],
|
||||
|
|
|
|||
|
|
@ -317,10 +317,13 @@ async function fetchReleases() {
|
|||
}
|
||||
|
||||
console.log(`${site.id}: Failed to fetch releases`);
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
console.error(`Cound not find scraper for '${site.name}' (${site.slug})`);
|
||||
|
||||
return [];
|
||||
}, {
|
||||
concurrency: 2,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,142 @@
|
|||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
|
||||
const { matchTags } = require('../tags');
|
||||
|
||||
async function scrape(json, site) {
|
||||
return Promise.all(json.map(async (scene) => {
|
||||
const {
|
||||
title,
|
||||
description,
|
||||
length,
|
||||
master_categories: rawTags,
|
||||
ratings_up: likes,
|
||||
ratings_down: dislikes,
|
||||
} = scene;
|
||||
|
||||
const entryId = scene.clip_id;
|
||||
const url = `https://evilangel.com/en/video/${scene.url_title}/${entryId}`;
|
||||
const date = moment(scene.release_date, 'YYYY-MM-DD').toDate();
|
||||
const actors = scene.actors.map(({ name }) => name);
|
||||
const director = scene.directors[0].name;
|
||||
|
||||
const tags = await matchTags(rawTags);
|
||||
const poster = `https://images-evilangel.gammacdn.com/movies${scene.pictures.resized}`;
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
description,
|
||||
length,
|
||||
actors,
|
||||
director,
|
||||
date,
|
||||
tags,
|
||||
poster,
|
||||
rating: {
|
||||
likes,
|
||||
dislikes,
|
||||
},
|
||||
site,
|
||||
};
|
||||
}));
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const json = $('script[type="application/ld+json"]').html();
|
||||
const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
|
||||
|
||||
const [data, data2] = JSON.parse(json);
|
||||
const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
|
||||
const entryId = new URL(url).pathname.split('/').slice(-1)[0];
|
||||
|
||||
const {
|
||||
name: title,
|
||||
description,
|
||||
} = data;
|
||||
// date in data object is not the release date of the scene, but the date the entry was added
|
||||
const date = moment.utc($('.updatedDate').first().text(), 'MM-DD-YYYY').toDate();
|
||||
|
||||
const actors = data.actor.map(actor => actor.name);
|
||||
|
||||
const director = (data.director && data.director[0].name) || (data2.director && data2.director[0].name) || null;
|
||||
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
|
||||
|
||||
const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
|
||||
|
||||
const rawTags = data.keywords.split(', ');
|
||||
const tags = await matchTags(rawTags);
|
||||
|
||||
const poster = videoData.picPreview;
|
||||
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
date,
|
||||
actors,
|
||||
director,
|
||||
description,
|
||||
duration,
|
||||
tags,
|
||||
poster,
|
||||
trailer: {
|
||||
src: trailer,
|
||||
quality: parseInt(videoData.sizeOnLoad, 10),
|
||||
},
|
||||
rating: {
|
||||
stars,
|
||||
},
|
||||
site,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1, upcoming = false) {
|
||||
const res = await bhttp.get('https://evilangel.com/en/videos');
|
||||
const body = res.body.toString();
|
||||
|
||||
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
|
||||
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
|
||||
const apiData = JSON.parse(apiSerial);
|
||||
|
||||
const { applicationID: appId, apiKey } = apiData.api.algolia;
|
||||
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
|
||||
|
||||
const apiRes = await bhttp.post(`https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_scenes',
|
||||
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? '' : 0}"]]`,
|
||||
},
|
||||
],
|
||||
}, {
|
||||
headers: {
|
||||
Referer: 'https://www.evilangel.com/en/videos',
|
||||
},
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
return scrape(apiRes.body.results[0].hits, site);
|
||||
}
|
||||
|
||||
async function fetchUpcoming(site) {
|
||||
return fetchLatest(site, 1, true);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return scrapeScene(res.body.toString(), url, site);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchUpcoming,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
@ -5,6 +5,7 @@ const bangbros = require('./bangbros');
|
|||
const blowpass = require('./blowpass');
|
||||
const brazzers = require('./brazzers');
|
||||
const ddfnetwork = require('./ddfnetwork');
|
||||
const evilangel = require('./evilangel');
|
||||
const julesjordan = require('./julesjordan');
|
||||
const kink = require('./kink');
|
||||
const legalporno = require('./legalporno');
|
||||
|
|
@ -22,6 +23,7 @@ module.exports = {
|
|||
blowpass,
|
||||
brazzers,
|
||||
ddfnetwork,
|
||||
evilangel,
|
||||
julesjordan,
|
||||
kink,
|
||||
legalporno,
|
||||
|
|
|
|||
Loading…
Reference in New Issue