Added generic Gamma photo and actor scraper for XEmpire, 21Sextury, Blowpass and Evil Angel.

This commit is contained in:
ThePendulum 2020-01-22 22:25:58 +01:00
parent 4e4323704a
commit f8175f6054
17 changed files with 347 additions and 290 deletions

View File

@ -26,7 +26,7 @@
<span
v-if="actor.gender"
class="bio-gender"
:class="{ male: actor.gender === 'male', female: actor.gender === 'female' }"
:class="{ [actor.gender]: true }"
><Icon :icon="actor.gender" /></span>
</li>
@ -392,6 +392,12 @@ export default {
&.male .icon {
fill: $male;
}
&.transsexual .icon {
fill: $text-contrast;
filter: drop-shadow(1px 0 0 $female) drop-shadow(-1px 0 0 $female) drop-shadow(0 1px 0 $female) drop-shadow(0 -1px 0 $female)
drop-shadow(1px 0 0 $male) drop-shadow(-1px 0 0 $male) drop-shadow(0 1px 0 $male) drop-shadow(0 -1px 0 $male);
}
}
.birthdate {

View File

@ -1,11 +1,18 @@
<template>
<div class="networks">
<input
:placeholder="`Find ${siteCount} sites in ${networks.length} networks`"
class="search"
>
<div class="network-tiles">
<Network
v-for="network in networks"
:key="`network-${network.id}`"
:key="`network-tile-${network.slug}`"
:network="network"
/>
</div>
</div>
</template>
<script>
@ -15,6 +22,10 @@ async function mounted() {
this.networks = await this.$store.dispatch('fetchNetworks');
}
function siteCount() {
return this.networks.map(network => network.sites).flat().length;
}
export default {
components: {
Network,
@ -24,6 +35,9 @@ export default {
networks: [],
};
},
computed: {
siteCount,
},
mounted,
};
</script>
@ -31,7 +45,23 @@ export default {
<style lang="scss" scoped>
@import 'theme';
.networks {
.search {
width: 40rem;
max-width: 100%;
box-sizing: border-box;
padding: 1rem;
border: none;
box-shadow: 0 0 3px $shadow-weak;
margin: 1rem;
font-size: 1rem;
outline: none;
&:focus {
box-shadow: 0 0 3px $primary;
}
}
.network-tiles {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
grid-gap: 1rem;

View File

@ -73,6 +73,7 @@ function initActorActions(store, _router) {
hasPiercings
tattoos
piercings
description
avatar: actorsAvatarByActorId {
media {
thumbnail

68
package-lock.json generated
View File

@ -1437,25 +1437,25 @@
"integrity": "sha512-OoIgewLowjegJzz3tpcRE5LpQKqsap1ETFaZtC1r9p36h5ieUJnWTxCTpB7XIsU9muMTt7MMt8x0E5apS47QIQ=="
},
"@tensorflow/tfjs": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.4.0.tgz",
"integrity": "sha512-pvc7arrWZ0NNOpEFVL4e99Bj933ty3tmUR0xLnjKao3psts9xr7w0sfJpuSn98HKjz+dj87UEXZIi0fwsAZdwQ==",
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.5.1.tgz",
"integrity": "sha512-WiE+JQ3ibr5LibGiBz6HWUqLJW8HiX6ywUSCA7ehZ67vFsw4mPuVjv0WEEUfD/l47PkXYVAmWd+RYOJiuZC7Eg==",
"requires": {
"@tensorflow/tfjs-converter": "1.4.0",
"@tensorflow/tfjs-core": "1.4.0",
"@tensorflow/tfjs-data": "1.4.0",
"@tensorflow/tfjs-layers": "1.4.0"
"@tensorflow/tfjs-converter": "1.5.1",
"@tensorflow/tfjs-core": "1.5.1",
"@tensorflow/tfjs-data": "1.5.1",
"@tensorflow/tfjs-layers": "1.5.1"
}
},
"@tensorflow/tfjs-converter": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.4.0.tgz",
"integrity": "sha512-vdZumj6zHcEALtQlonCBbVwGdEEDcPrF7prgzf4HF82QG4RpM1x/kVdvzsGvfUIPYjRImNn0ZSPC3WaKslNcXw=="
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.5.1.tgz",
"integrity": "sha512-M9tl2/ep8ntcZpmncHwKuvThsS7TaUWqJ9vJSgJmkazwTfAvlAJmZ8/p1miJ+m5sH1EJO4oTjiEmch6g8IA5IQ=="
},
"@tensorflow/tfjs-core": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.4.0.tgz",
"integrity": "sha512-a7dHhSsBbtaxp8/1UAeYKrjY1bQxsDy/Uhj57+mTuGLAcL8CDrTOXXZzucCgs5nvQErdscp7Gp/2VCcA1xp6XQ==",
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.5.1.tgz",
"integrity": "sha512-N4fsi8mLsRwRs8UJN2cARB1rYFxyVXkLyZ4wOusiR976BwwZbCwQrTTSIPzPqYT3rwiexEUzm7sM6ZaDl5dpXA==",
"requires": {
"@types/offscreencanvas": "~2019.3.0",
"@types/seedrandom": "2.4.27",
@ -1466,29 +1466,29 @@
}
},
"@tensorflow/tfjs-data": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.4.0.tgz",
"integrity": "sha512-00N/pe5IYkO3aDI9ZmavQ6mwY4VIepCteusnI7DLiCeqBO0NzGjlH3zH1LVlWIBRypW7JNqYbdl3oVvnd2wWwg==",
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.5.1.tgz",
"integrity": "sha512-eu4X0tHS1Tng+cvMO9gkMhUWX/UZQ//VpiaZfQJfa3zvUgxw6s1MHJFb0JC1T1FOnEgDVriZ8G758ysJZOybog==",
"requires": {
"@types/node-fetch": "^2.1.2",
"node-fetch": "~2.1.2"
}
},
"@tensorflow/tfjs-layers": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.4.0.tgz",
"integrity": "sha512-PXNOShZWDVW9OzX9botHJdDD6ClHEQtkfaFjoGZ2I2qYC9+WaVgxJPxwToTS2H2trwrf34q6hZ1lAjVlTuK0Gg=="
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.5.1.tgz",
"integrity": "sha512-DyuhifqflK+bdpBRLAj3RuWm1eTVe8yNX2+WH+W+wmhpjGg7Yagnar6/66JdS2h3WUFoiplCpZRAVMVw631E5g=="
},
"@tensorflow/tfjs-node": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-1.4.0.tgz",
"integrity": "sha512-6BVD4Jg72B2iPckz3lawB5UB4fR+pm4skhEEN0x98ykt08QF/3fgJX0xiT1K5otqfpwRDKWwcTO18ajOXgJt1Q==",
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-1.5.1.tgz",
"integrity": "sha512-gtIVn5zTLSgHlJUThN3knzKjycXOezmgL0W5MtxunJPznGmUZQive9efuCsgKcs5YKizTG+KprTILTSbi/2Owg==",
"requires": {
"@tensorflow/tfjs": "1.4.0",
"@tensorflow/tfjs": "1.5.1",
"adm-zip": "^0.4.11",
"google-protobuf": "^3.9.2",
"https-proxy-agent": "^2.2.1",
"node-pre-gyp": "0.13.0",
"node-pre-gyp": "0.14.0",
"progress": "^2.0.0",
"rimraf": "^2.6.2",
"tar": "^4.4.6"
@ -2977,9 +2977,9 @@
"integrity": "sha512-/xL2AbW/XWHNu1gnIrO8UitBGoFthcsDgU9VLK1/dpsoxbaD5LscHozKze05R6WLsBvLhqv78dAPozMFQBYLbQ=="
},
"canvas": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.6.0.tgz",
"integrity": "sha512-bEO9f1ThmbknLPxCa8Es7obPlN9W3stB1bo7njlhOFKIdUTldeTqXCh9YclCPAi2pSQs84XA0jq/QEZXSzgyMw==",
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.6.1.tgz",
"integrity": "sha512-S98rKsPcuhfTcYbtF53UIJhcbgIAK533d1kJKMwsMwAIFgfd58MOyxRud3kktlzWiEkFliaJtvyZCBtud/XVEA==",
"requires": {
"nan": "^2.14.0",
"node-pre-gyp": "^0.11.0",
@ -5933,9 +5933,9 @@
}
},
"google-protobuf": {
"version": "3.11.1",
"resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.11.1.tgz",
"integrity": "sha512-1fdyssLSa7Dip+MDdluLl0uK+2rspgU1PrbdPJKt7hywqSl3+R27TkLYKuBn9dvOyr0VQ0cliQWlfMheW5WKqA=="
"version": "3.11.2",
"resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.11.2.tgz",
"integrity": "sha512-T4fin7lcYLUPj2ChUZ4DvfuuHtg3xi1621qeRZt2J7SvOQusOzq+sDT4vbotWTCjUXJoR36CA016LlhtPy80uQ=="
},
"graceful-fs": {
"version": "4.1.15",
@ -7999,9 +7999,9 @@
"integrity": "sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA="
},
"node-pre-gyp": {
"version": "0.13.0",
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.13.0.tgz",
"integrity": "sha512-Md1D3xnEne8b/HGVQkZZwV27WUi1ZRuZBij24TNaZwUPU3ZAFtvT6xxJGaUVillfmMKnn5oD1HoGsp2Ftik7SQ==",
"version": "0.14.0",
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.14.0.tgz",
"integrity": "sha512-+CvDC7ZttU/sSt9rFjix/P05iS43qHCOOGzcr3Ry99bXG7VX953+vFyEuph/tfqoYu8dttBkE86JSKBO2OzcxA==",
"requires": {
"detect-libc": "^1.0.2",
"mkdirp": "^0.5.1",
@ -8012,7 +8012,7 @@
"rc": "^1.2.7",
"rimraf": "^2.6.1",
"semver": "^5.3.0",
"tar": "^4"
"tar": "^4.4.2"
},
"dependencies": {
"fs-minipass": {

View File

@ -68,13 +68,13 @@
"dependencies": {
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
"@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
"@tensorflow/tfjs-node": "^1.4.0",
"@tensorflow/tfjs-node": "^1.5.1",
"babel-polyfill": "^6.26.0",
"bhttp": "^1.2.4",
"blake2": "^4.0.0",
"bluebird": "^3.7.2",
"body-parser": "^1.19.0",
"canvas": "^2.6.0",
"canvas": "^2.6.1",
"cheerio": "^1.0.0-rc.3",
"cli-confirm": "^1.0.1",
"config": "^3.2.4",

View File

@ -936,7 +936,21 @@
/* $primary: #ff886c; */
/* $logo-highlight: drop-shadow(1px 0 0 $highlight-weak) drop-shadow(-1px 0 0 $highlight-weak) drop-shadow(0 1px 0 $highlight-weak) drop-shadow(0 -1px 0 $highlight-weak); */
.networks[data-v-4709d404] {
.search[data-v-4709d404] {
width: 40rem;
max-width: 100%;
box-sizing: border-box;
padding: 1rem;
border: none;
box-shadow: 0 0 3px rgba(0, 0, 0, 0.2);
margin: 1rem;
font-size: 1rem;
outline: none;
}
.search[data-v-4709d404]:focus {
box-shadow: 0 0 3px #ff6c88;
}
.network-tiles[data-v-4709d404] {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
grid-gap: 1rem;
@ -1105,6 +1119,11 @@
.bio-gender.male .icon[data-v-ea0483c2] {
fill: #0af;
}
.bio-gender.transsexual .icon[data-v-ea0483c2] {
fill: #fff;
-webkit-filter: drop-shadow(1px 0 0 #f0a) drop-shadow(-1px 0 0 #f0a) drop-shadow(0 1px 0 #f0a) drop-shadow(0 -1px 0 #f0a) drop-shadow(1px 0 0 #0af) drop-shadow(-1px 0 0 #0af) drop-shadow(0 1px 0 #0af) drop-shadow(0 -1px 0 #0af);
filter: drop-shadow(1px 0 0 #f0a) drop-shadow(-1px 0 0 #f0a) drop-shadow(0 1px 0 #f0a) drop-shadow(0 -1px 0 #f0a) drop-shadow(1px 0 0 #0af) drop-shadow(-1px 0 0 #0af) drop-shadow(0 1px 0 #0af) drop-shadow(0 -1px 0 #0af);
}
.birthdate[data-v-ea0483c2] {
display: block;
}

View File

@ -1611,18 +1611,20 @@ function getTagAliases(tagsMap) {
function getSiteTags() {
return {
teenallanal: ['anal', 'mff'],
allblackx: ['ebony', 'bbc'],
boundgods: ['gay'],
buttmachineboys: ['gay'],
cum4k: ['fake-cum', 'creampie', '4k'],
darkx: ['interracial'],
deepthroatlove: ['blowjob', 'deepthroat'],
divinebitches: ['femdom'],
dpparodies: ['parody'],
eighteenyearsold: ['teen'],
exotic4k: ['4k'],
givemepink: ['solo', 'masturbation'],
lubed: ['oil'],
familystrokes: ['family'],
givemepink: ['solo', 'masturbation'],
lesbianx: ['lesbian'],
lubed: ['oil'],
massagecreep: ['massage'],
menonedge: ['gay'],
povd: ['pov'],
@ -1631,6 +1633,7 @@ function getSiteTags() {
spyfam: ['family'],
submissived: ['bdsm'],
swallowed: ['blowjob', 'deepthroat', 'facefucking'],
teenallanal: ['anal', 'mff'],
teenbff: ['mff'],
tiny4k: ['4k'],
trueanal: ['anal'],

View File

@ -301,7 +301,7 @@ async function mergeProfiles(profiles, actor) {
piercings: prevProfile.piercings || profile.piercings,
tattoos: prevProfile.tattoos || profile.tattoos,
social: prevProfile.social.concat(profile.social || []),
avatars: prevProfile.avatars.concat(profile.avatar || []),
avatars: prevProfile.avatars.concat([profile.avatar] || []),
};
}, {
social: [],

View File

@ -175,7 +175,7 @@ async function storePhotos(photos, {
return;
}
const pluckedPhotos = pluckPhotos(photos);
const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration
const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label);
const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {

View File

@ -159,7 +159,7 @@ async function attachChannelSite(release) {
}
if (!release.channel) {
throw new Error(`Unable to derive channel site from generic URL: ${release.url}.`);
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
}
const [site] = await fetchSites({
@ -182,7 +182,7 @@ async function attachChannelSite(release) {
site: urlSite,
};
} catch (error) {
throw new Error(`Unable to derive channel site from generic URL: ${release.url}.`);
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
}
}

View File

@ -1,51 +1,10 @@
'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
async function fetchPhotos(photoPath) {
const res = await bhttp.get(`https://21sextury.com${photoPath}`);
return res.body.toString();
}
function scrapePhotos(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const unlockedPhotos = $('.preview .imgLink.pgUnlocked')
.map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
const lockedThumbnails = $('.preview .imgLink.lockedPicture img')
.map((photoIndex, photoElement) => $(photoElement)
.attr('src'))
// .replace('_tb.jpg', '.jpg')) does not always work
.toArray();
return unlockedPhotos.concat(lockedThumbnails);
}
async function getPhotos(photoPath) {
if (!photoPath || photoPath.match('join')) {
return [];
}
const html = await fetchPhotos(photoPath);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html);
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
const otherPhotos = await Promise.map(pages, async (pagePath) => {
const pageHtml = await fetchPhotos(pagePath);
return scrapePhotos(pageHtml);
}, {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
}
const { getPhotos, fetchProfile } = require('./gamma');
function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
@ -131,8 +90,7 @@ async function scrapeScene(html, url, site) {
const poster = videoData.picPreview;
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
const photoPath = $('.picturesItem a').attr('href');
const photos = await getPhotos(photoPath, site);
const photos = await getPhotos($('.picturesItem a').attr('href'), '21sextury.com', site);
const tags = data.keywords.split(', ');
const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
@ -181,8 +139,13 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.toString(), url, site);
}
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, '21sextury', true);
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchUpcoming,
fetchScene,
};

View File

@ -1,71 +1,11 @@
'use strict';
/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
async function fetchPhotos(url) {
const res = await bhttp.get(url);
return res.body.toString();
}
function scrapePhotos(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
return $('.preview .imgLink').toArray().map((linkEl) => {
const url = $(linkEl).attr('href');
if (url.match('/join')) {
// URL links to join page instead of full photo, extract thumbnail
const src = $(linkEl).find('img').attr('src');
if (src.match('previews/')) {
// resource often serves full photo at a modifier URL anyway, add as primary source
const highRes = src
.replace('previews/', '')
.replace('_tb.jpg', '.jpg');
// keep original thumbnail as fallback in case full photo is not available
return [highRes, src];
}
return src;
}
// URL links to full photo
return url;
});
}
async function getPhotos(albumPath, siteDomain) {
const albumUrl = `https://www.blowpass.com${albumPath}`;
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html);
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
const otherPhotos = await Promise.map(pages, async (page) => {
const pageUrl = `https://${siteDomain}${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml);
}, {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
} catch (error) {
console.error(`Failed to fetch Blowpass photos from ${albumPath}: ${error.message}`);
return [];
}
}
const { getPhotos, fetchProfile } = require('./gamma');
function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
@ -132,7 +72,7 @@ async function scrapeScene(html, url, site) {
const poster = playerData.picPreview;
const trailer = `${playerData.playerOptions.host}${playerData.url}`;
const photos = await getPhotos($('.picturesItem a').attr('href'), channel, site);
const photos = await getPhotos($('.picturesItem a').attr('href'), 'blowpass.com', site);
const duration = moment.duration(data.duration.slice(2)).asSeconds();
const tags = data.keywords.split(', ');
@ -180,8 +120,13 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.toString(), url, site);
}
async function blowpassFetchProfile(actorName) {
return fetchProfile(actorName, 'blowpass');
}
module.exports = {
fetchLatest,
fetchUpcoming,
fetchProfile: blowpassFetchProfile,
fetchScene,
fetchUpcoming,
};

View File

@ -4,6 +4,8 @@ const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const { getPhotos } = require('./gamma');
async function scrape(json, site) {
return Promise.all(json.map(async (scene) => {
const {
@ -75,6 +77,8 @@ async function scrapeScene(html, url, site) {
const poster = videoData.picPreview;
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
const photos = await getPhotos($('.picturesItem a').attr('href'), 'evilangel.com', site);
return {
url,
entryId,
@ -86,6 +90,7 @@ async function scrapeScene(html, url, site) {
duration,
tags,
poster,
photos,
trailer: {
src: trailer,
quality: parseInt(videoData.sizeOnLoad, 10),
@ -97,7 +102,26 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site, page = 1, upcoming = false) {
function scrapeActor(data) {
const actor = {};
if (data.male === 1) actor.gender = 'male';
if (data.female === 1) actor.gender = 'female';
if (data.shemale === 1 || data.trans === 1) actor.gender = 'transsexual';
if (data.description) actor.description = data.description.trim();
if (data.attributes.ethnicity) actor.ethnicity = data.attributes.ethnicity;
if (data.attributes.eye_color) actor.eyes = data.attributes.eye_color;
if (data.attributes.hair_color) actor.hair = data.attributes.hair_color;
const avatarPath = Object.values(data.pictures).reverse()[0];
actor.avatar = `https://images01-evilangel.gammacdn.com/actors${avatarPath}`;
return actor;
}
async function fetchApiCredentials() {
const res = await bhttp.get('https://evilangel.com/en/videos');
const body = res.body.toString();
@ -108,7 +132,20 @@ async function fetchLatest(site, page = 1, upcoming = false) {
const { applicationID: appId, apiKey } = apiData.api.algolia;
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
const apiRes = await bhttp.post(`https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`, {
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
return {
appId,
apiKey,
userAgent,
apiUrl,
};
}
async function fetchLatest(site, page = 1, upcoming = false) {
const { apiUrl } = await fetchApiCredentials();
const res = await bhttp.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
@ -122,7 +159,7 @@ async function fetchLatest(site, page = 1, upcoming = false) {
encodeJSON: true,
});
return scrape(apiRes.body.results[0].hits, site);
return scrape(res.body.results[0].hits, site);
}
async function fetchUpcoming(site) {
@ -135,8 +172,38 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName) {
const { apiUrl } = await fetchApiCredentials();
const actorSlug = encodeURI(actorName);
const res = await bhttp.post(apiUrl, {
requests: [
{
indexName: 'all_actors',
params: `query=${actorSlug}`,
},
],
}, {
headers: {
Referer: `https://www.evilangel.com/en/search?query=${actorSlug}&tab=actors`,
},
encodeJSON: true,
});
if (res.statusCode === 200 && res.body.results[0].hits.length > 0) {
const actorData = res.body.results[0].hits.find(actor => actor.name === actorName);
if (actorData) {
return scrapeActor(actorData);
}
}
return null;
}
module.exports = {
fetchLatest,
fetchUpcoming,
fetchProfile,
fetchScene,
fetchUpcoming,
};

136
src/scrapers/gamma.js Normal file
View File

@ -0,0 +1,136 @@
'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
async function fetchPhotos(url) {
const res = await bhttp.get(url);
return res.body.toString();
}
function scrapePhotos(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
return $('.preview .imgLink').toArray().map((linkEl) => {
const url = $(linkEl).attr('href');
if (url.match('/join')) {
// URL links to join page instead of full photo, extract thumbnail
const src = $(linkEl).find('img').attr('src');
if (src.match('previews/')) {
// resource often serves full photo at a modifier URL anyway, add as primary source
const highRes = src
.replace('previews/', '')
.replace('_tb.jpg', '.jpg');
// keep original thumbnail as fallback in case full photo is not available
return [highRes, src];
}
return src;
}
// URL links to full photo
return url;
});
}
async function getPhotos(albumPath, siteDomain) {
const albumUrl = `https://${siteDomain}${albumPath}`;
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html);
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
const otherPhotos = await Promise.map(pages, async (page) => {
const pageUrl = `https://${siteDomain}${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml);
}, {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
} catch (error) {
console.error(`Failed to fetch ${siteDomain} photos from ${albumPath}: ${error.message}`);
return [];
}
}
function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
return actorLink ? actorLink.href : null;
}
function scrapeProfile(html, url, actorName, siteSlug) {
const { document } = new JSDOM(html).window;
const avatarEl = document.querySelector('img.actorPicture');
const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
const profile = {
name: actorName,
};
if (avatarEl) {
// larger sizes usually available, provide fallbacks
const avatars = [
avatarEl.src.replace(/\d+x\d+/, '500x750'),
avatarEl.src.replace(/\d+x\d+/, '240x360'),
avatarEl.src.replace(/\d+x\d+/, '200x300'),
avatarEl.src,
];
profile.avatar = avatars;
}
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://${siteSlug}.com${el.href}`);
return profile;
}
async function fetchProfile(actorName, siteSlug, altSearchUrl) {
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
const searchUrl = altSearchUrl
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
: `https://www.${siteSlug}.com/en/search/${siteSlug}/actor/${actorSlug}`;
const searchRes = await bhttp.get(searchUrl);
if (searchRes.statusCode !== 200) {
return null;
}
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
if (actorUrl) {
const url = `https://${siteSlug}.com${actorUrl}`;
const actorRes = await bhttp.get(url);
if (actorRes.statusCode !== 200) {
return null;
}
return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug);
}
return null;
}
module.exports = {
getPhotos,
fetchProfile,
scrapeProfile,
};

View File

@ -1,14 +1,11 @@
'use strict';
// releases
const twentyonesextury = require('./21sextury');
const babes = require('./babes');
const bang = require('./bang');
const bangbros = require('./bangbros');
const blowpass = require('./blowpass');
const dogfart = require('./dogfart');
const digitalplayground = require('./digitalplayground');
const evilangel = require('./evilangel');
const fakehub = require('./fakehub');
const jayrock = require('./jayrock');
const kink = require('./kink');
@ -25,11 +22,14 @@ const teamskeet = require('./teamskeet');
const vixen = require('./vixen');
// releases and profiles
const ddfnetwork = require('./ddfnetwork');
const blowpass = require('./blowpass');
const brazzers = require('./brazzers');
const ddfnetwork = require('./ddfnetwork');
const evilangel = require('./evilangel');
const julesjordan = require('./julesjordan');
const kellymadison = require('./kellymadison');
const legalporno = require('./legalporno');
const twentyonesextury = require('./21sextury');
const xempire = require('./xempire');
// profiles
@ -71,7 +71,10 @@ module.exports = {
},
actors: {
// ordered by data priority
'21sextury': twentyonesextury,
evilangel,
xempire,
blowpass,
julesjordan,
brazzers,
legalporno,

View File

@ -1,79 +1,10 @@
'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const defaultTags = {
hardx: [],
darkx: ['interracial'],
eroticax: [],
lesbianx: ['lesbian'],
allblackx: ['ebony', 'bbc'],
};
async function fetchPhotos(url) {
const res = await bhttp.get(url);
return res.body.toString();
}
function scrapePhotos(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
return $('.preview .imgLink').toArray().map((linkEl) => {
const url = $(linkEl).attr('href');
if (url.match('/join')) {
// URL links to join page instead of full photo, extract thumbnail
const src = $(linkEl).find('img').attr('src');
if (src.match('previews/')) {
// resource often serves full photo at a modifier URL anyway, add as primary source
const highRes = src
.replace('previews/', '')
.replace('_tb.jpg', '.jpg');
// keep original thumbnail as fallback in case full photo is not available
return [highRes, src];
}
return src;
}
// URL links to full photo
return url;
});
}
async function getPhotos(albumPath, siteDomain) {
const albumUrl = `https://${siteDomain}${albumPath}`;
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html);
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
const otherPhotos = await Promise.map(pages, async (page) => {
const pageUrl = `https://${siteDomain}${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml);
}, {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
} catch (error) {
console.error(`Failed to fetch XEmpire photos from ${albumPath}: ${error.message}`);
return [];
}
}
const { getPhotos, fetchProfile } = require('./gamma');
function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
@ -154,8 +85,7 @@ async function scrapeScene(html, url, site) {
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
const rawTags = data.keywords.split(', ');
const tags = [...defaultTags[siteSlug], ...rawTags];
const tags = data.keywords.split(', ');
return {
url: `${siteUrl}/en/video/${new URL(url).pathname.split('/').slice(-2).join('/')}`,
@ -181,31 +111,6 @@ async function scrapeScene(html, url, site) {
};
}
function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
return actorLink ? actorLink.href : null;
}
function scrapeProfile(html, url, actorName) {
const { document } = new JSDOM(html).window;
const avatarEl = document.querySelector('img.actorPicture');
const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
const profile = {
name: actorName,
};
if (avatarEl) profile.avatar = avatarEl.src;
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://xempire.com${el.href}`);
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
@ -224,34 +129,13 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName) {
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
const searchUrl = `https://www.xempire.com/en/search/xempire/actor/${actorSlug}`;
const searchRes = await bhttp.get(searchUrl);
if (searchRes.statusCode !== 200) {
return null;
}
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
if (actorUrl) {
const url = `https://xempire.com${actorUrl}`;
const actorRes = await bhttp.get(url);
if (actorRes.statusCode !== 200) {
return null;
}
return scrapeProfile(actorRes.body.toString(), url, actorName);
}
return null;
async function xEmpireFetchProfile(actorName) {
return fetchProfile(actorName, 'xempire');
}
module.exports = {
fetchLatest,
fetchProfile,
fetchProfile: xEmpireFetchProfile,
fetchUpcoming,
fetchScene,
};

View File

@ -60,7 +60,7 @@ function destructConfigNetworks(networks) {
}
async function findSiteByUrl(url) {
const { hostname, origin } = new URL(url);
const { hostname } = new URL(url);
const domain = hostname.replace(/www.|tour./, '');
const sites = await knex('sites')
@ -69,8 +69,8 @@ async function findSiteByUrl(url) {
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.where('sites.url', 'like', `${domain}`)
.orWhere('sites.url', 'like', `${origin}`)
.where('sites.url', 'like', `%${domain}`)
.orWhere('sites.url', 'like', url)
.orWhere('sites.url', url);
if (sites.length > 0) {