Added generic Gamma photo and actor scraper for XEmpire, 21Sextury, Blowpass and Evil Angel.
This commit is contained in:
parent
4e4323704a
commit
f8175f6054
|
|
@ -26,7 +26,7 @@
|
|||
<span
|
||||
v-if="actor.gender"
|
||||
class="bio-gender"
|
||||
:class="{ male: actor.gender === 'male', female: actor.gender === 'female' }"
|
||||
:class="{ [actor.gender]: true }"
|
||||
><Icon :icon="actor.gender" /></span>
|
||||
</li>
|
||||
|
||||
|
|
@ -392,6 +392,12 @@ export default {
|
|||
&.male .icon {
|
||||
fill: $male;
|
||||
}
|
||||
|
||||
&.transsexual .icon {
|
||||
fill: $text-contrast;
|
||||
filter: drop-shadow(1px 0 0 $female) drop-shadow(-1px 0 0 $female) drop-shadow(0 1px 0 $female) drop-shadow(0 -1px 0 $female)
|
||||
drop-shadow(1px 0 0 $male) drop-shadow(-1px 0 0 $male) drop-shadow(0 1px 0 $male) drop-shadow(0 -1px 0 $male);
|
||||
}
|
||||
}
|
||||
|
||||
.birthdate {
|
||||
|
|
|
|||
|
|
@ -1,10 +1,17 @@
|
|||
<template>
|
||||
<div class="networks">
|
||||
<Network
|
||||
v-for="network in networks"
|
||||
:key="`network-${network.id}`"
|
||||
:network="network"
|
||||
/>
|
||||
<input
|
||||
:placeholder="`Find ${siteCount} sites in ${networks.length} networks`"
|
||||
class="search"
|
||||
>
|
||||
|
||||
<div class="network-tiles">
|
||||
<Network
|
||||
v-for="network in networks"
|
||||
:key="`network-tile-${network.slug}`"
|
||||
:network="network"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
|
|
@ -15,6 +22,10 @@ async function mounted() {
|
|||
this.networks = await this.$store.dispatch('fetchNetworks');
|
||||
}
|
||||
|
||||
function siteCount() {
|
||||
return this.networks.map(network => network.sites).flat().length;
|
||||
}
|
||||
|
||||
export default {
|
||||
components: {
|
||||
Network,
|
||||
|
|
@ -24,6 +35,9 @@ export default {
|
|||
networks: [],
|
||||
};
|
||||
},
|
||||
computed: {
|
||||
siteCount,
|
||||
},
|
||||
mounted,
|
||||
};
|
||||
</script>
|
||||
|
|
@ -31,7 +45,23 @@ export default {
|
|||
<style lang="scss" scoped>
|
||||
@import 'theme';
|
||||
|
||||
.networks {
|
||||
.search {
|
||||
width: 40rem;
|
||||
max-width: 100%;
|
||||
box-sizing: border-box;
|
||||
padding: 1rem;
|
||||
border: none;
|
||||
box-shadow: 0 0 3px $shadow-weak;
|
||||
margin: 1rem;
|
||||
font-size: 1rem;
|
||||
outline: none;
|
||||
|
||||
&:focus {
|
||||
box-shadow: 0 0 3px $primary;
|
||||
}
|
||||
}
|
||||
|
||||
.network-tiles {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
|
||||
grid-gap: 1rem;
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ function initActorActions(store, _router) {
|
|||
hasPiercings
|
||||
tattoos
|
||||
piercings
|
||||
description
|
||||
avatar: actorsAvatarByActorId {
|
||||
media {
|
||||
thumbnail
|
||||
|
|
|
|||
|
|
@ -1437,25 +1437,25 @@
|
|||
"integrity": "sha512-OoIgewLowjegJzz3tpcRE5LpQKqsap1ETFaZtC1r9p36h5ieUJnWTxCTpB7XIsU9muMTt7MMt8x0E5apS47QIQ=="
|
||||
},
|
||||
"@tensorflow/tfjs": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.4.0.tgz",
|
||||
"integrity": "sha512-pvc7arrWZ0NNOpEFVL4e99Bj933ty3tmUR0xLnjKao3psts9xr7w0sfJpuSn98HKjz+dj87UEXZIi0fwsAZdwQ==",
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.5.1.tgz",
|
||||
"integrity": "sha512-WiE+JQ3ibr5LibGiBz6HWUqLJW8HiX6ywUSCA7ehZ67vFsw4mPuVjv0WEEUfD/l47PkXYVAmWd+RYOJiuZC7Eg==",
|
||||
"requires": {
|
||||
"@tensorflow/tfjs-converter": "1.4.0",
|
||||
"@tensorflow/tfjs-core": "1.4.0",
|
||||
"@tensorflow/tfjs-data": "1.4.0",
|
||||
"@tensorflow/tfjs-layers": "1.4.0"
|
||||
"@tensorflow/tfjs-converter": "1.5.1",
|
||||
"@tensorflow/tfjs-core": "1.5.1",
|
||||
"@tensorflow/tfjs-data": "1.5.1",
|
||||
"@tensorflow/tfjs-layers": "1.5.1"
|
||||
}
|
||||
},
|
||||
"@tensorflow/tfjs-converter": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.4.0.tgz",
|
||||
"integrity": "sha512-vdZumj6zHcEALtQlonCBbVwGdEEDcPrF7prgzf4HF82QG4RpM1x/kVdvzsGvfUIPYjRImNn0ZSPC3WaKslNcXw=="
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.5.1.tgz",
|
||||
"integrity": "sha512-M9tl2/ep8ntcZpmncHwKuvThsS7TaUWqJ9vJSgJmkazwTfAvlAJmZ8/p1miJ+m5sH1EJO4oTjiEmch6g8IA5IQ=="
|
||||
},
|
||||
"@tensorflow/tfjs-core": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.4.0.tgz",
|
||||
"integrity": "sha512-a7dHhSsBbtaxp8/1UAeYKrjY1bQxsDy/Uhj57+mTuGLAcL8CDrTOXXZzucCgs5nvQErdscp7Gp/2VCcA1xp6XQ==",
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.5.1.tgz",
|
||||
"integrity": "sha512-N4fsi8mLsRwRs8UJN2cARB1rYFxyVXkLyZ4wOusiR976BwwZbCwQrTTSIPzPqYT3rwiexEUzm7sM6ZaDl5dpXA==",
|
||||
"requires": {
|
||||
"@types/offscreencanvas": "~2019.3.0",
|
||||
"@types/seedrandom": "2.4.27",
|
||||
|
|
@ -1466,29 +1466,29 @@
|
|||
}
|
||||
},
|
||||
"@tensorflow/tfjs-data": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.4.0.tgz",
|
||||
"integrity": "sha512-00N/pe5IYkO3aDI9ZmavQ6mwY4VIepCteusnI7DLiCeqBO0NzGjlH3zH1LVlWIBRypW7JNqYbdl3oVvnd2wWwg==",
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.5.1.tgz",
|
||||
"integrity": "sha512-eu4X0tHS1Tng+cvMO9gkMhUWX/UZQ//VpiaZfQJfa3zvUgxw6s1MHJFb0JC1T1FOnEgDVriZ8G758ysJZOybog==",
|
||||
"requires": {
|
||||
"@types/node-fetch": "^2.1.2",
|
||||
"node-fetch": "~2.1.2"
|
||||
}
|
||||
},
|
||||
"@tensorflow/tfjs-layers": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.4.0.tgz",
|
||||
"integrity": "sha512-PXNOShZWDVW9OzX9botHJdDD6ClHEQtkfaFjoGZ2I2qYC9+WaVgxJPxwToTS2H2trwrf34q6hZ1lAjVlTuK0Gg=="
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.5.1.tgz",
|
||||
"integrity": "sha512-DyuhifqflK+bdpBRLAj3RuWm1eTVe8yNX2+WH+W+wmhpjGg7Yagnar6/66JdS2h3WUFoiplCpZRAVMVw631E5g=="
|
||||
},
|
||||
"@tensorflow/tfjs-node": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-1.4.0.tgz",
|
||||
"integrity": "sha512-6BVD4Jg72B2iPckz3lawB5UB4fR+pm4skhEEN0x98ykt08QF/3fgJX0xiT1K5otqfpwRDKWwcTO18ajOXgJt1Q==",
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-1.5.1.tgz",
|
||||
"integrity": "sha512-gtIVn5zTLSgHlJUThN3knzKjycXOezmgL0W5MtxunJPznGmUZQive9efuCsgKcs5YKizTG+KprTILTSbi/2Owg==",
|
||||
"requires": {
|
||||
"@tensorflow/tfjs": "1.4.0",
|
||||
"@tensorflow/tfjs": "1.5.1",
|
||||
"adm-zip": "^0.4.11",
|
||||
"google-protobuf": "^3.9.2",
|
||||
"https-proxy-agent": "^2.2.1",
|
||||
"node-pre-gyp": "0.13.0",
|
||||
"node-pre-gyp": "0.14.0",
|
||||
"progress": "^2.0.0",
|
||||
"rimraf": "^2.6.2",
|
||||
"tar": "^4.4.6"
|
||||
|
|
@ -2977,9 +2977,9 @@
|
|||
"integrity": "sha512-/xL2AbW/XWHNu1gnIrO8UitBGoFthcsDgU9VLK1/dpsoxbaD5LscHozKze05R6WLsBvLhqv78dAPozMFQBYLbQ=="
|
||||
},
|
||||
"canvas": {
|
||||
"version": "2.6.0",
|
||||
"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.6.0.tgz",
|
||||
"integrity": "sha512-bEO9f1ThmbknLPxCa8Es7obPlN9W3stB1bo7njlhOFKIdUTldeTqXCh9YclCPAi2pSQs84XA0jq/QEZXSzgyMw==",
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.6.1.tgz",
|
||||
"integrity": "sha512-S98rKsPcuhfTcYbtF53UIJhcbgIAK533d1kJKMwsMwAIFgfd58MOyxRud3kktlzWiEkFliaJtvyZCBtud/XVEA==",
|
||||
"requires": {
|
||||
"nan": "^2.14.0",
|
||||
"node-pre-gyp": "^0.11.0",
|
||||
|
|
@ -5933,9 +5933,9 @@
|
|||
}
|
||||
},
|
||||
"google-protobuf": {
|
||||
"version": "3.11.1",
|
||||
"resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.11.1.tgz",
|
||||
"integrity": "sha512-1fdyssLSa7Dip+MDdluLl0uK+2rspgU1PrbdPJKt7hywqSl3+R27TkLYKuBn9dvOyr0VQ0cliQWlfMheW5WKqA=="
|
||||
"version": "3.11.2",
|
||||
"resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.11.2.tgz",
|
||||
"integrity": "sha512-T4fin7lcYLUPj2ChUZ4DvfuuHtg3xi1621qeRZt2J7SvOQusOzq+sDT4vbotWTCjUXJoR36CA016LlhtPy80uQ=="
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "4.1.15",
|
||||
|
|
@ -7999,9 +7999,9 @@
|
|||
"integrity": "sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA="
|
||||
},
|
||||
"node-pre-gyp": {
|
||||
"version": "0.13.0",
|
||||
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.13.0.tgz",
|
||||
"integrity": "sha512-Md1D3xnEne8b/HGVQkZZwV27WUi1ZRuZBij24TNaZwUPU3ZAFtvT6xxJGaUVillfmMKnn5oD1HoGsp2Ftik7SQ==",
|
||||
"version": "0.14.0",
|
||||
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.14.0.tgz",
|
||||
"integrity": "sha512-+CvDC7ZttU/sSt9rFjix/P05iS43qHCOOGzcr3Ry99bXG7VX953+vFyEuph/tfqoYu8dttBkE86JSKBO2OzcxA==",
|
||||
"requires": {
|
||||
"detect-libc": "^1.0.2",
|
||||
"mkdirp": "^0.5.1",
|
||||
|
|
@ -8012,7 +8012,7 @@
|
|||
"rc": "^1.2.7",
|
||||
"rimraf": "^2.6.1",
|
||||
"semver": "^5.3.0",
|
||||
"tar": "^4"
|
||||
"tar": "^4.4.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"fs-minipass": {
|
||||
|
|
|
|||
|
|
@ -68,13 +68,13 @@
|
|||
"dependencies": {
|
||||
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
|
||||
"@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
|
||||
"@tensorflow/tfjs-node": "^1.4.0",
|
||||
"@tensorflow/tfjs-node": "^1.5.1",
|
||||
"babel-polyfill": "^6.26.0",
|
||||
"bhttp": "^1.2.4",
|
||||
"blake2": "^4.0.0",
|
||||
"bluebird": "^3.7.2",
|
||||
"body-parser": "^1.19.0",
|
||||
"canvas": "^2.6.0",
|
||||
"canvas": "^2.6.1",
|
||||
"cheerio": "^1.0.0-rc.3",
|
||||
"cli-confirm": "^1.0.1",
|
||||
"config": "^3.2.4",
|
||||
|
|
|
|||
|
|
@ -936,7 +936,21 @@
|
|||
|
||||
/* $primary: #ff886c; */
|
||||
/* $logo-highlight: drop-shadow(1px 0 0 $highlight-weak) drop-shadow(-1px 0 0 $highlight-weak) drop-shadow(0 1px 0 $highlight-weak) drop-shadow(0 -1px 0 $highlight-weak); */
|
||||
.networks[data-v-4709d404] {
|
||||
.search[data-v-4709d404] {
|
||||
width: 40rem;
|
||||
max-width: 100%;
|
||||
box-sizing: border-box;
|
||||
padding: 1rem;
|
||||
border: none;
|
||||
box-shadow: 0 0 3px rgba(0, 0, 0, 0.2);
|
||||
margin: 1rem;
|
||||
font-size: 1rem;
|
||||
outline: none;
|
||||
}
|
||||
.search[data-v-4709d404]:focus {
|
||||
box-shadow: 0 0 3px #ff6c88;
|
||||
}
|
||||
.network-tiles[data-v-4709d404] {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
|
||||
grid-gap: 1rem;
|
||||
|
|
@ -1105,6 +1119,11 @@
|
|||
.bio-gender.male .icon[data-v-ea0483c2] {
|
||||
fill: #0af;
|
||||
}
|
||||
.bio-gender.transsexual .icon[data-v-ea0483c2] {
|
||||
fill: #fff;
|
||||
-webkit-filter: drop-shadow(1px 0 0 #f0a) drop-shadow(-1px 0 0 #f0a) drop-shadow(0 1px 0 #f0a) drop-shadow(0 -1px 0 #f0a) drop-shadow(1px 0 0 #0af) drop-shadow(-1px 0 0 #0af) drop-shadow(0 1px 0 #0af) drop-shadow(0 -1px 0 #0af);
|
||||
filter: drop-shadow(1px 0 0 #f0a) drop-shadow(-1px 0 0 #f0a) drop-shadow(0 1px 0 #f0a) drop-shadow(0 -1px 0 #f0a) drop-shadow(1px 0 0 #0af) drop-shadow(-1px 0 0 #0af) drop-shadow(0 1px 0 #0af) drop-shadow(0 -1px 0 #0af);
|
||||
}
|
||||
.birthdate[data-v-ea0483c2] {
|
||||
display: block;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1611,18 +1611,20 @@ function getTagAliases(tagsMap) {
|
|||
|
||||
function getSiteTags() {
|
||||
return {
|
||||
teenallanal: ['anal', 'mff'],
|
||||
allblackx: ['ebony', 'bbc'],
|
||||
boundgods: ['gay'],
|
||||
buttmachineboys: ['gay'],
|
||||
cum4k: ['fake-cum', 'creampie', '4k'],
|
||||
darkx: ['interracial'],
|
||||
deepthroatlove: ['blowjob', 'deepthroat'],
|
||||
divinebitches: ['femdom'],
|
||||
dpparodies: ['parody'],
|
||||
eighteenyearsold: ['teen'],
|
||||
exotic4k: ['4k'],
|
||||
givemepink: ['solo', 'masturbation'],
|
||||
lubed: ['oil'],
|
||||
familystrokes: ['family'],
|
||||
givemepink: ['solo', 'masturbation'],
|
||||
lesbianx: ['lesbian'],
|
||||
lubed: ['oil'],
|
||||
massagecreep: ['massage'],
|
||||
menonedge: ['gay'],
|
||||
povd: ['pov'],
|
||||
|
|
@ -1631,6 +1633,7 @@ function getSiteTags() {
|
|||
spyfam: ['family'],
|
||||
submissived: ['bdsm'],
|
||||
swallowed: ['blowjob', 'deepthroat', 'facefucking'],
|
||||
teenallanal: ['anal', 'mff'],
|
||||
teenbff: ['mff'],
|
||||
tiny4k: ['4k'],
|
||||
trueanal: ['anal'],
|
||||
|
|
|
|||
|
|
@ -301,7 +301,7 @@ async function mergeProfiles(profiles, actor) {
|
|||
piercings: prevProfile.piercings || profile.piercings,
|
||||
tattoos: prevProfile.tattoos || profile.tattoos,
|
||||
social: prevProfile.social.concat(profile.social || []),
|
||||
avatars: prevProfile.avatars.concat(profile.avatar || []),
|
||||
avatars: prevProfile.avatars.concat([profile.avatar] || []),
|
||||
};
|
||||
}, {
|
||||
social: [],
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ async function storePhotos(photos, {
|
|||
return;
|
||||
}
|
||||
|
||||
const pluckedPhotos = pluckPhotos(photos);
|
||||
const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration
|
||||
const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label);
|
||||
|
||||
const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {
|
||||
|
|
|
|||
|
|
@ -159,7 +159,7 @@ async function attachChannelSite(release) {
|
|||
}
|
||||
|
||||
if (!release.channel) {
|
||||
throw new Error(`Unable to derive channel site from generic URL: ${release.url}.`);
|
||||
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
|
||||
}
|
||||
|
||||
const [site] = await fetchSites({
|
||||
|
|
@ -182,7 +182,7 @@ async function attachChannelSite(release) {
|
|||
site: urlSite,
|
||||
};
|
||||
} catch (error) {
|
||||
throw new Error(`Unable to derive channel site from generic URL: ${release.url}.`);
|
||||
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,51 +1,10 @@
|
|||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
|
||||
async function fetchPhotos(photoPath) {
|
||||
const res = await bhttp.get(`https://21sextury.com${photoPath}`);
|
||||
|
||||
return res.body.toString();
|
||||
}
|
||||
|
||||
function scrapePhotos(html) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
||||
const unlockedPhotos = $('.preview .imgLink.pgUnlocked')
|
||||
.map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
|
||||
|
||||
const lockedThumbnails = $('.preview .imgLink.lockedPicture img')
|
||||
.map((photoIndex, photoElement) => $(photoElement)
|
||||
.attr('src'))
|
||||
// .replace('_tb.jpg', '.jpg')) does not always work
|
||||
.toArray();
|
||||
|
||||
return unlockedPhotos.concat(lockedThumbnails);
|
||||
}
|
||||
|
||||
async function getPhotos(photoPath) {
|
||||
if (!photoPath || photoPath.match('join')) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const html = await fetchPhotos(photoPath);
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const photos = scrapePhotos(html);
|
||||
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
|
||||
|
||||
const otherPhotos = await Promise.map(pages, async (pagePath) => {
|
||||
const pageHtml = await fetchPhotos(pagePath);
|
||||
|
||||
return scrapePhotos(pageHtml);
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
|
||||
return photos.concat(otherPhotos.flat());
|
||||
}
|
||||
const { getPhotos, fetchProfile } = require('./gamma');
|
||||
|
||||
function scrape(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
|
@ -131,8 +90,7 @@ async function scrapeScene(html, url, site) {
|
|||
const poster = videoData.picPreview;
|
||||
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
||||
|
||||
const photoPath = $('.picturesItem a').attr('href');
|
||||
const photos = await getPhotos(photoPath, site);
|
||||
const photos = await getPhotos($('.picturesItem a').attr('href'), '21sextury.com', site);
|
||||
|
||||
const tags = data.keywords.split(', ');
|
||||
const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
|
||||
|
|
@ -181,8 +139,13 @@ async function fetchScene(url, site) {
|
|||
return scrapeScene(res.body.toString(), url, site);
|
||||
}
|
||||
|
||||
async function networkFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, '21sextury', true);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchUpcoming,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,71 +1,11 @@
|
|||
'use strict';
|
||||
|
||||
/* eslint-disable newline-per-chained-call */
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
|
||||
async function fetchPhotos(url) {
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return res.body.toString();
|
||||
}
|
||||
|
||||
function scrapePhotos(html) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
||||
return $('.preview .imgLink').toArray().map((linkEl) => {
|
||||
const url = $(linkEl).attr('href');
|
||||
|
||||
if (url.match('/join')) {
|
||||
// URL links to join page instead of full photo, extract thumbnail
|
||||
const src = $(linkEl).find('img').attr('src');
|
||||
|
||||
if (src.match('previews/')) {
|
||||
// resource often serves full photo at a modifier URL anyway, add as primary source
|
||||
const highRes = src
|
||||
.replace('previews/', '')
|
||||
.replace('_tb.jpg', '.jpg');
|
||||
|
||||
// keep original thumbnail as fallback in case full photo is not available
|
||||
return [highRes, src];
|
||||
}
|
||||
|
||||
return src;
|
||||
}
|
||||
|
||||
// URL links to full photo
|
||||
return url;
|
||||
});
|
||||
}
|
||||
|
||||
async function getPhotos(albumPath, siteDomain) {
|
||||
const albumUrl = `https://www.blowpass.com${albumPath}`;
|
||||
|
||||
try {
|
||||
const html = await fetchPhotos(albumUrl);
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const photos = scrapePhotos(html);
|
||||
|
||||
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
|
||||
|
||||
const otherPhotos = await Promise.map(pages, async (page) => {
|
||||
const pageUrl = `https://${siteDomain}${page}`;
|
||||
const pageHtml = await fetchPhotos(pageUrl);
|
||||
|
||||
return scrapePhotos(pageHtml);
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
|
||||
return photos.concat(otherPhotos.flat());
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch Blowpass photos from ${albumPath}: ${error.message}`);
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
const { getPhotos, fetchProfile } = require('./gamma');
|
||||
|
||||
function scrape(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
|
@ -132,7 +72,7 @@ async function scrapeScene(html, url, site) {
|
|||
|
||||
const poster = playerData.picPreview;
|
||||
const trailer = `${playerData.playerOptions.host}${playerData.url}`;
|
||||
const photos = await getPhotos($('.picturesItem a').attr('href'), channel, site);
|
||||
const photos = await getPhotos($('.picturesItem a').attr('href'), 'blowpass.com', site);
|
||||
|
||||
const duration = moment.duration(data.duration.slice(2)).asSeconds();
|
||||
const tags = data.keywords.split(', ');
|
||||
|
|
@ -180,8 +120,13 @@ async function fetchScene(url, site) {
|
|||
return scrapeScene(res.body.toString(), url, site);
|
||||
}
|
||||
|
||||
async function blowpassFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, 'blowpass');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchUpcoming,
|
||||
fetchProfile: blowpassFetchProfile,
|
||||
fetchScene,
|
||||
fetchUpcoming,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ const bhttp = require('bhttp');
|
|||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
|
||||
const { getPhotos } = require('./gamma');
|
||||
|
||||
async function scrape(json, site) {
|
||||
return Promise.all(json.map(async (scene) => {
|
||||
const {
|
||||
|
|
@ -75,6 +77,8 @@ async function scrapeScene(html, url, site) {
|
|||
const poster = videoData.picPreview;
|
||||
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
||||
|
||||
const photos = await getPhotos($('.picturesItem a').attr('href'), 'evilangel.com', site);
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
|
|
@ -86,6 +90,7 @@ async function scrapeScene(html, url, site) {
|
|||
duration,
|
||||
tags,
|
||||
poster,
|
||||
photos,
|
||||
trailer: {
|
||||
src: trailer,
|
||||
quality: parseInt(videoData.sizeOnLoad, 10),
|
||||
|
|
@ -97,7 +102,26 @@ async function scrapeScene(html, url, site) {
|
|||
};
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1, upcoming = false) {
|
||||
function scrapeActor(data) {
|
||||
const actor = {};
|
||||
|
||||
if (data.male === 1) actor.gender = 'male';
|
||||
if (data.female === 1) actor.gender = 'female';
|
||||
if (data.shemale === 1 || data.trans === 1) actor.gender = 'transsexual';
|
||||
|
||||
if (data.description) actor.description = data.description.trim();
|
||||
|
||||
if (data.attributes.ethnicity) actor.ethnicity = data.attributes.ethnicity;
|
||||
if (data.attributes.eye_color) actor.eyes = data.attributes.eye_color;
|
||||
if (data.attributes.hair_color) actor.hair = data.attributes.hair_color;
|
||||
|
||||
const avatarPath = Object.values(data.pictures).reverse()[0];
|
||||
actor.avatar = `https://images01-evilangel.gammacdn.com/actors${avatarPath}`;
|
||||
|
||||
return actor;
|
||||
}
|
||||
|
||||
async function fetchApiCredentials() {
|
||||
const res = await bhttp.get('https://evilangel.com/en/videos');
|
||||
const body = res.body.toString();
|
||||
|
||||
|
|
@ -108,7 +132,20 @@ async function fetchLatest(site, page = 1, upcoming = false) {
|
|||
const { applicationID: appId, apiKey } = apiData.api.algolia;
|
||||
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
|
||||
|
||||
const apiRes = await bhttp.post(`https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`, {
|
||||
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
|
||||
|
||||
return {
|
||||
appId,
|
||||
apiKey,
|
||||
userAgent,
|
||||
apiUrl,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1, upcoming = false) {
|
||||
const { apiUrl } = await fetchApiCredentials();
|
||||
|
||||
const res = await bhttp.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_scenes',
|
||||
|
|
@ -122,7 +159,7 @@ async function fetchLatest(site, page = 1, upcoming = false) {
|
|||
encodeJSON: true,
|
||||
});
|
||||
|
||||
return scrape(apiRes.body.results[0].hits, site);
|
||||
return scrape(res.body.results[0].hits, site);
|
||||
}
|
||||
|
||||
async function fetchUpcoming(site) {
|
||||
|
|
@ -135,8 +172,38 @@ async function fetchScene(url, site) {
|
|||
return scrapeScene(res.body.toString(), url, site);
|
||||
}
|
||||
|
||||
async function fetchProfile(actorName) {
|
||||
const { apiUrl } = await fetchApiCredentials();
|
||||
const actorSlug = encodeURI(actorName);
|
||||
|
||||
const res = await bhttp.post(apiUrl, {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_actors',
|
||||
params: `query=${actorSlug}`,
|
||||
},
|
||||
],
|
||||
}, {
|
||||
headers: {
|
||||
Referer: `https://www.evilangel.com/en/search?query=${actorSlug}&tab=actors`,
|
||||
},
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.statusCode === 200 && res.body.results[0].hits.length > 0) {
|
||||
const actorData = res.body.results[0].hits.find(actor => actor.name === actorName);
|
||||
|
||||
if (actorData) {
|
||||
return scrapeActor(actorData);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchUpcoming,
|
||||
fetchProfile,
|
||||
fetchScene,
|
||||
fetchUpcoming,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
async function fetchPhotos(url) {
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return res.body.toString();
|
||||
}
|
||||
|
||||
function scrapePhotos(html) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
||||
return $('.preview .imgLink').toArray().map((linkEl) => {
|
||||
const url = $(linkEl).attr('href');
|
||||
|
||||
if (url.match('/join')) {
|
||||
// URL links to join page instead of full photo, extract thumbnail
|
||||
const src = $(linkEl).find('img').attr('src');
|
||||
|
||||
if (src.match('previews/')) {
|
||||
// resource often serves full photo at a modifier URL anyway, add as primary source
|
||||
const highRes = src
|
||||
.replace('previews/', '')
|
||||
.replace('_tb.jpg', '.jpg');
|
||||
|
||||
// keep original thumbnail as fallback in case full photo is not available
|
||||
return [highRes, src];
|
||||
}
|
||||
|
||||
return src;
|
||||
}
|
||||
|
||||
// URL links to full photo
|
||||
return url;
|
||||
});
|
||||
}
|
||||
|
||||
async function getPhotos(albumPath, siteDomain) {
|
||||
const albumUrl = `https://${siteDomain}${albumPath}`;
|
||||
|
||||
try {
|
||||
const html = await fetchPhotos(albumUrl);
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const photos = scrapePhotos(html);
|
||||
|
||||
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
|
||||
|
||||
const otherPhotos = await Promise.map(pages, async (page) => {
|
||||
const pageUrl = `https://${siteDomain}${page}`;
|
||||
const pageHtml = await fetchPhotos(pageUrl);
|
||||
|
||||
return scrapePhotos(pageHtml);
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
|
||||
return photos.concat(otherPhotos.flat());
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch ${siteDomain} photos from ${albumPath}: ${error.message}`);
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function scrapeActorSearch(html, url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
|
||||
|
||||
return actorLink ? actorLink.href : null;
|
||||
}
|
||||
|
||||
function scrapeProfile(html, url, actorName, siteSlug) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
|
||||
const avatarEl = document.querySelector('img.actorPicture');
|
||||
const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
|
||||
|
||||
const profile = {
|
||||
name: actorName,
|
||||
};
|
||||
|
||||
if (avatarEl) {
|
||||
// larger sizes usually available, provide fallbacks
|
||||
const avatars = [
|
||||
avatarEl.src.replace(/\d+x\d+/, '500x750'),
|
||||
avatarEl.src.replace(/\d+x\d+/, '240x360'),
|
||||
avatarEl.src.replace(/\d+x\d+/, '200x300'),
|
||||
avatarEl.src,
|
||||
];
|
||||
|
||||
profile.avatar = avatars;
|
||||
}
|
||||
|
||||
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
|
||||
|
||||
profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://${siteSlug}.com${el.href}`);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile(actorName, siteSlug, altSearchUrl) {
|
||||
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
|
||||
const searchUrl = altSearchUrl
|
||||
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
|
||||
: `https://www.${siteSlug}.com/en/search/${siteSlug}/actor/${actorSlug}`;
|
||||
const searchRes = await bhttp.get(searchUrl);
|
||||
|
||||
if (searchRes.statusCode !== 200) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
|
||||
|
||||
if (actorUrl) {
|
||||
const url = `https://${siteSlug}.com${actorUrl}`;
|
||||
const actorRes = await bhttp.get(url);
|
||||
|
||||
if (actorRes.statusCode !== 200) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getPhotos,
|
||||
fetchProfile,
|
||||
scrapeProfile,
|
||||
};
|
||||
|
|
@ -1,14 +1,11 @@
|
|||
'use strict';
|
||||
|
||||
// releases
|
||||
const twentyonesextury = require('./21sextury');
|
||||
const babes = require('./babes');
|
||||
const bang = require('./bang');
|
||||
const bangbros = require('./bangbros');
|
||||
const blowpass = require('./blowpass');
|
||||
const dogfart = require('./dogfart');
|
||||
const digitalplayground = require('./digitalplayground');
|
||||
const evilangel = require('./evilangel');
|
||||
const fakehub = require('./fakehub');
|
||||
const jayrock = require('./jayrock');
|
||||
const kink = require('./kink');
|
||||
|
|
@ -25,11 +22,14 @@ const teamskeet = require('./teamskeet');
|
|||
const vixen = require('./vixen');
|
||||
|
||||
// releases and profiles
|
||||
const ddfnetwork = require('./ddfnetwork');
|
||||
const blowpass = require('./blowpass');
|
||||
const brazzers = require('./brazzers');
|
||||
const ddfnetwork = require('./ddfnetwork');
|
||||
const evilangel = require('./evilangel');
|
||||
const julesjordan = require('./julesjordan');
|
||||
const kellymadison = require('./kellymadison');
|
||||
const legalporno = require('./legalporno');
|
||||
const twentyonesextury = require('./21sextury');
|
||||
const xempire = require('./xempire');
|
||||
|
||||
// profiles
|
||||
|
|
@ -71,7 +71,10 @@ module.exports = {
|
|||
},
|
||||
actors: {
|
||||
// ordered by data priority
|
||||
'21sextury': twentyonesextury,
|
||||
evilangel,
|
||||
xempire,
|
||||
blowpass,
|
||||
julesjordan,
|
||||
brazzers,
|
||||
legalporno,
|
||||
|
|
|
|||
|
|
@ -1,79 +1,10 @@
|
|||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const moment = require('moment');
|
||||
|
||||
const defaultTags = {
|
||||
hardx: [],
|
||||
darkx: ['interracial'],
|
||||
eroticax: [],
|
||||
lesbianx: ['lesbian'],
|
||||
allblackx: ['ebony', 'bbc'],
|
||||
};
|
||||
|
||||
async function fetchPhotos(url) {
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return res.body.toString();
|
||||
}
|
||||
|
||||
function scrapePhotos(html) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
||||
return $('.preview .imgLink').toArray().map((linkEl) => {
|
||||
const url = $(linkEl).attr('href');
|
||||
|
||||
if (url.match('/join')) {
|
||||
// URL links to join page instead of full photo, extract thumbnail
|
||||
const src = $(linkEl).find('img').attr('src');
|
||||
|
||||
if (src.match('previews/')) {
|
||||
// resource often serves full photo at a modifier URL anyway, add as primary source
|
||||
const highRes = src
|
||||
.replace('previews/', '')
|
||||
.replace('_tb.jpg', '.jpg');
|
||||
|
||||
// keep original thumbnail as fallback in case full photo is not available
|
||||
return [highRes, src];
|
||||
}
|
||||
|
||||
return src;
|
||||
}
|
||||
|
||||
// URL links to full photo
|
||||
return url;
|
||||
});
|
||||
}
|
||||
|
||||
async function getPhotos(albumPath, siteDomain) {
|
||||
const albumUrl = `https://${siteDomain}${albumPath}`;
|
||||
|
||||
try {
|
||||
const html = await fetchPhotos(albumUrl);
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const photos = scrapePhotos(html);
|
||||
|
||||
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
|
||||
|
||||
const otherPhotos = await Promise.map(pages, async (page) => {
|
||||
const pageUrl = `https://${siteDomain}${page}`;
|
||||
const pageHtml = await fetchPhotos(pageUrl);
|
||||
|
||||
return scrapePhotos(pageHtml);
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
|
||||
return photos.concat(otherPhotos.flat());
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch XEmpire photos from ${albumPath}: ${error.message}`);
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
const { getPhotos, fetchProfile } = require('./gamma');
|
||||
|
||||
function scrape(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
|
@ -154,8 +85,7 @@ async function scrapeScene(html, url, site) {
|
|||
|
||||
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
|
||||
|
||||
const rawTags = data.keywords.split(', ');
|
||||
const tags = [...defaultTags[siteSlug], ...rawTags];
|
||||
const tags = data.keywords.split(', ');
|
||||
|
||||
return {
|
||||
url: `${siteUrl}/en/video/${new URL(url).pathname.split('/').slice(-2).join('/')}`,
|
||||
|
|
@ -181,31 +111,6 @@ async function scrapeScene(html, url, site) {
|
|||
};
|
||||
}
|
||||
|
||||
function scrapeActorSearch(html, url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
|
||||
|
||||
return actorLink ? actorLink.href : null;
|
||||
}
|
||||
|
||||
function scrapeProfile(html, url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
|
||||
const avatarEl = document.querySelector('img.actorPicture');
|
||||
const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
|
||||
|
||||
const profile = {
|
||||
name: actorName,
|
||||
};
|
||||
|
||||
if (avatarEl) profile.avatar = avatarEl.src;
|
||||
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
|
||||
|
||||
profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://xempire.com${el.href}`);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
|
||||
|
||||
|
|
@ -224,34 +129,13 @@ async function fetchScene(url, site) {
|
|||
return scrapeScene(res.body.toString(), url, site);
|
||||
}
|
||||
|
||||
async function fetchProfile(actorName) {
|
||||
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
|
||||
const searchUrl = `https://www.xempire.com/en/search/xempire/actor/${actorSlug}`;
|
||||
const searchRes = await bhttp.get(searchUrl);
|
||||
|
||||
if (searchRes.statusCode !== 200) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
|
||||
|
||||
if (actorUrl) {
|
||||
const url = `https://xempire.com${actorUrl}`;
|
||||
const actorRes = await bhttp.get(url);
|
||||
|
||||
if (actorRes.statusCode !== 200) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return scrapeProfile(actorRes.body.toString(), url, actorName);
|
||||
}
|
||||
|
||||
return null;
|
||||
async function xEmpireFetchProfile(actorName) {
|
||||
return fetchProfile(actorName, 'xempire');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile,
|
||||
fetchProfile: xEmpireFetchProfile,
|
||||
fetchUpcoming,
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ function destructConfigNetworks(networks) {
|
|||
}
|
||||
|
||||
async function findSiteByUrl(url) {
|
||||
const { hostname, origin } = new URL(url);
|
||||
const { hostname } = new URL(url);
|
||||
const domain = hostname.replace(/www.|tour./, '');
|
||||
|
||||
const sites = await knex('sites')
|
||||
|
|
@ -69,8 +69,8 @@ async function findSiteByUrl(url) {
|
|||
'sites.*',
|
||||
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
|
||||
)
|
||||
.where('sites.url', 'like', `${domain}`)
|
||||
.orWhere('sites.url', 'like', `${origin}`)
|
||||
.where('sites.url', 'like', `%${domain}`)
|
||||
.orWhere('sites.url', 'like', url)
|
||||
.orWhere('sites.url', url);
|
||||
|
||||
if (sites.length > 0) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue