Added generic Gamma photo and actor scraper for XEmpire, 21Sextury, Blowpass and Evil Angel.
This commit is contained in:
parent
4e4323704a
commit
f8175f6054
|
|
@ -26,7 +26,7 @@
|
||||||
<span
|
<span
|
||||||
v-if="actor.gender"
|
v-if="actor.gender"
|
||||||
class="bio-gender"
|
class="bio-gender"
|
||||||
:class="{ male: actor.gender === 'male', female: actor.gender === 'female' }"
|
:class="{ [actor.gender]: true }"
|
||||||
><Icon :icon="actor.gender" /></span>
|
><Icon :icon="actor.gender" /></span>
|
||||||
</li>
|
</li>
|
||||||
|
|
||||||
|
|
@ -392,6 +392,12 @@ export default {
|
||||||
&.male .icon {
|
&.male .icon {
|
||||||
fill: $male;
|
fill: $male;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
&.transsexual .icon {
|
||||||
|
fill: $text-contrast;
|
||||||
|
filter: drop-shadow(1px 0 0 $female) drop-shadow(-1px 0 0 $female) drop-shadow(0 1px 0 $female) drop-shadow(0 -1px 0 $female)
|
||||||
|
drop-shadow(1px 0 0 $male) drop-shadow(-1px 0 0 $male) drop-shadow(0 1px 0 $male) drop-shadow(0 -1px 0 $male);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.birthdate {
|
.birthdate {
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,17 @@
|
||||||
<template>
|
<template>
|
||||||
<div class="networks">
|
<div class="networks">
|
||||||
<Network
|
<input
|
||||||
v-for="network in networks"
|
:placeholder="`Find ${siteCount} sites in ${networks.length} networks`"
|
||||||
:key="`network-${network.id}`"
|
class="search"
|
||||||
:network="network"
|
>
|
||||||
/>
|
|
||||||
|
<div class="network-tiles">
|
||||||
|
<Network
|
||||||
|
v-for="network in networks"
|
||||||
|
:key="`network-tile-${network.slug}`"
|
||||||
|
:network="network"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
|
|
@ -15,6 +22,10 @@ async function mounted() {
|
||||||
this.networks = await this.$store.dispatch('fetchNetworks');
|
this.networks = await this.$store.dispatch('fetchNetworks');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function siteCount() {
|
||||||
|
return this.networks.map(network => network.sites).flat().length;
|
||||||
|
}
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
components: {
|
components: {
|
||||||
Network,
|
Network,
|
||||||
|
|
@ -24,6 +35,9 @@ export default {
|
||||||
networks: [],
|
networks: [],
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
computed: {
|
||||||
|
siteCount,
|
||||||
|
},
|
||||||
mounted,
|
mounted,
|
||||||
};
|
};
|
||||||
</script>
|
</script>
|
||||||
|
|
@ -31,7 +45,23 @@ export default {
|
||||||
<style lang="scss" scoped>
|
<style lang="scss" scoped>
|
||||||
@import 'theme';
|
@import 'theme';
|
||||||
|
|
||||||
.networks {
|
.search {
|
||||||
|
width: 40rem;
|
||||||
|
max-width: 100%;
|
||||||
|
box-sizing: border-box;
|
||||||
|
padding: 1rem;
|
||||||
|
border: none;
|
||||||
|
box-shadow: 0 0 3px $shadow-weak;
|
||||||
|
margin: 1rem;
|
||||||
|
font-size: 1rem;
|
||||||
|
outline: none;
|
||||||
|
|
||||||
|
&:focus {
|
||||||
|
box-shadow: 0 0 3px $primary;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-tiles {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
|
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
|
||||||
grid-gap: 1rem;
|
grid-gap: 1rem;
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,7 @@ function initActorActions(store, _router) {
|
||||||
hasPiercings
|
hasPiercings
|
||||||
tattoos
|
tattoos
|
||||||
piercings
|
piercings
|
||||||
|
description
|
||||||
avatar: actorsAvatarByActorId {
|
avatar: actorsAvatarByActorId {
|
||||||
media {
|
media {
|
||||||
thumbnail
|
thumbnail
|
||||||
|
|
|
||||||
|
|
@ -1437,25 +1437,25 @@
|
||||||
"integrity": "sha512-OoIgewLowjegJzz3tpcRE5LpQKqsap1ETFaZtC1r9p36h5ieUJnWTxCTpB7XIsU9muMTt7MMt8x0E5apS47QIQ=="
|
"integrity": "sha512-OoIgewLowjegJzz3tpcRE5LpQKqsap1ETFaZtC1r9p36h5ieUJnWTxCTpB7XIsU9muMTt7MMt8x0E5apS47QIQ=="
|
||||||
},
|
},
|
||||||
"@tensorflow/tfjs": {
|
"@tensorflow/tfjs": {
|
||||||
"version": "1.4.0",
|
"version": "1.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.5.1.tgz",
|
||||||
"integrity": "sha512-pvc7arrWZ0NNOpEFVL4e99Bj933ty3tmUR0xLnjKao3psts9xr7w0sfJpuSn98HKjz+dj87UEXZIi0fwsAZdwQ==",
|
"integrity": "sha512-WiE+JQ3ibr5LibGiBz6HWUqLJW8HiX6ywUSCA7ehZ67vFsw4mPuVjv0WEEUfD/l47PkXYVAmWd+RYOJiuZC7Eg==",
|
||||||
"requires": {
|
"requires": {
|
||||||
"@tensorflow/tfjs-converter": "1.4.0",
|
"@tensorflow/tfjs-converter": "1.5.1",
|
||||||
"@tensorflow/tfjs-core": "1.4.0",
|
"@tensorflow/tfjs-core": "1.5.1",
|
||||||
"@tensorflow/tfjs-data": "1.4.0",
|
"@tensorflow/tfjs-data": "1.5.1",
|
||||||
"@tensorflow/tfjs-layers": "1.4.0"
|
"@tensorflow/tfjs-layers": "1.5.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"@tensorflow/tfjs-converter": {
|
"@tensorflow/tfjs-converter": {
|
||||||
"version": "1.4.0",
|
"version": "1.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.5.1.tgz",
|
||||||
"integrity": "sha512-vdZumj6zHcEALtQlonCBbVwGdEEDcPrF7prgzf4HF82QG4RpM1x/kVdvzsGvfUIPYjRImNn0ZSPC3WaKslNcXw=="
|
"integrity": "sha512-M9tl2/ep8ntcZpmncHwKuvThsS7TaUWqJ9vJSgJmkazwTfAvlAJmZ8/p1miJ+m5sH1EJO4oTjiEmch6g8IA5IQ=="
|
||||||
},
|
},
|
||||||
"@tensorflow/tfjs-core": {
|
"@tensorflow/tfjs-core": {
|
||||||
"version": "1.4.0",
|
"version": "1.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.5.1.tgz",
|
||||||
"integrity": "sha512-a7dHhSsBbtaxp8/1UAeYKrjY1bQxsDy/Uhj57+mTuGLAcL8CDrTOXXZzucCgs5nvQErdscp7Gp/2VCcA1xp6XQ==",
|
"integrity": "sha512-N4fsi8mLsRwRs8UJN2cARB1rYFxyVXkLyZ4wOusiR976BwwZbCwQrTTSIPzPqYT3rwiexEUzm7sM6ZaDl5dpXA==",
|
||||||
"requires": {
|
"requires": {
|
||||||
"@types/offscreencanvas": "~2019.3.0",
|
"@types/offscreencanvas": "~2019.3.0",
|
||||||
"@types/seedrandom": "2.4.27",
|
"@types/seedrandom": "2.4.27",
|
||||||
|
|
@ -1466,29 +1466,29 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"@tensorflow/tfjs-data": {
|
"@tensorflow/tfjs-data": {
|
||||||
"version": "1.4.0",
|
"version": "1.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.5.1.tgz",
|
||||||
"integrity": "sha512-00N/pe5IYkO3aDI9ZmavQ6mwY4VIepCteusnI7DLiCeqBO0NzGjlH3zH1LVlWIBRypW7JNqYbdl3oVvnd2wWwg==",
|
"integrity": "sha512-eu4X0tHS1Tng+cvMO9gkMhUWX/UZQ//VpiaZfQJfa3zvUgxw6s1MHJFb0JC1T1FOnEgDVriZ8G758ysJZOybog==",
|
||||||
"requires": {
|
"requires": {
|
||||||
"@types/node-fetch": "^2.1.2",
|
"@types/node-fetch": "^2.1.2",
|
||||||
"node-fetch": "~2.1.2"
|
"node-fetch": "~2.1.2"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"@tensorflow/tfjs-layers": {
|
"@tensorflow/tfjs-layers": {
|
||||||
"version": "1.4.0",
|
"version": "1.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.5.1.tgz",
|
||||||
"integrity": "sha512-PXNOShZWDVW9OzX9botHJdDD6ClHEQtkfaFjoGZ2I2qYC9+WaVgxJPxwToTS2H2trwrf34q6hZ1lAjVlTuK0Gg=="
|
"integrity": "sha512-DyuhifqflK+bdpBRLAj3RuWm1eTVe8yNX2+WH+W+wmhpjGg7Yagnar6/66JdS2h3WUFoiplCpZRAVMVw631E5g=="
|
||||||
},
|
},
|
||||||
"@tensorflow/tfjs-node": {
|
"@tensorflow/tfjs-node": {
|
||||||
"version": "1.4.0",
|
"version": "1.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-1.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-1.5.1.tgz",
|
||||||
"integrity": "sha512-6BVD4Jg72B2iPckz3lawB5UB4fR+pm4skhEEN0x98ykt08QF/3fgJX0xiT1K5otqfpwRDKWwcTO18ajOXgJt1Q==",
|
"integrity": "sha512-gtIVn5zTLSgHlJUThN3knzKjycXOezmgL0W5MtxunJPznGmUZQive9efuCsgKcs5YKizTG+KprTILTSbi/2Owg==",
|
||||||
"requires": {
|
"requires": {
|
||||||
"@tensorflow/tfjs": "1.4.0",
|
"@tensorflow/tfjs": "1.5.1",
|
||||||
"adm-zip": "^0.4.11",
|
"adm-zip": "^0.4.11",
|
||||||
"google-protobuf": "^3.9.2",
|
"google-protobuf": "^3.9.2",
|
||||||
"https-proxy-agent": "^2.2.1",
|
"https-proxy-agent": "^2.2.1",
|
||||||
"node-pre-gyp": "0.13.0",
|
"node-pre-gyp": "0.14.0",
|
||||||
"progress": "^2.0.0",
|
"progress": "^2.0.0",
|
||||||
"rimraf": "^2.6.2",
|
"rimraf": "^2.6.2",
|
||||||
"tar": "^4.4.6"
|
"tar": "^4.4.6"
|
||||||
|
|
@ -2977,9 +2977,9 @@
|
||||||
"integrity": "sha512-/xL2AbW/XWHNu1gnIrO8UitBGoFthcsDgU9VLK1/dpsoxbaD5LscHozKze05R6WLsBvLhqv78dAPozMFQBYLbQ=="
|
"integrity": "sha512-/xL2AbW/XWHNu1gnIrO8UitBGoFthcsDgU9VLK1/dpsoxbaD5LscHozKze05R6WLsBvLhqv78dAPozMFQBYLbQ=="
|
||||||
},
|
},
|
||||||
"canvas": {
|
"canvas": {
|
||||||
"version": "2.6.0",
|
"version": "2.6.1",
|
||||||
"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.6.0.tgz",
|
"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.6.1.tgz",
|
||||||
"integrity": "sha512-bEO9f1ThmbknLPxCa8Es7obPlN9W3stB1bo7njlhOFKIdUTldeTqXCh9YclCPAi2pSQs84XA0jq/QEZXSzgyMw==",
|
"integrity": "sha512-S98rKsPcuhfTcYbtF53UIJhcbgIAK533d1kJKMwsMwAIFgfd58MOyxRud3kktlzWiEkFliaJtvyZCBtud/XVEA==",
|
||||||
"requires": {
|
"requires": {
|
||||||
"nan": "^2.14.0",
|
"nan": "^2.14.0",
|
||||||
"node-pre-gyp": "^0.11.0",
|
"node-pre-gyp": "^0.11.0",
|
||||||
|
|
@ -5933,9 +5933,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"google-protobuf": {
|
"google-protobuf": {
|
||||||
"version": "3.11.1",
|
"version": "3.11.2",
|
||||||
"resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.11.1.tgz",
|
"resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.11.2.tgz",
|
||||||
"integrity": "sha512-1fdyssLSa7Dip+MDdluLl0uK+2rspgU1PrbdPJKt7hywqSl3+R27TkLYKuBn9dvOyr0VQ0cliQWlfMheW5WKqA=="
|
"integrity": "sha512-T4fin7lcYLUPj2ChUZ4DvfuuHtg3xi1621qeRZt2J7SvOQusOzq+sDT4vbotWTCjUXJoR36CA016LlhtPy80uQ=="
|
||||||
},
|
},
|
||||||
"graceful-fs": {
|
"graceful-fs": {
|
||||||
"version": "4.1.15",
|
"version": "4.1.15",
|
||||||
|
|
@ -7999,9 +7999,9 @@
|
||||||
"integrity": "sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA="
|
"integrity": "sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA="
|
||||||
},
|
},
|
||||||
"node-pre-gyp": {
|
"node-pre-gyp": {
|
||||||
"version": "0.13.0",
|
"version": "0.14.0",
|
||||||
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.13.0.tgz",
|
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.14.0.tgz",
|
||||||
"integrity": "sha512-Md1D3xnEne8b/HGVQkZZwV27WUi1ZRuZBij24TNaZwUPU3ZAFtvT6xxJGaUVillfmMKnn5oD1HoGsp2Ftik7SQ==",
|
"integrity": "sha512-+CvDC7ZttU/sSt9rFjix/P05iS43qHCOOGzcr3Ry99bXG7VX953+vFyEuph/tfqoYu8dttBkE86JSKBO2OzcxA==",
|
||||||
"requires": {
|
"requires": {
|
||||||
"detect-libc": "^1.0.2",
|
"detect-libc": "^1.0.2",
|
||||||
"mkdirp": "^0.5.1",
|
"mkdirp": "^0.5.1",
|
||||||
|
|
@ -8012,7 +8012,7 @@
|
||||||
"rc": "^1.2.7",
|
"rc": "^1.2.7",
|
||||||
"rimraf": "^2.6.1",
|
"rimraf": "^2.6.1",
|
||||||
"semver": "^5.3.0",
|
"semver": "^5.3.0",
|
||||||
"tar": "^4"
|
"tar": "^4.4.2"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"fs-minipass": {
|
"fs-minipass": {
|
||||||
|
|
|
||||||
|
|
@ -68,13 +68,13 @@
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
|
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
|
||||||
"@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
|
"@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
|
||||||
"@tensorflow/tfjs-node": "^1.4.0",
|
"@tensorflow/tfjs-node": "^1.5.1",
|
||||||
"babel-polyfill": "^6.26.0",
|
"babel-polyfill": "^6.26.0",
|
||||||
"bhttp": "^1.2.4",
|
"bhttp": "^1.2.4",
|
||||||
"blake2": "^4.0.0",
|
"blake2": "^4.0.0",
|
||||||
"bluebird": "^3.7.2",
|
"bluebird": "^3.7.2",
|
||||||
"body-parser": "^1.19.0",
|
"body-parser": "^1.19.0",
|
||||||
"canvas": "^2.6.0",
|
"canvas": "^2.6.1",
|
||||||
"cheerio": "^1.0.0-rc.3",
|
"cheerio": "^1.0.0-rc.3",
|
||||||
"cli-confirm": "^1.0.1",
|
"cli-confirm": "^1.0.1",
|
||||||
"config": "^3.2.4",
|
"config": "^3.2.4",
|
||||||
|
|
|
||||||
|
|
@ -936,7 +936,21 @@
|
||||||
|
|
||||||
/* $primary: #ff886c; */
|
/* $primary: #ff886c; */
|
||||||
/* $logo-highlight: drop-shadow(1px 0 0 $highlight-weak) drop-shadow(-1px 0 0 $highlight-weak) drop-shadow(0 1px 0 $highlight-weak) drop-shadow(0 -1px 0 $highlight-weak); */
|
/* $logo-highlight: drop-shadow(1px 0 0 $highlight-weak) drop-shadow(-1px 0 0 $highlight-weak) drop-shadow(0 1px 0 $highlight-weak) drop-shadow(0 -1px 0 $highlight-weak); */
|
||||||
.networks[data-v-4709d404] {
|
.search[data-v-4709d404] {
|
||||||
|
width: 40rem;
|
||||||
|
max-width: 100%;
|
||||||
|
box-sizing: border-box;
|
||||||
|
padding: 1rem;
|
||||||
|
border: none;
|
||||||
|
box-shadow: 0 0 3px rgba(0, 0, 0, 0.2);
|
||||||
|
margin: 1rem;
|
||||||
|
font-size: 1rem;
|
||||||
|
outline: none;
|
||||||
|
}
|
||||||
|
.search[data-v-4709d404]:focus {
|
||||||
|
box-shadow: 0 0 3px #ff6c88;
|
||||||
|
}
|
||||||
|
.network-tiles[data-v-4709d404] {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
|
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
|
||||||
grid-gap: 1rem;
|
grid-gap: 1rem;
|
||||||
|
|
@ -1105,6 +1119,11 @@
|
||||||
.bio-gender.male .icon[data-v-ea0483c2] {
|
.bio-gender.male .icon[data-v-ea0483c2] {
|
||||||
fill: #0af;
|
fill: #0af;
|
||||||
}
|
}
|
||||||
|
.bio-gender.transsexual .icon[data-v-ea0483c2] {
|
||||||
|
fill: #fff;
|
||||||
|
-webkit-filter: drop-shadow(1px 0 0 #f0a) drop-shadow(-1px 0 0 #f0a) drop-shadow(0 1px 0 #f0a) drop-shadow(0 -1px 0 #f0a) drop-shadow(1px 0 0 #0af) drop-shadow(-1px 0 0 #0af) drop-shadow(0 1px 0 #0af) drop-shadow(0 -1px 0 #0af);
|
||||||
|
filter: drop-shadow(1px 0 0 #f0a) drop-shadow(-1px 0 0 #f0a) drop-shadow(0 1px 0 #f0a) drop-shadow(0 -1px 0 #f0a) drop-shadow(1px 0 0 #0af) drop-shadow(-1px 0 0 #0af) drop-shadow(0 1px 0 #0af) drop-shadow(0 -1px 0 #0af);
|
||||||
|
}
|
||||||
.birthdate[data-v-ea0483c2] {
|
.birthdate[data-v-ea0483c2] {
|
||||||
display: block;
|
display: block;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1611,18 +1611,20 @@ function getTagAliases(tagsMap) {
|
||||||
|
|
||||||
function getSiteTags() {
|
function getSiteTags() {
|
||||||
return {
|
return {
|
||||||
teenallanal: ['anal', 'mff'],
|
allblackx: ['ebony', 'bbc'],
|
||||||
boundgods: ['gay'],
|
boundgods: ['gay'],
|
||||||
buttmachineboys: ['gay'],
|
buttmachineboys: ['gay'],
|
||||||
cum4k: ['fake-cum', 'creampie', '4k'],
|
cum4k: ['fake-cum', 'creampie', '4k'],
|
||||||
|
darkx: ['interracial'],
|
||||||
deepthroatlove: ['blowjob', 'deepthroat'],
|
deepthroatlove: ['blowjob', 'deepthroat'],
|
||||||
divinebitches: ['femdom'],
|
divinebitches: ['femdom'],
|
||||||
dpparodies: ['parody'],
|
dpparodies: ['parody'],
|
||||||
eighteenyearsold: ['teen'],
|
eighteenyearsold: ['teen'],
|
||||||
exotic4k: ['4k'],
|
exotic4k: ['4k'],
|
||||||
givemepink: ['solo', 'masturbation'],
|
|
||||||
lubed: ['oil'],
|
|
||||||
familystrokes: ['family'],
|
familystrokes: ['family'],
|
||||||
|
givemepink: ['solo', 'masturbation'],
|
||||||
|
lesbianx: ['lesbian'],
|
||||||
|
lubed: ['oil'],
|
||||||
massagecreep: ['massage'],
|
massagecreep: ['massage'],
|
||||||
menonedge: ['gay'],
|
menonedge: ['gay'],
|
||||||
povd: ['pov'],
|
povd: ['pov'],
|
||||||
|
|
@ -1631,6 +1633,7 @@ function getSiteTags() {
|
||||||
spyfam: ['family'],
|
spyfam: ['family'],
|
||||||
submissived: ['bdsm'],
|
submissived: ['bdsm'],
|
||||||
swallowed: ['blowjob', 'deepthroat', 'facefucking'],
|
swallowed: ['blowjob', 'deepthroat', 'facefucking'],
|
||||||
|
teenallanal: ['anal', 'mff'],
|
||||||
teenbff: ['mff'],
|
teenbff: ['mff'],
|
||||||
tiny4k: ['4k'],
|
tiny4k: ['4k'],
|
||||||
trueanal: ['anal'],
|
trueanal: ['anal'],
|
||||||
|
|
|
||||||
|
|
@ -301,7 +301,7 @@ async function mergeProfiles(profiles, actor) {
|
||||||
piercings: prevProfile.piercings || profile.piercings,
|
piercings: prevProfile.piercings || profile.piercings,
|
||||||
tattoos: prevProfile.tattoos || profile.tattoos,
|
tattoos: prevProfile.tattoos || profile.tattoos,
|
||||||
social: prevProfile.social.concat(profile.social || []),
|
social: prevProfile.social.concat(profile.social || []),
|
||||||
avatars: prevProfile.avatars.concat(profile.avatar || []),
|
avatars: prevProfile.avatars.concat([profile.avatar] || []),
|
||||||
};
|
};
|
||||||
}, {
|
}, {
|
||||||
social: [],
|
social: [],
|
||||||
|
|
|
||||||
|
|
@ -175,7 +175,7 @@ async function storePhotos(photos, {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const pluckedPhotos = pluckPhotos(photos);
|
const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration
|
||||||
const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label);
|
const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label);
|
||||||
|
|
||||||
const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {
|
const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {
|
||||||
|
|
|
||||||
|
|
@ -159,7 +159,7 @@ async function attachChannelSite(release) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!release.channel) {
|
if (!release.channel) {
|
||||||
throw new Error(`Unable to derive channel site from generic URL: ${release.url}.`);
|
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const [site] = await fetchSites({
|
const [site] = await fetchSites({
|
||||||
|
|
@ -182,7 +182,7 @@ async function attachChannelSite(release) {
|
||||||
site: urlSite,
|
site: urlSite,
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new Error(`Unable to derive channel site from generic URL: ${release.url}.`);
|
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,51 +1,10 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const Promise = require('bluebird');
|
|
||||||
const bhttp = require('bhttp');
|
const bhttp = require('bhttp');
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
async function fetchPhotos(photoPath) {
|
const { getPhotos, fetchProfile } = require('./gamma');
|
||||||
const res = await bhttp.get(`https://21sextury.com${photoPath}`);
|
|
||||||
|
|
||||||
return res.body.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapePhotos(html) {
|
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
|
||||||
|
|
||||||
const unlockedPhotos = $('.preview .imgLink.pgUnlocked')
|
|
||||||
.map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
|
|
||||||
|
|
||||||
const lockedThumbnails = $('.preview .imgLink.lockedPicture img')
|
|
||||||
.map((photoIndex, photoElement) => $(photoElement)
|
|
||||||
.attr('src'))
|
|
||||||
// .replace('_tb.jpg', '.jpg')) does not always work
|
|
||||||
.toArray();
|
|
||||||
|
|
||||||
return unlockedPhotos.concat(lockedThumbnails);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getPhotos(photoPath) {
|
|
||||||
if (!photoPath || photoPath.match('join')) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
const html = await fetchPhotos(photoPath);
|
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
|
||||||
const photos = scrapePhotos(html);
|
|
||||||
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
|
|
||||||
|
|
||||||
const otherPhotos = await Promise.map(pages, async (pagePath) => {
|
|
||||||
const pageHtml = await fetchPhotos(pagePath);
|
|
||||||
|
|
||||||
return scrapePhotos(pageHtml);
|
|
||||||
}, {
|
|
||||||
concurrency: 2,
|
|
||||||
});
|
|
||||||
|
|
||||||
return photos.concat(otherPhotos.flat());
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrape(html, site) {
|
function scrape(html, site) {
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||||
|
|
@ -131,8 +90,7 @@ async function scrapeScene(html, url, site) {
|
||||||
const poster = videoData.picPreview;
|
const poster = videoData.picPreview;
|
||||||
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
||||||
|
|
||||||
const photoPath = $('.picturesItem a').attr('href');
|
const photos = await getPhotos($('.picturesItem a').attr('href'), '21sextury.com', site);
|
||||||
const photos = await getPhotos(photoPath, site);
|
|
||||||
|
|
||||||
const tags = data.keywords.split(', ');
|
const tags = data.keywords.split(', ');
|
||||||
const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
|
const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
|
||||||
|
|
@ -181,8 +139,13 @@ async function fetchScene(url, site) {
|
||||||
return scrapeScene(res.body.toString(), url, site);
|
return scrapeScene(res.body.toString(), url, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function networkFetchProfile(actorName) {
|
||||||
|
return fetchProfile(actorName, '21sextury', true);
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
fetchLatest,
|
fetchLatest,
|
||||||
|
fetchProfile: networkFetchProfile,
|
||||||
fetchUpcoming,
|
fetchUpcoming,
|
||||||
fetchScene,
|
fetchScene,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,71 +1,11 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
/* eslint-disable newline-per-chained-call */
|
/* eslint-disable newline-per-chained-call */
|
||||||
const Promise = require('bluebird');
|
|
||||||
const bhttp = require('bhttp');
|
const bhttp = require('bhttp');
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
async function fetchPhotos(url) {
|
const { getPhotos, fetchProfile } = require('./gamma');
|
||||||
const res = await bhttp.get(url);
|
|
||||||
|
|
||||||
return res.body.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapePhotos(html) {
|
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
|
||||||
|
|
||||||
return $('.preview .imgLink').toArray().map((linkEl) => {
|
|
||||||
const url = $(linkEl).attr('href');
|
|
||||||
|
|
||||||
if (url.match('/join')) {
|
|
||||||
// URL links to join page instead of full photo, extract thumbnail
|
|
||||||
const src = $(linkEl).find('img').attr('src');
|
|
||||||
|
|
||||||
if (src.match('previews/')) {
|
|
||||||
// resource often serves full photo at a modifier URL anyway, add as primary source
|
|
||||||
const highRes = src
|
|
||||||
.replace('previews/', '')
|
|
||||||
.replace('_tb.jpg', '.jpg');
|
|
||||||
|
|
||||||
// keep original thumbnail as fallback in case full photo is not available
|
|
||||||
return [highRes, src];
|
|
||||||
}
|
|
||||||
|
|
||||||
return src;
|
|
||||||
}
|
|
||||||
|
|
||||||
// URL links to full photo
|
|
||||||
return url;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getPhotos(albumPath, siteDomain) {
|
|
||||||
const albumUrl = `https://www.blowpass.com${albumPath}`;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const html = await fetchPhotos(albumUrl);
|
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
|
||||||
const photos = scrapePhotos(html);
|
|
||||||
|
|
||||||
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
|
|
||||||
|
|
||||||
const otherPhotos = await Promise.map(pages, async (page) => {
|
|
||||||
const pageUrl = `https://${siteDomain}${page}`;
|
|
||||||
const pageHtml = await fetchPhotos(pageUrl);
|
|
||||||
|
|
||||||
return scrapePhotos(pageHtml);
|
|
||||||
}, {
|
|
||||||
concurrency: 2,
|
|
||||||
});
|
|
||||||
|
|
||||||
return photos.concat(otherPhotos.flat());
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Failed to fetch Blowpass photos from ${albumPath}: ${error.message}`);
|
|
||||||
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrape(html, site) {
|
function scrape(html, site) {
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||||
|
|
@ -132,7 +72,7 @@ async function scrapeScene(html, url, site) {
|
||||||
|
|
||||||
const poster = playerData.picPreview;
|
const poster = playerData.picPreview;
|
||||||
const trailer = `${playerData.playerOptions.host}${playerData.url}`;
|
const trailer = `${playerData.playerOptions.host}${playerData.url}`;
|
||||||
const photos = await getPhotos($('.picturesItem a').attr('href'), channel, site);
|
const photos = await getPhotos($('.picturesItem a').attr('href'), 'blowpass.com', site);
|
||||||
|
|
||||||
const duration = moment.duration(data.duration.slice(2)).asSeconds();
|
const duration = moment.duration(data.duration.slice(2)).asSeconds();
|
||||||
const tags = data.keywords.split(', ');
|
const tags = data.keywords.split(', ');
|
||||||
|
|
@ -180,8 +120,13 @@ async function fetchScene(url, site) {
|
||||||
return scrapeScene(res.body.toString(), url, site);
|
return scrapeScene(res.body.toString(), url, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function blowpassFetchProfile(actorName) {
|
||||||
|
return fetchProfile(actorName, 'blowpass');
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
fetchLatest,
|
fetchLatest,
|
||||||
fetchUpcoming,
|
fetchProfile: blowpassFetchProfile,
|
||||||
fetchScene,
|
fetchScene,
|
||||||
|
fetchUpcoming,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ const bhttp = require('bhttp');
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
|
const { getPhotos } = require('./gamma');
|
||||||
|
|
||||||
async function scrape(json, site) {
|
async function scrape(json, site) {
|
||||||
return Promise.all(json.map(async (scene) => {
|
return Promise.all(json.map(async (scene) => {
|
||||||
const {
|
const {
|
||||||
|
|
@ -75,6 +77,8 @@ async function scrapeScene(html, url, site) {
|
||||||
const poster = videoData.picPreview;
|
const poster = videoData.picPreview;
|
||||||
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
||||||
|
|
||||||
|
const photos = await getPhotos($('.picturesItem a').attr('href'), 'evilangel.com', site);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
url,
|
url,
|
||||||
entryId,
|
entryId,
|
||||||
|
|
@ -86,6 +90,7 @@ async function scrapeScene(html, url, site) {
|
||||||
duration,
|
duration,
|
||||||
tags,
|
tags,
|
||||||
poster,
|
poster,
|
||||||
|
photos,
|
||||||
trailer: {
|
trailer: {
|
||||||
src: trailer,
|
src: trailer,
|
||||||
quality: parseInt(videoData.sizeOnLoad, 10),
|
quality: parseInt(videoData.sizeOnLoad, 10),
|
||||||
|
|
@ -97,7 +102,26 @@ async function scrapeScene(html, url, site) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatest(site, page = 1, upcoming = false) {
|
function scrapeActor(data) {
|
||||||
|
const actor = {};
|
||||||
|
|
||||||
|
if (data.male === 1) actor.gender = 'male';
|
||||||
|
if (data.female === 1) actor.gender = 'female';
|
||||||
|
if (data.shemale === 1 || data.trans === 1) actor.gender = 'transsexual';
|
||||||
|
|
||||||
|
if (data.description) actor.description = data.description.trim();
|
||||||
|
|
||||||
|
if (data.attributes.ethnicity) actor.ethnicity = data.attributes.ethnicity;
|
||||||
|
if (data.attributes.eye_color) actor.eyes = data.attributes.eye_color;
|
||||||
|
if (data.attributes.hair_color) actor.hair = data.attributes.hair_color;
|
||||||
|
|
||||||
|
const avatarPath = Object.values(data.pictures).reverse()[0];
|
||||||
|
actor.avatar = `https://images01-evilangel.gammacdn.com/actors${avatarPath}`;
|
||||||
|
|
||||||
|
return actor;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchApiCredentials() {
|
||||||
const res = await bhttp.get('https://evilangel.com/en/videos');
|
const res = await bhttp.get('https://evilangel.com/en/videos');
|
||||||
const body = res.body.toString();
|
const body = res.body.toString();
|
||||||
|
|
||||||
|
|
@ -108,7 +132,20 @@ async function fetchLatest(site, page = 1, upcoming = false) {
|
||||||
const { applicationID: appId, apiKey } = apiData.api.algolia;
|
const { applicationID: appId, apiKey } = apiData.api.algolia;
|
||||||
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
|
const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
|
||||||
|
|
||||||
const apiRes = await bhttp.post(`https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`, {
|
const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
|
||||||
|
|
||||||
|
return {
|
||||||
|
appId,
|
||||||
|
apiKey,
|
||||||
|
userAgent,
|
||||||
|
apiUrl,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchLatest(site, page = 1, upcoming = false) {
|
||||||
|
const { apiUrl } = await fetchApiCredentials();
|
||||||
|
|
||||||
|
const res = await bhttp.post(apiUrl, {
|
||||||
requests: [
|
requests: [
|
||||||
{
|
{
|
||||||
indexName: 'all_scenes',
|
indexName: 'all_scenes',
|
||||||
|
|
@ -122,7 +159,7 @@ async function fetchLatest(site, page = 1, upcoming = false) {
|
||||||
encodeJSON: true,
|
encodeJSON: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
return scrape(apiRes.body.results[0].hits, site);
|
return scrape(res.body.results[0].hits, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchUpcoming(site) {
|
async function fetchUpcoming(site) {
|
||||||
|
|
@ -135,8 +172,38 @@ async function fetchScene(url, site) {
|
||||||
return scrapeScene(res.body.toString(), url, site);
|
return scrapeScene(res.body.toString(), url, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function fetchProfile(actorName) {
|
||||||
|
const { apiUrl } = await fetchApiCredentials();
|
||||||
|
const actorSlug = encodeURI(actorName);
|
||||||
|
|
||||||
|
const res = await bhttp.post(apiUrl, {
|
||||||
|
requests: [
|
||||||
|
{
|
||||||
|
indexName: 'all_actors',
|
||||||
|
params: `query=${actorSlug}`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}, {
|
||||||
|
headers: {
|
||||||
|
Referer: `https://www.evilangel.com/en/search?query=${actorSlug}&tab=actors`,
|
||||||
|
},
|
||||||
|
encodeJSON: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (res.statusCode === 200 && res.body.results[0].hits.length > 0) {
|
||||||
|
const actorData = res.body.results[0].hits.find(actor => actor.name === actorName);
|
||||||
|
|
||||||
|
if (actorData) {
|
||||||
|
return scrapeActor(actorData);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
fetchLatest,
|
fetchLatest,
|
||||||
fetchUpcoming,
|
fetchProfile,
|
||||||
fetchScene,
|
fetchScene,
|
||||||
|
fetchUpcoming,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,136 @@
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const Promise = require('bluebird');
|
||||||
|
const bhttp = require('bhttp');
|
||||||
|
const { JSDOM } = require('jsdom');
|
||||||
|
const cheerio = require('cheerio');
|
||||||
|
|
||||||
|
async function fetchPhotos(url) {
|
||||||
|
const res = await bhttp.get(url);
|
||||||
|
|
||||||
|
return res.body.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapePhotos(html) {
|
||||||
|
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||||
|
|
||||||
|
return $('.preview .imgLink').toArray().map((linkEl) => {
|
||||||
|
const url = $(linkEl).attr('href');
|
||||||
|
|
||||||
|
if (url.match('/join')) {
|
||||||
|
// URL links to join page instead of full photo, extract thumbnail
|
||||||
|
const src = $(linkEl).find('img').attr('src');
|
||||||
|
|
||||||
|
if (src.match('previews/')) {
|
||||||
|
// resource often serves full photo at a modifier URL anyway, add as primary source
|
||||||
|
const highRes = src
|
||||||
|
.replace('previews/', '')
|
||||||
|
.replace('_tb.jpg', '.jpg');
|
||||||
|
|
||||||
|
// keep original thumbnail as fallback in case full photo is not available
|
||||||
|
return [highRes, src];
|
||||||
|
}
|
||||||
|
|
||||||
|
return src;
|
||||||
|
}
|
||||||
|
|
||||||
|
// URL links to full photo
|
||||||
|
return url;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getPhotos(albumPath, siteDomain) {
|
||||||
|
const albumUrl = `https://${siteDomain}${albumPath}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const html = await fetchPhotos(albumUrl);
|
||||||
|
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||||
|
const photos = scrapePhotos(html);
|
||||||
|
|
||||||
|
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
|
||||||
|
|
||||||
|
const otherPhotos = await Promise.map(pages, async (page) => {
|
||||||
|
const pageUrl = `https://${siteDomain}${page}`;
|
||||||
|
const pageHtml = await fetchPhotos(pageUrl);
|
||||||
|
|
||||||
|
return scrapePhotos(pageHtml);
|
||||||
|
}, {
|
||||||
|
concurrency: 2,
|
||||||
|
});
|
||||||
|
|
||||||
|
return photos.concat(otherPhotos.flat());
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Failed to fetch ${siteDomain} photos from ${albumPath}: ${error.message}`);
|
||||||
|
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeActorSearch(html, url, actorName) {
|
||||||
|
const { document } = new JSDOM(html).window;
|
||||||
|
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
|
||||||
|
|
||||||
|
return actorLink ? actorLink.href : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeProfile(html, url, actorName, siteSlug) {
|
||||||
|
const { document } = new JSDOM(html).window;
|
||||||
|
|
||||||
|
const avatarEl = document.querySelector('img.actorPicture');
|
||||||
|
const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
|
||||||
|
|
||||||
|
const profile = {
|
||||||
|
name: actorName,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (avatarEl) {
|
||||||
|
// larger sizes usually available, provide fallbacks
|
||||||
|
const avatars = [
|
||||||
|
avatarEl.src.replace(/\d+x\d+/, '500x750'),
|
||||||
|
avatarEl.src.replace(/\d+x\d+/, '240x360'),
|
||||||
|
avatarEl.src.replace(/\d+x\d+/, '200x300'),
|
||||||
|
avatarEl.src,
|
||||||
|
];
|
||||||
|
|
||||||
|
profile.avatar = avatars;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
|
||||||
|
|
||||||
|
profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://${siteSlug}.com${el.href}`);
|
||||||
|
|
||||||
|
return profile;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchProfile(actorName, siteSlug, altSearchUrl) {
|
||||||
|
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
|
||||||
|
const searchUrl = altSearchUrl
|
||||||
|
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
|
||||||
|
: `https://www.${siteSlug}.com/en/search/${siteSlug}/actor/${actorSlug}`;
|
||||||
|
const searchRes = await bhttp.get(searchUrl);
|
||||||
|
|
||||||
|
if (searchRes.statusCode !== 200) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
|
||||||
|
|
||||||
|
if (actorUrl) {
|
||||||
|
const url = `https://${siteSlug}.com${actorUrl}`;
|
||||||
|
const actorRes = await bhttp.get(url);
|
||||||
|
|
||||||
|
if (actorRes.statusCode !== 200) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
getPhotos,
|
||||||
|
fetchProfile,
|
||||||
|
scrapeProfile,
|
||||||
|
};
|
||||||
|
|
@ -1,14 +1,11 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
// releases
|
// releases
|
||||||
const twentyonesextury = require('./21sextury');
|
|
||||||
const babes = require('./babes');
|
const babes = require('./babes');
|
||||||
const bang = require('./bang');
|
const bang = require('./bang');
|
||||||
const bangbros = require('./bangbros');
|
const bangbros = require('./bangbros');
|
||||||
const blowpass = require('./blowpass');
|
|
||||||
const dogfart = require('./dogfart');
|
const dogfart = require('./dogfart');
|
||||||
const digitalplayground = require('./digitalplayground');
|
const digitalplayground = require('./digitalplayground');
|
||||||
const evilangel = require('./evilangel');
|
|
||||||
const fakehub = require('./fakehub');
|
const fakehub = require('./fakehub');
|
||||||
const jayrock = require('./jayrock');
|
const jayrock = require('./jayrock');
|
||||||
const kink = require('./kink');
|
const kink = require('./kink');
|
||||||
|
|
@ -25,11 +22,14 @@ const teamskeet = require('./teamskeet');
|
||||||
const vixen = require('./vixen');
|
const vixen = require('./vixen');
|
||||||
|
|
||||||
// releases and profiles
|
// releases and profiles
|
||||||
const ddfnetwork = require('./ddfnetwork');
|
const blowpass = require('./blowpass');
|
||||||
const brazzers = require('./brazzers');
|
const brazzers = require('./brazzers');
|
||||||
|
const ddfnetwork = require('./ddfnetwork');
|
||||||
|
const evilangel = require('./evilangel');
|
||||||
const julesjordan = require('./julesjordan');
|
const julesjordan = require('./julesjordan');
|
||||||
const kellymadison = require('./kellymadison');
|
const kellymadison = require('./kellymadison');
|
||||||
const legalporno = require('./legalporno');
|
const legalporno = require('./legalporno');
|
||||||
|
const twentyonesextury = require('./21sextury');
|
||||||
const xempire = require('./xempire');
|
const xempire = require('./xempire');
|
||||||
|
|
||||||
// profiles
|
// profiles
|
||||||
|
|
@ -71,7 +71,10 @@ module.exports = {
|
||||||
},
|
},
|
||||||
actors: {
|
actors: {
|
||||||
// ordered by data priority
|
// ordered by data priority
|
||||||
|
'21sextury': twentyonesextury,
|
||||||
|
evilangel,
|
||||||
xempire,
|
xempire,
|
||||||
|
blowpass,
|
||||||
julesjordan,
|
julesjordan,
|
||||||
brazzers,
|
brazzers,
|
||||||
legalporno,
|
legalporno,
|
||||||
|
|
|
||||||
|
|
@ -1,79 +1,10 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const Promise = require('bluebird');
|
|
||||||
const bhttp = require('bhttp');
|
const bhttp = require('bhttp');
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const { JSDOM } = require('jsdom');
|
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
const defaultTags = {
|
const { getPhotos, fetchProfile } = require('./gamma');
|
||||||
hardx: [],
|
|
||||||
darkx: ['interracial'],
|
|
||||||
eroticax: [],
|
|
||||||
lesbianx: ['lesbian'],
|
|
||||||
allblackx: ['ebony', 'bbc'],
|
|
||||||
};
|
|
||||||
|
|
||||||
async function fetchPhotos(url) {
|
|
||||||
const res = await bhttp.get(url);
|
|
||||||
|
|
||||||
return res.body.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapePhotos(html) {
|
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
|
||||||
|
|
||||||
return $('.preview .imgLink').toArray().map((linkEl) => {
|
|
||||||
const url = $(linkEl).attr('href');
|
|
||||||
|
|
||||||
if (url.match('/join')) {
|
|
||||||
// URL links to join page instead of full photo, extract thumbnail
|
|
||||||
const src = $(linkEl).find('img').attr('src');
|
|
||||||
|
|
||||||
if (src.match('previews/')) {
|
|
||||||
// resource often serves full photo at a modifier URL anyway, add as primary source
|
|
||||||
const highRes = src
|
|
||||||
.replace('previews/', '')
|
|
||||||
.replace('_tb.jpg', '.jpg');
|
|
||||||
|
|
||||||
// keep original thumbnail as fallback in case full photo is not available
|
|
||||||
return [highRes, src];
|
|
||||||
}
|
|
||||||
|
|
||||||
return src;
|
|
||||||
}
|
|
||||||
|
|
||||||
// URL links to full photo
|
|
||||||
return url;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getPhotos(albumPath, siteDomain) {
|
|
||||||
const albumUrl = `https://${siteDomain}${albumPath}`;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const html = await fetchPhotos(albumUrl);
|
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
|
||||||
const photos = scrapePhotos(html);
|
|
||||||
|
|
||||||
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
|
|
||||||
|
|
||||||
const otherPhotos = await Promise.map(pages, async (page) => {
|
|
||||||
const pageUrl = `https://${siteDomain}${page}`;
|
|
||||||
const pageHtml = await fetchPhotos(pageUrl);
|
|
||||||
|
|
||||||
return scrapePhotos(pageHtml);
|
|
||||||
}, {
|
|
||||||
concurrency: 2,
|
|
||||||
});
|
|
||||||
|
|
||||||
return photos.concat(otherPhotos.flat());
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Failed to fetch XEmpire photos from ${albumPath}: ${error.message}`);
|
|
||||||
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrape(html, site) {
|
function scrape(html, site) {
|
||||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||||
|
|
@ -154,8 +85,7 @@ async function scrapeScene(html, url, site) {
|
||||||
|
|
||||||
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
|
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
|
||||||
|
|
||||||
const rawTags = data.keywords.split(', ');
|
const tags = data.keywords.split(', ');
|
||||||
const tags = [...defaultTags[siteSlug], ...rawTags];
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
url: `${siteUrl}/en/video/${new URL(url).pathname.split('/').slice(-2).join('/')}`,
|
url: `${siteUrl}/en/video/${new URL(url).pathname.split('/').slice(-2).join('/')}`,
|
||||||
|
|
@ -181,31 +111,6 @@ async function scrapeScene(html, url, site) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeActorSearch(html, url, actorName) {
|
|
||||||
const { document } = new JSDOM(html).window;
|
|
||||||
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
|
|
||||||
|
|
||||||
return actorLink ? actorLink.href : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeProfile(html, url, actorName) {
|
|
||||||
const { document } = new JSDOM(html).window;
|
|
||||||
|
|
||||||
const avatarEl = document.querySelector('img.actorPicture');
|
|
||||||
const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)');
|
|
||||||
|
|
||||||
const profile = {
|
|
||||||
name: actorName,
|
|
||||||
};
|
|
||||||
|
|
||||||
if (avatarEl) profile.avatar = avatarEl.src;
|
|
||||||
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
|
|
||||||
|
|
||||||
profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://xempire.com${el.href}`);
|
|
||||||
|
|
||||||
return profile;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchLatest(site, page = 1) {
|
async function fetchLatest(site, page = 1) {
|
||||||
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
|
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
|
||||||
|
|
||||||
|
|
@ -224,34 +129,13 @@ async function fetchScene(url, site) {
|
||||||
return scrapeScene(res.body.toString(), url, site);
|
return scrapeScene(res.body.toString(), url, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function xEmpireFetchProfile(actorName) {
|
||||||
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
|
return fetchProfile(actorName, 'xempire');
|
||||||
const searchUrl = `https://www.xempire.com/en/search/xempire/actor/${actorSlug}`;
|
|
||||||
const searchRes = await bhttp.get(searchUrl);
|
|
||||||
|
|
||||||
if (searchRes.statusCode !== 200) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
|
|
||||||
|
|
||||||
if (actorUrl) {
|
|
||||||
const url = `https://xempire.com${actorUrl}`;
|
|
||||||
const actorRes = await bhttp.get(url);
|
|
||||||
|
|
||||||
if (actorRes.statusCode !== 200) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return scrapeProfile(actorRes.body.toString(), url, actorName);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
fetchLatest,
|
fetchLatest,
|
||||||
fetchProfile,
|
fetchProfile: xEmpireFetchProfile,
|
||||||
fetchUpcoming,
|
fetchUpcoming,
|
||||||
fetchScene,
|
fetchScene,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,7 @@ function destructConfigNetworks(networks) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function findSiteByUrl(url) {
|
async function findSiteByUrl(url) {
|
||||||
const { hostname, origin } = new URL(url);
|
const { hostname } = new URL(url);
|
||||||
const domain = hostname.replace(/www.|tour./, '');
|
const domain = hostname.replace(/www.|tour./, '');
|
||||||
|
|
||||||
const sites = await knex('sites')
|
const sites = await knex('sites')
|
||||||
|
|
@ -69,8 +69,8 @@ async function findSiteByUrl(url) {
|
||||||
'sites.*',
|
'sites.*',
|
||||||
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
|
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
|
||||||
)
|
)
|
||||||
.where('sites.url', 'like', `${domain}`)
|
.where('sites.url', 'like', `%${domain}`)
|
||||||
.orWhere('sites.url', 'like', `${origin}`)
|
.orWhere('sites.url', 'like', url)
|
||||||
.orWhere('sites.url', url);
|
.orWhere('sites.url', url);
|
||||||
|
|
||||||
if (sites.length > 0) {
|
if (sites.length > 0) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue