forked from DebaucheryLibrarian/traxxx
Resolving actor birth and residence place before storage. Layout improvements.
This commit is contained in:
parent
4be508b388
commit
0dbe853f39
|
|
@ -33,7 +33,7 @@
|
|||
v-if="actor.aliases.length"
|
||||
class="bio-item"
|
||||
>
|
||||
<dfn class="bio-heading">Also known as</dfn>
|
||||
<dfn class="bio-label">Also known as</dfn>
|
||||
<span>{{ actor.aliases.join(', ') }}</span>
|
||||
</li>
|
||||
|
||||
|
|
@ -41,7 +41,7 @@
|
|||
v-if="actor.birthdate"
|
||||
class="bio-item"
|
||||
>
|
||||
<dfn class="bio-heading"><Icon icon="cake" />Birthdate</dfn>
|
||||
<dfn class="bio-label"><Icon icon="cake" />Birthdate</dfn>
|
||||
|
||||
<span
|
||||
v-if="actor.birthdate"
|
||||
|
|
@ -50,50 +50,55 @@
|
|||
</li>
|
||||
|
||||
<li
|
||||
v-if="actor.birthCountry || actor.birthPlace"
|
||||
v-if="actor.origin"
|
||||
class="bio-item birth"
|
||||
>
|
||||
<dfn class="bio-heading"><Icon icon="home2" />Born in</dfn>
|
||||
<dfn class="bio-label"><Icon icon="home2" />Born in</dfn>
|
||||
|
||||
<span>
|
||||
<span
|
||||
v-if="actor.birthPlace"
|
||||
class="place"
|
||||
>{{ actor.birthPlace }}</span>
|
||||
v-if="actor.origin.city"
|
||||
class="city"
|
||||
>{{ actor.origin.city }}</span><span
|
||||
v-if="actor.origin.state && actor.origin.country && actor.origin.country.alpha2 === 'US'"
|
||||
class="state"
|
||||
>{{ actor.origin.city ? `, ${actor.origin.state}` : actor.origin.state }}</span>
|
||||
|
||||
<span
|
||||
v-if="actor.birthCountry"
|
||||
v-if="actor.origin.country"
|
||||
class="country birthcountry"
|
||||
>
|
||||
<img
|
||||
class="flag"
|
||||
:src="`/img/flags/${actor.birthCountry.alpha2.toLowerCase()}.png`"
|
||||
>{{ actor.birthCountry.name }}
|
||||
:src="`/img/flags/${actor.origin.country.alpha2.toLowerCase()}.png`"
|
||||
>{{ actor.origin.country.name }}
|
||||
</span>
|
||||
</span>
|
||||
</li>
|
||||
|
||||
<li
|
||||
v-if="actor.residenceCountry || actor.residencePlace"
|
||||
v-if="actor.residence"
|
||||
class="bio-item residence"
|
||||
>
|
||||
<dfn class="bio-heading"><Icon icon="location" />Lives in</dfn>
|
||||
<dfn class="bio-label"><Icon icon="location" />Lives in</dfn>
|
||||
|
||||
<span>
|
||||
<span
|
||||
v-if="actor.residencePlace"
|
||||
class="place"
|
||||
>{{ actor.residencePlace }}</span>
|
||||
v-if="actor.residence.city"
|
||||
class="city"
|
||||
>{{ actor.residence.city }}</span><span
|
||||
v-if="actor.residence.state && actor.residence.country && actor.residence.country.alpha2 === 'US'"
|
||||
class="state"
|
||||
>{{ actor.residence.city ? `, ${actor.residence.state}` : actor.residence.state }}</span>
|
||||
|
||||
<span
|
||||
v-if="actor.residenceCountry"
|
||||
v-if="actor.residence.country"
|
||||
class="country"
|
||||
>
|
||||
<img
|
||||
v-if="actor.residenceCountry"
|
||||
class="flag"
|
||||
:src="`/img/flags/${actor.residenceCountry.alpha2.toLowerCase()}.png`"
|
||||
>{{ actor.residenceCountry.name }}
|
||||
:src="`/img/flags/${actor.residence.country.alpha2.toLowerCase()}.png`"
|
||||
>{{ actor.residence.country.name }}
|
||||
</span>
|
||||
</span>
|
||||
</li>
|
||||
|
|
@ -104,7 +109,7 @@
|
|||
v-if="actor.ethnicity"
|
||||
class="bio-item ethnicity"
|
||||
>
|
||||
<dfn class="bio-heading"><Icon icon="earth2" />Ethnicity</dfn>
|
||||
<dfn class="bio-label"><Icon icon="earth2" />Ethnicity</dfn>
|
||||
<span>{{ actor.ethnicity }}</span>
|
||||
</li>
|
||||
|
||||
|
|
@ -113,7 +118,7 @@
|
|||
title="bust-waist-hip"
|
||||
class="bio-item"
|
||||
>
|
||||
<dfn class="bio-heading"><Icon icon="ruler" />Sizes</dfn>
|
||||
<dfn class="bio-label"><Icon icon="ruler" />Sizes</dfn>
|
||||
<span>
|
||||
<Icon
|
||||
v-if="actor.naturalBoobs === false"
|
||||
|
|
@ -128,7 +133,7 @@
|
|||
v-if="actor.height"
|
||||
class="bio-item height"
|
||||
>
|
||||
<dfn class="bio-heading"><Icon icon="height" />Height</dfn>
|
||||
<dfn class="bio-label"><Icon icon="height" />Height</dfn>
|
||||
<span>
|
||||
<span class="height-metric">{{ actor.height }} cm</span>
|
||||
<span class="height-imperial">{{ imperialHeight.feet }}' {{ imperialHeight.inches }}"</span>
|
||||
|
|
@ -139,7 +144,7 @@
|
|||
v-if="actor.weight"
|
||||
class="bio-item weight"
|
||||
>
|
||||
<dfn class="bio-heading"><Icon icon="scale" />Weight</dfn>
|
||||
<dfn class="bio-label"><Icon icon="scale" />Weight</dfn>
|
||||
|
||||
<span>
|
||||
<span class="weight-metric">{{ actor.weight }} kg</span>
|
||||
|
|
@ -147,6 +152,34 @@
|
|||
</span>
|
||||
</li>
|
||||
|
||||
<li
|
||||
v-if="actor.hasTattoos"
|
||||
class="bio-item tattoos"
|
||||
>
|
||||
<dfn class="bio-label"><Icon icon="flower" />Tattoos</dfn>
|
||||
|
||||
<span
|
||||
v-if="actor.tattoos"
|
||||
v-tooltip="actor.tattoos"
|
||||
class="bio-value"
|
||||
>{{ actor.tattoos }}</span>
|
||||
<span v-else>Yes</span>
|
||||
</li>
|
||||
|
||||
<li
|
||||
v-if="actor.hasPiercings"
|
||||
class="bio-item piercings"
|
||||
>
|
||||
<dfn class="bio-label"><Icon icon="trophy4" />Piercings</dfn>
|
||||
|
||||
<span
|
||||
v-if="actor.piercings"
|
||||
v-tooltip="actor.piercings"
|
||||
class="bio-value"
|
||||
>{{ actor.piercings }}</span>
|
||||
<span v-else>Yes</span>
|
||||
</li>
|
||||
|
||||
<li class="bio-item scraped">Updated on {{ formatDate(actor.scrapedAt, 'YYYY-MM-DD HH:mm') }}</li>
|
||||
</ul>
|
||||
|
||||
|
|
@ -154,8 +187,7 @@
|
|||
<p
|
||||
v-if="actor.description"
|
||||
class="description"
|
||||
:class="{ expanded }"
|
||||
@click="expanded = !expanded"
|
||||
@wheel.prevent="scrollDescription"
|
||||
>{{ actor.description }}</p>
|
||||
|
||||
<li
|
||||
|
|
@ -190,10 +222,10 @@
|
|||
<div
|
||||
v-if="actor.photos && actor.photos.length > 0"
|
||||
class="photos-container"
|
||||
@wheel.prevent="scrollPhotos"
|
||||
>
|
||||
<div
|
||||
class="photos"
|
||||
:class="{ wide: actor.photos.length > 2 }"
|
||||
@wheel.prevent="scrollPhotos"
|
||||
>
|
||||
<a
|
||||
|
|
@ -259,13 +291,19 @@ function scrollPhotos(event) {
|
|||
event.currentTarget.scrollLeft += event.deltaY; // eslint-disable-line no-param-reassign
|
||||
}
|
||||
|
||||
function scrollDescription(event) {
|
||||
event.currentTarget.scrollTop += event.deltaY; // eslint-disable-line no-param-reassign
|
||||
}
|
||||
|
||||
async function mounted() {
|
||||
[[this.actor]] = await Promise.all([
|
||||
this.$store.dispatch('fetchActors', this.$route.params.actorSlug),
|
||||
this.fetchReleases(),
|
||||
]);
|
||||
|
||||
this.pageTitle = this.actor.name;
|
||||
if (this.actor) {
|
||||
this.pageTitle = this.actor.name;
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
|
|
@ -289,6 +327,7 @@ export default {
|
|||
mounted,
|
||||
methods: {
|
||||
fetchReleases,
|
||||
scrollDescription,
|
||||
scrollPhotos,
|
||||
},
|
||||
};
|
||||
|
|
@ -315,12 +354,12 @@ export default {
|
|||
|
||||
.avatar-link {
|
||||
font-size: 0;
|
||||
padding: 1rem;
|
||||
padding: 1rem 0 1rem 1rem;
|
||||
}
|
||||
|
||||
.avatar {
|
||||
height: 15rem;
|
||||
width: 15rem;
|
||||
height: 12rem;
|
||||
width: 12rem;
|
||||
flex-shrink: 0;
|
||||
margin: 0 1rem 0 0;
|
||||
object-fit: cover;
|
||||
|
|
@ -330,10 +369,10 @@ export default {
|
|||
|
||||
.bio {
|
||||
flex-grow: 1;
|
||||
min-width: 18rem;
|
||||
min-width: 20rem;
|
||||
box-sizing: border-box;
|
||||
padding: 1rem;
|
||||
margin: 0 3rem 0 0;
|
||||
margin: 0 2rem 0 0;
|
||||
}
|
||||
|
||||
.bio-header {
|
||||
|
|
@ -350,20 +389,29 @@ export default {
|
|||
line-height: 1.75;
|
||||
text-align: right;
|
||||
font-size: .9rem;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.bio-heading {
|
||||
.bio-label {
|
||||
color: $highlight;
|
||||
font-weight: normal;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
margin: 0 1rem 0 0;
|
||||
flex-shrink: 0;
|
||||
font-style: normal;
|
||||
font-weight: bold;
|
||||
|
||||
.icon {
|
||||
fill: $highlight;
|
||||
margin: 0 .5rem 0 0;
|
||||
margin: 0 .5rem .5rem 0;
|
||||
}
|
||||
}
|
||||
|
||||
.bio-value {
|
||||
white-space: nowrap;
|
||||
text-overflow: ellipsis;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.flag {
|
||||
margin: 0 .25rem 0 0;
|
||||
}
|
||||
|
|
@ -403,21 +451,6 @@ export default {
|
|||
padding: 0 0 0 .5rem;
|
||||
border-left: solid 1px $highlight-weak;
|
||||
margin: 0 0 0 .5rem;
|
||||
|
||||
/*
|
||||
&::before {
|
||||
content: ' (';
|
||||
}
|
||||
|
||||
&::after {
|
||||
content: ')';
|
||||
}
|
||||
|
||||
&::before,
|
||||
&::after {
|
||||
color: $highlight;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
.country {
|
||||
|
|
@ -440,18 +473,19 @@ export default {
|
|||
max-height: 10rem;
|
||||
flex-grow: 1;
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
display: block;
|
||||
box-sizing: border-box;
|
||||
padding: 1rem 0 0 0;
|
||||
margin: 0 2rem 0 0;
|
||||
line-height: 1.5;
|
||||
text-overflow: ellipsis;
|
||||
font-size: .9rem;
|
||||
overflow: hidden;
|
||||
cursor: pointer;
|
||||
overflow: auto;
|
||||
scrollbar-width: none;
|
||||
|
||||
&.expanded {
|
||||
overflow: visible;
|
||||
&::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -501,7 +535,6 @@ export default {
|
|||
}
|
||||
|
||||
.photos {
|
||||
max-width: 35vw;
|
||||
display: inline-grid;
|
||||
grid-template-columns: repeat(auto-fit, 15rem);
|
||||
grid-gap: .5rem;
|
||||
|
|
@ -535,6 +568,12 @@ export default {
|
|||
}
|
||||
}
|
||||
|
||||
@media(min-width: $breakpoint3) {
|
||||
.photos.wide {
|
||||
max-width: 35vw;
|
||||
}
|
||||
}
|
||||
|
||||
@media(max-width: $breakpoint3) {
|
||||
.profile .avatar-link,
|
||||
.social {
|
||||
|
|
@ -591,10 +630,13 @@ export default {
|
|||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
.place,
|
||||
.city,
|
||||
.state,
|
||||
.ethnicity,
|
||||
.residence,
|
||||
.weight,
|
||||
.tattoos,
|
||||
.piercings,
|
||||
.scraped {
|
||||
display: none;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
z-index: 10000;
|
||||
|
||||
.tooltip-inner {
|
||||
max-width: 20rem;
|
||||
background: #222;
|
||||
color: white;
|
||||
border-radius: 16px;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
<!-- Generated by IcoMoon.io -->
|
||||
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
|
||||
<title>flower</title>
|
||||
<path d="M15.801 5.902c-0.835-2.568-3.368-2.828-5.323-0.876 1.252-2.463 0.222-4.792-2.478-4.792s-3.73 2.329-2.478 4.792c-1.955-1.951-4.488-1.692-5.323 0.876s1.062 4.267 3.791 3.838c-2.46 1.257-2.996 3.746-0.811 5.333s4.386 0.308 4.821-2.42c0.435 2.728 2.637 4.007 4.822 2.42s1.648-4.077-0.812-5.333c2.729 0.429 4.626-1.27 3.792-3.838zM8 10.717c-1.277 0-2.313-1.035-2.313-2.313s1.035-2.313 2.313-2.313c1.277 0 2.313 1.035 2.313 2.313s-1.036 2.313-2.313 2.313z"></path>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 628 B |
|
|
@ -0,0 +1,5 @@
|
|||
<!-- Generated by IcoMoon.io -->
|
||||
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
|
||||
<title>trophy4</title>
|
||||
<path d="M13 2h-10l-3 4 8 9-2.222-8h4.444l-2.222 8 8-9-3-4zM1.25 6l2.282-3.043 1.69 3.043h-3.972zM5.778 6l1.667-3h1.111l1.667 3h-4.444zM12.468 2.957l2.282 3.043h-3.972l1.69-3.043z"></path>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 350 B |
|
|
@ -20,18 +20,19 @@ exports.up = knex => Promise.resolve()
|
|||
table.string('gender', 18);
|
||||
table.text('description');
|
||||
|
||||
table.string('birth_city');
|
||||
table.string('birth_state');
|
||||
table.string('birth_country_alpha2', 2)
|
||||
.references('alpha2')
|
||||
.inTable('countries');
|
||||
|
||||
table.string('ethnicity');
|
||||
table.string('birth_place');
|
||||
|
||||
table.string('residence_city');
|
||||
table.string('residence_state');
|
||||
table.string('residence_country_alpha2', 2)
|
||||
.references('alpha2')
|
||||
.inTable('countries');
|
||||
|
||||
table.string('residence_place');
|
||||
table.string('ethnicity');
|
||||
|
||||
table.string('bust', 10);
|
||||
table.integer('waist', 3);
|
||||
|
|
|
|||
|
|
@ -599,11 +599,11 @@
|
|||
}
|
||||
.profile .avatar-link[data-v-677a8360] {
|
||||
font-size: 0;
|
||||
padding: 1rem;
|
||||
padding: 1rem 0 1rem 1rem;
|
||||
}
|
||||
.profile .avatar[data-v-677a8360] {
|
||||
height: 15rem;
|
||||
width: 15rem;
|
||||
height: 12rem;
|
||||
width: 12rem;
|
||||
flex-shrink: 0;
|
||||
margin: 0 1rem 0 0;
|
||||
-o-object-fit: cover;
|
||||
|
|
@ -613,10 +613,10 @@
|
|||
}
|
||||
.bio[data-v-677a8360] {
|
||||
flex-grow: 1;
|
||||
min-width: 18rem;
|
||||
min-width: 20rem;
|
||||
box-sizing: border-box;
|
||||
padding: 1rem;
|
||||
margin: 0 3rem 0 0;
|
||||
margin: 0 2rem 0 0;
|
||||
}
|
||||
.bio-header[data-v-677a8360] {
|
||||
display: flex;
|
||||
|
|
@ -631,16 +631,24 @@
|
|||
line-height: 1.75;
|
||||
text-align: right;
|
||||
font-size: .9rem;
|
||||
}
|
||||
.bio-label[data-v-677a8360] {
|
||||
color: rgba(255, 255, 255, 0.5);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
margin: 0 1rem 0 0;
|
||||
flex-shrink: 0;
|
||||
font-style: normal;
|
||||
font-weight: bold;
|
||||
}
|
||||
.bio-heading[data-v-677a8360] {
|
||||
color: rgba(255, 255, 255, 0.5);
|
||||
font-weight: normal;
|
||||
font-style: normal;
|
||||
}
|
||||
.bio-heading .icon[data-v-677a8360] {
|
||||
.bio-label .icon[data-v-677a8360] {
|
||||
fill: rgba(255, 255, 255, 0.5);
|
||||
margin: 0 .5rem 0 0;
|
||||
margin: 0 .5rem .5rem 0;
|
||||
}
|
||||
.bio-value[data-v-677a8360] {
|
||||
white-space: nowrap;
|
||||
text-overflow: ellipsis;
|
||||
overflow: hidden;
|
||||
}
|
||||
.flag[data-v-677a8360] {
|
||||
margin: 0 .25rem 0 0;
|
||||
|
|
@ -674,20 +682,6 @@
|
|||
padding: 0 0 0 .5rem;
|
||||
border-left: solid 1px rgba(255, 255, 255, 0.2);
|
||||
margin: 0 0 0 .5rem;
|
||||
/*
|
||||
&::before {
|
||||
content: ' (';
|
||||
}
|
||||
|
||||
&::after {
|
||||
content: ')';
|
||||
}
|
||||
|
||||
&::before,
|
||||
&::after {
|
||||
color: $highlight;
|
||||
}
|
||||
*/
|
||||
}
|
||||
.country[data-v-677a8360] {
|
||||
display: block;
|
||||
|
|
@ -706,18 +700,19 @@
|
|||
max-height: 10rem;
|
||||
flex-grow: 1;
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
display: block;
|
||||
box-sizing: border-box;
|
||||
padding: 1rem 0 0 0;
|
||||
margin: 0 2rem 0 0;
|
||||
line-height: 1.5;
|
||||
text-overflow: ellipsis;
|
||||
font-size: .9rem;
|
||||
overflow: hidden;
|
||||
cursor: pointer;
|
||||
overflow: auto;
|
||||
scrollbar-width: none;
|
||||
}
|
||||
.description.expanded[data-v-677a8360] {
|
||||
overflow: visible;
|
||||
.description[data-v-677a8360]::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
.social[data-v-677a8360] {
|
||||
display: block;
|
||||
|
|
@ -757,7 +752,6 @@
|
|||
margin: 0 .5rem 0 0;
|
||||
}
|
||||
.photos[data-v-677a8360] {
|
||||
max-width: 35vw;
|
||||
display: inline-grid;
|
||||
grid-template-columns: repeat(auto-fit, 15rem);
|
||||
grid-gap: .5rem;
|
||||
|
|
@ -787,6 +781,11 @@
|
|||
display: none;
|
||||
}
|
||||
}
|
||||
@media (min-width: 1200px) {
|
||||
.photos.wide[data-v-677a8360] {
|
||||
max-width: 35vw;
|
||||
}
|
||||
}
|
||||
@media (max-width: 1200px) {
|
||||
.profile .avatar-link[data-v-677a8360],
|
||||
.social[data-v-677a8360] {
|
||||
|
|
@ -833,10 +832,13 @@
|
|||
.bio-header[data-v-677a8360] {
|
||||
margin: 1rem 0;
|
||||
}
|
||||
.place[data-v-677a8360],
|
||||
.city[data-v-677a8360],
|
||||
.state[data-v-677a8360],
|
||||
.ethnicity[data-v-677a8360],
|
||||
.residence[data-v-677a8360],
|
||||
.weight[data-v-677a8360],
|
||||
.tattoos[data-v-677a8360],
|
||||
.piercings[data-v-677a8360],
|
||||
.scraped[data-v-677a8360] {
|
||||
display: none;
|
||||
}
|
||||
|
|
@ -901,6 +903,7 @@
|
|||
display: block !important;
|
||||
z-index: 10000; }
|
||||
.tooltip .tooltip-inner {
|
||||
max-width: 20rem;
|
||||
background: #222;
|
||||
color: white;
|
||||
border-radius: 16px;
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB After Width: | Height: | Size: 59 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB After Width: | Height: | Size: 59 KiB |
|
|
@ -912,7 +912,7 @@ const countries = [
|
|||
alpha2: 'RO',
|
||||
},
|
||||
{
|
||||
name: 'Russian Federation',
|
||||
name: 'Russia',
|
||||
code: '643',
|
||||
alpha2: 'RU',
|
||||
},
|
||||
|
|
|
|||
130
src/actors.js
130
src/actors.js
|
|
@ -6,6 +6,7 @@ const knex = require('./knex');
|
|||
const argv = require('./argv');
|
||||
const scrapers = require('./scrapers/scrapers');
|
||||
const whereOr = require('./utils/where-or');
|
||||
const resolvePlace = require('./utils/resolve-place');
|
||||
const { createActorMediaDirectory, storeAvatars } = require('./media');
|
||||
|
||||
async function curateActor(actor) {
|
||||
|
|
@ -18,27 +19,15 @@ async function curateActor(actor) {
|
|||
.where({ domain: 'actors', target_id: actor.id }),
|
||||
]);
|
||||
|
||||
return {
|
||||
const curatedActor = {
|
||||
id: actor.id,
|
||||
gender: actor.gender,
|
||||
name: actor.name,
|
||||
description: actor.description,
|
||||
birthdate: actor.birthdate && new Date(actor.birthdate),
|
||||
country: actor.country_alpha2,
|
||||
residencePlace: actor.residence_place,
|
||||
residenceCountry: actor.residence_country_alpha2
|
||||
? {
|
||||
alpha2: actor.residence_country_alpha2,
|
||||
name: actor.residence_country_name,
|
||||
}
|
||||
: null,
|
||||
birthPlace: actor.birth_place,
|
||||
birthCountry: actor.birth_country_alpha2
|
||||
? {
|
||||
alpha2: actor.birth_country_alpha2,
|
||||
name: actor.birth_country_name,
|
||||
}
|
||||
: null,
|
||||
origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null,
|
||||
residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null,
|
||||
ethnicity: actor.ethnicity,
|
||||
height: actor.height,
|
||||
weight: actor.weight,
|
||||
|
|
@ -50,9 +39,35 @@ async function curateActor(actor) {
|
|||
slug: actor.slug,
|
||||
avatar: photos.find(photo => photo.role === 'avatar'),
|
||||
photos: photos.filter(photo => photo.role === 'photo'),
|
||||
hasTattoos: actor.has_tattoos,
|
||||
hasPiercings: actor.has_piercings,
|
||||
tattoos: actor.tattoos,
|
||||
piercings: actor.piercings,
|
||||
social,
|
||||
scrapedAt: actor.scraped_at,
|
||||
};
|
||||
|
||||
if (actor.birth_city) curatedActor.origin.city = actor.birth_city;
|
||||
if (actor.birth_state) curatedActor.origin.state = actor.birth_state;
|
||||
|
||||
if (actor.birth_country_alpha2) {
|
||||
curatedActor.origin.country = {
|
||||
alpha2: actor.birth_country_alpha2,
|
||||
name: actor.birth_country_name,
|
||||
};
|
||||
}
|
||||
|
||||
if (actor.residence_city) curatedActor.residence.city = actor.residence_city;
|
||||
if (actor.residence_state) curatedActor.residence.state = actor.residence_state;
|
||||
|
||||
if (actor.residence_country_alpha2) {
|
||||
curatedActor.residence.country = {
|
||||
alpha2: actor.residence_country_alpha2,
|
||||
name: actor.residence_country_name,
|
||||
};
|
||||
}
|
||||
|
||||
return curatedActor;
|
||||
}
|
||||
|
||||
function curateActors(releases) {
|
||||
|
|
@ -70,10 +85,6 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
|
|||
description: actor.description,
|
||||
gender: actor.gender,
|
||||
ethnicity: actor.ethnicity,
|
||||
birth_country_alpha2: actor.birthCountry,
|
||||
residence_country_alpha2: actor.residenceCountry,
|
||||
birth_place: actor.birthPlace,
|
||||
residence_place: actor.residencePlace,
|
||||
bust: actor.bust,
|
||||
waist: actor.waist,
|
||||
hip: actor.hip,
|
||||
|
|
@ -92,6 +103,18 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
|
|||
curatedActor.id = actor.id;
|
||||
}
|
||||
|
||||
if (actor.birthPlace) {
|
||||
curatedActor.birth_city = actor.birthPlace.city;
|
||||
curatedActor.birth_state = actor.birthPlace.state;
|
||||
curatedActor.birth_country_alpha2 = actor.birthPlace.country;
|
||||
}
|
||||
|
||||
if (actor.residencePlace) {
|
||||
curatedActor.residence_city = actor.residencePlace.city;
|
||||
curatedActor.residence_state = actor.residencePlace.state;
|
||||
curatedActor.residence_country_alpha2 = actor.residencePlace.country;
|
||||
}
|
||||
|
||||
if (scraped) {
|
||||
curatedActor.scraped_at = new Date();
|
||||
curatedActor.scrape_success = scrapeSuccess;
|
||||
|
|
@ -102,7 +125,7 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
|
|||
|
||||
function curateSocialEntry(url, actorId) {
|
||||
const { hostname, origin, pathname } = new URL(url);
|
||||
const platform = ['facebook', 'twitter', 'instagram', 'tumblr', 'snapchat', 'amazon', 'youtube'].find(platformName => hostname.match(platformName));
|
||||
const platform = ['facebook', 'twitter', 'instagram', 'tumblr', 'snapchat', 'amazon', 'youtube', 'fancentro'].find(platformName => hostname.match(platformName));
|
||||
|
||||
return {
|
||||
url: `${origin}${pathname}`,
|
||||
|
|
@ -184,8 +207,8 @@ async function updateActor(actor, scraped = false, scrapeSuccess = false) {
|
|||
return actorEntry;
|
||||
}
|
||||
|
||||
function mergeProfiles(profiles, actor) {
|
||||
return profiles.reduce((prevProfile, profile) => {
|
||||
async function mergeProfiles(profiles, actor) {
|
||||
const mergedProfile = profiles.reduce((prevProfile, profile) => {
|
||||
if (profile === null) {
|
||||
return prevProfile;
|
||||
}
|
||||
|
|
@ -196,21 +219,19 @@ function mergeProfiles(profiles, actor) {
|
|||
description: prevProfile.description || profile.description,
|
||||
gender: prevProfile.gender || profile.gender,
|
||||
birthdate: Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
|
||||
birthCountry: prevProfile.birthCountry || profile.birthCountry,
|
||||
residenceCountry: prevProfile.residenceCountry || profile.residenceCountry,
|
||||
birthPlace: prevProfile.birthPlace || profile.birthPlace,
|
||||
residencePlace: prevProfile.residencePlace || profile.residencePlace,
|
||||
ethnicity: prevProfile.ethnicity || profile.ethnicity,
|
||||
bust: prevProfile.bust || profile.bust,
|
||||
waist: prevProfile.waist || profile.waist,
|
||||
hip: prevProfile.hip || profile.hip,
|
||||
naturalBoobs: prevProfile.naturalBoobs || profile.naturalBoobs,
|
||||
naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs,
|
||||
height: prevProfile.height || profile.height,
|
||||
weight: prevProfile.weight || profile.weight,
|
||||
hair: prevProfile.hair || profile.hair,
|
||||
eyes: prevProfile.eyes || profile.eyes,
|
||||
hasPiercings: prevProfile.hasPiercings || profile.hasPiercings,
|
||||
hasTattoos: prevProfile.hasTattoos || profile.hasTattoos,
|
||||
hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings,
|
||||
hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos,
|
||||
piercings: prevProfile.piercings || profile.piercings,
|
||||
tattoos: prevProfile.tattoos || profile.tattoos,
|
||||
social: prevProfile.social.concat(profile.social || []),
|
||||
|
|
@ -220,6 +241,16 @@ function mergeProfiles(profiles, actor) {
|
|||
social: [],
|
||||
avatars: [],
|
||||
});
|
||||
|
||||
const [birthPlace, residencePlace] = await Promise.all([
|
||||
resolvePlace(mergedProfile.birthPlace),
|
||||
resolvePlace(mergedProfile.residencePlace),
|
||||
]);
|
||||
|
||||
mergedProfile.birthPlace = birthPlace;
|
||||
mergedProfile.residencePlace = residencePlace;
|
||||
|
||||
return mergedProfile;
|
||||
}
|
||||
|
||||
async function scrapeActors(actorNames) {
|
||||
|
|
@ -228,35 +259,44 @@ async function scrapeActors(actorNames) {
|
|||
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
|
||||
|
||||
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
|
||||
const profiles = await Promise.all(
|
||||
Object.values(scrapers.actors)
|
||||
.map(scraper => scraper.fetchProfile(actorEntry ? actorEntry.name : actorName)),
|
||||
);
|
||||
const profiles = await Promise.map(Object.entries(scrapers.actors), async ([scraperSlug, scraper]) => {
|
||||
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName);
|
||||
|
||||
const profile = mergeProfiles(profiles, actorEntry);
|
||||
return {
|
||||
scraper: scraperSlug,
|
||||
...profile,
|
||||
};
|
||||
});
|
||||
|
||||
const profile = await mergeProfiles(profiles, actorEntry);
|
||||
|
||||
if (profile === null) {
|
||||
console.log(`Could not find profile for actor '${actorName}'`);
|
||||
await updateActor(profile, true, false);
|
||||
|
||||
if (argv.save) {
|
||||
await updateActor(profile, true, false);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (actorEntry && profile) {
|
||||
await createActorMediaDirectory(profile, actorEntry);
|
||||
if (argv.save) {
|
||||
if (actorEntry && profile) {
|
||||
await createActorMediaDirectory(profile, actorEntry);
|
||||
|
||||
await Promise.all([
|
||||
updateActor(profile, true, true),
|
||||
storeAvatars(profile, actorEntry),
|
||||
]);
|
||||
await Promise.all([
|
||||
updateActor(profile, true, true),
|
||||
storeAvatars(profile, actorEntry),
|
||||
]);
|
||||
|
||||
return;
|
||||
return;
|
||||
}
|
||||
|
||||
const newActorEntry = await storeActor(profile, true, true);
|
||||
|
||||
await createActorMediaDirectory(profile, newActorEntry);
|
||||
await storeAvatars(profile, newActorEntry);
|
||||
}
|
||||
|
||||
const newActorEntry = await storeActor(profile, true, true);
|
||||
|
||||
await createActorMediaDirectory(profile, newActorEntry);
|
||||
await storeAvatars(profile, newActorEntry);
|
||||
} catch (error) {
|
||||
console.warn(actorName, error);
|
||||
}
|
||||
|
|
|
|||
21
src/media.js
21
src/media.js
|
|
@ -188,10 +188,10 @@ async function storeAvatars(profile, actor) {
|
|||
console.log(`Storing ${profile.avatars.length} avatars for '${profile.name}'`);
|
||||
|
||||
const files = await Promise.map(profile.avatars, async (avatarUrl, index) => {
|
||||
const { pathname } = new URL(avatarUrl);
|
||||
const mimetype = mime.getType(pathname);
|
||||
|
||||
try {
|
||||
const { pathname } = new URL(avatarUrl);
|
||||
const mimetype = mime.getType(pathname);
|
||||
|
||||
const res = await bhttp.get(avatarUrl);
|
||||
|
||||
if (res.statusCode === 200) {
|
||||
|
|
@ -220,7 +220,7 @@ async function storeAvatars(profile, actor) {
|
|||
|
||||
throw new Error(`Response ${res.statusCode} not OK`);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to store avatar ${index + 1} for '${profile.name}'`);
|
||||
console.warn(`Failed to store avatar ${index + 1} for '${profile.name}': ${avatarUrl}`);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
|
@ -228,17 +228,12 @@ async function storeAvatars(profile, actor) {
|
|||
concurrency: 2,
|
||||
});
|
||||
|
||||
const avatars = files.filter(file => file);
|
||||
|
||||
const existingAvatars = await knex('media')
|
||||
.whereIn('hash', files.map(file => file.hash));
|
||||
|
||||
const newAvatars = files.filter((file) => {
|
||||
if (!file) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !existingAvatars.some(avatar => file.hash === avatar.hash);
|
||||
});
|
||||
.whereIn('hash', avatars.map(file => file.hash));
|
||||
|
||||
const newAvatars = avatars.filter(file => !existingAvatars.some(avatar => file.hash === avatar.hash));
|
||||
const hasAvatar = existingAvatars.some(avatar => avatar.role === 'avatar');
|
||||
|
||||
await knex('media')
|
||||
|
|
|
|||
|
|
@ -157,10 +157,8 @@ function scrapeProfile(html, url, actorName) {
|
|||
if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]);
|
||||
if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();
|
||||
|
||||
if (bio['Body Art']) {
|
||||
profile.hasTattoo = !!bio['Body Art'].match('Tattoo');
|
||||
profile.hasPiercing = !!bio['Body Art'].match('Piercing');
|
||||
}
|
||||
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
|
||||
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
|
||||
|
||||
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
|
||||
if (avatarEl) profile.avatar = `https:${avatarEl.src}`;
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ const bhttp = require('bhttp');
|
|||
const { JSDOM } = require('jsdom');
|
||||
const moment = require('moment');
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
async function scrapeProfileFrontpage(html, url, name) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const bioEl = document.querySelector('.dashboard-bio-list');
|
||||
|
|
@ -18,55 +16,47 @@ async function scrapeProfileFrontpage(html, url, name) {
|
|||
|
||||
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
|
||||
|
||||
const profile = {
|
||||
name,
|
||||
gender: 'female',
|
||||
};
|
||||
|
||||
const birthdateString = bio['Date of Birth:'];
|
||||
const birthdate = birthdateString && birthdateString !== 'Unknown (Add)'
|
||||
? moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate()
|
||||
: null;
|
||||
|
||||
const measurementsString = bio['Measurements:'];
|
||||
const [bust, waist, hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
|
||||
const naturalBoobs = bio['Fake Boobs:'] === 'No';
|
||||
|
||||
const residenceCountryName = bio['Country of Origin:'];
|
||||
const countryEntry = await knex('countries').where({ name: residenceCountryName }).first();
|
||||
const residenceCountry = countryEntry ? countryEntry.alpha2 : null;
|
||||
const birthPlace = bio['Place of Birth:'];
|
||||
const birthCityString = bio['Place of Birth:'];
|
||||
const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString;
|
||||
|
||||
const hair = bio['Hair Color:'].toLowerCase();
|
||||
const eyes = bio['Eye Color:'].toLowerCase();
|
||||
const birthCountryString = bio['Country of Origin:'];
|
||||
const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString;
|
||||
|
||||
const piercingsString = bio['Piercings:'];
|
||||
const hasPiercings = !!(piercingsString !== undefined && piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
|
||||
const piercings = hasPiercings && piercingsString;
|
||||
|
||||
const tattoosString = bio['Tattoos:'];
|
||||
const hasTattoos = !!(tattoosString !== undefined && tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
|
||||
const tattoos = hasTattoos && tattoosString;
|
||||
|
||||
const social = Array.from(bioEl.querySelectorAll('.dashboard-socialmedia a'), el => el.href);
|
||||
if (birthdateString && birthdateString !== 'Unknown (add)') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate();
|
||||
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
|
||||
|
||||
if (bio['Fake Boobs:']) profile.naturalBoobs = bio['Fake Boobs:'] === 'No';
|
||||
profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`;
|
||||
|
||||
profile.hair = bio['Hair Color:'].toLowerCase();
|
||||
profile.eyes = bio['Eye Color:'].toLowerCase();
|
||||
|
||||
if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
|
||||
if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
|
||||
|
||||
if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString;
|
||||
if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString;
|
||||
|
||||
profile.social = Array.from(bioEl.querySelectorAll('.dashboard-socialmedia a'), el => el.href);
|
||||
|
||||
return {
|
||||
bio: {
|
||||
name,
|
||||
gender: 'female',
|
||||
birthdate,
|
||||
residenceCountry,
|
||||
birthPlace,
|
||||
naturalBoobs,
|
||||
bust,
|
||||
waist,
|
||||
hip,
|
||||
hair,
|
||||
eyes,
|
||||
piercings,
|
||||
tattoos,
|
||||
social,
|
||||
},
|
||||
profile,
|
||||
url: bioUrl,
|
||||
};
|
||||
}
|
||||
|
||||
async function scrapeProfileBio(html, frontpageBio, url, name) {
|
||||
async function scrapeProfileBio(html, frontpageProfile, url, name) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const bioEl = document.querySelector('#biographyTable');
|
||||
|
||||
|
|
@ -75,58 +65,46 @@ async function scrapeProfileBio(html, frontpageBio, url, name) {
|
|||
|
||||
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
|
||||
|
||||
const birthdateString = bio['Date of Birth:'];
|
||||
const birthdate = birthdateString && birthdateString !== 'Unknown'
|
||||
? moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate()
|
||||
: null;
|
||||
|
||||
const measurementsString = bio['Measurements:'];
|
||||
const [bust, waist, hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
|
||||
const boobsNatural = bio['Fake boobs:'] === 'No';
|
||||
const ethnicity = bio['Ethnicity:'];
|
||||
|
||||
const residenceCountryName = bio['Country of Origin:'];
|
||||
const countryEntry = await knex('countries').where({ name: residenceCountryName }).first();
|
||||
const residenceCountry = countryEntry ? countryEntry.alpha2 : null;
|
||||
const birthPlace = bio['Place of Birth:'];
|
||||
|
||||
const hair = bio['Hair Color:'].toLowerCase();
|
||||
const eyes = bio['Eye Color:'].toLowerCase();
|
||||
const height = Number(bio['Height:'].match(/\d+/)[0]);
|
||||
const weight = Number(bio['Weight:'].match(/\d+/)[0]);
|
||||
|
||||
const piercingsString = bio['Piercings:'];
|
||||
const hasPiercings = !!(piercingsString !== undefined && piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
|
||||
const piercings = hasPiercings && piercingsString;
|
||||
|
||||
const tattoosString = bio['Tattoos:'];
|
||||
const hasTattoos = !!(tattoosString !== undefined && tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
|
||||
const tattoos = hasTattoos && tattoosString;
|
||||
|
||||
const social = Array.from(bioEl.querySelectorAll('#socialmedia a'), el => el.href);
|
||||
|
||||
return {
|
||||
...frontpageBio,
|
||||
const profile = {
|
||||
...frontpageProfile,
|
||||
name,
|
||||
gender: 'female',
|
||||
birthdate,
|
||||
residenceCountry,
|
||||
birthPlace,
|
||||
ethnicity,
|
||||
naturalBoobs: boobsNatural,
|
||||
bust,
|
||||
waist,
|
||||
hip,
|
||||
height,
|
||||
weight,
|
||||
hair,
|
||||
eyes,
|
||||
hasPiercings,
|
||||
hasTattoos,
|
||||
piercings,
|
||||
tattoos,
|
||||
social,
|
||||
};
|
||||
|
||||
const birthdateString = bio['Date of Birth:'];
|
||||
const measurementsString = bio['Measurements:'];
|
||||
|
||||
const birthCityString = bio['Place of Birth:'];
|
||||
const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString;
|
||||
|
||||
const birthCountryString = bio['Country of Origin:'];
|
||||
const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString;
|
||||
|
||||
const piercingsString = bio['Piercings:'];
|
||||
const tattoosString = bio['Tattoos:'];
|
||||
|
||||
if (birthdateString && birthdateString !== 'Unknown') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate();
|
||||
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
|
||||
|
||||
if (bio['Fake boobs']) profile.naturalBoobs = bio['Fake boobs:'] === 'No';
|
||||
profile.ethnicity = bio['Ethnicity:'];
|
||||
|
||||
profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`;
|
||||
|
||||
profile.hair = bio['Hair Color:'].toLowerCase();
|
||||
profile.eyes = bio['Eye Color:'].toLowerCase();
|
||||
profile.height = Number(bio['Height:'].match(/\d+/)[0]);
|
||||
profile.weight = Number(bio['Weight:'].match(/\d+/)[0]);
|
||||
|
||||
if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
|
||||
if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
|
||||
|
||||
if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString;
|
||||
if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString;
|
||||
|
||||
profile.social = Array.from(bioEl.querySelectorAll('#socialmedia a'), el => el.href);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile(actorName) {
|
||||
|
|
@ -148,10 +126,10 @@ async function fetchProfile(actorName) {
|
|||
const resFallback = await bhttp.get(fallbackUrl);
|
||||
|
||||
if (resFallback.statusCode === 200) {
|
||||
const { url, bio } = await scrapeProfileFrontpage(resFallback.body.toString(), fallbackUrl, actorName);
|
||||
const { url, profile } = await scrapeProfileFrontpage(resFallback.body.toString(), fallbackUrl, actorName);
|
||||
const resBio = await bhttp.get(url);
|
||||
|
||||
return scrapeProfileBio(resBio.body.toString(), bio, url, actorName);
|
||||
return scrapeProfileBio(resBio.body.toString(), profile, url, actorName);
|
||||
}
|
||||
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -224,13 +224,15 @@ function scrapeProfile(html, url, actorName) {
|
|||
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString[0].split('-');
|
||||
|
||||
if (avatarEl) {
|
||||
const src = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src') + 5, avatarEl.innerHTML.indexOf('set.jpg') + 7);
|
||||
const src0 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0') + 6, avatarEl.innerHTML.indexOf('set.jpg') + 7);
|
||||
const src1 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_1x') + 9, avatarEl.innerHTML.indexOf('1x.jpg') + 6);
|
||||
const src2 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_2x') + 9, avatarEl.innerHTML.indexOf('2x.jpg') + 6);
|
||||
const src3 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_3x') + 9, avatarEl.innerHTML.indexOf('3x.jpg') + 6);
|
||||
const src = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src') + 5, avatarEl.innerHTML.indexOf('set.jpg') + 7).trim();
|
||||
const src0 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0') + 6, avatarEl.innerHTML.indexOf('set.jpg') + 7).trim();
|
||||
const src1 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_1x') + 9, avatarEl.innerHTML.indexOf('1x.jpg') + 6).trim();
|
||||
const src2 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_2x') + 9, avatarEl.innerHTML.indexOf('2x.jpg') + 6).trim();
|
||||
const src3 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_3x') + 9, avatarEl.innerHTML.indexOf('3x.jpg') + 6).trim();
|
||||
|
||||
profile.avatar = src3 || src2 || src1 || src0 || src;
|
||||
const avatar = src3 || src2 || src1 || src0 || src;
|
||||
|
||||
if (avatar) profile.avatar = avatar;
|
||||
}
|
||||
|
||||
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href);
|
||||
|
|
|
|||
|
|
@ -70,34 +70,6 @@ function scrapeLatest(html, site) {
|
|||
});
|
||||
}
|
||||
|
||||
async function scrapeProfile(html, _url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
|
||||
const profile = {
|
||||
name: actorName,
|
||||
};
|
||||
|
||||
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
|
||||
const entries = Array.from(document.querySelectorAll('.model--description tr'), el => el.textContent.replace(/\n/g, '').split(':'));
|
||||
|
||||
const bio = entries
|
||||
.filter(entry => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
|
||||
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
|
||||
|
||||
const birthCountryName = bio.Nationality;
|
||||
|
||||
if (birthCountryName) {
|
||||
const countryEntry = await knex('countries').where({ name: birthCountryName }).first();
|
||||
|
||||
if (countryEntry) profile.birthCountry = countryEntry.alpha2;
|
||||
}
|
||||
|
||||
if (bio.Age) profile.age = bio.Age;
|
||||
if (avatarEl) profile.avatar = avatarEl.src;
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site, useGallery) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const playerObject = $('script:contains("new VideoPlayer")').html();
|
||||
|
|
@ -158,6 +130,28 @@ async function scrapeScene(html, url, site, useGallery) {
|
|||
};
|
||||
}
|
||||
|
||||
async function scrapeProfile(html, _url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
|
||||
const profile = {
|
||||
name: actorName,
|
||||
};
|
||||
|
||||
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
|
||||
const entries = Array.from(document.querySelectorAll('.model--description tr'), el => el.textContent.replace(/\n/g, '').split(':'));
|
||||
|
||||
const bio = entries
|
||||
.filter(entry => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
|
||||
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
|
||||
|
||||
profile.birthPlace = bio.Nationality;
|
||||
|
||||
if (bio.Age) profile.age = bio.Age;
|
||||
if (avatarEl) profile.avatar = avatarEl.src;
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const res = await bhttp.get(`${site.url}/new-videos/${page}`);
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ const bhttp = require('bhttp');
|
|||
const { JSDOM } = require('jsdom');
|
||||
const moment = require('moment');
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
const ethnicityMap = {
|
||||
White: 'Caucasian',
|
||||
};
|
||||
|
|
@ -14,10 +12,6 @@ const hairMap = {
|
|||
Brunette: 'brown',
|
||||
};
|
||||
|
||||
const countryMap = {
|
||||
'United States of America': 'United States',
|
||||
};
|
||||
|
||||
async function scrapeProfile(html, _url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
|
||||
|
|
@ -28,9 +22,7 @@ async function scrapeProfile(html, _url, actorName) {
|
|||
name: actorName,
|
||||
};
|
||||
|
||||
const descriptionString = document.querySelector('div[itemprop="description"]');
|
||||
const birthPlaceString = bio['Birth Place'] || bio.Birthplace;
|
||||
const residencePlaceString = bio['City and Country'];
|
||||
const descriptionString = document.querySelector('div[itemprop="description"]') || document.querySelector('.longBio');
|
||||
const avatarEl = document.querySelector('#getAvatar') || document.querySelector('.thumbImage img');
|
||||
|
||||
if (bio.Gender) profile.gender = bio.Gender.toLowerCase();
|
||||
|
|
@ -38,35 +30,20 @@ async function scrapeProfile(html, _url, actorName) {
|
|||
|
||||
if (descriptionString) profile.description = descriptionString.textContent;
|
||||
|
||||
if (bio.Birthday) bio.birthdate = moment.utc(bio.Birthday, 'MMM D, YYYY').toDate();
|
||||
if (bio.Born) bio.birthdate = moment.utc(bio.Born, 'YYYY-MM-DD').toDate();
|
||||
if (bio.Birthday) profile.birthdate = moment.utc(bio.Birthday, 'MMM D, YYYY').toDate();
|
||||
if (bio.Born) profile.birthdate = moment.utc(bio.Born, 'YYYY-MM-DD').toDate();
|
||||
|
||||
if (birthPlaceString) {
|
||||
const birthPlaceSegments = birthPlaceString.split(',');
|
||||
const birthCountryName = birthPlaceSegments.slice(-1)[0].trim();
|
||||
const birthCountryEntry = await knex('countries').where('name', countryMap[birthCountryName] || birthCountryName).first();
|
||||
profile.birthPlace = bio['Birth Place'] || bio.Birthplace;
|
||||
profile.residencePlace = bio['City and Country'];
|
||||
|
||||
profile.birthPlace = birthPlaceSegments.slice(0, -1).join(',').trim();
|
||||
profile.birthCountry = birthCountryEntry ? birthCountryEntry.alpha2 : null;
|
||||
}
|
||||
|
||||
if (residencePlaceString) {
|
||||
const residencePlaceSegments = residencePlaceString.split(',');
|
||||
const residenceCountryAlpha2 = residencePlaceSegments.slice(-1)[0].trim();
|
||||
const residenceCountryEntry = await knex('countries').where('alpha2', residenceCountryAlpha2).first();
|
||||
|
||||
profile.residencePlace = residencePlaceSegments.slice(0, -1).join(',').trim();
|
||||
profile.residenceCountry = residenceCountryEntry ? residenceCountryEntry.alpha2 : null;
|
||||
}
|
||||
|
||||
if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-').map(measurement => parseInt(measurement, 10) || null);
|
||||
if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
|
||||
if (bio['Fake Boobs']) profile.naturalBoobs = bio['Fake Boobs'] === 'No';
|
||||
|
||||
if (bio.Height) profile.height = Number(bio.Height.match(/\(\d+/)[0].slice(1));
|
||||
if (bio.Weight) profile.weight = Number(bio.Weight.match(/\(\d+/)[0].slice(1));
|
||||
if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();
|
||||
if (bio.Piercings) profile.hasPiercings = bio.Piercings === 'Yes';
|
||||
if (bio.Tattoos) profile.hasTattoos = bio.hasTattoos === 'Yes';
|
||||
if (bio.Tattoos) profile.hasTattoos = bio.Tattoos === 'Yes';
|
||||
|
||||
if (avatarEl) profile.avatar = avatarEl.src;
|
||||
profile.social = Array.from(document.querySelectorAll('.socialList a'), el => el.href).filter(link => link !== 'https://www.twitter.com/'); // PH links to Twitter itself for some reason
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
|
||||
async function resolvePlace(query) {
|
||||
if (!query) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const res = await bhttp.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`);
|
||||
const [item] = res.body;
|
||||
|
||||
if (item && item.address) {
|
||||
const rawPlace = item.address;
|
||||
const place = {};
|
||||
|
||||
if (rawPlace.city) place.city = rawPlace.city;
|
||||
if (rawPlace.state) place.state = rawPlace.state;
|
||||
if (rawPlace.country_code) place.country = rawPlace.country_code.toUpperCase();
|
||||
if (rawPlace.continent) place.continent = rawPlace.continent;
|
||||
|
||||
return place;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = resolvePlace;
|
||||
Loading…
Reference in New Issue