diff --git a/assets/components/actor/actor.vue b/assets/components/actor/actor.vue index bf7b1c62..450f21a7 100644 --- a/assets/components/actor/actor.vue +++ b/assets/components/actor/actor.vue @@ -33,7 +33,7 @@ v-if="actor.aliases.length" class="bio-item" > - Also known as + Also known as {{ actor.aliases.join(', ') }} @@ -41,7 +41,7 @@ v-if="actor.birthdate" class="bio-item" > - Birthdate + Birthdate
  • - Born in + Born in {{ actor.birthPlace }} + v-if="actor.origin.city" + class="city" + >{{ actor.origin.city }}{{ actor.origin.city ? `, ${actor.origin.state}` : actor.origin.state }} {{ actor.birthCountry.name }} + :src="`/img/flags/${actor.origin.country.alpha2.toLowerCase()}.png`" + >{{ actor.origin.country.name }}
  • - Lives in + Lives in {{ actor.residencePlace }} + v-if="actor.residence.city" + class="city" + >{{ actor.residence.city }}{{ actor.residence.city ? `, ${actor.residence.state}` : actor.residence.state }} {{ actor.residenceCountry.name }} + :src="`/img/flags/${actor.residence.country.alpha2.toLowerCase()}.png`" + >{{ actor.residence.country.name }}
    • @@ -104,7 +109,7 @@ v-if="actor.ethnicity" class="bio-item ethnicity" > - Ethnicity + Ethnicity {{ actor.ethnicity }} @@ -113,7 +118,7 @@ title="bust-waist-hip" class="bio-item" > - Sizes + Sizes - Height + Height {{ actor.height }} cm {{ imperialHeight.feet }}' {{ imperialHeight.inches }}" @@ -139,7 +144,7 @@ v-if="actor.weight" class="bio-item weight" > - Weight + Weight {{ actor.weight }} kg @@ -147,6 +152,34 @@ +
  • + Tattoos + + {{ actor.tattoos }} + Yes +
    • + +
  • + Piercings + + {{ actor.piercings }} + Yes +
    • +
  • Updated on {{ formatDate(actor.scrapedAt, 'YYYY-MM-DD HH:mm') }}
    • @@ -154,8 +187,7 @@

    {{ actor.description }}

  • + +flower + + diff --git a/assets/img/trophy4.svg b/assets/img/trophy4.svg new file mode 100644 index 00000000..09aecd4d --- /dev/null +++ b/assets/img/trophy4.svg @@ -0,0 +1,5 @@ + + +trophy4 + + diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index bb527a37..dad14f33 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -20,18 +20,19 @@ exports.up = knex => Promise.resolve() table.string('gender', 18); table.text('description'); + table.string('birth_city'); + table.string('birth_state'); table.string('birth_country_alpha2', 2) .references('alpha2') .inTable('countries'); - table.string('ethnicity'); - table.string('birth_place'); - + table.string('residence_city'); + table.string('residence_state'); table.string('residence_country_alpha2', 2) .references('alpha2') .inTable('countries'); - table.string('residence_place'); + table.string('ethnicity'); table.string('bust', 10); table.integer('waist', 3); diff --git a/public/css/style.css b/public/css/style.css index 9f86c386..962887af 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -599,11 +599,11 @@ } .profile .avatar-link[data-v-677a8360] { font-size: 0; - padding: 1rem; + padding: 1rem 0 1rem 1rem; } .profile .avatar[data-v-677a8360] { - height: 15rem; - width: 15rem; + height: 12rem; + width: 12rem; flex-shrink: 0; margin: 0 1rem 0 0; -o-object-fit: cover; @@ -613,10 +613,10 @@ } .bio[data-v-677a8360] { flex-grow: 1; - min-width: 18rem; + min-width: 20rem; box-sizing: border-box; padding: 1rem; - margin: 0 3rem 0 0; + margin: 0 2rem 0 0; } .bio-header[data-v-677a8360] { display: flex; @@ -631,16 +631,24 @@ line-height: 1.75; text-align: right; font-size: .9rem; +} +.bio-label[data-v-677a8360] { + color: rgba(255, 255, 255, 0.5); + display: flex; + align-items: center; + margin: 0 1rem 0 0; + flex-shrink: 0; + font-style: normal; font-weight: bold; } -.bio-heading[data-v-677a8360] { - color: rgba(255, 255, 255, 0.5); - font-weight: normal; - font-style: normal; -} -.bio-heading .icon[data-v-677a8360] { +.bio-label .icon[data-v-677a8360] { fill: rgba(255, 255, 255, 0.5); - margin: 0 .5rem 0 0; + margin: 0 .5rem .5rem 0; +} +.bio-value[data-v-677a8360] { + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; } .flag[data-v-677a8360] { margin: 0 .25rem 0 0; @@ -674,20 +682,6 @@ padding: 0 0 0 .5rem; border-left: solid 1px rgba(255, 255, 255, 0.2); margin: 0 0 0 .5rem; - /* - &::before { - content: ' ('; - } - - &::after { - content: ')'; - } - - &::before, - &::after { - color: $highlight; - } - */ } .country[data-v-677a8360] { display: block; @@ -706,18 +700,19 @@ max-height: 10rem; flex-grow: 1; position: relative; - display: inline-block; + display: block; box-sizing: border-box; padding: 1rem 0 0 0; margin: 0 2rem 0 0; line-height: 1.5; text-overflow: ellipsis; font-size: .9rem; - overflow: hidden; cursor: pointer; + overflow: auto; + scrollbar-width: none; } -.description.expanded[data-v-677a8360] { - overflow: visible; +.description[data-v-677a8360]::-webkit-scrollbar { + display: none; } .social[data-v-677a8360] { display: block; @@ -757,7 +752,6 @@ margin: 0 .5rem 0 0; } .photos[data-v-677a8360] { - max-width: 35vw; display: inline-grid; grid-template-columns: repeat(auto-fit, 15rem); grid-gap: .5rem; @@ -787,6 +781,11 @@ display: none; } } +@media (min-width: 1200px) { +.photos.wide[data-v-677a8360] { + max-width: 35vw; +} +} @media (max-width: 1200px) { .profile .avatar-link[data-v-677a8360], .social[data-v-677a8360] { @@ -833,10 +832,13 @@ .bio-header[data-v-677a8360] { margin: 1rem 0; } -.place[data-v-677a8360], +.city[data-v-677a8360], + .state[data-v-677a8360], .ethnicity[data-v-677a8360], .residence[data-v-677a8360], .weight[data-v-677a8360], + .tattoos[data-v-677a8360], + .piercings[data-v-677a8360], .scraped[data-v-677a8360] { display: none; } @@ -901,6 +903,7 @@ display: block !important; z-index: 10000; } .tooltip .tooltip-inner { + max-width: 20rem; background: #222; color: white; border-radius: 16px; diff --git a/public/img/logos/julesjordan/julesjordan.png b/public/img/logos/julesjordan/julesjordan.png index 0f96f5de..59153a4f 100644 Binary files a/public/img/logos/julesjordan/julesjordan.png and b/public/img/logos/julesjordan/julesjordan.png differ diff --git a/public/img/logos/julesjordan/network.png b/public/img/logos/julesjordan/network.png index 0f96f5de..0ee758ca 100644 Binary files a/public/img/logos/julesjordan/network.png and b/public/img/logos/julesjordan/network.png differ diff --git a/seeds/04_countries.js b/seeds/04_countries.js index f95bb516..8d0298a1 100644 --- a/seeds/04_countries.js +++ b/seeds/04_countries.js @@ -912,7 +912,7 @@ const countries = [ alpha2: 'RO', }, { - name: 'Russian Federation', + name: 'Russia', code: '643', alpha2: 'RU', }, diff --git a/src/actors.js b/src/actors.js index e563b65f..c2035903 100644 --- a/src/actors.js +++ b/src/actors.js @@ -6,6 +6,7 @@ const knex = require('./knex'); const argv = require('./argv'); const scrapers = require('./scrapers/scrapers'); const whereOr = require('./utils/where-or'); +const resolvePlace = require('./utils/resolve-place'); const { createActorMediaDirectory, storeAvatars } = require('./media'); async function curateActor(actor) { @@ -18,27 +19,15 @@ async function curateActor(actor) { .where({ domain: 'actors', target_id: actor.id }), ]); - return { + const curatedActor = { id: actor.id, gender: actor.gender, name: actor.name, description: actor.description, birthdate: actor.birthdate && new Date(actor.birthdate), country: actor.country_alpha2, - residencePlace: actor.residence_place, - residenceCountry: actor.residence_country_alpha2 - ? { - alpha2: actor.residence_country_alpha2, - name: actor.residence_country_name, - } - : null, - birthPlace: actor.birth_place, - birthCountry: actor.birth_country_alpha2 - ? { - alpha2: actor.birth_country_alpha2, - name: actor.birth_country_name, - } - : null, + origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null, + residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null, ethnicity: actor.ethnicity, height: actor.height, weight: actor.weight, @@ -50,9 +39,35 @@ async function curateActor(actor) { slug: actor.slug, avatar: photos.find(photo => photo.role === 'avatar'), photos: photos.filter(photo => photo.role === 'photo'), + hasTattoos: actor.has_tattoos, + hasPiercings: actor.has_piercings, + tattoos: actor.tattoos, + piercings: actor.piercings, social, scrapedAt: actor.scraped_at, }; + + if (actor.birth_city) curatedActor.origin.city = actor.birth_city; + if (actor.birth_state) curatedActor.origin.state = actor.birth_state; + + if (actor.birth_country_alpha2) { + curatedActor.origin.country = { + alpha2: actor.birth_country_alpha2, + name: actor.birth_country_name, + }; + } + + if (actor.residence_city) curatedActor.residence.city = actor.residence_city; + if (actor.residence_state) curatedActor.residence.state = actor.residence_state; + + if (actor.residence_country_alpha2) { + curatedActor.residence.country = { + alpha2: actor.residence_country_alpha2, + name: actor.residence_country_name, + }; + } + + return curatedActor; } function curateActors(releases) { @@ -70,10 +85,6 @@ function curateActorEntry(actor, scraped, scrapeSuccess) { description: actor.description, gender: actor.gender, ethnicity: actor.ethnicity, - birth_country_alpha2: actor.birthCountry, - residence_country_alpha2: actor.residenceCountry, - birth_place: actor.birthPlace, - residence_place: actor.residencePlace, bust: actor.bust, waist: actor.waist, hip: actor.hip, @@ -92,6 +103,18 @@ function curateActorEntry(actor, scraped, scrapeSuccess) { curatedActor.id = actor.id; } + if (actor.birthPlace) { + curatedActor.birth_city = actor.birthPlace.city; + curatedActor.birth_state = actor.birthPlace.state; + curatedActor.birth_country_alpha2 = actor.birthPlace.country; + } + + if (actor.residencePlace) { + curatedActor.residence_city = actor.residencePlace.city; + curatedActor.residence_state = actor.residencePlace.state; + curatedActor.residence_country_alpha2 = actor.residencePlace.country; + } + if (scraped) { curatedActor.scraped_at = new Date(); curatedActor.scrape_success = scrapeSuccess; @@ -102,7 +125,7 @@ function curateActorEntry(actor, scraped, scrapeSuccess) { function curateSocialEntry(url, actorId) { const { hostname, origin, pathname } = new URL(url); - const platform = ['facebook', 'twitter', 'instagram', 'tumblr', 'snapchat', 'amazon', 'youtube'].find(platformName => hostname.match(platformName)); + const platform = ['facebook', 'twitter', 'instagram', 'tumblr', 'snapchat', 'amazon', 'youtube', 'fancentro'].find(platformName => hostname.match(platformName)); return { url: `${origin}${pathname}`, @@ -184,8 +207,8 @@ async function updateActor(actor, scraped = false, scrapeSuccess = false) { return actorEntry; } -function mergeProfiles(profiles, actor) { - return profiles.reduce((prevProfile, profile) => { +async function mergeProfiles(profiles, actor) { + const mergedProfile = profiles.reduce((prevProfile, profile) => { if (profile === null) { return prevProfile; } @@ -196,21 +219,19 @@ function mergeProfiles(profiles, actor) { description: prevProfile.description || profile.description, gender: prevProfile.gender || profile.gender, birthdate: Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate, - birthCountry: prevProfile.birthCountry || profile.birthCountry, - residenceCountry: prevProfile.residenceCountry || profile.residenceCountry, birthPlace: prevProfile.birthPlace || profile.birthPlace, residencePlace: prevProfile.residencePlace || profile.residencePlace, ethnicity: prevProfile.ethnicity || profile.ethnicity, bust: prevProfile.bust || profile.bust, waist: prevProfile.waist || profile.waist, hip: prevProfile.hip || profile.hip, - naturalBoobs: prevProfile.naturalBoobs || profile.naturalBoobs, + naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs, height: prevProfile.height || profile.height, weight: prevProfile.weight || profile.weight, hair: prevProfile.hair || profile.hair, eyes: prevProfile.eyes || profile.eyes, - hasPiercings: prevProfile.hasPiercings || profile.hasPiercings, - hasTattoos: prevProfile.hasTattoos || profile.hasTattoos, + hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings, + hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos, piercings: prevProfile.piercings || profile.piercings, tattoos: prevProfile.tattoos || profile.tattoos, social: prevProfile.social.concat(profile.social || []), @@ -220,6 +241,16 @@ function mergeProfiles(profiles, actor) { social: [], avatars: [], }); + + const [birthPlace, residencePlace] = await Promise.all([ + resolvePlace(mergedProfile.birthPlace), + resolvePlace(mergedProfile.residencePlace), + ]); + + mergedProfile.birthPlace = birthPlace; + mergedProfile.residencePlace = residencePlace; + + return mergedProfile; } async function scrapeActors(actorNames) { @@ -228,35 +259,44 @@ async function scrapeActors(actorNames) { const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-'); const actorEntry = await knex('actors').where({ slug: actorSlug }).first(); - const profiles = await Promise.all( - Object.values(scrapers.actors) - .map(scraper => scraper.fetchProfile(actorEntry ? actorEntry.name : actorName)), - ); + const profiles = await Promise.map(Object.entries(scrapers.actors), async ([scraperSlug, scraper]) => { + const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName); - const profile = mergeProfiles(profiles, actorEntry); + return { + scraper: scraperSlug, + ...profile, + }; + }); + + const profile = await mergeProfiles(profiles, actorEntry); if (profile === null) { console.log(`Could not find profile for actor '${actorName}'`); - await updateActor(profile, true, false); + + if (argv.save) { + await updateActor(profile, true, false); + } return; } - if (actorEntry && profile) { - await createActorMediaDirectory(profile, actorEntry); + if (argv.save) { + if (actorEntry && profile) { + await createActorMediaDirectory(profile, actorEntry); - await Promise.all([ - updateActor(profile, true, true), - storeAvatars(profile, actorEntry), - ]); + await Promise.all([ + updateActor(profile, true, true), + storeAvatars(profile, actorEntry), + ]); - return; + return; + } + + const newActorEntry = await storeActor(profile, true, true); + + await createActorMediaDirectory(profile, newActorEntry); + await storeAvatars(profile, newActorEntry); } - - const newActorEntry = await storeActor(profile, true, true); - - await createActorMediaDirectory(profile, newActorEntry); - await storeAvatars(profile, newActorEntry); } catch (error) { console.warn(actorName, error); } diff --git a/src/media.js b/src/media.js index e3c4a75e..b10aa65a 100644 --- a/src/media.js +++ b/src/media.js @@ -188,10 +188,10 @@ async function storeAvatars(profile, actor) { console.log(`Storing ${profile.avatars.length} avatars for '${profile.name}'`); const files = await Promise.map(profile.avatars, async (avatarUrl, index) => { - const { pathname } = new URL(avatarUrl); - const mimetype = mime.getType(pathname); - try { + const { pathname } = new URL(avatarUrl); + const mimetype = mime.getType(pathname); + const res = await bhttp.get(avatarUrl); if (res.statusCode === 200) { @@ -220,7 +220,7 @@ async function storeAvatars(profile, actor) { throw new Error(`Response ${res.statusCode} not OK`); } catch (error) { - console.warn(`Failed to store avatar ${index + 1} for '${profile.name}'`); + console.warn(`Failed to store avatar ${index + 1} for '${profile.name}': ${avatarUrl}`); return null; } @@ -228,17 +228,12 @@ async function storeAvatars(profile, actor) { concurrency: 2, }); + const avatars = files.filter(file => file); + const existingAvatars = await knex('media') - .whereIn('hash', files.map(file => file.hash)); - - const newAvatars = files.filter((file) => { - if (!file) { - return false; - } - - return !existingAvatars.some(avatar => file.hash === avatar.hash); - }); + .whereIn('hash', avatars.map(file => file.hash)); + const newAvatars = avatars.filter(file => !existingAvatars.some(avatar => file.hash === avatar.hash)); const hasAvatar = existingAvatars.some(avatar => avatar.role === 'avatar'); await knex('media') diff --git a/src/scrapers/brazzers.js b/src/scrapers/brazzers.js index b96c5c6d..59e66084 100644 --- a/src/scrapers/brazzers.js +++ b/src/scrapers/brazzers.js @@ -157,10 +157,8 @@ function scrapeProfile(html, url, actorName) { if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]); if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase(); - if (bio['Body Art']) { - profile.hasTattoo = !!bio['Body Art'].match('Tattoo'); - profile.hasPiercing = !!bio['Body Art'].match('Piercing'); - } + if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true; + if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true; if (descriptionEl) profile.description = descriptionEl.textContent.trim(); if (avatarEl) profile.avatar = `https:${avatarEl.src}`; diff --git a/src/scrapers/freeones.js b/src/scrapers/freeones.js index e64ae4f0..7711a1c3 100644 --- a/src/scrapers/freeones.js +++ b/src/scrapers/freeones.js @@ -5,8 +5,6 @@ const bhttp = require('bhttp'); const { JSDOM } = require('jsdom'); const moment = require('moment'); -const knex = require('../knex'); - async function scrapeProfileFrontpage(html, url, name) { const { document } = new JSDOM(html).window; const bioEl = document.querySelector('.dashboard-bio-list'); @@ -18,55 +16,47 @@ async function scrapeProfileFrontpage(html, url, name) { const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {}); + const profile = { + name, + gender: 'female', + }; + const birthdateString = bio['Date of Birth:']; - const birthdate = birthdateString && birthdateString !== 'Unknown (Add)' - ? moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate() - : null; - const measurementsString = bio['Measurements:']; - const [bust, waist, hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement)); - const naturalBoobs = bio['Fake Boobs:'] === 'No'; - const residenceCountryName = bio['Country of Origin:']; - const countryEntry = await knex('countries').where({ name: residenceCountryName }).first(); - const residenceCountry = countryEntry ? countryEntry.alpha2 : null; - const birthPlace = bio['Place of Birth:']; + const birthCityString = bio['Place of Birth:']; + const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString; - const hair = bio['Hair Color:'].toLowerCase(); - const eyes = bio['Eye Color:'].toLowerCase(); + const birthCountryString = bio['Country of Origin:']; + const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString; const piercingsString = bio['Piercings:']; - const hasPiercings = !!(piercingsString !== undefined && piercingsString !== 'Unknown (add)' && piercingsString !== 'None'); - const piercings = hasPiercings && piercingsString; - const tattoosString = bio['Tattoos:']; - const hasTattoos = !!(tattoosString !== undefined && tattoosString !== 'Unknown (add)' && tattoosString !== 'None'); - const tattoos = hasTattoos && tattoosString; - const social = Array.from(bioEl.querySelectorAll('.dashboard-socialmedia a'), el => el.href); + if (birthdateString && birthdateString !== 'Unknown (add)') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate(); + if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement)); + + if (bio['Fake Boobs:']) profile.naturalBoobs = bio['Fake Boobs:'] === 'No'; + profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`; + + profile.hair = bio['Hair Color:'].toLowerCase(); + profile.eyes = bio['Eye Color:'].toLowerCase(); + + if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None'); + if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None'); + + if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString; + if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString; + + profile.social = Array.from(bioEl.querySelectorAll('.dashboard-socialmedia a'), el => el.href); return { - bio: { - name, - gender: 'female', - birthdate, - residenceCountry, - birthPlace, - naturalBoobs, - bust, - waist, - hip, - hair, - eyes, - piercings, - tattoos, - social, - }, + profile, url: bioUrl, }; } -async function scrapeProfileBio(html, frontpageBio, url, name) { +async function scrapeProfileBio(html, frontpageProfile, url, name) { const { document } = new JSDOM(html).window; const bioEl = document.querySelector('#biographyTable'); @@ -75,58 +65,46 @@ async function scrapeProfileBio(html, frontpageBio, url, name) { const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {}); - const birthdateString = bio['Date of Birth:']; - const birthdate = birthdateString && birthdateString !== 'Unknown' - ? moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate() - : null; - - const measurementsString = bio['Measurements:']; - const [bust, waist, hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement)); - const boobsNatural = bio['Fake boobs:'] === 'No'; - const ethnicity = bio['Ethnicity:']; - - const residenceCountryName = bio['Country of Origin:']; - const countryEntry = await knex('countries').where({ name: residenceCountryName }).first(); - const residenceCountry = countryEntry ? countryEntry.alpha2 : null; - const birthPlace = bio['Place of Birth:']; - - const hair = bio['Hair Color:'].toLowerCase(); - const eyes = bio['Eye Color:'].toLowerCase(); - const height = Number(bio['Height:'].match(/\d+/)[0]); - const weight = Number(bio['Weight:'].match(/\d+/)[0]); - - const piercingsString = bio['Piercings:']; - const hasPiercings = !!(piercingsString !== undefined && piercingsString !== 'Unknown (add)' && piercingsString !== 'None'); - const piercings = hasPiercings && piercingsString; - - const tattoosString = bio['Tattoos:']; - const hasTattoos = !!(tattoosString !== undefined && tattoosString !== 'Unknown (add)' && tattoosString !== 'None'); - const tattoos = hasTattoos && tattoosString; - - const social = Array.from(bioEl.querySelectorAll('#socialmedia a'), el => el.href); - - return { - ...frontpageBio, + const profile = { + ...frontpageProfile, name, gender: 'female', - birthdate, - residenceCountry, - birthPlace, - ethnicity, - naturalBoobs: boobsNatural, - bust, - waist, - hip, - height, - weight, - hair, - eyes, - hasPiercings, - hasTattoos, - piercings, - tattoos, - social, }; + + const birthdateString = bio['Date of Birth:']; + const measurementsString = bio['Measurements:']; + + const birthCityString = bio['Place of Birth:']; + const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString; + + const birthCountryString = bio['Country of Origin:']; + const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString; + + const piercingsString = bio['Piercings:']; + const tattoosString = bio['Tattoos:']; + + if (birthdateString && birthdateString !== 'Unknown') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate(); + if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement)); + + if (bio['Fake boobs']) profile.naturalBoobs = bio['Fake boobs:'] === 'No'; + profile.ethnicity = bio['Ethnicity:']; + + profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`; + + profile.hair = bio['Hair Color:'].toLowerCase(); + profile.eyes = bio['Eye Color:'].toLowerCase(); + profile.height = Number(bio['Height:'].match(/\d+/)[0]); + profile.weight = Number(bio['Weight:'].match(/\d+/)[0]); + + if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None'); + if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None'); + + if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString; + if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString; + + profile.social = Array.from(bioEl.querySelectorAll('#socialmedia a'), el => el.href); + + return profile; } async function fetchProfile(actorName) { @@ -148,10 +126,10 @@ async function fetchProfile(actorName) { const resFallback = await bhttp.get(fallbackUrl); if (resFallback.statusCode === 200) { - const { url, bio } = await scrapeProfileFrontpage(resFallback.body.toString(), fallbackUrl, actorName); + const { url, profile } = await scrapeProfileFrontpage(resFallback.body.toString(), fallbackUrl, actorName); const resBio = await bhttp.get(url); - return scrapeProfileBio(resBio.body.toString(), bio, url, actorName); + return scrapeProfileBio(resBio.body.toString(), profile, url, actorName); } return null; diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index a2c649c2..e9b6c8ad 100644 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -224,13 +224,15 @@ function scrapeProfile(html, url, actorName) { if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString[0].split('-'); if (avatarEl) { - const src = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src') + 5, avatarEl.innerHTML.indexOf('set.jpg') + 7); - const src0 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0') + 6, avatarEl.innerHTML.indexOf('set.jpg') + 7); - const src1 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_1x') + 9, avatarEl.innerHTML.indexOf('1x.jpg') + 6); - const src2 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_2x') + 9, avatarEl.innerHTML.indexOf('2x.jpg') + 6); - const src3 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_3x') + 9, avatarEl.innerHTML.indexOf('3x.jpg') + 6); + const src = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src') + 5, avatarEl.innerHTML.indexOf('set.jpg') + 7).trim(); + const src0 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0') + 6, avatarEl.innerHTML.indexOf('set.jpg') + 7).trim(); + const src1 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_1x') + 9, avatarEl.innerHTML.indexOf('1x.jpg') + 6).trim(); + const src2 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_2x') + 9, avatarEl.innerHTML.indexOf('2x.jpg') + 6).trim(); + const src3 = avatarEl.innerHTML.slice(avatarEl.innerHTML.indexOf('src0_3x') + 9, avatarEl.innerHTML.indexOf('3x.jpg') + 6).trim(); - profile.avatar = src3 || src2 || src1 || src0 || src; + const avatar = src3 || src2 || src1 || src0 || src; + + if (avatar) profile.avatar = avatar; } profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href); diff --git a/src/scrapers/legalporno.js b/src/scrapers/legalporno.js index 54738bfe..00a29b87 100644 --- a/src/scrapers/legalporno.js +++ b/src/scrapers/legalporno.js @@ -70,34 +70,6 @@ function scrapeLatest(html, site) { }); } -async function scrapeProfile(html, _url, actorName) { - const { document } = new JSDOM(html).window; - - const profile = { - name: actorName, - }; - - const avatarEl = document.querySelector('.model--avatar img[src^="http"]'); - const entries = Array.from(document.querySelectorAll('.model--description tr'), el => el.textContent.replace(/\n/g, '').split(':')); - - const bio = entries - .filter(entry => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry) - .reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {}); - - const birthCountryName = bio.Nationality; - - if (birthCountryName) { - const countryEntry = await knex('countries').where({ name: birthCountryName }).first(); - - if (countryEntry) profile.birthCountry = countryEntry.alpha2; - } - - if (bio.Age) profile.age = bio.Age; - if (avatarEl) profile.avatar = avatarEl.src; - - return profile; -} - async function scrapeScene(html, url, site, useGallery) { const $ = cheerio.load(html, { normalizeWhitespace: true }); const playerObject = $('script:contains("new VideoPlayer")').html(); @@ -158,6 +130,28 @@ async function scrapeScene(html, url, site, useGallery) { }; } +async function scrapeProfile(html, _url, actorName) { + const { document } = new JSDOM(html).window; + + const profile = { + name: actorName, + }; + + const avatarEl = document.querySelector('.model--avatar img[src^="http"]'); + const entries = Array.from(document.querySelectorAll('.model--description tr'), el => el.textContent.replace(/\n/g, '').split(':')); + + const bio = entries + .filter(entry => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry) + .reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {}); + + profile.birthPlace = bio.Nationality; + + if (bio.Age) profile.age = bio.Age; + if (avatarEl) profile.avatar = avatarEl.src; + + return profile; +} + async function fetchLatest(site, page = 1) { const res = await bhttp.get(`${site.url}/new-videos/${page}`); diff --git a/src/scrapers/pornhub.js b/src/scrapers/pornhub.js index 3f3710e2..27cdc4bb 100644 --- a/src/scrapers/pornhub.js +++ b/src/scrapers/pornhub.js @@ -4,8 +4,6 @@ const bhttp = require('bhttp'); const { JSDOM } = require('jsdom'); const moment = require('moment'); -const knex = require('../knex'); - const ethnicityMap = { White: 'Caucasian', }; @@ -14,10 +12,6 @@ const hairMap = { Brunette: 'brown', }; -const countryMap = { - 'United States of America': 'United States', -}; - async function scrapeProfile(html, _url, actorName) { const { document } = new JSDOM(html).window; @@ -28,9 +22,7 @@ async function scrapeProfile(html, _url, actorName) { name: actorName, }; - const descriptionString = document.querySelector('div[itemprop="description"]'); - const birthPlaceString = bio['Birth Place'] || bio.Birthplace; - const residencePlaceString = bio['City and Country']; + const descriptionString = document.querySelector('div[itemprop="description"]') || document.querySelector('.longBio'); const avatarEl = document.querySelector('#getAvatar') || document.querySelector('.thumbImage img'); if (bio.Gender) profile.gender = bio.Gender.toLowerCase(); @@ -38,35 +30,20 @@ async function scrapeProfile(html, _url, actorName) { if (descriptionString) profile.description = descriptionString.textContent; - if (bio.Birthday) bio.birthdate = moment.utc(bio.Birthday, 'MMM D, YYYY').toDate(); - if (bio.Born) bio.birthdate = moment.utc(bio.Born, 'YYYY-MM-DD').toDate(); + if (bio.Birthday) profile.birthdate = moment.utc(bio.Birthday, 'MMM D, YYYY').toDate(); + if (bio.Born) profile.birthdate = moment.utc(bio.Born, 'YYYY-MM-DD').toDate(); - if (birthPlaceString) { - const birthPlaceSegments = birthPlaceString.split(','); - const birthCountryName = birthPlaceSegments.slice(-1)[0].trim(); - const birthCountryEntry = await knex('countries').where('name', countryMap[birthCountryName] || birthCountryName).first(); + profile.birthPlace = bio['Birth Place'] || bio.Birthplace; + profile.residencePlace = bio['City and Country']; - profile.birthPlace = birthPlaceSegments.slice(0, -1).join(',').trim(); - profile.birthCountry = birthCountryEntry ? birthCountryEntry.alpha2 : null; - } - - if (residencePlaceString) { - const residencePlaceSegments = residencePlaceString.split(','); - const residenceCountryAlpha2 = residencePlaceSegments.slice(-1)[0].trim(); - const residenceCountryEntry = await knex('countries').where('alpha2', residenceCountryAlpha2).first(); - - profile.residencePlace = residencePlaceSegments.slice(0, -1).join(',').trim(); - profile.residenceCountry = residenceCountryEntry ? residenceCountryEntry.alpha2 : null; - } - - if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-').map(measurement => parseInt(measurement, 10) || null); + if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-'); if (bio['Fake Boobs']) profile.naturalBoobs = bio['Fake Boobs'] === 'No'; if (bio.Height) profile.height = Number(bio.Height.match(/\(\d+/)[0].slice(1)); if (bio.Weight) profile.weight = Number(bio.Weight.match(/\(\d+/)[0].slice(1)); if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase(); if (bio.Piercings) profile.hasPiercings = bio.Piercings === 'Yes'; - if (bio.Tattoos) profile.hasTattoos = bio.hasTattoos === 'Yes'; + if (bio.Tattoos) profile.hasTattoos = bio.Tattoos === 'Yes'; if (avatarEl) profile.avatar = avatarEl.src; profile.social = Array.from(document.querySelectorAll('.socialList a'), el => el.href).filter(link => link !== 'https://www.twitter.com/'); // PH links to Twitter itself for some reason diff --git a/src/utils/resolve-place.js b/src/utils/resolve-place.js new file mode 100644 index 00000000..09de6073 --- /dev/null +++ b/src/utils/resolve-place.js @@ -0,0 +1,28 @@ +'use strict'; + +const bhttp = require('bhttp'); + +async function resolvePlace(query) { + if (!query) { + return null; + } + + const res = await bhttp.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`); + const [item] = res.body; + + if (item && item.address) { + const rawPlace = item.address; + const place = {}; + + if (rawPlace.city) place.city = rawPlace.city; + if (rawPlace.state) place.state = rawPlace.state; + if (rawPlace.country_code) place.country = rawPlace.country_code.toUpperCase(); + if (rawPlace.continent) place.continent = rawPlace.continent; + + return place; + } + + return null; +} + +module.exports = resolvePlace;