DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added Kelly Madison profile scraper.

This commit is contained in:
ThePendulum 2019-12-10 22:35:00 +01:00
parent 8802bb4317
commit b9bac6d8f9
11 changed files with 133 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
assets/components/actors/actor.vue
View File

@ -47,7 +47,7 @@
<span
v-if="actor.birthdate"
class="birthdate"
>{{ formatDate(actor.birthdate, 'MMMM D, YYYY') }}<span class="age">{{ age }}</span></span>
>{{ formatDate(actor.birthdate, 'MMMM D, YYYY') }}<span class="age">{{ actor.age }}</span></span>
</li>
<li
@ -239,18 +239,12 @@
</template>
<script>
import dayjs from 'dayjs';
import { cmToFeetInches, kgToLbs } from '../../../src/utils/convert';
import Photos from './photos.vue';
import FilterBar from '../header/filter-bar.vue';
import Releases from '../releases/releases.vue';
function age() {
return dayjs(new Date()).diff(this.actor.birthdate, 'years');
}
async function fetchReleases() {
this.releases = await this.$store.dispatch('fetchActorReleases', this.$route.params.actorSlug);
}
@ -293,7 +287,6 @@ export default {
};
},
computed: {
age,
imperialHeight,
imperialWeight,
},

3
assets/components/release/release.vue
View File

@ -293,7 +293,7 @@ export default {
.logo {
display: inline-block;
filter: drop-shadow(0 0 1px $shadow);
filter: $logo-outline;
}
.logo-site {
@ -301,7 +301,6 @@ export default {
max-width: 15rem;
object-fit: contain;
object-position: 100% 50%;
filter: drop-shadow(0 0 1px $shadow);
}
.logo-network {

32
assets/components/tile/actor.vue
View File

@ -22,6 +22,23 @@
v-else
class="avatar"
>No photo</span>
<span
v-if="actor.age || actor.origin"
class="details"
>
<span class="age">{{ actor.age }}</span>
<span
v-if="actor.origin"
class="country"
>
{{ actor.origin.country.alpha2 }}
<img
class="flag"
:src="`/img/flags/${actor.origin.country.alpha2.toLowerCase()}.png`"
>
</span>
</span>
</a>
</div>
</template>
@ -45,6 +62,7 @@ export default {
background: $background;
display: inline-block;
margin: 0 .5rem .5rem 0;
position: relative;
box-shadow: 0 0 3px $shadow-weak;
}
@ -78,4 +96,18 @@ export default {
object-fit: cover;
object-position: 50% 0;
}
.details {
background: $shadow;
color: $text-contrast;
width: 100%;
display: flex;
justify-content: space-between;
box-sizing: border-box;
padding: .5rem;
position: absolute;
bottom: 0;
font-size: .8rem;
font-weight: bold;
}
</style>

3
assets/components/tile/network.vue
View File

@ -33,7 +33,6 @@ export default {
align-items: center;
box-sizing: border-box;
padding: .5rem 1rem;
border-radius: .25rem;
box-shadow: 0 0 3px rgba(0, 0, 0, .25);
height: 100%;
text-align: center;
@ -53,7 +52,7 @@ export default {
object-fit: contain;
font-size: 1rem;
font-weight: bold;
filter: drop-shadow(0 0 1px $shadow);
filter: $logo-outline;
}
.title {

2
assets/components/tile/site.vue
View File

@ -53,7 +53,7 @@ export default {
object-fit: contain;
font-size: 1rem;
font-weight: bold;
filter: drop-shadow(0 0 1px $shadow);
filter: $logo-outline;
}
.title {

2
assets/css/_theme.scss
View File

@ -23,6 +23,8 @@ $highlight-strong: rgba(255, 255, 255, .7);
$highlight-weak: rgba(255, 255, 255, .2);
$highlight-hint: rgba(255, 255, 255, .075);
$logo-outline: drop-shadow(1px 0 0 $shadow-weak) drop-shadow(-1px 0 0 $shadow-weak) drop-shadow(0 1px 0 $shadow-weak) drop-shadow(0 -1px 0 $shadow-weak);
$profile: #222;
$link: #cc4466;

31
public/css/style.css
View File

@ -276,6 +276,7 @@
background: #fff;
display: inline-block;
margin: 0 .5rem .5rem 0;
position: relative;
box-shadow: 0 0 3px rgba(0, 0, 0, 0.2);
}
.link[data-v-6989dc6f] {
@ -310,6 +311,21 @@
-o-object-position: 50% 0;
object-position: 50% 0;
}
.details[data-v-6989dc6f] {
background: rgba(0, 0, 0, 0.5);
color: #fff;
width: 100%;
display: -webkit-box;
display: flex;
-webkit-box-pack: justify;
justify-content: space-between;
box-sizing: border-box;
padding: .5rem;
position: absolute;
bottom: 0;
font-size: .8rem;
font-weight: bold;
}
/* $primary: #ff886c; */
.banner[data-v-cbb14462] {
@ -400,8 +416,8 @@
}
.logo[data-v-2bc41e74] {
display: inline-block;
-webkit-filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
-webkit-filter: drop-shadow(1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(-1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 1px 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 -1px 0 rgba(0, 0, 0, 0.2));
filter: drop-shadow(1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(-1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 1px 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 -1px 0 rgba(0, 0, 0, 0.2));
}
.logo-site[data-v-2bc41e74] {
height: 3rem;
@ -410,8 +426,6 @@
object-fit: contain;
-o-object-position: 100% 50%;
object-position: 100% 50%;
-webkit-filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
}
.logo-network[data-v-2bc41e74] {
height: 1.5rem;
@ -592,8 +606,8 @@
object-fit: contain;
font-size: 1rem;
font-weight: bold;
-webkit-filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
-webkit-filter: drop-shadow(1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(-1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 1px 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 -1px 0 rgba(0, 0, 0, 0.2));
filter: drop-shadow(1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(-1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 1px 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 -1px 0 rgba(0, 0, 0, 0.2));
}
.title[data-v-f4958086] {
color: #222;
@ -659,7 +673,6 @@
align-items: center;
box-sizing: border-box;
padding: .5rem 1rem;
border-radius: .25rem;
box-shadow: 0 0 3px rgba(0, 0, 0, 0.25);
height: 100%;
text-align: center;
@ -681,8 +694,8 @@
object-fit: contain;
font-size: 1rem;
font-weight: bold;
-webkit-filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
-webkit-filter: drop-shadow(1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(-1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 1px 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 -1px 0 rgba(0, 0, 0, 0.2));
filter: drop-shadow(1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(-1px 0 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 1px 0 rgba(0, 0, 0, 0.2)) drop-shadow(0 -1px 0 rgba(0, 0, 0, 0.2));
}
.title[data-v-8b4c90b0] {
color: #222;

5
src/actors.js
View File

@ -2,6 +2,7 @@
const Promise = require('bluebird');
const UrlPattern = require('url-pattern');
const moment = require('moment');
const knex = require('./knex');
const argv = require('./argv');
@ -49,6 +50,10 @@ async function curateActor(actor) {
scrapedAt: actor.scraped_at,
};
if (curatedActor.birthdate) {
curatedActor.age = moment().diff(curatedActor.birthdate, 'years');
}
if (actor.birth_city) curatedActor.origin.city = actor.birth_city;
if (actor.birth_state) curatedActor.origin.state = actor.birth_state;

25
src/releases.js
View File

@ -1,6 +1,8 @@
'use strict';
const Promise = require('bluebird');
const moment = require('moment');
const knex = require('./knex');
const argv = require('./argv');
const whereOr = require('./utils/where-or');
@ -17,9 +19,14 @@ const { fetchSites, findSiteByUrl } = require('./sites');
async function curateRelease(release) {
const [actors, tags, media] = await Promise.all([
knex('actors_associated')
.select('actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'media.thumbnail as avatar')
.select(
'actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'actors.birthdate',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'media.thumbnail as avatar',
)
.where({ release_id: release.id })
.leftJoin('actors', 'actors.id', 'actors_associated.actor_id')
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('media', (builder) => {
builder
.on('media.target_id', 'actors.id')
@ -52,7 +59,21 @@ async function curateRelease(release) {
url: release.url,
shootId: release.shoot_id,
entryId: release.entry_id,
actors,
actors: actors.map(actor => ({
id: actor.id,
slug: actor.slug,
name: actor.name,
gender: actor.gender,
birthdate: actor.birthdate,
age: moment().diff(actor.birthdate, 'years'),
avatar: actor.avatar,
origin: {
country: {
name: actor.birth_country_alias,
alpha2: actor.birth_country_alpha2,
},
},
})),
director: release.director,
tags,
duration: release.duration,

43
src/scrapers/kellymadison.js
View File

@ -4,6 +4,8 @@ const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const { feetInchesToCm } = require('../utils/convert');
const siteMapByKey = {
PF: 'pornfidelity',
TF: 'teenfidelity',
@ -112,6 +114,31 @@ function scrapeScene(html, url, site, shallowRelease) {
return release;
}
function scrapeProfile(html, actorName) {
const { document } = new JSDOM(html).window;
const profile = { name: actorName };
const bioKeys = Array.from(document.querySelectorAll('table.table td:nth-child(1)'), el => el.textContent.slice(0, -1));
const bioValues = Array.from(document.querySelectorAll('table.table td:nth-child(2)'), el => el.textContent);
const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key]: bioValues[index] }), {});
if (bio.Measurements) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio.Birthplace) profile.birthPlace = bio.Birthplace;
if (bio.Height) {
const [feet, inches] = bio.Height.match(/\d+/g);
profile.height = feetInchesToCm(feet, inches);
}
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
const avatarEl = Array.from(document.querySelectorAll('img')).find(photo => photo.src.match('model'));
if (avatarEl) profile.avatar = avatarEl.src;
return profile;
}
async function fetchLatest(site, page = 1) {
const url = `https://kellymadison.com/episodes/search?page=${page}`; // TLS issues with teenfidelity.com, same overview on all sites
const res = await bhttp.get(url, {
@ -139,7 +166,23 @@ async function fetchScene(url, site, shallowRelease) {
return scrapeScene(res.body.toString(), url, site, shallowRelease);
}
async function fetchProfile(actorName) {
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
const res = await bhttp.get(`https://www.kellymadison.com/models/${actorSlug}`, {
headers: {
'X-Requested-With': 'XMLHttpRequest',
},
});
if (res.statusCode === 200) {
return scrapeProfile(res.body.toString(), actorName);
}
return null;
}
module.exports = {
fetchLatest,
fetchProfile,
fetchScene,
};

3
src/scrapers/scrapers.js
View File

@ -6,7 +6,6 @@ const bangbros = require('./bangbros');
const blowpass = require('./blowpass');
const dogfart = require('./dogfart');
const evilangel = require('./evilangel');
const kellymadison = require('./kellymadison');
const kink = require('./kink');
const mikeadriano = require('./mikeadriano');
const mofos = require('./mofos');
@ -20,6 +19,7 @@ const vixen = require('./vixen');
const ddfnetwork = require('./ddfnetwork');
const brazzers = require('./brazzers');
const julesjordan = require('./julesjordan');
const kellymadison = require('./kellymadison');
const legalporno = require('./legalporno');
const xempire = require('./xempire');
@ -55,6 +55,7 @@ module.exports = {
xempire,
brazzers,
freeones,
kellymadison,
julesjordan,
legalporno,
pornhub,