DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added profile interpolation.

This commit is contained in:
ThePendulum 2020-05-17 03:00:44 +02:00
parent 05ee57378a
commit 985ab9d2dc
16 changed files with 252 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
assets/components/actors/actor.vue
View File

@ -90,7 +90,7 @@
>
<img
class="flag"
:src="`/img/flags/svg-simple/${actor.origin.country.alpha2.toLowerCase()}.svg`"
:src="`/img/flags/${actor.origin.country.alpha2.toLowerCase()}.svg`"
>{{ actor.origin.country.alias || actor.origin.country.name }}
</span>
</span>
@ -117,7 +117,7 @@
>
<img
class="flag"
:src="`/img/flags/${actor.residence.country.alpha2.toLowerCase()}.png`"
:src="`/img/flags/${actor.residence.country.alpha2.toLowerCase()}.svg`"
>{{ actor.residence.country.alias || actor.residence.country.name }}
</span>
</span>
@ -134,16 +134,16 @@
<li
v-if="actor.bust || actor.waist || actor.hip"
title="bust-waist-hip"
class="bio-item"
class="bio-item figure"
>
<dfn class="bio-label"><Icon icon="ruler" />Figure</dfn>
<span>
<span class="bio-value">
<Icon
v-if="actor.naturalBoobs === false"
v-tooltip="'Boobs enhanced'"
icon="magic-wand"
v-tooltip="'Enhanced boobs'"
icon="star"
class="enhanced"
/>{{ actor.bust || '??' }}-{{ actor.waist || '??' }}-{{ actor.hip || '??' }}
/>{{ actor.bust || '??' }}{{ actor.cup || '?' }}-{{ actor.waist || '??' }}-{{ actor.hip || '??' }}
</span>
</li>
@ -412,6 +412,12 @@ export default {
}
}
.bio-label,
.bio-value {
display: flex;
align-items: center;
}
.bio-label {
color: $highlight;
margin: 0 1rem 0 0;
@ -421,7 +427,7 @@ export default {
.icon {
fill: $highlight;
margin: 0 .5rem 0 0;
margin: -.25rem .5rem 0 0;
}
}
@ -430,6 +436,10 @@ export default {
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
.icon {
margin: -.25rem 0 0 0;
}
}
.flag {
@ -456,6 +466,11 @@ export default {
.country {
display: flex;
justify-content: flex-end;
}
.figure .bio-label .icon {
margin: -.5rem .5rem 0 0;
}
.height-imperial,

7
assets/img/icons/cash.svg Normal file
View File

@ -0,0 +1,7 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="17" height="16" viewBox="0 0 17 16">
<title>cash</title>
<path d="M7 7h1v1h-1v-1z"></path>
<path d="M0 4v9h17v-9h-17zM3 12h-2v-2h1v1h1v1zM3 6h-1v1h-1v-2h2v1zM10.5 8c0.276 0 0.5 0.224 0.5 0.5v2c0 0.276-0.224 0.5-0.5 0.5h-1.5v0.5c0 0.276-0.224 0.5-0.5 0.5s-0.5-0.224-0.5-0.5v-0.5h-1.5c-0.276 0-0.5-0.224-0.5-0.5s0.224-0.5 0.5-0.5h1.5v-1h-1.5c-0.276 0-0.5-0.224-0.5-0.5v-2c0-0.276 0.224-0.5 0.5-0.5h1.5v-0.5c0-0.276 0.224-0.5 0.5-0.5s0.5 0.224 0.5 0.5v0.5h1.5c0.276 0 0.5 0.224 0.5 0.5s-0.224 0.5-0.5 0.5h-1.5v1h1.5zM16 12h-2v-1h1v-1h1v2zM16 7h-1v-1h-1v-1h2v2z"></path>
<path d="M9 9h1v1h-1v-1z"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 702 B

9
assets/img/icons/cash3.svg Normal file
View File

@ -0,0 +1,9 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="17" height="16" viewBox="0 0 17 16">
<title>cash3</title>
<path d="M7 9h1v1h-1v-1z"></path>
<path d="M0 6v9h17v-9h-17zM3 14h-2v-2h1v1h1v1zM3 8h-1v1h-1v-2h2v1zM10.5 10c0.276 0 0.5 0.224 0.5 0.5v2c0 0.276-0.224 0.5-0.5 0.5h-1.5v0.5c0 0.276-0.224 0.5-0.5 0.5s-0.5-0.224-0.5-0.5v-0.5h-1.5c-0.276 0-0.5-0.224-0.5-0.5s0.224-0.5 0.5-0.5h1.5v-1h-1.5c-0.276 0-0.5-0.224-0.5-0.5v-2c0-0.276 0.224-0.5 0.5-0.5h1.5v-0.5c0-0.276 0.224-0.5 0.5-0.5s0.5 0.224 0.5 0.5v0.5h1.5c0.276 0 0.5 0.224 0.5 0.5s-0.224 0.5-0.5 0.5h-1.5v1h1.5zM16 14h-2v-1h1v-1h1v2zM16 9h-1v-1h-1v-1h2v2z"></path>
<path d="M9 11h1v1h-1v-1z"></path>
<path d="M1 4h15v1.5h-15v-1.5z"></path>
<path d="M2 2h13v1.5h-13v-1.5z"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 785 B

5
assets/img/icons/coin-dollar.svg Normal file
View File

@ -0,0 +1,5 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
<title>coin-dollar</title>
<path d="M7.5 1c-4.142 0-7.5 3.358-7.5 7.5s3.358 7.5 7.5 7.5c4.142 0 7.5-3.358 7.5-7.5s-3.358-7.5-7.5-7.5zM7.5 14.5c-3.314 0-6-2.686-6-6s2.686-6 6-6c3.314 0 6 2.686 6 6s-2.686 6-6 6zM8 8v-2h2v-1h-2v-1h-1v1h-2v4h2v2h-2v1h2v1h1v-1h2l-0-4h-2zM7 8h-1v-2h1v2zM9 11h-1v-2h1v2z"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 445 B

5
assets/img/icons/plus-circle.svg Normal file
View File

@ -0,0 +1,5 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
<title>plus-circle</title>
<path d="M8 0c-4.418 0-8 3.582-8 8s3.582 8 8 8 8-3.582 8-8-3.582-8-8-8zM8 14c-3.314 0-6-2.686-6-6s2.686-6 6-6c3.314 0 6 2.686 6 6s-2.686 6-6 6zM12 9h-3v3h-2v-3h-3v-2h3v-3h2v3h3z"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 352 B

5
assets/img/icons/price-tag.svg Normal file
View File

@ -0,0 +1,5 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
<title>price-tag</title>
<path d="M6 8h1v2h-1zM8 11h1v2h-1zM12.514 4.47l-3.611-3.939c-0.267-0.292-0.796-0.53-1.174-0.53h-0.458c-0.378 0-0.906 0.239-1.174 0.53l-3.611 3.939c-0.267 0.292-0.486 0.868-0.486 1.28v9.5c0 0.412 0.309 0.75 0.688 0.75h9.625c0.378 0 0.688-0.338 0.688-0.75v-9.5c0-0.412-0.219-0.989-0.486-1.28zM10 8h-2v2h2v4h-2v1h-1v-1h-2v-1h2v-2h-2v-4h2v-1h1v1h2v1zM8.281 2.5c0 0.431-0.35 0.781-0.781 0.781s-0.781-0.35-0.781-0.781 0.35-0.781 0.781-0.781 0.781 0.35 0.781 0.781z"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 631 B

7
assets/img/icons/vector.svg Normal file
View File

@ -0,0 +1,7 @@
<!-- Generated by IcoMoon.io -->
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
<title>vector</title>
<path d="M5 5v2.339c-1.879 2.383-3 5.391-3 8.661h1c0-1.755 0.344-3.458 1.021-5.060 0.447-1.058 1.027-2.042 1.73-2.94h2.249v-2.249c0.898-0.703 1.882-1.283 2.94-1.73 1.602-0.678 3.304-1.021 5.060-1.021v-1c-3.27 0-6.278 1.121-8.661 3h-2.339zM5 15h2v1h-2v-1zM9 15h2v1h-2v-1zM15 13v2h-2v1h3v-3h-1zM15 5h1v2h-1v-2zM15 9h1v2h-1v-2z"></path>
<path d="M1 5c-0.552 0-1 0.448-1 1s0.448 1 1 1v9h1v-10c0-0.552-0.448-1-1-1z"></path>
<path d="M7 1c0-0.552-0.448-1-1-1s-1 0.448-1 1 0.448 1 1 1h10v-1h-9z"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 657 B

26
assets/js/actors/actions.js
View File

@ -37,8 +37,11 @@ function curateActor(actor) {
};
if (actor.profiles && actor.profiles.length > 0) {
curatedActor.avatar = actor.profiles.slice(0, 1)[0].avatar;
curatedActor.photos = actor.profiles.slice(1).map(profile => profile.avatar);
const photos = actor.profiles
.map(profile => profile.avatar)
.filter(avatar => avatar && (!curatedActor.avatar || avatar.hash !== curatedActor.avatar.hash));
curatedActor.photos = Object.values(photos.reduce((acc, photo) => ({ ...acc, [photo.hash]: photo }), {}));
}
if (actor.releases) {
@ -76,6 +79,7 @@ function initActorActions(store, _router) {
birthdate: dateOfBirth
age
ethnicity
cup
bust
waist
hip
@ -95,6 +99,15 @@ function initActorActions(store, _router) {
id
name
slug
}
avatar: avatarMedia {
id
path
thumbnail
lazy
hash
comment
copyright
}
profiles: actorsProfiles {
description
@ -103,6 +116,7 @@ function initActorActions(store, _router) {
path
thumbnail
lazy
hash
comment
copyright
}
@ -226,6 +240,14 @@ function initActorActions(store, _router) {
name
slug
}
avatar: avatarMedia {
id
path
thumbnail
lazy
comment
copyright
}
actorsProfiles {
actorsAvatarByProfileId {
media {

2
config/default.js
View File

@ -69,7 +69,7 @@ module.exports = {
'famedigital',
],
[
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), Burning Angel and Wicked have their own assets
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), (sometimes) Burning Angel and Wicked have their own assets
'xempire',
'blowpass',
],

4
migrations/20190325001339_releases.js
View File

@ -312,6 +312,10 @@ exports.up = knex => Promise.resolve()
table.string('piercings');
table.string('tattoos');
table.string('avatar_media_id', 21)
.references('id')
.inTable('media');
table.integer('batch_id', 12)
.references('id')
.inTable('batches');

143
src/actors.js
View File

@ -2,6 +2,7 @@
const config = require('config');
const Promise = require('bluebird');
const moment = require('moment');
// const logger = require('./logger')(__filename);
const knex = require('./knex');
@ -10,12 +11,46 @@ const scrapers = require('./scrapers/scrapers').actors;
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const logger = require('./logger')(__filename);
const { toBaseReleases } = require('./deep');
const { associateAvatars } = require('./media');
const slugify = require('./utils/slugify');
const capitalize = require('./utils/capitalize');
const resolvePlace = require('./utils/resolve-place');
const { associateAvatars } = require('./media');
const { toBaseReleases } = require('./deep');
function getMostFrequent(items) {
const { mostFrequent } = items.reduce((acc, item) => {
acc.counts[item] = (acc.counts[item] || 0) + 1;
if (!acc.mostFrequent || acc.counts[item] > acc.counts[acc.mostFrequent]) {
acc.mostFrequent = item;
}
return acc;
}, {
counts: {},
mostFrequent: null,
});
return mostFrequent;
}
function getMostFrequentDate(dates) {
const year = getMostFrequent(dates.map(dateX => dateX.getFullYear()));
const month = getMostFrequent(dates.map(dateX => dateX.getMonth()));
const date = getMostFrequent(dates.map(dateX => dateX.getDate()));
return moment({ year, month, date }).toDate();
}
function getLongest(items) {
return items.sort((itemA, itemB) => itemB.length - itemA.length)[0] || null;
}
function getAverage(items) {
return Math.round(items.reduce((acc, item) => acc + item, 0) / items.length);
}
function toBaseActors(actorsOrNames, release) {
return actorsOrNames.map((actorOrName) => {
@ -64,10 +99,10 @@ function curateProfileEntry(profile) {
description: profile.description,
birth_city: profile.placeOfBirth?.city || null,
birth_state: profile.placeOfBirth?.state || null,
birth_country_alpha2: profile.placeOfBirth?.country?.alpha2 || null,
birth_country_alpha2: profile.placeOfBirth?.country || null,
residence_city: profile.placeOfResidence?.city || null,
residence_state: profile.placeOfResidence?.state || null,
residence_country_alpha2: profile.placeOfResidence?.country?.alpha2 || null,
residence_country_alpha2: profile.placeOfResidence?.country || null,
cup: profile.cup,
bust: profile.bust,
waist: profile.waist,
@ -131,6 +166,7 @@ async function curateProfile(profile) {
curatedProfile.hasTattoos = typeof profile.hasTattoos === 'boolean' ? profile.hasTattoos : null;
curatedProfile.hasPiercings = typeof profile.hasPiercings === 'boolean' ? profile.hasPiercings : null;
if (argv.resolvePlace) {
const [placeOfBirth, placeOfResidence] = await Promise.all([
resolvePlace(profile.birthPlace),
resolvePlace(profile.residencePlace),
@ -138,6 +174,7 @@ async function curateProfile(profile) {
curatedProfile.placeOfBirth = placeOfBirth;
curatedProfile.placeOfResidence = placeOfResidence;
}
if (!curatedProfile.placeOfBirth && curatedProfile.nationality) {
const country = await knex('countries')
@ -164,6 +201,10 @@ async function curateProfile(profile) {
curatedProfile.releases = toBaseReleases(profile.releases);
if (argv.inspect) {
console.log(curatedProfile);
}
return curatedProfile;
} catch (error) {
logger.error(`Failed to curate '${profile.name}': ${error.message}`);
@ -172,6 +213,91 @@ async function curateProfile(profile) {
}
}
async function interpolateProfiles(actors) {
const profiles = await knex('actors_profiles')
.select(['actors_profiles.*', 'media.width as avatar_width', 'media.height as avatar_height', 'media.size as avatar_size'])
.whereIn('actor_id', actors.map(actor => actor.id))
.leftJoin('media', 'actors_profiles.avatar_media_id', 'media.id');
const profilesByActorId = profiles.reduce((acc, profile) => ({
...acc,
[profile.actor_id]: [
...(acc[profile.actor_id] || []),
profile,
],
}), {});
const interpolatedProfiles = Object.entries(profilesByActorId).map(([actorId, actorProfiles]) => {
const valuesByProperty = actorProfiles.reduce((acc, profile) => Object
.entries(profile)
.reduce((profileAcc, [property, value]) => ({
...profileAcc,
[property]: [
...(acc[property] || []),
...(value === null ? [] : [value]),
],
}), {}), {});
const avatars = actorProfiles.map(profile => profile.avatar_media_id && ({
id: profile.avatar_media_id,
width: profile.avatar_width,
height: profile.avatar_height,
size: profile.avatar_size,
})).filter(Boolean);
const profile = {
id: actorId,
};
profile.gender = getMostFrequent(valuesByProperty.gender);
profile.ethnicity = getMostFrequent(valuesByProperty.ethnicity.map(ethnicity => ethnicity.toLowerCase()));
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
profile.birth_city = getMostFrequent(valuesByProperty.birth_city);
profile.birth_state = getMostFrequent(valuesByProperty.birth_state);
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.birth_country_alpha2);
profile.residence_city = getMostFrequent(valuesByProperty.residence_city);
profile.residence_state = getMostFrequent(valuesByProperty.residence_state);
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence_country_alpha2);
profile.cup = getMostFrequent(valuesByProperty.cup);
profile.bust = getMostFrequent(valuesByProperty.bust);
profile.waist = getMostFrequent(valuesByProperty.waist);
profile.hip = getMostFrequent(valuesByProperty.hip);
profile.natural_boobs = getMostFrequent(valuesByProperty.natural_boobs);
profile.hair = getMostFrequent(valuesByProperty.hair.map(hair => hair.toLowerCase()));
profile.eyes = getMostFrequent(valuesByProperty.eyes.map(eyes => eyes.toLowerCase()));
profile.weight = getAverage(valuesByProperty.weight);
profile.height = getMostFrequent(valuesByProperty.height);
profile.has_tattoos = getMostFrequent(valuesByProperty.has_tattoos);
profile.has_piercings = getMostFrequent(valuesByProperty.has_piercings);
profile.tattoos = getLongest(valuesByProperty.tattoos);
profile.piercings = getLongest(valuesByProperty.piercings);
profile.avatar_media_id = avatars.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0].id;
return profile;
});
const transaction = await knex.transaction();
const queries = interpolatedProfiles.map(profile => knex('actors')
.where('id', profile.id)
.update(profile)
.transacting(transaction));
await Promise.all(queries)
.then(transaction.commit)
.catch(transaction.rollback);
}
async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
const profiles = Promise.map(sources, async (source) => {
try {
@ -217,7 +343,9 @@ async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
return profiles.filter(Boolean);
}
async function upsertProfiles(curatedProfileEntries) {
async function upsertProfiles(profiles) {
const curatedProfileEntries = profiles.map(profile => curateProfileEntry(profile));
const existingProfiles = await knex('actors_profiles')
.whereIn(['actor_id', 'network_id'], curatedProfileEntries.map(entry => [entry.actor_id, entry.network_id]))
.orWhereIn(['actor_id', 'site_id'], curatedProfileEntries.map(entry => [entry.actor_id, entry.site_id]));
@ -311,9 +439,8 @@ async function scrapeActors(actorNames) {
const profiles = await Promise.all(profilesPerActor.flat().map(profile => curateProfile(profile)));
const profilesWithAvatarIds = await associateAvatars(profiles);
const curatedProfileEntries = profilesWithAvatarIds.map(profile => curateProfileEntry(profile));
await upsertProfiles(curatedProfileEntries);
await upsertProfiles(profilesWithAvatarIds);
await interpolateProfiles(actors);
}
async function getOrCreateActors(baseActors, batchId) {

5
src/argv.js
View File

@ -177,6 +177,11 @@ const { argv } = yargs
type: 'string',
default: process.env.NODE_ENV === 'development' ? 'silly' : 'info',
})
.option('resolve-place', {
describe: 'Call OSM Nominatim API for actor place of birth and residence. Raw value discarded if disabled.',
type: 'boolean',
default: true,
})
.option('debug', {
describe: 'Show error stack traces',
type: 'boolean',

4
src/deep.js
View File

@ -52,6 +52,10 @@ async function findSites(baseReleases) {
}
function toBaseReleases(baseReleasesOrUrls) {
if (!baseReleasesOrUrls) {
return [];
}
return baseReleasesOrUrls
.map((baseReleaseOrUrl) => {
if (baseReleaseOrUrl.url) {

8
src/scrapers/brazzers.js
View File

@ -141,7 +141,7 @@ async function fetchActorReleases({ qu, html }, accReleases = []) {
return accReleases.concat(releases);
}
async function scrapeProfile(html, url, actorName) {
async function scrapeProfile(html, url, actorName, include) {
const qProfile = ex(html);
const { q, qa } = qProfile;
@ -175,7 +175,9 @@ async function scrapeProfile(html, url, actorName) {
const avatarEl = q('.big-pic-model-container img');
if (avatarEl) profile.avatar = `https:${avatarEl.src}`;
if (include.releases) {
profile.releases = await fetchActorReleases(qProfile);
}
return profile;
}
@ -198,7 +200,7 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName) {
async function fetchProfile(actorName, scraperSlug, siteOrNetwork, include) {
const searchUrl = 'https://brazzers.com/pornstars-search/';
const searchRes = await bhttp.get(searchUrl, {
headers: {
@ -212,7 +214,7 @@ async function fetchProfile(actorName) {
const url = `https://brazzers.com${actorLink}`;
const res = await bhttp.get(url);
return scrapeProfile(res.body.toString(), url, actorName);
return scrapeProfile(res.body.toString(), url, actorName, include);
}
return null;

6
src/scrapers/gamma.js
View File

@ -368,7 +368,7 @@ function scrapeApiProfile(data, releases, siteSlug) {
const avatarPaths = Object.values(data.pictures).reverse();
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map(avatarPath => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
if (releases) profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
return profile;
}
@ -579,7 +579,7 @@ async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesU
return null;
}
async function fetchApiProfile(actorName, siteSlug) {
async function fetchApiProfile(actorName, siteSlug, site, include) {
const actorSlug = encodeURI(actorName);
const referer = `https://www.${siteSlug}.com/en/search`;
@ -603,7 +603,7 @@ async function fetchApiProfile(actorName, siteSlug) {
const actorData = res.body.results[0].hits.find(actor => slugify(actor.name) === slugify(actorName));
if (actorData) {
const actorScenes = await fetchActorScenes(actorData.name, apiUrl, siteSlug);
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);
return scrapeApiProfile(actorData, actorScenes, siteSlug);
}

6
src/web/plugins/actors.js
View File

@ -12,7 +12,7 @@ const schemaExtender = makeExtendSchemaPlugin(_build => ({
}
extend type Actor {
age: Int @requires(columns: ["date_of_birth"])
age: Int @requires(columns: ["dateOfBirth"])
height(units:Units): String @requires(columns: ["height"])
weight(units:Units): String @requires(columns: ["weight"])
}
@ -20,9 +20,9 @@ const schemaExtender = makeExtendSchemaPlugin(_build => ({
resolvers: {
Actor: {
age(parent, _args, _context, _info) {
if (!parent.birthdate) return null;
if (!parent.dateOfBirth) return null;
return moment().diff(parent.birthdate, 'years');
return moment().diff(parent.dateOfBirth, 'years');
},
height(parent, args, _context, _info) {
if (!parent.height) return null;