Storing actor profile URL when provided from scene page.

This commit is contained in:
DebaucheryLibrarian 2020-08-31 02:43:41 +02:00
parent 1bfdf4b232
commit f6353ca14c
9 changed files with 99 additions and 3 deletions

View File

@ -325,6 +325,8 @@ exports.up = knex => Promise.resolve()
table.integer('priority', 4) table.integer('priority', 4)
.defaultTo(1); .defaultTo(1);
table.text('url');
table.text('real_name'); table.text('real_name');
table.text('gender', 18); table.text('gender', 18);

BIN
public/img/tags/milf/1.jpeg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 613 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 43 KiB

After

Width:  |  Height:  |  Size: 34 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 40 KiB

View File

@ -628,7 +628,7 @@ const tagPosters = [
['latina', 2, 'Veronica Leal for Her Limit'], ['latina', 2, 'Veronica Leal for Her Limit'],
['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'], ['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'],
['maid', 0, 'Whitney Wright in "Dredd Up Your Ass 2" for Jules Jordan'], ['maid', 0, 'Whitney Wright in "Dredd Up Your Ass 2" for Jules Jordan'],
['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'], ['milf', 1, 'Francesca Le for Evil Angel'],
['mff', 1, 'Anikka Albrite, Kelsi Monroe and Mick Blue for HardX'], ['mff', 1, 'Anikka Albrite, Kelsi Monroe and Mick Blue for HardX'],
['mfm', 0, 'Vina Sky in "Jules Jordan\'s Three Ways" for Jules Jordan'], ['mfm', 0, 'Vina Sky in "Jules Jordan\'s Three Ways" for Jules Jordan'],
['natural-boobs', 1, 'Nia Nacci for First Class POV'], ['natural-boobs', 1, 'Nia Nacci for First Class POV'],
@ -723,6 +723,7 @@ const tagPhotos = [
['gaping', 2, 'Alex Grey in "DP Masters 5" for Jules Jordan'], ['gaping', 2, 'Alex Grey in "DP Masters 5" for Jules Jordan'],
['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'], ['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'],
['latina', 0, 'Vienna Black for Spizoo'], ['latina', 0, 'Vienna Black for Spizoo'],
// ['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'],
['mff', 0, 'Madison Ivy, Adriana Chechik and Keiran Lee in "Day With A Pornstar" for Brazzers'], ['mff', 0, 'Madison Ivy, Adriana Chechik and Keiran Lee in "Day With A Pornstar" for Brazzers'],
['mfm', 6, 'Honey Gold in "Slut Puppies 12" for Jules Jordan'], ['mfm', 6, 'Honey Gold in "Slut Puppies 12" for Jules Jordan'],
['natural-boobs', 0, 'Valentina Nappi in "Hypnotic Curves" for LesbianX'], ['natural-boobs', 0, 'Valentina Nappi in "Hypnotic Curves" for LesbianX'],

View File

@ -148,6 +148,7 @@ function curateActor(actor, withDetails = false, isProfile = false) {
id: actor.id, id: actor.id,
name: actor.name, name: actor.name,
slug: actor.slug, slug: actor.slug,
url: actor.url,
gender: actor.gender, gender: actor.gender,
entityId: actor.entity_id, entityId: actor.entity_id,
aliasFor: actor.alias_for, aliasFor: actor.alias_for,
@ -238,6 +239,7 @@ function curateProfileEntry(profile) {
entity_id: profile.entity?.id || null, entity_id: profile.entity?.id || null,
date_of_birth: profile.dateOfBirth, date_of_birth: profile.dateOfBirth,
date_of_death: profile.dateOfDeath, date_of_death: profile.dateOfDeath,
url: profile.url,
gender: profile.gender, gender: profile.gender,
ethnicity: profile.ethnicity, ethnicity: profile.ethnicity,
description: profile.description, description: profile.description,
@ -552,7 +554,10 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
logger.verbose(`Searching profile for '${actor.name}' on '${label}'`); logger.verbose(`Searching profile for '${actor.name}' on '${label}'`);
const profile = await scraper.fetchProfile(actor, context, include); const profile = await scraper.fetchProfile(curateActor({
...existingProfile,
...actor,
}), context, include);
if (!profile || typeof profile === 'number') { // scraper returns HTTP code on request failure if (!profile || typeof profile === 'number') { // scraper returns HTTP code on request failure
logger.verbose(`Profile for '${actor.name}' not available on ${label}, scraper returned ${profile}`); logger.verbose(`Profile for '${actor.name}' not available on ${label}, scraper returned ${profile}`);
@ -657,7 +662,11 @@ async function scrapeActors(argNames) {
const actors = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []); const actors = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []);
const existingProfiles = await knex('actors_profiles').whereIn('actor_id', actors.map(actor => actor.id)); const existingProfiles = await knex('actors_profiles')
.select(knex.raw('actors_profiles.*, row_to_json(avatars) as avatar'))
.whereIn('actor_id', actors.map(actor => actor.id))
.leftJoin('media as avatars', 'avatars.id', 'actors_profiles.avatar_media_id');
const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({ const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({
...acc, ...acc,
[profile.actor_id]: { [profile.actor_id]: {
@ -724,6 +733,8 @@ async function getOrCreateActors(baseActors, batchId) {
})) }))
.filter(actor => !!actor.id); .filter(actor => !!actor.id);
console.log(newActorIdsByEntityIdAndSlug, newActorProfiles);
await storeProfiles(newActorProfiles); await storeProfiles(newActorProfiles);
if (Array.isArray(newActors)) { if (Array.isArray(newActors)) {

View File

@ -2,6 +2,7 @@
const qu = require('../utils/q'); const qu = require('../utils/q');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
const { feetInchesToCm, lbsToKg } = require('../utils/convert');
function scrapeAll(scenes, channel) { function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => { return scenes.map(({ query }) => {
@ -58,6 +59,7 @@ async function scrapeRelease({ query, html }, url, channel, baseRelease, type =
return { return {
name: qu.query.cnt(el, 'span'), name: qu.query.cnt(el, 'span'),
url: qu.query.url(el, 'a', 'href', { origin: channel.url }),
avatar: [ avatar: [
avatar.replace(/\/actor\/\d+/, '/actor/1600'), avatar.replace(/\/actor\/\d+/, '/actor/1600'),
avatar, avatar,
@ -124,6 +126,68 @@ function scrapeMovies(movies, channel) {
}); });
} }
function scrapeActorScenes(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
release.title = query.cnt('.grid-item-title');
const poster = query.img('a img');
release.poster = [
poster.replace(/\/\d+\//, '/1600/'),
poster,
];
return release;
});
}
async function scrapeProfile({ query }, url, channel, include) {
const profile = {};
const bio = query.cnts('.performer-page-header li').reduce((acc, info) => {
const [key, value] = info.split(':');
return {
...acc,
[slugify(key, '_')]: value.trim(),
};
}, {});
const measurements = bio.meas?.match(/(\d+)(\w+)-(\d+)-(\d+)/);
if (measurements) {
[profile.bust, profile.cup, profile.waist, profile.hip] = measurements.slice(1);
}
profile.hair = bio.hair;
profile.eyes = bio.eyes;
profile.ethnicity = bio.ethnicity;
profile.height = feetInchesToCm(bio.height);
profile.weight = lbsToKg(bio.weight);
profile.avatar = query.img('picture img');
if (include) {
const actorId = new URL(url).pathname.match(/\/(\d+)/)[1];
const res = await qu.getAll(`https://www.elegantangel.com/streaming-video-by-scene.html?cast=${actorId}`, '.grid-item', null, {
rejectUnauthorized: false,
});
if (res.ok) {
profile.releases = scrapeActorScenes(res.items, channel);
}
}
console.log(profile);
return profile;
}
async function fetchLatest(channel, page = 1) { async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/tour?page=${page}`; const url = `${channel.url}/tour?page=${page}`;
const res = await qu.getAll(url, '.scene-update', null, { const res = await qu.getAll(url, '.scene-update', null, {
@ -177,9 +241,26 @@ async function fetchMovies(channel, page = 1) {
return res.status; return res.status;
} }
async function fetchProfile(baseActor, channel, include) {
if (!baseActor.url) {
return null;
}
const res = await qu.get(baseActor.url, '.performer-page', null, {
rejectUnauthorized: false,
});
if (res.ok) {
return scrapeProfile(res.item, baseActor.url, channel, include);
}
return res.status;
}
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchMovies, fetchMovies,
fetchMovie, fetchMovie,
fetchProfile,
}; };

View File

@ -178,6 +178,7 @@ module.exports = {
devilsfilm: famedigital, devilsfilm: famedigital,
digitalplayground, digitalplayground,
dtfsluts: fullpornnetwork, dtfsluts: fullpornnetwork,
elegantangel,
evilangel, evilangel,
eyeontheguy: hush, eyeontheguy: hush,
fakehub, fakehub,