diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index bbad1083..42e11fb5 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -325,6 +325,8 @@ exports.up = knex => Promise.resolve() table.integer('priority', 4) .defaultTo(1); + table.text('url'); + table.text('real_name'); table.text('gender', 18); diff --git a/public/img/tags/milf/1.jpeg b/public/img/tags/milf/1.jpeg new file mode 100644 index 00000000..be54e07c Binary files /dev/null and b/public/img/tags/milf/1.jpeg differ diff --git a/public/img/tags/milf/lazy/1.jpeg b/public/img/tags/milf/lazy/1.jpeg new file mode 100644 index 00000000..525b834b Binary files /dev/null and b/public/img/tags/milf/lazy/1.jpeg differ diff --git a/public/img/tags/milf/thumbs/0.jpeg b/public/img/tags/milf/thumbs/0.jpeg index ffded92d..f325544e 100644 Binary files a/public/img/tags/milf/thumbs/0.jpeg and b/public/img/tags/milf/thumbs/0.jpeg differ diff --git a/public/img/tags/milf/thumbs/1.jpeg b/public/img/tags/milf/thumbs/1.jpeg new file mode 100644 index 00000000..d8767cc8 Binary files /dev/null and b/public/img/tags/milf/thumbs/1.jpeg differ diff --git a/seeds/04_media.js b/seeds/04_media.js index 87f97e35..391c2e73 100644 --- a/seeds/04_media.js +++ b/seeds/04_media.js @@ -628,7 +628,7 @@ const tagPosters = [ ['latina', 2, 'Veronica Leal for Her Limit'], ['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'], ['maid', 0, 'Whitney Wright in "Dredd Up Your Ass 2" for Jules Jordan'], - ['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'], + ['milf', 1, 'Francesca Le for Evil Angel'], ['mff', 1, 'Anikka Albrite, Kelsi Monroe and Mick Blue for HardX'], ['mfm', 0, 'Vina Sky in "Jules Jordan\'s Three Ways" for Jules Jordan'], ['natural-boobs', 1, 'Nia Nacci for First Class POV'], @@ -723,6 +723,7 @@ const tagPhotos = [ ['gaping', 2, 'Alex Grey in "DP Masters 5" for Jules Jordan'], ['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'], ['latina', 0, 'Vienna Black for Spizoo'], + // ['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'], ['mff', 0, 'Madison Ivy, Adriana Chechik and Keiran Lee in "Day With A Pornstar" for Brazzers'], ['mfm', 6, 'Honey Gold in "Slut Puppies 12" for Jules Jordan'], ['natural-boobs', 0, 'Valentina Nappi in "Hypnotic Curves" for LesbianX'], diff --git a/src/actors.js b/src/actors.js index c73ed5ef..55b19042 100644 --- a/src/actors.js +++ b/src/actors.js @@ -148,6 +148,7 @@ function curateActor(actor, withDetails = false, isProfile = false) { id: actor.id, name: actor.name, slug: actor.slug, + url: actor.url, gender: actor.gender, entityId: actor.entity_id, aliasFor: actor.alias_for, @@ -238,6 +239,7 @@ function curateProfileEntry(profile) { entity_id: profile.entity?.id || null, date_of_birth: profile.dateOfBirth, date_of_death: profile.dateOfDeath, + url: profile.url, gender: profile.gender, ethnicity: profile.ethnicity, description: profile.description, @@ -552,7 +554,10 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy logger.verbose(`Searching profile for '${actor.name}' on '${label}'`); - const profile = await scraper.fetchProfile(actor, context, include); + const profile = await scraper.fetchProfile(curateActor({ + ...existingProfile, + ...actor, + }), context, include); if (!profile || typeof profile === 'number') { // scraper returns HTTP code on request failure logger.verbose(`Profile for '${actor.name}' not available on ${label}, scraper returned ${profile}`); @@ -657,7 +662,11 @@ async function scrapeActors(argNames) { const actors = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []); - const existingProfiles = await knex('actors_profiles').whereIn('actor_id', actors.map(actor => actor.id)); + const existingProfiles = await knex('actors_profiles') + .select(knex.raw('actors_profiles.*, row_to_json(avatars) as avatar')) + .whereIn('actor_id', actors.map(actor => actor.id)) + .leftJoin('media as avatars', 'avatars.id', 'actors_profiles.avatar_media_id'); + const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({ ...acc, [profile.actor_id]: { @@ -724,6 +733,8 @@ async function getOrCreateActors(baseActors, batchId) { })) .filter(actor => !!actor.id); + console.log(newActorIdsByEntityIdAndSlug, newActorProfiles); + await storeProfiles(newActorProfiles); if (Array.isArray(newActors)) { diff --git a/src/scrapers/elegantangel.js b/src/scrapers/elegantangel.js index da4f3a31..75487ad9 100644 --- a/src/scrapers/elegantangel.js +++ b/src/scrapers/elegantangel.js @@ -2,6 +2,7 @@ const qu = require('../utils/q'); const slugify = require('../utils/slugify'); +const { feetInchesToCm, lbsToKg } = require('../utils/convert'); function scrapeAll(scenes, channel) { return scenes.map(({ query }) => { @@ -58,6 +59,7 @@ async function scrapeRelease({ query, html }, url, channel, baseRelease, type = return { name: qu.query.cnt(el, 'span'), + url: qu.query.url(el, 'a', 'href', { origin: channel.url }), avatar: [ avatar.replace(/\/actor\/\d+/, '/actor/1600'), avatar, @@ -124,6 +126,68 @@ function scrapeMovies(movies, channel) { }); } +function scrapeActorScenes(scenes, channel) { + return scenes.map(({ query }) => { + const release = {}; + + release.url = query.url('a', 'href', { origin: channel.url }); + release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1]; + + release.title = query.cnt('.grid-item-title'); + + const poster = query.img('a img'); + release.poster = [ + poster.replace(/\/\d+\//, '/1600/'), + poster, + ]; + + return release; + }); +} + +async function scrapeProfile({ query }, url, channel, include) { + const profile = {}; + + const bio = query.cnts('.performer-page-header li').reduce((acc, info) => { + const [key, value] = info.split(':'); + + return { + ...acc, + [slugify(key, '_')]: value.trim(), + }; + }, {}); + + const measurements = bio.meas?.match(/(\d+)(\w+)-(\d+)-(\d+)/); + + if (measurements) { + [profile.bust, profile.cup, profile.waist, profile.hip] = measurements.slice(1); + } + + profile.hair = bio.hair; + profile.eyes = bio.eyes; + profile.ethnicity = bio.ethnicity; + + profile.height = feetInchesToCm(bio.height); + profile.weight = lbsToKg(bio.weight); + + profile.avatar = query.img('picture img'); + + if (include) { + const actorId = new URL(url).pathname.match(/\/(\d+)/)[1]; + const res = await qu.getAll(`https://www.elegantangel.com/streaming-video-by-scene.html?cast=${actorId}`, '.grid-item', null, { + rejectUnauthorized: false, + }); + + if (res.ok) { + profile.releases = scrapeActorScenes(res.items, channel); + } + } + + console.log(profile); + + return profile; +} + async function fetchLatest(channel, page = 1) { const url = `${channel.url}/tour?page=${page}`; const res = await qu.getAll(url, '.scene-update', null, { @@ -177,9 +241,26 @@ async function fetchMovies(channel, page = 1) { return res.status; } +async function fetchProfile(baseActor, channel, include) { + if (!baseActor.url) { + return null; + } + + const res = await qu.get(baseActor.url, '.performer-page', null, { + rejectUnauthorized: false, + }); + + if (res.ok) { + return scrapeProfile(res.item, baseActor.url, channel, include); + } + + return res.status; +} + module.exports = { fetchLatest, fetchScene, fetchMovies, fetchMovie, + fetchProfile, }; diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index ba748fa2..86368d89 100644 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -178,6 +178,7 @@ module.exports = { devilsfilm: famedigital, digitalplayground, dtfsluts: fullpornnetwork, + elegantangel, evilangel, eyeontheguy: hush, fakehub,