diff --git a/seeds/00_tags.js b/seeds/00_tags.js index 94cb0332..b1bb0a52 100755 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -95,6 +95,14 @@ const tags = [ slug: '69', group: 'position', }, + { + name: 'AI generated', + slug: 'ai', + }, + { + name: 'AI enhanced', + slug: 'aienhanced', + }, { name: 'airtight', slug: 'airtight', diff --git a/seeds/01_networks.js b/seeds/01_networks.js index bac867c1..1ad71d00 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -621,6 +621,11 @@ const networks = [ url: 'https://mamacitaz.com', parent: 'porndoe', }, + { + slug: 'manyvids', + name: 'ManyVids', + url: 'https://www.manyvids.com', + }, { slug: 'men', name: 'Men', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index bae25e0c..1cd41c4c 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -7691,6 +7691,14 @@ const sites = [ delete: true, parent: 'mamacitaz', }, + // MANYVIDS + { + name: 'nicoledoshi', + slug: 'mv.nicoledoshi', + url: 'https://nicoledoshi.manyvids.com', + hasLogo: false, + parent: 'manyvids', + }, // MARISKA X { name: 'MariskaX', diff --git a/src/actors.js b/src/actors.js index 7530de38..163c0062 100755 --- a/src/actors.js +++ b/src/actors.js @@ -214,6 +214,7 @@ function curateActor(actor, withDetails = false, isProfile = false) { name: actor.name, slug: actor.slug, url: actor.url, + entryId: actor.entry_id, gender: actor.gender, orientation: actor.orientation, entityId: actor.entity_id, @@ -693,7 +694,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy } }), Promise.reject(new Error())); } catch (error) { - console.log(error); + // console.log(error); if (error.code !== 'PROFILE_NOT_AVAILABLE') { logger.error(`Failed to fetch profile for '${actor.name}': ${error.message}`); diff --git a/src/scrapers/actors.js b/src/scrapers/actors.js index 646365e1..8a285a20 100644 --- a/src/scrapers/actors.js +++ b/src/scrapers/actors.js @@ -34,6 +34,7 @@ const pornbox = require('./pornbox'); const littlecapricedreams = require('./littlecapricedreams'); const loveherfilms = require('./loveherfilms'); const bluedonkeymedia = require('./bluedonkeymedia'); +const manyvids = require('./manyvids'); const mikeadriano = require('./mikeadriano'); const aylo = require('./aylo'); const missax = require('./missax'); @@ -206,14 +207,6 @@ module.exports = { jerkaoke: modelmedia, modelmediaasia: modelmedia, delphine: modelmedia, - // etc - '18vr': badoink, - theflourishxxx: theflourish, - pierrewoodman, - exploitedx, // only from known URL that will specify site - fullpornnetwork, - adultempire, - allherluv: missax, // americanpornstar, angelogodshackoriginal, babevr: badoink, @@ -283,4 +276,13 @@ module.exports = { slayed: vixen, wifey: vixen, vrcosplayx: badoink, + // etc + '18vr': badoink, + theflourishxxx: theflourish, + pierrewoodman, + exploitedx, // only from known URL that will specify site + fullpornnetwork, + adultempire, + allherluv: missax, + manyvids, }; diff --git a/src/scrapers/jaxslayher.js b/src/scrapers/jaxslayher.js index dc7f1cf4..866f93ce 100755 --- a/src/scrapers/jaxslayher.js +++ b/src/scrapers/jaxslayher.js @@ -206,12 +206,13 @@ function scrapeProfile(payload, actorName, channel, url) { profile.entryId = actorData?.id; - profile.birthplace = getBioItem(bioData, 'born'); + profile.birthPlace = getBioItem(bioData, 'born'); profile.dateOfBirth = getBioItem(bioData, 'birthdate'); profile.age = getBioItem(bioData, 'age'); profile.eyes = getBioItem(bioData, 'eyes'); + profile.hairColor = getBioItem(bioData, 'hair'); profile.weight = convert(getBioItem(bioData, 'weight'), 'lb', 'kg'); profile.height = convert(getBioItem(bioData, 'height'), 'cm'); profile.measurements = getBioItem(bioData, 'measurements'); diff --git a/src/scrapers/manyvids.js b/src/scrapers/manyvids.js new file mode 100755 index 00000000..f36268f6 --- /dev/null +++ b/src/scrapers/manyvids.js @@ -0,0 +1,213 @@ +'use strict'; + +const unprint = require('unprint'); + +const slugify = require('../utils/slugify'); + +function splitTag(tag) { + return tag + .replace(/([a-z])([A-Z])/g, '$1 $2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') // handle sequence of caps + .toLowerCase() + .trim(); +} + +async function getTrailer(videoId) { + if (!videoId) { + return null; + } + + const res = await unprint.get(`https://api.manyvids.com/store/video/${videoId}/private`); + + if (res.ok && res.data?.statusCode === 200) { + return res.data.data.teaser.filepath; + } + + return null; +} + +function getModelUrl(modelData) { + const slug = modelData.slug || modelData.profileUrl?.match(/\/profile\/\d+\/(\w+)/i)?.[1] || null; + + if (slug) { + return `https://${slug}.manyvids.com`; + } + + return null; +} + +async function scrapeScene(data, context) { + const release = {}; + + release.url = data.url + ? `https://www.manyvids.com${data.url}` + : `https://www.manyvids.com/Video/${data.id}/${data.slug || ''}`; + + release.entryId = data.id; + + release.attributes = { + price: data.price?.regular, + discount: data.price?.discountedPrice, + isCustomOrder: data.isCustomOrder, + isPremium: data.isPremium, + isPrivate: data.isPrivate, + isExclusive: data.isExclusive, + isStreamOnly: data.isStreamOnly, + }; + + release.title = data.title; + release.description = data.description; + + release.date = new Date(data.launchDate); + release.duration = unprint.extractDuration(data.duration || data.videoDuration); + + const model = data.creator || data.model; + + release.actors = [{ + name: model.stageName || model.displayName, + entryId: model.id, + avatar: model.avatar?.url || model.avatar, + url: getModelUrl(model), + }]; + + release.tags = [ + data.is3D && '3d', + data.isAiGenerated && 'ai', + ...data.tagList?.map((tag) => splitTag(tag.label)) || [], + ].filter(Boolean); + + release.poster = data.thumbnail?.url || data.screenshot || data.thumbnail; // thumbnail URL is full size, thumbnail direct is thumb sized + + if (data.preview?.url) { + release.trailer = data.preview?.url; + } else if (context.includeTrailers) { + release.trailer = await getTrailer(data.id); + } + + release.qualities = [data.height].filter(Boolean); + + console.log(release); + + return release; +} + +async function fetchLatest(channel, page = 1) { + const idRes = await unprint.request(channel.url, { + method: 'head', + followRedirects: 0, + }); + + const profileId = idRes.headers.get('location')?.match(/\/profile\/(\d+)/i)?.[1]; + + if (profileId) { + const res = await unprint.get(`https://api.manyvids.com/store/videos/${profileId}?sort=newest&limit=100&page=${page}`); + + if (res.ok && res.data?.statusCode === 200) { + return Promise.all(res.data.data.map(async (data) => scrapeScene(data, channel))); + } + } + + return null; +} + +async function fetchScene(url, channel, baseRelease, context) { + const videoId = new URL(url).pathname.match(/\/video\/(\d+)/i)?.[1]; + + if (!videoId) { + return null; + } + + const res = await unprint.get(`https://api.manyvids.com/store/video/${videoId}`); + + if (res.ok && res.data?.statusCode === 200) { + return scrapeScene(res.data.data, { + channel, + includeTrailers: context.includeTrailers, + isDeep: !!baseRelease.entryId, + }); + } + + return res.status; +} + +function scrapeProfile(data) { + const profile = {}; + + profile.description = [data.bio, data.description].filter(Boolean).join(' | ') || null; + + profile.placeOfResidence = data.location || null; + profile.ethnicity = data.ethnicity || null; + profile.hairColor = data.hairColor || null; + + profile.gender = data.identification?.toLowerCase() || null; + profile.orientation = data.orientation; + + profile.avatar = data.avatar || null; + profile.photos = [data.portrait].filter(Boolean); + + profile.socials = [data.socLinkTwitter, data.socLinkInstagram, data.socLinkReddit].filter(Boolean); + + const dateOfBirth = data.dob && unprint.extractDate(data.dob); + + if (dateOfBirth && new Date().getFullYear() - dateOfBirth.getFullYear() > 18) { + // date of births that would not be legal are likely bogus + profile.dateOfBirth = dateOfBirth; + } + + console.log(profile); + + return profile; +} + +async function getActorUrl(actor) { + if (actor.url) { + const idRes = await unprint.request(actor.url, { + method: 'head', + followRedirects: 0, + }); + + const profileId = idRes.headers.get('location')?.match(/\/profile\/(\d+)/i)?.[1]; + + if (profileId) { + return `https://api.manyvids.com/profile/profile/${profileId}`; + } + } + + const res = await unprint.get(`https://api.manyvids.com/search/autocomplete?keywords=${slugify(actor.name, '+')}`); + + if (res.ok) { + const model = res.data?.models?.find((modelResult) => modelResult.stage_name === actor.name || modelResult.username === actor.name); + + if (model) { + const profileId = model.url?.match(/\/profile\/(\d+)/i)?.[1]; + + if (profileId) { + return `https://api.manyvids.com/profile/profile/${profileId}`; + } + } + } + + return null; +} + +async function fetchProfile(actor, entity) { + const actorUrl = await getActorUrl(actor); + + if (actorUrl) { + const res = await unprint.get(actorUrl); + + if (res.ok && res.data?.username) { + return scrapeProfile(res.data, entity); + } + + return res.status; + } + + return actorUrl; +} + +module.exports = { + fetchLatest, + fetchProfile, + fetchScene, +}; diff --git a/src/scrapers/releases.js b/src/scrapers/releases.js index c9473d80..24b0e55f 100644 --- a/src/scrapers/releases.js +++ b/src/scrapers/releases.js @@ -44,6 +44,7 @@ const pornbox = require('./pornbox'); const littlecapricedreams = require('./littlecapricedreams'); const loveherfilms = require('./loveherfilms'); const bluedonkeymedia = require('./bluedonkeymedia'); +const manyvids = require('./manyvids'); const mikeadriano = require('./mikeadriano'); const aylo = require('./aylo'); const missax = require('./missax'); @@ -150,6 +151,7 @@ module.exports = { littlecapricedreams, loveherfilms, mamacitaz: porndoe, + manyvids, mariskax, mikeadriano, missax, diff --git a/tests/profiles.js b/tests/profiles.js index e9a52e04..ed185eff 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -265,6 +265,7 @@ const actors = [ { entity: 'tokyohot', name: 'Mai Kawana', url: 'https://my.tokyo-hot.com/cast/2099/', fields: ['avatar', 'birthPlace', 'height', 'cup', 'bust', 'waist', 'hip', 'hairStyle', 'shoeSize', 'bloodType'] }, { entity: 'wakeupnfuck', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] }, { entity: 'darkkotv', name: 'Aidra Fox', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'measurements', 'naturalBoobs', 'hasTattoos', 'hasPiercings'] }, + { entity: 'jaxslayher', name: 'Eliza Ibarra', fields: ['avatar', 'description', 'eyes', 'height', 'weight', 'birthPlace', 'measurements', 'hairColor'] }, ]; const actorScrapers = scrapers.actors;