diff --git a/src/actors.js b/src/actors.js index 08ac79e6..9fb59a18 100755 --- a/src/actors.js +++ b/src/actors.js @@ -833,7 +833,7 @@ async function scrapeActors(argNames) { const entitySlugs = sources.flat(); const [entitiesBySlug, existingActorEntries] = await Promise.all([ - fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'], prefer: argv.prefer }), + fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'], prefer: argv.prefer || 'channel' }), knex('actors') .select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity')) .whereIn('actors.slug', baseActors.map((baseActor) => baseActor.slug)) diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index f8dc94cf..84137c2e 100755 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -218,7 +218,7 @@ function curateTitle(title, channel) { } async function scrapeApiReleases(json, site, options) { - return json.slice(0, 6).reduce((acc, scene) => { + return json.reduce((acc, scene) => { if (options.parameters?.extract && scene.sitename !== options.parameters.extract) { return acc; } @@ -658,7 +658,7 @@ function scrapeApiProfile(data, releases, siteSlug) { return profile; } -async function fetchLatestApi(site, page = 1, options, preData, upcoming = false) { +async function fetchLatestApi(site, page = 1, options, _preData, upcoming = false) { const referer = options.parameters?.referer || `${options.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`; const { apiUrl } = await fetchApiCredentials(referer, site); const slug = options.parameters.querySlug || site.slug; diff --git a/src/scrapers/porndoe.js b/src/scrapers/porndoe.js index 194924b7..f910d7cb 100755 --- a/src/scrapers/porndoe.js +++ b/src/scrapers/porndoe.js @@ -136,7 +136,7 @@ async function fetchScene(url, channel) { } async function fetchProfile({ name: actorName }, entity, include) { - const url = `${entity.url}/models/${slugify(actorName)}.en.html`; + const url = `${new URL(entity.url).origin}/models/${slugify(actorName)}.en.html`; const res = await unprint.get(url); return res.ok ? scrapeProfile(res.context, url, include) : res.status; diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index 975915c9..354c441b 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -277,19 +277,36 @@ const scrapers = { interracialpovs: hush, povpornstars: hush, seehimfuck: hush, + // wankzvr + wankzvr, + tranzvr: wankzvr, + milfvr: wankzvr, + // nubilus + anilos: nubiles, + brattysis: nubiles, + deeplush: nubiles, + hotcrazymess: nubiles, + nfbusty: nubiles, + nubilefilms: nubiles, + nubiles, + thatsitcomshow: nubiles, + // porndoe + amateureuro: porndoe, + forbondage: porndoe, + mamacitaz: porndoe, + transbella: porndoe, + vipsexvault: porndoe, // etc '18vr': badoink, theflourishxxx: theflourish, adultempire, archangel, allherluv: missax, - amateureuro: porndoe, americanpornstar, analbbc: fullpornnetwork, analized: fullpornnetwork, analviolation: fullpornnetwork, angelogodshackoriginal, - anilos: nubiles, asiam: modelmedia, aziani, '2poles1hole': aziani, @@ -305,23 +322,19 @@ const scrapers = { vurigvlaanderen: bluedonkeymedia, boobpedia, bradmontana, - brattysis: nubiles, cherrypimps, cumlouder, - deeplush: nubiles, dorcelclub: dorcel, doubleviewcasting: firstanalquest, dtfsluts: fullpornnetwork, exploitedx, // only from known URL that will specify site firstanalquest, - forbondage: porndoe, freeones, girlfaction: fullpornnetwork, hergape: fullpornnetwork, hitzefrei, homemadeanalwhores: fullpornnetwork, hookuphotshot, - hotcrazymess: nubiles, inthecrack, jamesdeen: fullpornnetwork, jerkaoke: modelmedia, @@ -339,18 +352,13 @@ const scrapers = { // analvids, analvids: pornbox, littlecapricedreams, - mamacitaz: porndoe, mariskax, - milfvr: wankzvr, missax, mylf: teamskeet, mugfucked: fullpornnetwork, naughtyamerica, tonightsgirlfriend: naughtyamerica, nebraskacoeds: elevatedx, - nfbusty: nubiles, - nubilefilms: nubiles, - nubiles, onlyprince: fullpornnetwork, pascalssubsluts, pervcity, @@ -383,11 +391,7 @@ const scrapers = { teencoreclub, teenmegaworld, testedefudelidade, - thatsitcomshow: nubiles, tokyohot, - transbella: porndoe, - tranzvr: wankzvr, - vipsexvault: porndoe, virtualtaboo, darkroomvr: virtualtaboo, onlytarts: virtualtaboo, @@ -403,7 +407,6 @@ const scrapers = { slayed: vixen, wifey: vixen, vrcosplayx: badoink, - wankzvr, wildoncam: cherrypimps, }, }; diff --git a/src/scrapers/virtualtaboo.js b/src/scrapers/virtualtaboo.js index 192dd325..79b3e70c 100755 --- a/src/scrapers/virtualtaboo.js +++ b/src/scrapers/virtualtaboo.js @@ -105,9 +105,9 @@ function scrapeScene({ query, window }, { url }) { function scrapeProfile({ query }) { const profile = {}; - const bioKeys = query.contents('.pornstar-detail__params--top strong, .actor-detail__param-name'); - const bioValues = query.exists('.actor-detail__param-value') - ? query.contents('.actor-detail__param-value') + const bioKeys = query.contents('.pornstar-detail__params--top strong, .actor-detail__param-name, td.pornstar-detail__info--title'); + const bioValues = query.exists('.actor-detail__param-value, .pornstar-detail__info--title') + ? query.contents('.actor-detail__param-value, .pornstar-detail__info--title + td') : query.text('.pornstar-detail__params--top', { join: false })?.map((text) => text.split('•')[0].replace(':', '').trim()); const bio = Object.fromEntries(bioKeys.map((key, index) => [slugify(key, '_'), bioValues[index]])); diff --git a/src/scrapers/wankzvr.js b/src/scrapers/wankzvr.js index 31461ba8..e6698e4f 100755 --- a/src/scrapers/wankzvr.js +++ b/src/scrapers/wankzvr.js @@ -133,7 +133,7 @@ async function scrapeProfile({ query }, url, entity, options) { profile.description = query.cnt('.person__content'); profile.gender = entity.slug === 'tranzvr' ? 'transsexual' : 'female'; - profile.age = bio.age; + profile.age = Number(bio.age) || null; profile.birthPlace = bio.birthplace; diff --git a/tests/profiles.js b/tests/profiles.js index b685035b..8a7a8210 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -124,6 +124,27 @@ const actors = [ { entity: 'archangel', name: 'Summer Brielle', fields: ['avatar', 'description', 'dateOfBirth', 'age', 'measurements', 'height', 'aliases'] }, { entity: 'theflourishxxx', name: 'XWifeKaren', fields: ['avatar', 'description'] }, { entity: 'hookuphotshot', name: 'Kenzie Reeves', fields: ['avatar', 'description'] }, + { entity: 'tokyohot', name: 'Mai Kawana', url: 'https://my.tokyo-hot.com/cast/2099/', fields: ['avatar', 'birthPlace', 'height', 'cup', 'bust', 'waist', 'hip', 'hairStyle', 'shoeSize', 'bloodType'] }, + { entity: 'teenmegaworld', name: 'Sheri Vi', fields: ['avatar', 'description', 'hairColor', 'eyes'] }, + // wankz + { entity: 'wankzvr', name: 'Melody Marks', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'measurements', 'age'] }, + { entity: 'milfvr', name: 'Ember Snow', fields: ['avatar', 'gender', 'description', 'measurements', 'birthPlace', 'height', 'age'] }, + { entity: 'tranzvr', name: 'Thayssa Fadinha', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'age'] }, + // virtual taboo + { entity: 'virtualtaboo', name: 'Kama Oxi', fields: ['avatar', 'birthPlace', 'dateOfBirth', 'naturalBoobs', 'hasTattoos'] }, + { entity: 'onlytarts', name: 'Kama Oxi', fields: ['avatar', 'birthPlace', 'dateOfBirth', 'measurements', 'height', 'weight'] }, + { entity: 'oopsfamily', name: 'Angel Windell', fields: ['avatar', 'birthPlace', 'dateOfBirth', 'measurements', 'height', 'weight'] }, + { entity: 'darkroomvr', name: 'Alexa Flexy', fields: ['avatar', 'birthPlace', 'dateOfBirth', 'measurements', 'height', 'weight'] }, + // nubiles + { entity: 'nubiles', name: 'Lolli Pop', fields: ['avatar', 'age', 'description', 'residencePlace', 'height', 'measurements', 'photos'] }, + { entity: 'anilos', name: 'Rina Helen', fields: ['avatar', 'age', 'description', 'residencePlace', 'height', 'measurements', 'photos'] }, + { entity: 'deeplush', name: 'Penelope Woods', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] }, + { entity: 'hotcrazymess', name: 'Eliza Ibarra', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] }, + { entity: 'nfbusty', name: 'Ella Reese', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] }, + { entity: 'nubilefilms', name: 'Jade Kimiko', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] }, + { entity: 'thatsitcomshow', name: 'Casey Calvert', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] }, + // porndoe + { entity: 'vipsexvault', name: 'Amirah Adara', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'hairColor', 'description'] }, ]; const actorScrapers = scrapers.actors; @@ -158,7 +179,7 @@ const validators = { birthCountry: (value) => typeof value === 'string' && value.length > 1, nationality: (value) => typeof value === 'string' && value.length > 3, // height: (value) => !!Number(value) || /\d'\d{1,2}"/.test(value), // ft in needs to be converted - height: (value) => !!Number(value) && value > 150, + height: (value) => !!Number(value) && value > 130, weight: (value) => !!Number(value) && value > 40, eyes: (value) => typeof value === 'string' && value.length > 3, hairColor: (value) => typeof value === 'string' && value.length > 3, @@ -188,7 +209,7 @@ const validators = { // profiler in this context is shorthand for profile scraper async function init() { - const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: argv.prefer }); + const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: 'channel' }); Object.entries(actorScrapers).reduce(async (chain, [entitySlug, scraper]) => { await chain; @@ -218,6 +239,7 @@ async function init() { const profile = await fetchProfile({ name: actor.name, slug: slugify(actor.name), + url: actor.url, }, { ...entity, entity, @@ -226,13 +248,13 @@ async function init() { parameters: getRecursiveParameters(entity), }, include); - console.log(profile); - console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => `'${field}'`).join(', ')); - if (!profile) { assert.fail('profile not found'); } + console.log(profile); + console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => `'${field}'`).join(', ')); + await Promise.all(actor.fields.map(async (field) => { assert.ok( validators[field]