diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 22e02936..52202be5 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -358,6 +358,7 @@ const networks = [ description: 'Watch and download thousands of the best porn videos at FameDigital.com, the largest porn network on the web! The hottest teens, MILFs and more pornstars are all here!', parameters: { mobile: 'https://m.dpfanatics.com/en/video', + layout: 'api', }, parent: 'gamma', }, @@ -794,6 +795,9 @@ const networks = [ url: 'https://www.vivid.com', description: 'Home of the Kim Kardashian Sex Tape, Porn Parodies, and over 30,000 XXX Movies from The World Leader In Adult Entertainment.', parent: 'gamma', + parameters: { + layout: 'api', + }, }, { slug: 'vixen', @@ -827,7 +831,7 @@ const networks = [ slug: 'zerotolerance', name: 'Zero Tolerance', alias: ['ztod'], - url: 'http://www.ztod.com', + url: 'https://www.zerotolerancefilms.com', parameters: { layout: 'api', }, diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 9444cca1..18729123 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -568,18 +568,6 @@ const sites = [ }, */ // ADULT TIME INDEPENDENT - { - name: 'Blake Mason', - slug: 'blakemason', - url: 'https://www.blakemason.com', - parent: 'gamma', - tags: ['gay'], - independent: true, - parameters: { - layout: 'api', - referer: 'https://freetour.adulttime.com/en/join', - }, - }, { name: 'ChaosMen', slug: 'chaosmen', @@ -975,6 +963,7 @@ const sites = [ independent: true, parameters: { scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie', + layout: 'api', }, }, { @@ -986,6 +975,7 @@ const sites = [ independent: true, parameters: { scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets', + layout: 'api', }, }, /* different layout @@ -998,6 +988,7 @@ const sites = [ independent: true, parameters: { scene: 'https://www.portagloryhole.com/scenes', + layout: 'api', }, }, */ @@ -3949,7 +3940,6 @@ const sites = [ name: 'Devil\'s Film', url: 'https://www.devilsfilm.com', description: 'Welcome to the best porn network, DevilsFilm.com, featuring teens, MILFs, trans and interracial porn with all of your favorite pornstars in 4k ultra HD!', - parameters: { api: true }, parent: 'famedigital', }, { @@ -3969,7 +3959,10 @@ const sites = [ url: 'https://www.daringsexhd.com/', description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.', parent: 'famedigital', - parameters: { api: true }, + parameters: { + api: true, + layout: null, + }, }, { slug: 'peternorth', @@ -3987,7 +3980,6 @@ const sites = [ name: 'Rocco Siffredi', url: 'https://www.roccosiffredi.com', description: 'Welcome to the official RoccoSiffredi.com, the Italian Stallion, with hardcore anal fucking and rough sex from the man himself who has coined the term hardcore.', - parameters: { api: true }, parent: 'famedigital', }, { @@ -4994,6 +4986,14 @@ const sites = [ name: 'InTheCrack', url: 'https://inthecrack.com', }, + // TODO: INDIE https://nats.indiebucks.com/external.php?page=sites + { + // used to be Gamma + name: 'Blake Mason', + slug: 'blakemason', + url: 'https://www.blakemason.com', + tags: ['gay'], + }, // INN OF SIN { slug: 'richardmannevents', diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index b4413676..f8dc94cf 100755 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -624,7 +624,7 @@ async function scrapeProfile({ query }, url, actorName, _siteSlug, getActorRelea profile.description = query.cnt('.actorBio p:not(.bioTitle)'); - if (hair) profile.hair = hair.split(':')[1].trim(); + if (hair) profile.hairColor = hair.split(':')[1].trim(); if (height) profile.height = Number(height.match(/\d+/)[0]); if (weight) profile.weight = Number(weight.match(/\d+/)[0]); if (alias) profile.aliases = alias.split(':')[1].trim().split(', '); @@ -648,7 +648,7 @@ function scrapeApiProfile(data, releases, siteSlug) { if (data.attributes.ethnicity) profile.ethnicity = data.attributes.ethnicity; if (data.attributes.eye_color) profile.eyes = data.attributes.eye_color; - if (data.attributes.hair_color) profile.hair = data.attributes.hair_color; + if (data.attributes.hair_color) profile.hairColor = data.attributes.hair_color; const avatarPaths = Object.values(data.pictures).reverse(); if (avatarPaths.length > 0) profile.avatar = avatarPaths.map((avatarPath) => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`); @@ -929,7 +929,8 @@ async function fetchApiProfile({ name: actorName }, context, include) { const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug; const actorSlug = encodeURI(actorName); - const referer = `https://www.${siteSlug}.com/en/search`; + // const referer = `https://www.${siteSlug}.com/en/search`; + const referer = `${new URL(context.entity.url).origin}/en/search`; const { apiUrl } = await fetchApiCredentials(referer); diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index 34089088..9a24ce32 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -20,7 +20,7 @@ const czechav = require('./czechav'); const modelmedia = require('./modelmedia'); const dorcel = require('./dorcel'); const fabulouscash = require('./fabulouscash'); -const famedigital = require('./famedigital'); +// const famedigital = require('./famedigital'); const firstanalquest = require('./firstanalquest'); const elevatedx = require('./elevatedx'); const exploitedx = require('./exploitedx'); @@ -89,6 +89,10 @@ const freeones = require('./freeones'); const scrapers = { releases: { + // gamma + gamma, + // daringsex, + // etc amateurallure, americanpornstar, amateureuro: porndoe, @@ -114,13 +118,11 @@ const scrapers = { delphine: modelmedia, dorcel, elegantangel: adultempire, - famedigital, exploitedx, fabulouscash, firstanalquest, forbondage: porndoe, fullpornnetwork, - gamma, gaywire: aylo, girlgirl: julesjordan, hitzefrei, @@ -220,9 +222,40 @@ const scrapers = { transangels: aylo, trueamateurs: aylo, twistys: aylo, + // gamma + '21sextury': gamma, + biphoria: gamma, + blakemason: gamma, + blowpass: gamma, + burningangel: gamma, + chaosmen: gamma, + devilsfilm: gamma, + diabolic: gamma, + dogfartnetwork: gamma, + evilangel: gamma, + famedigital: gamma, + fantasymassage: gamma, + filthykings: gamma, + gangbangcreampie: gamma, + girlsway: gamma, + gloryholesecrets: gamma, + peternorth: gamma, + pridestudios: gamma, + puretaboo: gamma, + roccosiffredi: gamma, + silverstonedvd: gamma, + silviasaint: gamma, + tabooheat: gamma, + vivid: gamma, + whiteghetto: gamma, + wicked: gamma, + xempire: gamma, + zerotolerancefilms: gamma, + '3rddegreefilms': gamma, + addicted2girls: gamma, + genderxfilms: gamma, // etc '18vr': badoink, - '21sextury': gamma, adultempire, allanal: mikeadriano, allherluv: missax, @@ -252,25 +285,19 @@ const scrapers = { boobpedia, bradmontana, brattysis: nubiles, - burningangel: gamma, cherrypimps, cumlouder, deeplush: nubiles, - devilsfilm: famedigital, dirtyauditions: mikeadriano, - dogfartnetwork: gamma, dorcelclub: dorcel, doubleviewcasting: firstanalquest, dtfsluts: fullpornnetwork, - evilangel: gamma, exploitedx, // only from known URL that will specify site eyeontheguy: hush, firstanalquest, forbondage: porndoe, freeones, - gangbangcreampie: gamma, girlfaction: fullpornnetwork, - gloryholesecrets: gamma, gotfilled: radical, hergape: fullpornnetwork, hitzefrei, @@ -316,7 +343,6 @@ const scrapers = { pascalssubsluts, pervcity, pervertgallery: fullpornnetwork, - peternorth: famedigital, pierrewoodman, pimpxxx: cherrypimps, porncz, @@ -328,13 +354,10 @@ const scrapers = { purgatoryx, realvr: badoink, rickysroom, - roccosiffredi: famedigital, sayuncle: teamskeet, score, seehimfuck: hush, sexlikereal, - silverstonedvd: famedigital, - silviasaint: famedigital, spermmania: snowvalley, handjobjapan: snowvalley, fellatiojapan: snowvalley, @@ -378,9 +401,7 @@ const scrapers = { wifey: vixen, vrcosplayx: badoink, wankzvr, - wicked: gamma, wildoncam: cherrypimps, - xempire: gamma, }, }; diff --git a/tests/profiles.js b/tests/profiles.js index 28e153df..7833cb09 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -15,9 +15,6 @@ const knex = require('../src/knex'); const actors = [ // jules jordan { entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'description', 'avatar'] }, - // gamma - { entity: 'wicked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, - { entity: 'xempire', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, // vixen { entity: 'vixen', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, { entity: 'tushy', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, @@ -81,6 +78,38 @@ const actors = [ { entity: 'sweetsinner', name: 'Anna Claire Clouds', fields: ['avatar', 'gender'] }, // bangros { entity: 'bangbros', name: 'Kira Perez', fields: ['avatar', 'gender', 'ethnicity', 'hairColor'] }, + // gamma + { entity: '21sextury', name: 'Aletta Ocean', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] }, + { entity: 'biphoria', name: 'Cherry Kiss', fields: ['avatar', 'gender'] }, + { entity: 'blowpass', name: 'Jynx Maze', fields: ['avatar', 'description', 'gender'] }, + { entity: 'burningangel', name: 'Joanna Angel', fields: ['avatar', 'gender'] }, + { entity: 'chaosmen', name: 'Kenzo Alvarez', fields: ['avatar', 'gender'] }, + { entity: 'dogfartnetwork', name: 'Liz Jordan', fields: ['avatar', 'gender'] }, + { entity: 'diabolic', name: 'Kira Noir', fields: ['avatar', 'gender'] }, + { entity: 'evilangel', name: 'Francesca Le', fields: ['avatar', 'gender'] }, + { entity: 'fantasymassage', name: 'Cherry Kiss', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] }, + { entity: 'filthykings', name: 'Armani Black', fields: ['avatar', 'gender'] }, + { entity: 'gangbangcreampie', name: 'Luna Lovely', fields: ['avatar', 'gender', 'description'] }, + { entity: 'girlsway', name: 'Adriana Chechik', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] }, + { entity: 'gloryholesecrets', name: 'Maya Bijou', fields: ['avatar', 'gender'] }, + { entity: 'pridestudios', name: 'Peter Hooke', fields: ['avatar', 'gender'] }, + { entity: 'puretaboo', name: 'Jane Wilde', fields: ['avatar', 'gender'] }, + { entity: 'tabooheat', name: 'Cory Chase', fields: ['avatar', 'gender'] }, + { entity: 'vivid', name: 'Avi Love', fields: ['avatar', 'gender'] }, + { entity: 'wicked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, + { entity: 'xempire', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, + // gamma > zero tolerance / stod + { entity: '3rddegreefilms', name: 'Angel Dark', fields: ['gender', 'avatar'] }, + { entity: 'addicted2girls', name: 'Abigail Mac', fields: ['gender', 'avatar'] }, + { entity: 'zerotolerancefilms', name: 'Courtney Cummz', fields: ['gender', 'avatar'] }, + { entity: 'genderxfilms', name: 'Kasey Kei', fields: ['gender', 'avatar'] }, + // gamma > famedigital + { entity: 'roccosiffredi', name: 'Malena Nazionale', fields: ['avatar', 'gender'] }, + { entity: 'famedigital', name: 'Hime Marie', fields: ['avatar', 'gender'] }, + { entity: 'peternorth', name: 'Carmen Caliente', fields: ['avatar', 'gender'] }, + { entity: 'silverstonedvd', name: 'Leanni Lei', fields: ['avatar', 'gender'] }, + { entity: 'silviasaint', name: 'Silvia Saint', fields: ['avatar', 'gender', 'description'] }, + { entity: 'whiteghetto', name: 'Proxy Paige', fields: ['avatar', 'gender', 'description'] }, ]; const actorScrapers = scrapers.actors; @@ -183,7 +212,7 @@ async function init() { }, include); console.log(profile); - console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => field).join(', ')); + console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => `'${field}'`).join(', ')); if (!profile) { assert.fail('profile not found');