diff --git a/config/default.js b/config/default.js index 6b2f9ce7..7fe1e32b 100755 --- a/config/default.js +++ b/config/default.js @@ -73,6 +73,9 @@ module.exports = { }, exclude: { channels: [ + // aylo + 'pornhub', + 'spicevids', // 21sextreme, no longer updated 'mightymistress', 'dominatedgirls', @@ -125,8 +128,6 @@ module.exports = { 'ncversocinema', 'pussybabes', 'uksinners', - // mindgeek - 'pornhub', // insex 'paintoy', 'aganmedon', diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 28c92616..22e02936 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -557,10 +557,11 @@ const networks = [ url: 'https://www.mikeadriano.com', }, { - slug: 'milehighmedia', - name: 'Mile High Media', - url: 'https://www.milehighmedia.com', - description: 'MileHighMedia.com is the only niche porn network you need! Watch lesbian sex, hardcore fucking and family porn stories with the hottest teens & MILFs!', + slug: 'adultmobile', + rename: 'milehighmedia', + name: 'Adult Mobile', + alias: ['mile high media', 'mile high'], + url: 'https://www.adultmobile.com', parent: 'aylo', parameters: { forceDeep: true, // Mile High Media has movie and series information not available in the latest updates API diff --git a/seeds/02_sites.js b/seeds/02_sites.js index c892aa31..9444cca1 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -6903,9 +6903,10 @@ const sites = [ parent: 'aylo', }, { - slug: 'tube8vip', - name: 'Tube8Vip', - url: 'https://www.tube8vip.com', + slug: 'spicevids', + rename: 'tube8vip', + name: 'Spicevids', + url: 'https://www.spicevids.com', description: '', parameters: { native: true }, parent: 'aylo', @@ -6996,23 +6997,24 @@ const sites = [ parent: 'milehighmedia', }, { - slug: 'milehighmedia', - name: 'Mile High Media', + slug: 'adultmobile', + rename: 'milehighmedia', + name: 'Adult Mobile', url: 'https://www.milehighmedia.com/scenes?site=323', - alias: ['mhm'], + alias: ['mhm', 'mile high media', 'mile high'], parameters: { siteId: 323 }, - parent: 'milehighmedia', + parent: 'adultmobile', }, { slug: 'realityjunkies', name: 'Reality Junkies', - url: 'https://www.realityjunkies.com', + url: 'https://www.adultmobile.com/scenes?site=324', alias: ['rj'], parameters: { siteId: 324, native: true, }, - parent: 'milehighmedia', + parent: 'adultmobile', }, { slug: 'sweetheartvideo', @@ -7023,7 +7025,7 @@ const sites = [ siteId: 325, native: true, }, - parent: 'milehighmedia', + parent: 'adultmobile', }, { slug: 'sweetsinner', @@ -7034,7 +7036,7 @@ const sites = [ siteId: 326, native: true, }, - parent: 'milehighmedia', + parent: 'adultmobile', }, { slug: 'familysinners', @@ -7046,7 +7048,7 @@ const sites = [ siteId: 317, native: true, }, - parent: 'milehighmedia', + parent: 'adultmobile', }, { slug: 'iconmale', @@ -7058,7 +7060,7 @@ const sites = [ siteId: 328, native: true, }, - parent: 'milehighmedia', + parent: 'adultmobile', }, // MISSA X { diff --git a/src/scrapers/aylo.js b/src/scrapers/aylo.js index 733c6bc8..147389e7 100755 --- a/src/scrapers/aylo.js +++ b/src/scrapers/aylo.js @@ -363,7 +363,7 @@ function scrapeProfile(data, networkName, _releases = []) { profile.gender = data.gender === 'other' ? 'transsexual' : data.gender; if (profile.gender === 'male') { - profile.penisLength = Number(data.measurements); + profile.penisLength = Number(data.measurements) || null; } else { profile.measurements = data.measurements; } diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index 54be7b67..73bf6128 100755 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -81,6 +81,8 @@ function scrapeAll(scenes, site, entryIdFromTitle) { || query.element('.rating_box')?.dataset.id || query.attribute('a img', 'id')?.match(/set-target-(\d+)/)?.[1]; + console.log(release.entryId); + return release; }); } diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index 4390c096..34089088 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -194,6 +194,33 @@ const scrapers = { // xempire, }, actors: { + // aylo + adultmobile: aylo, + babes: aylo, + bangbros: aylo, + brazzers: aylo, + digitalplayground: aylo, + doghousedigital: aylo, + fakehub: aylo, + familysinners: aylo, + gaywire: aylo, + iconmale: aylo, + letsdoeit: aylo, + men: aylo, + metrohd: aylo, + mofos: aylo, + propertysex: aylo, + realityjunkies: aylo, + realitykings: aylo, + sexyhub: aylo, + spicevids: aylo, + squirted: aylo, + sweetheartvideo: aylo, + sweetsinner: aylo, + transangels: aylo, + trueamateurs: aylo, + twistys: aylo, + // etc '18vr': badoink, '21sextury': gamma, adultempire, @@ -212,13 +239,11 @@ const scrapers = { aziani, '2poles1hole': aziani, creampiled: aziani, - babes: aylo, babevr: badoink, baddaddypov: fullpornnetwork, badoinkvr: badoink, bamvisions, bang, - bangbros: aylo, bjraw: radical, bluedonkeymedia, delphine: modelmedia, @@ -227,13 +252,11 @@ const scrapers = { boobpedia, bradmontana, brattysis: nubiles, - brazzers: aylo, burningangel: gamma, cherrypimps, cumlouder, deeplush: nubiles, devilsfilm: famedigital, - digitalplayground: aylo, dirtyauditions: mikeadriano, dogfartnetwork: gamma, dorcelclub: dorcel, @@ -242,12 +265,10 @@ const scrapers = { evilangel: gamma, exploitedx, // only from known URL that will specify site eyeontheguy: hush, - fakehub: aylo, firstanalquest, forbondage: porndoe, freeones, gangbangcreampie: gamma, - gaywire: aylo, girlfaction: fullpornnetwork, gloryholesecrets: gamma, gotfilled: radical, @@ -258,7 +279,6 @@ const scrapers = { hotcrazymess: nubiles, hushpass: hush, hussiepass: hush, - iconmale: aylo, inserted: radical, interracialpass: hush, interracialpovs: hush, @@ -273,7 +293,6 @@ const scrapers = { kink, kinkmen: kink, kinkvr: kink, - letsdoeit: aylo, loveherfilms, loveherfeet: loveherfilms, shelovesblack: loveherfilms, @@ -281,13 +300,9 @@ const scrapers = { analvids: pornbox, littlecapricedreams, mamacitaz: porndoe, - men: aylo, mariskax, - metrohd: aylo, - milehighmedia: aylo, milfvr: wankzvr, missax, - mofos: aylo, mylf: teamskeet, mugfucked: fullpornnetwork, naughtyamerica, @@ -311,7 +326,6 @@ const scrapers = { povpornstars: hush, private: privateNetwork, purgatoryx, - realitykings: aylo, realvr: badoink, rickysroom, roccosiffredi: famedigital, @@ -319,7 +333,6 @@ const scrapers = { score, seehimfuck: hush, sexlikereal, - sexyhub: aylo, silverstonedvd: famedigital, silviasaint: famedigital, spermmania: snowvalley, @@ -345,11 +358,9 @@ const scrapers = { testedefudelidade, thatsitcomshow: nubiles, tokyohot, - transangels: aylo, transbella: porndoe, tranzvr: wankzvr, trueanal: mikeadriano, - twistys: aylo, vipsexvault: porndoe, virtualtaboo, darkroomvr: virtualtaboo, diff --git a/src/utils/slugify.js b/src/utils/slugify.js index 3b4fab9a..5c7451d2 100755 --- a/src/utils/slugify.js +++ b/src/utils/slugify.js @@ -42,7 +42,7 @@ const accentMap = { }; const plainCharRegex = /[a-zA-Z0-9]/; -const defaultPunctuationRegex = /[.,?!:;&'‘’"“”…()[]{}<>\/*—]/; +const defaultPunctuationRegex = /[.,?!:;&'‘’"“”…()[\]{}<>/*—]/; const defaultSymbolRegex = /[@$€£#%^+=\\~]/; function slugify(strings, delimiter = '-', { @@ -50,7 +50,7 @@ function slugify(strings, delimiter = '-', { lower = true, encode = false, accents: keepAccents = false, - punctuation: keepPunctuation = false, + punctuation: keepPunctuation = 'split', punctuationRegex = defaultPunctuationRegex, symbols: keepSymbols = false, symbolRegex = defaultSymbolRegex, diff --git a/tests/profiles.js b/tests/profiles.js index 6d030e04..28e153df 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -62,6 +62,25 @@ const actors = [ { entity: 'babes', name: 'Alina Lopez', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] }, { entity: 'letsdoeit', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] }, { entity: 'men', name: 'Cade Maddox', fields: ['avatar', 'description', 'gender', 'height', 'ethnicity', 'penisLength', 'dateOfBirth', 'weight', 'hairColor', 'hasTattoos'] }, + { entity: 'metrohd', name: 'April Olsen', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight'] }, + { entity: 'mofos', name: 'Ariana Starr', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] }, + { entity: 'propertysex', name: 'Desiree Dulce', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasPiercings'] }, + { entity: 'sexyhub', name: 'Angie Lynx', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] }, + { entity: 'squirted', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] }, + { entity: 'transangels', name: 'Aubrey Kate', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos'] }, + { entity: 'trueamateurs', name: 'Natalie FLowers', fields: ['avatar', 'gender'] }, + { entity: 'spicevids', name: 'Remy LaCroix', fields: ['avatar', 'gender', 'description', 'height', 'measurements', 'dateOfBirth', 'weight'] }, + { entity: 'twistys', name: 'Remy LaCroix', fields: ['avatar', 'gender', 'description', 'height', 'measurements', 'dateOfBirth', 'weight', 'birthPlace', 'hairColor', 'ethnicity', 'naturalBoobs', 'hasPiercings'] }, + // aylo > adult mobile + { entity: 'adultmobile', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] }, + { entity: 'doghousedigital', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] }, + { entity: 'familysinners', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] }, + { entity: 'iconmale', name: 'Troy Accola', fields: ['avatar', 'gender', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] }, + { entity: 'realityjunkies', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] }, + { entity: 'sweetheartvideo', name: 'Lexi Lore', fields: ['avatar', 'gender', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] }, + { entity: 'sweetsinner', name: 'Anna Claire Clouds', fields: ['avatar', 'gender'] }, + // bangros + { entity: 'bangbros', name: 'Kira Perez', fields: ['avatar', 'gender', 'ethnicity', 'hairColor'] }, ]; const actorScrapers = scrapers.actors; @@ -90,9 +109,9 @@ async function validateUrl(url, mime = 'image/') { const validators = { age: (value) => !!Number(value), - gender: (value) => value && ['female', 'male', 'transsexual'].includes(value.toLowerCase()), + gender: (value) => value && ['female', 'male', 'transsexual', 'trans'].includes(value.toLowerCase()), description: (value) => typeof value === 'string' && value.length > 3, - birthPlace: (value) => typeof value === 'string' && value.length > 3, + birthPlace: (value) => typeof value === 'string' && value.length > 1, // may return US or USA birthCountry: (value) => typeof value === 'string' && value.length > 1, nationality: (value) => typeof value === 'string' && value.length > 3, height: (value) => !!Number(value) || /\d'\d{1,2}"/.test(value), @@ -164,7 +183,7 @@ async function init() { }, include); console.log(profile); - console.log('Untested fields', Object.keys(profile).filter((field) => !actor.fields.includes(field)).join(', ')); + console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => field).join(', ')); if (!profile) { assert.fail('profile not found');