'use strict'; const test = require('node:test'); const assert = require('node:assert/strict'); const argv = require('../src/argv'); const include = require('../src/utils/argv-include')(argv); const slugify = require('../src/utils/slugify'); const scrapers = require('../src/scrapers/scrapers'); const { fetchEntitiesBySlug } = require('../src/entities'); const { resolveLayoutScraper } = require('../src/scrapers/resolve'); const getRecursiveParameters = require('../src/utils/get-recursive-parameters'); const knex = require('../src/knex'); const actors = [ // jules jordan { entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'description', 'avatar'] }, // vixen { entity: 'vixen', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, { entity: 'tushy', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, { entity: 'tushyraw', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, { entity: 'blacked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, { entity: 'blackedraw', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, { entity: 'slayed', name: 'Vanna Bardot', fields: ['gender', 'avatar', 'description'] }, { entity: 'deeper', name: 'Vanna Bardot', fields: ['gender', 'avatar', 'description'] }, { entity: 'milfy', name: 'Clea Gaultier', fields: ['gender', 'avatar', 'description'] }, { entity: 'wifey', name: 'Danielle Renae', fields: ['gender', 'avatar', 'description'] }, // teamskeet { entity: 'teamskeet', name: 'Abella Danger', fields: ['description', 'avatar', 'measurements', 'birthPlace', 'nationality', 'ethnicity', 'height', 'weight', 'hairColor', 'hasPiercings'] }, { entity: 'teamskeet', name: 'Kali Roses', fields: ['description', 'avatar', 'measurements', 'nationality', 'ethnicity', 'hairColor', 'hasPiercings', 'hasTattoos'] }, // tattoos // analvids { entity: 'analvids', name: 'Veronica Leal', fields: ['avatar', 'gender', 'birthCountry', 'nationality', 'age', 'aliases', 'nationality'] }, // mike adriano { entity: 'trueanal', name: 'Brenna McKenna', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor', 'hasTattoos'] }, { entity: 'analonly', name: 'Lilith Grace', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] }, { entity: 'allanal', name: 'Lexi Lore', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] }, { entity: 'swallowed', name: 'Brooklyn Gray', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor', 'hasTattoos'] }, { entity: 'nympho', name: 'Gianna Dior', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] }, { entity: 'dirtyauditions', name: 'Nicole Kitt', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] }, // spizoo { entity: 'spizoo', name: 'Charlotte Sins', fields: ['description', 'avatar', 'dateOfBirth', 'ethnicity', 'nationality', 'height', 'measurements', 'hasTattoos', 'hasPiercings', 'hairColor', 'eyes', 'butt', 'pussy'] }, { entity: 'rawattack', name: 'Kitana Montana', fields: ['avatar', 'dateOfBirth', 'nationality', 'measurements', 'eyes', 'height', 'hairColor', 'hasTattoos'] }, // hush / hussiepass { entity: 'hussiepass', name: 'Roxie Sinner', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'foot', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] }, { entity: 'eyeontheguy', name: 'Tommy Gunn', fields: ['avatar'] }, { entity: 'interracialpovs', name: 'Nia Nacci', fields: ['avatar', 'aliases', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] }, { entity: 'povpornstars', name: 'Anna Bell Peaks', fields: ['avatar', 'aliases', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] }, { entity: 'seehimfuck', name: 'Sheem The Dream', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'penisLength', 'circumcised', 'socials'] }, { entity: 'hushpass', name: 'Dylan Ryder', fields: ['avatar'] }, { entity: 'interracialpass', name: 'Aidra Fox', fields: ['avatar', 'height', 'measurements'] }, // kelly madison / 8K { entity: 'kellymadison', name: 'Ava Addams', fields: ['avatar', 'description', 'age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] }, { entity: '8kmembers', name: 'Angie Lynx', fields: ['age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] }, // aylo { entity: 'brazzers', name: 'Lexi Lore', fields: ['avatar', 'description', 'gender', 'height', 'weight', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity', 'hairColor', 'hasTattoos', 'hasPiercings'] }, { entity: 'digitalplayground', name: 'Elly Clutch', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] }, { entity: 'realitykings', name: 'Abella Danger', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity'] }, { entity: 'fakehub', name: 'Abella Danger', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity'] }, { entity: 'babes', name: 'Alina Lopez', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] }, { entity: 'letsdoeit', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] }, { entity: 'men', name: 'Cade Maddox', fields: ['avatar', 'description', 'gender', 'height', 'ethnicity', 'penisLength', 'dateOfBirth', 'weight', 'hairColor', 'hasTattoos'] }, { entity: 'metrohd', name: 'April Olsen', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight'] }, { entity: 'mofos', name: 'Ariana Starr', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] }, { entity: 'propertysex', name: 'Desiree Dulce', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasPiercings'] }, { entity: 'sexyhub', name: 'Angie Lynx', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] }, { entity: 'squirted', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] }, { entity: 'transangels', name: 'Aubrey Kate', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos'] }, { entity: 'trueamateurs', name: 'Natalie FLowers', fields: ['avatar', 'gender'] }, { entity: 'spicevids', name: 'Remy LaCroix', fields: ['avatar', 'gender', 'description', 'height', 'measurements', 'dateOfBirth', 'weight'] }, { entity: 'twistys', name: 'Remy LaCroix', fields: ['avatar', 'gender', 'description', 'height', 'measurements', 'dateOfBirth', 'weight', 'birthPlace', 'hairColor', 'ethnicity', 'naturalBoobs', 'hasPiercings'] }, { entity: 'mypervyfamily', name: 'Anissa Kate', fields: ['avatar', 'gender'] }, // aylo > adult mobile { entity: 'adultmobile', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] }, { entity: 'doghousedigital', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] }, { entity: 'familysinners', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] }, { entity: 'iconmale', name: 'Troy Accola', fields: ['avatar', 'gender', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] }, { entity: 'realityjunkies', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] }, { entity: 'sweetheartvideo', name: 'Lexi Lore', fields: ['avatar', 'gender', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] }, { entity: 'sweetsinner', name: 'Anna Claire Clouds', fields: ['avatar', 'gender'] }, // bangros { entity: 'bangbros', name: 'Kira Perez', fields: ['avatar', 'gender', 'ethnicity', 'hairColor'] }, // gamma { entity: '21sextury', name: 'Aletta Ocean', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] }, { entity: 'biphoria', name: 'Cherry Kiss', fields: ['avatar', 'gender'] }, { entity: 'blowpass', name: 'Jynx Maze', fields: ['avatar', 'description', 'gender'] }, { entity: 'burningangel', name: 'Joanna Angel', fields: ['avatar', 'gender'] }, { entity: 'chaosmen', name: 'Kenzo Alvarez', fields: ['avatar', 'gender'] }, { entity: 'dogfartnetwork', name: 'Liz Jordan', fields: ['avatar', 'gender'] }, { entity: 'diabolic', name: 'Kira Noir', fields: ['avatar', 'gender'] }, { entity: 'evilangel', name: 'Francesca Le', fields: ['avatar', 'gender'] }, { entity: 'fantasymassage', name: 'Cherry Kiss', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] }, { entity: 'filthykings', name: 'Armani Black', fields: ['avatar', 'gender'] }, { entity: 'gangbangcreampie', name: 'Luna Lovely', fields: ['avatar', 'gender', 'description'] }, { entity: 'girlsway', name: 'Adriana Chechik', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] }, { entity: 'gloryholesecrets', name: 'Maya Bijou', fields: ['avatar', 'gender'] }, { entity: 'pridestudios', name: 'Peter Hooke', fields: ['avatar', 'gender'] }, { entity: 'puretaboo', name: 'Jane Wilde', fields: ['avatar', 'gender'] }, { entity: 'tabooheat', name: 'Cory Chase', fields: ['avatar', 'gender'] }, { entity: 'vivid', name: 'Avi Love', fields: ['avatar', 'gender'] }, { entity: 'wicked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, { entity: 'xempire', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] }, // gamma > zero tolerance / stod { entity: '3rddegreefilms', name: 'Angel Dark', fields: ['gender', 'avatar'] }, { entity: 'addicted2girls', name: 'Abigail Mac', fields: ['gender', 'avatar'] }, { entity: 'zerotolerancefilms', name: 'Courtney Cummz', fields: ['gender', 'avatar'] }, { entity: 'genderxfilms', name: 'Kasey Kei', fields: ['gender', 'avatar'] }, // gamma > famedigital { entity: 'roccosiffredi', name: 'Malena Nazionale', fields: ['avatar', 'gender'] }, { entity: 'famedigital', name: 'Hime Marie', fields: ['avatar', 'gender'] }, { entity: 'peternorth', name: 'Carmen Caliente', fields: ['avatar', 'gender'] }, { entity: 'silverstonedvd', name: 'Leanni Lei', fields: ['avatar', 'gender'] }, { entity: 'silviasaint', name: 'Silvia Saint', fields: ['avatar', 'gender', 'description'] }, { entity: 'whiteghetto', name: 'Proxy Paige', fields: ['avatar', 'gender', 'description'] }, // perv city { entity: 'pervcity', name: 'Brooklyn Gray', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'dpdiva', name: 'Liz Jordan', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] }, // radical { entity: 'bjraw', name: 'Nikki Knightly', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'gotfilled', name: 'Alexa Chains', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'inserted', name: 'Anissa Kate', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, // etc. { entity: 'archangel', name: 'Summer Brielle', fields: ['avatar', 'description', 'dateOfBirth', 'age', 'measurements', 'height', 'aliases'] }, { entity: 'theflourishxxx', name: 'XWifeKaren', fields: ['avatar', 'description'] }, { entity: 'hookuphotshot', name: 'Kenzie Reeves', fields: ['avatar', 'description'] }, ]; const actorScrapers = scrapers.actors; const source = argv.source?.[0] || null; async function validateUrl(url, mime = 'image/') { if (!url) { return false; } const href = url.src || url; try { new URL(href); // eslint-disable-line no-new } catch (_error) { return false; } const res = await fetch(href); const type = res.headers.get('content-type'); const resolvedType = url.expectType?.[type] || type; return resolvedType.includes(mime); } const validators = { age: (value) => !!Number(value), gender: (value) => value && ['female', 'male', 'transsexual', 'trans'].includes(value.toLowerCase()), description: (value) => typeof value === 'string' && value.length > 3, birthPlace: (value) => typeof value === 'string' && value.length > 1, // may return US or USA birthCountry: (value) => typeof value === 'string' && value.length > 1, nationality: (value) => typeof value === 'string' && value.length > 3, // height: (value) => !!Number(value) || /\d'\d{1,2}"/.test(value), // ft in needs to be converted height: (value) => !!Number(value) && value > 150, weight: (value) => !!Number(value) && value > 40, eyes: (value) => typeof value === 'string' && value.length > 3, hairColor: (value) => typeof value === 'string' && value.length > 3, measurements: (value) => /(\d+)([a-z]+)?(?:\s*[-x]\s*(\d+)\s*[-x]\s*(\d+))?/i.test(value), // from actors module dateOfBirth: (value) => value instanceof Date && !Number.isNaN(value.getFullYear()), hasTattoos: (value) => typeof value === 'boolean', hasPiercings: (value) => typeof value === 'boolean', avatar: async (value) => [].concat(value).reduce(async (chain, url) => { const acc = await chain; if (!acc) { return acc; } return validateUrl(url); }, Promise.resolve(true)), socials: async (value) => [].concat(value).reduce(async (chain, url) => { const acc = await chain; if (!acc) { return acc; } return validateUrl(url, 'text/html'); }, Promise.resolve(true)), }; // profiler in this context is shorthand for profile scraper async function init() { const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: argv.prefer }); Object.entries(actorScrapers).reduce(async (chain, [entitySlug, scraper]) => { await chain; const entity = entitiesBySlug[entitySlug] || null; const fetchProfile = resolveLayoutScraper(entity, scraper)?.fetchProfile; const tests = actors.filter((actor) => actor.entity === entitySlug); // TODO: remove when all tests are written if (tests.length === 0) { console.log('TODO', entitySlug); return; } if (source && source !== entitySlug) { // console.log('____', entitySlug); return; } await test(`${entitySlug} (${entity?.name})`, async () => { await test(`${entitySlug} has scraper`, () => assert.notEqual(fetchProfile, null)); await test(`${entitySlug} has entity`, () => assert.notEqual(entity, null)); await test(`${entitySlug} has tests`, () => assert.notEqual(tests.length, 0)); await test(`${entitySlug} has valid fields`, async () => Promise.all(tests.map(async (actor) => { const profile = await fetchProfile({ name: actor.name, slug: slugify(actor.name), }, { ...entity, entity, channel: entity, network: entity.parent, parameters: getRecursiveParameters(entity), }, include); console.log(profile); console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => `'${field}'`).join(', ')); if (!profile) { assert.fail('profile not found'); } await Promise.all(actor.fields.map(async (field) => { assert.ok( validators[field] ? await validators[field](profile[field]) : typeof profile[field] !== 'undefined', `broken field ${field}, got ${profile[field]}`, ); })); }))); }); }, Promise.resolve()); await knex.destroy(); } init();