208 lines
12 KiB
JavaScript
208 lines
12 KiB
JavaScript
'use strict';
|
|
|
|
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
|
|
const argv = require('../src/argv');
|
|
const include = require('../src/utils/argv-include')(argv);
|
|
const slugify = require('../src/utils/slugify');
|
|
const scrapers = require('../src/scrapers/scrapers');
|
|
const { fetchEntitiesBySlug } = require('../src/entities');
|
|
const { resolveLayoutScraper } = require('../src/scrapers/resolve');
|
|
const getRecursiveParameters = require('../src/utils/get-recursive-parameters');
|
|
const knex = require('../src/knex');
|
|
|
|
const actors = [
|
|
// jules jordan
|
|
{ entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'description', 'avatar'] },
|
|
// gamma
|
|
{ entity: 'wicked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'xempire', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
|
// vixen
|
|
{ entity: 'vixen', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'tushy', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'tushyraw', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'blacked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'blackedraw', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'slayed', name: 'Vanna Bardot', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'deeper', name: 'Vanna Bardot', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'milfy', name: 'Clea Gaultier', fields: ['gender', 'avatar', 'description'] },
|
|
{ entity: 'wifey', name: 'Danielle Renae', fields: ['gender', 'avatar', 'description'] },
|
|
// teamskeet
|
|
{ entity: 'teamskeet', name: 'Abella Danger', fields: ['description', 'avatar', 'measurements', 'birthPlace', 'nationality', 'ethnicity', 'height', 'weight', 'hairColor', 'hasPiercings'] },
|
|
{ entity: 'teamskeet', name: 'Kali Roses', fields: ['description', 'avatar', 'measurements', 'nationality', 'ethnicity', 'hairColor', 'hasPiercings', 'hasTattoos'] }, // tattoos
|
|
// analvids
|
|
{ entity: 'analvids', name: 'Veronica Leal', fields: ['avatar', 'gender', 'birthCountry', 'nationality', 'age', 'aliases', 'nationality'] },
|
|
// mike adriano
|
|
{ entity: 'trueanal', name: 'Brenna McKenna', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor', 'hasTattoos'] },
|
|
{ entity: 'analonly', name: 'Lilith Grace', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },
|
|
{ entity: 'allanal', name: 'Lexi Lore', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },
|
|
{ entity: 'swallowed', name: 'Brooklyn Gray', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor', 'hasTattoos'] },
|
|
{ entity: 'nympho', name: 'Gianna Dior', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },
|
|
{ entity: 'dirtyauditions', name: 'Nicole Kitt', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },
|
|
// spizoo
|
|
{ entity: 'spizoo', name: 'Charlotte Sins', fields: ['description', 'avatar', 'dateOfBirth', 'ethnicity', 'nationality', 'height', 'measurements', 'hasTattoos', 'hasPiercings', 'hairColor', 'eyes', 'butt', 'pussy'] },
|
|
{ entity: 'rawattack', name: 'Kitana Montana', fields: ['avatar', 'dateOfBirth', 'nationality', 'measurements', 'eyes', 'height', 'hairColor', 'hasTattoos'] },
|
|
// hush / hussiepass
|
|
{ entity: 'hussiepass', name: 'Roxie Sinner', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'foot', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
|
|
{ entity: 'eyeontheguy', name: 'Tommy Gunn', fields: ['avatar'] },
|
|
{ entity: 'interracialpovs', name: 'Nia Nacci', fields: ['avatar', 'aliases', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
|
|
{ entity: 'povpornstars', name: 'Anna Bell Peaks', fields: ['avatar', 'aliases', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
|
|
{ entity: 'seehimfuck', name: 'Sheem The Dream', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'penisLength', 'circumcised', 'socials'] },
|
|
{ entity: 'hushpass', name: 'Dylan Ryder', fields: ['avatar'] },
|
|
{ entity: 'interracialpass', name: 'Aidra Fox', fields: ['avatar', 'height', 'measurements'] },
|
|
// kelly madison / 8K
|
|
{ entity: 'kellymadison', name: 'Ava Addams', fields: ['avatar', 'description', 'age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] },
|
|
{ entity: '8kmembers', name: 'Angie Lynx', fields: ['age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] },
|
|
// aylo
|
|
{ entity: 'brazzers', name: 'Lexi Lore', fields: ['avatar', 'description', 'gender', 'height', 'weight', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity', 'hairColor', 'hasTattoos', 'hasPiercings'] },
|
|
{ entity: 'digitalplayground', name: 'Elly Clutch', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] },
|
|
{ entity: 'realitykings', name: 'Abella Danger', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity'] },
|
|
{ entity: 'fakehub', name: 'Abella Danger', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity'] },
|
|
{ entity: 'babes', name: 'Alina Lopez', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] },
|
|
{ entity: 'letsdoeit', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] },
|
|
{ entity: 'men', name: 'Cade Maddox', fields: ['avatar', 'description', 'gender', 'height', 'ethnicity', 'penisLength', 'dateOfBirth', 'weight', 'hairColor', 'hasTattoos'] },
|
|
{ entity: 'metrohd', name: 'April Olsen', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight'] },
|
|
{ entity: 'mofos', name: 'Ariana Starr', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] },
|
|
{ entity: 'propertysex', name: 'Desiree Dulce', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasPiercings'] },
|
|
{ entity: 'sexyhub', name: 'Angie Lynx', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] },
|
|
{ entity: 'squirted', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] },
|
|
{ entity: 'transangels', name: 'Aubrey Kate', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos'] },
|
|
{ entity: 'trueamateurs', name: 'Natalie FLowers', fields: ['avatar', 'gender'] },
|
|
{ entity: 'spicevids', name: 'Remy LaCroix', fields: ['avatar', 'gender', 'description', 'height', 'measurements', 'dateOfBirth', 'weight'] },
|
|
{ entity: 'twistys', name: 'Remy LaCroix', fields: ['avatar', 'gender', 'description', 'height', 'measurements', 'dateOfBirth', 'weight', 'birthPlace', 'hairColor', 'ethnicity', 'naturalBoobs', 'hasPiercings'] },
|
|
// aylo > adult mobile
|
|
{ entity: 'adultmobile', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] },
|
|
{ entity: 'doghousedigital', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] },
|
|
{ entity: 'familysinners', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] },
|
|
{ entity: 'iconmale', name: 'Troy Accola', fields: ['avatar', 'gender', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] },
|
|
{ entity: 'realityjunkies', name: 'Scarlett Alexis', fields: ['avatar', 'gender'] },
|
|
{ entity: 'sweetheartvideo', name: 'Lexi Lore', fields: ['avatar', 'gender', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] },
|
|
{ entity: 'sweetsinner', name: 'Anna Claire Clouds', fields: ['avatar', 'gender'] },
|
|
// bangros
|
|
{ entity: 'bangbros', name: 'Kira Perez', fields: ['avatar', 'gender', 'ethnicity', 'hairColor'] },
|
|
];
|
|
|
|
const actorScrapers = scrapers.actors;
|
|
const source = argv.source?.[0] || null;
|
|
|
|
async function validateUrl(url, mime = 'image/') {
|
|
if (!url) {
|
|
return false;
|
|
}
|
|
|
|
const href = url.src || url;
|
|
|
|
try {
|
|
new URL(href); // eslint-disable-line no-new
|
|
} catch (_error) {
|
|
return false;
|
|
}
|
|
|
|
const res = await fetch(href);
|
|
|
|
const type = res.headers.get('content-type');
|
|
const resolvedType = url.expectType?.[type] || type;
|
|
|
|
return resolvedType.includes(mime);
|
|
}
|
|
|
|
const validators = {
|
|
age: (value) => !!Number(value),
|
|
gender: (value) => value && ['female', 'male', 'transsexual', 'trans'].includes(value.toLowerCase()),
|
|
description: (value) => typeof value === 'string' && value.length > 3,
|
|
birthPlace: (value) => typeof value === 'string' && value.length > 1, // may return US or USA
|
|
birthCountry: (value) => typeof value === 'string' && value.length > 1,
|
|
nationality: (value) => typeof value === 'string' && value.length > 3,
|
|
height: (value) => !!Number(value) || /\d'\d{1,2}"/.test(value),
|
|
weight: (value) => !!Number(value),
|
|
eyes: (value) => typeof value === 'string' && value.length > 3,
|
|
hairColor: (value) => typeof value === 'string' && value.length > 3,
|
|
measurements: (value) => /(\d+)([a-z]+)?(?:\s*[-x]\s*(\d+)\s*[-x]\s*(\d+))?/i.test(value), // from actors module
|
|
dateOfBirth: (value) => value instanceof Date && !Number.isNaN(value.getFullYear()),
|
|
hasTattoos: (value) => typeof value === 'boolean',
|
|
hasPiercings: (value) => typeof value === 'boolean',
|
|
avatar: async (value) => [].concat(value).reduce(async (chain, url) => {
|
|
const acc = await chain;
|
|
|
|
if (!acc) {
|
|
return acc;
|
|
}
|
|
|
|
return validateUrl(url);
|
|
}, Promise.resolve(true)),
|
|
socials: async (value) => [].concat(value).reduce(async (chain, url) => {
|
|
const acc = await chain;
|
|
|
|
if (!acc) {
|
|
return acc;
|
|
}
|
|
|
|
return validateUrl(url, 'text/html');
|
|
}, Promise.resolve(true)),
|
|
};
|
|
|
|
// profiler in this context is shorthand for profile scraper
|
|
async function init() {
|
|
const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: argv.prefer });
|
|
|
|
Object.entries(actorScrapers).reduce(async (chain, [entitySlug, scraper]) => {
|
|
await chain;
|
|
|
|
const entity = entitiesBySlug[entitySlug] || null;
|
|
const fetchProfile = resolveLayoutScraper(entity, scraper)?.fetchProfile;
|
|
|
|
const tests = actors.filter((actor) => actor.entity === entitySlug);
|
|
|
|
// TODO: remove when all tests are written
|
|
if (tests.length === 0) {
|
|
console.log('TODO', entitySlug);
|
|
return;
|
|
}
|
|
|
|
if (source && source !== entitySlug) {
|
|
console.log('____', entitySlug);
|
|
return;
|
|
}
|
|
|
|
await test(`${entitySlug} (${entity?.name})`, async () => {
|
|
await test(`${entitySlug} has scraper`, () => assert.notEqual(fetchProfile, null));
|
|
await test(`${entitySlug} has entity`, () => assert.notEqual(entity, null));
|
|
await test(`${entitySlug} has tests`, () => assert.notEqual(tests.length, 0));
|
|
|
|
await test(`${entitySlug} has valid fields`, async () => Promise.all(tests.map(async (actor) => {
|
|
const profile = await fetchProfile({
|
|
name: actor.name,
|
|
slug: slugify(actor.name),
|
|
}, {
|
|
...entity,
|
|
entity,
|
|
channel: entity,
|
|
network: entity.parent,
|
|
parameters: getRecursiveParameters(entity),
|
|
}, include);
|
|
|
|
console.log(profile);
|
|
console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => field).join(', '));
|
|
|
|
if (!profile) {
|
|
assert.fail('profile not found');
|
|
}
|
|
|
|
await Promise.all(actor.fields.map(async (field) => {
|
|
assert.ok(
|
|
validators[field]
|
|
? await validators[field](profile[field])
|
|
: typeof profile[field] !== 'undefined',
|
|
`broken field ${field}, got ${profile[field]}`,
|
|
);
|
|
}));
|
|
})));
|
|
});
|
|
}, Promise.resolve());
|
|
|
|
await knex.destroy();
|
|
}
|
|
|
|
init();
|