Added profile scraper tests (WIP), fixed some profile scrapers. Fixed slugify not breaking existing slugs.
This commit is contained in:
DebaucheryLibrarian
@@ -4,10 +4,124 @@ const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const argv = require('../src/argv');
|
||||
const include = require('../src/utils/argv-include')(argv);
|
||||
const slugify = require('../src/utils/slugify');
|
||||
const scrapers = require('../src/scrapers/scrapers');
|
||||
const { fetchEntitiesBySlug } = require('../src/entities');
|
||||
const { resolveLayoutScraper } = require('../src/scrapers/resolve');
|
||||
const getRecursiveParameters = require('../src/utils/get-recursive-parameters');
|
||||
const knex = require('../src/knex');
|
||||
|
||||
const actors = [
|
||||
// jules jordan
|
||||
{ entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'description', 'avatar'] },
|
||||
// gamma
|
||||
{ entity: 'wicked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'xempire', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
||||
// vixen
|
||||
{ entity: 'vixen', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'tushy', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'tushyraw', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'blacked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'blackedraw', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'slayed', name: 'Vanna Bardot', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'deeper', name: 'Vanna Bardot', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'milfy', name: 'Clea Gaultier', fields: ['gender', 'avatar', 'description'] },
|
||||
{ entity: 'wifey', name: 'Danielle Renae', fields: ['gender', 'avatar', 'description'] },
|
||||
// teamskeet
|
||||
{ entity: 'teamskeet', name: 'Abella Danger', fields: ['description', 'avatar', 'measurements', 'birthPlace', 'nationality', 'ethnicity', 'height', 'weight', 'hairColor', 'hasPiercings'] },
|
||||
{ entity: 'teamskeet', name: 'Kali Roses', fields: ['description', 'avatar', 'measurements', 'nationality', 'ethnicity', 'hairColor', 'hasPiercings', 'hasTattoos'] }, // tattoos
|
||||
// analvids
|
||||
{ entity: 'analvids', name: 'Veronica Leal', fields: ['avatar', 'gender', 'birthCountry', 'nationality', 'age', 'aliases', 'nationality'] },
|
||||
// mike adriano
|
||||
{ entity: 'trueanal', name: 'Brenna McKenna', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor', 'hasTattoos'] },
|
||||
{ entity: 'analonly', name: 'Lilith Grace', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },
|
||||
{ entity: 'allanal', name: 'Lexi Lore', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },
|
||||
{ entity: 'swallowed', name: 'Brooklyn Gray', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor', 'hasTattoos'] },
|
||||
{ entity: 'nympho', name: 'Gianna Dior', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },
|
||||
{ entity: 'dirtyauditions', name: 'Nicole Kitt', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },
|
||||
// spizoo
|
||||
{ entity: 'spizoo', name: 'Charlotte Sins', fields: ['description', 'avatar', 'dateOfBirth', 'ethnicity', 'nationality', 'height', 'measurements', 'hasTattoos', 'hasPiercings', 'hairColor', 'eyes', 'butt', 'pussy'] },
|
||||
{ entity: 'rawattack', name: 'Kitana Montana', fields: ['avatar', 'dateOfBirth', 'nationality', 'measurements', 'eyes', 'height', 'hairColor', 'hasTattoos'] },
|
||||
// hush / hussiepass
|
||||
{ entity: 'hussiepass', name: 'Roxie Sinner', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'foot', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
|
||||
{ entity: 'eyeontheguy', name: 'Tommy Gunn', fields: ['avatar'] },
|
||||
{ entity: 'interracialpovs', name: 'Nia Nacci', fields: ['avatar', 'aliases', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
|
||||
{ entity: 'povpornstars', name: 'Anna Bell Peaks', fields: ['avatar', 'aliases', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
|
||||
{ entity: 'seehimfuck', name: 'Sheem The Dream', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'penisLength', 'circumcised', 'socials'] },
|
||||
{ entity: 'hushpass', name: 'Dylan Ryder', fields: ['avatar'] },
|
||||
{ entity: 'interracialpass', name: 'Aidra Fox', fields: ['avatar', 'height', 'measurements'] },
|
||||
// kelly madison / 8K
|
||||
{ entity: 'kellymadison', name: 'Ava Addams', fields: ['avatar', 'description', 'age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] },
|
||||
{ entity: '8kmembers', name: 'Angie Lynx', fields: ['age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] },
|
||||
// aylo
|
||||
{ entity: 'brazzers', name: 'Lexi Lore', fields: ['avatar', 'description', 'gender', 'height', 'weight', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity', 'hairColor', 'hasTattoos', 'hasPiercings'] },
|
||||
{ entity: 'digitalplayground', name: 'Elly Clutch', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] },
|
||||
{ entity: 'realitykings', name: 'Abella Danger', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity'] },
|
||||
{ entity: 'fakehub', name: 'Abella Danger', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity'] },
|
||||
{ entity: 'babes', name: 'Alina Lopez', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos', 'hasPiercings'] },
|
||||
{ entity: 'letsdoeit', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] },
|
||||
{ entity: 'men', name: 'Cade Maddox', fields: ['avatar', 'description', 'gender', 'height', 'ethnicity', 'penisLength', 'dateOfBirth', 'weight', 'hairColor', 'hasTattoos'] },
|
||||
];
|
||||
|
||||
const actorScrapers = scrapers.actors;
|
||||
const source = argv.source?.[0] || null;
|
||||
|
||||
async function validateUrl(url, mime = 'image/') {
|
||||
if (!url) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const href = url.src || url;
|
||||
|
||||
try {
|
||||
new URL(href); // eslint-disable-line no-new
|
||||
} catch (_error) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const res = await fetch(href);
|
||||
|
||||
const type = res.headers.get('content-type');
|
||||
const resolvedType = url.expectType?.[type] || type;
|
||||
|
||||
return resolvedType.includes(mime);
|
||||
}
|
||||
|
||||
const validators = {
|
||||
age: (value) => !!Number(value),
|
||||
gender: (value) => value && ['female', 'male', 'transsexual'].includes(value.toLowerCase()),
|
||||
description: (value) => typeof value === 'string' && value.length > 3,
|
||||
birthPlace: (value) => typeof value === 'string' && value.length > 3,
|
||||
birthCountry: (value) => typeof value === 'string' && value.length > 1,
|
||||
nationality: (value) => typeof value === 'string' && value.length > 3,
|
||||
height: (value) => !!Number(value) || /\d'\d{1,2}"/.test(value),
|
||||
weight: (value) => !!Number(value),
|
||||
eyes: (value) => typeof value === 'string' && value.length > 3,
|
||||
hairColor: (value) => typeof value === 'string' && value.length > 3,
|
||||
measurements: (value) => /(\d+)([a-z]+)?(?:\s*[-x]\s*(\d+)\s*[-x]\s*(\d+))?/i.test(value), // from actors module
|
||||
dateOfBirth: (value) => value instanceof Date && !Number.isNaN(value.getFullYear()),
|
||||
hasTattoos: (value) => typeof value === 'boolean',
|
||||
hasPiercings: (value) => typeof value === 'boolean',
|
||||
avatar: async (value) => [].concat(value).reduce(async (chain, url) => {
|
||||
const acc = await chain;
|
||||
|
||||
if (!acc) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
return validateUrl(url);
|
||||
}, Promise.resolve(true)),
|
||||
socials: async (value) => [].concat(value).reduce(async (chain, url) => {
|
||||
const acc = await chain;
|
||||
|
||||
if (!acc) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
return validateUrl(url, 'text/html');
|
||||
}, Promise.resolve(true)),
|
||||
};
|
||||
|
||||
// profiler in this context is shorthand for profile scraper
|
||||
async function init() {
|
||||
@@ -17,31 +131,58 @@ async function init() {
|
||||
await chain;
|
||||
|
||||
const entity = entitiesBySlug[entitySlug] || null;
|
||||
const fetchProfile = resolveLayoutScraper(entity, scraper)?.fetchProfile;
|
||||
|
||||
const profilers = Array.from(new Set(Object.entries(scraper) // some layouts will use the same profiler
|
||||
.flatMap(([fnKey, fnOrLayout]) => {
|
||||
if (fnOrLayout.fetchProfile) {
|
||||
// layout
|
||||
return fnOrLayout.fetchProfile;
|
||||
}
|
||||
const tests = actors.filter((actor) => actor.entity === entitySlug);
|
||||
|
||||
if (fnKey === 'fetchProfile') {
|
||||
// primary
|
||||
return fnOrLayout;
|
||||
}
|
||||
// TODO: remove when all tests are written
|
||||
if (tests.length === 0) {
|
||||
console.log('TODO', entitySlug);
|
||||
return;
|
||||
}
|
||||
|
||||
return null;
|
||||
}).filter(Boolean)));
|
||||
if (source && source !== entitySlug) {
|
||||
console.log('____', entitySlug);
|
||||
return;
|
||||
}
|
||||
|
||||
await test(`${entitySlug} (${entity?.name})`, async () => {
|
||||
await test('has entity', () => assert.notEqual(entity, null));
|
||||
await test('has profilers', () => assert.ok(profilers.length > 0));
|
||||
await test(`${entitySlug} has scraper`, () => assert.notEqual(fetchProfile, null));
|
||||
await test(`${entitySlug} has entity`, () => assert.notEqual(entity, null));
|
||||
await test(`${entitySlug} has tests`, () => assert.notEqual(tests.length, 0));
|
||||
|
||||
await test('foo', () => {
|
||||
assert.strictEqual(5, 5);
|
||||
});
|
||||
await test(`${entitySlug} has valid fields`, async () => Promise.all(tests.map(async (actor) => {
|
||||
const profile = await fetchProfile({
|
||||
name: actor.name,
|
||||
slug: slugify(actor.name),
|
||||
}, {
|
||||
...entity,
|
||||
entity,
|
||||
channel: entity,
|
||||
network: entity.parent,
|
||||
parameters: getRecursiveParameters(entity),
|
||||
}, include);
|
||||
|
||||
console.log(profile);
|
||||
console.log('Untested fields', Object.keys(profile).filter((field) => !actor.fields.includes(field)).join(', '));
|
||||
|
||||
if (!profile) {
|
||||
assert.fail('profile not found');
|
||||
}
|
||||
|
||||
await Promise.all(actor.fields.map(async (field) => {
|
||||
assert.ok(
|
||||
validators[field]
|
||||
? await validators[field](profile[field])
|
||||
: typeof profile[field] !== 'undefined',
|
||||
`broken field ${field}, got ${profile[field]}`,
|
||||
);
|
||||
}));
|
||||
})));
|
||||
});
|
||||
}, Promise.resolve());
|
||||
|
||||
await knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
|
||||
Reference in New Issue
Block a user