DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Renamed traxxx utils to common. Added natural lips column.

This commit is contained in:
DebaucheryLibrarian 2024-10-22 02:51:00 +02:00
parent 1b0101f20a
commit eea8d6cc68
4 changed files with 254 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "common"]
path = common
url = git@unknown.name:DebaucheryLibrarian/traxxx-common.git

1
common Submodule

@ -0,0 +1 @@
Subproject commit 32f811c24d217d15ccb6c65e421231b0a642210b

13
migrations/20241019013040_actor_revisions.js
View File

@ -53,12 +53,12 @@ exports.up = async (knex) => {
table.enum('boobs_placement', ['over', 'under']); table.enum('boobs_placement', ['over', 'under']);
table.string('boobs_surgeon'); table.string('boobs_surgeon');
table.boolean('natural_butt');
table.integer('butt_volume'); table.integer('butt_volume');
table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']); table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']);
table.boolean('natural_lips');
table.integer('lips_volume'); table.integer('lips_volume');
table.boolean('natural_butt');
}); });
await knex.schema.alterTable('actors_profiles', (table) => { await knex.schema.alterTable('actors_profiles', (table) => {
@ -67,11 +67,12 @@ exports.up = async (knex) => {
table.enum('boobs_placement', ['over', 'under']); table.enum('boobs_placement', ['over', 'under']);
table.string('boobs_surgeon'); table.string('boobs_surgeon');
table.boolean('natural_butt');
table.integer('butt_volume'); table.integer('butt_volume');
table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']); table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']);
table.boolean('natural_lips');
table.integer('lips_volume'); table.integer('lips_volume');
table.boolean('natural_butt');
}); });
}; };
@ -86,11 +87,12 @@ exports.down = async (knex) => {
table.dropColumn('boobs_placement'); table.dropColumn('boobs_placement');
table.dropColumn('boobs_surgeon'); table.dropColumn('boobs_surgeon');
table.dropColumn('natural_butt');
table.dropColumn('butt_volume'); table.dropColumn('butt_volume');
table.dropColumn('butt_implant'); table.dropColumn('butt_implant');
table.dropColumn('natural_lips');
table.dropColumn('lips_volume'); table.dropColumn('lips_volume');
table.dropColumn('natural_butt');
}); });
await knex.schema.alterTable('actors_profiles', (table) => { await knex.schema.alterTable('actors_profiles', (table) => {
@ -99,10 +101,11 @@ exports.down = async (knex) => {
table.dropColumn('boobs_placement'); table.dropColumn('boobs_placement');
table.dropColumn('boobs_surgeon'); table.dropColumn('boobs_surgeon');
table.dropColumn('natural_butt');
table.dropColumn('butt_volume'); table.dropColumn('butt_volume');
table.dropColumn('butt_implant'); table.dropColumn('butt_implant');
table.dropColumn('natural_lips');
table.dropColumn('lips_volume'); table.dropColumn('lips_volume');
table.dropColumn('natural_butt');
}); });
}; };

454
src/actors.js
View File

@ -27,6 +27,7 @@ const { toBaseReleases } = require('./deep');
const { associateAvatars, flushOrphanedMedia } = require('./media'); const { associateAvatars, flushOrphanedMedia } = require('./media');
const { fetchEntitiesBySlug } = require('./entities'); const { fetchEntitiesBySlug } = require('./entities');
const { deleteScenes } = require('./releases'); const { deleteScenes } = require('./releases');
const { interpolateProfiles: interpolateProfilesUtil } = require('../common/actors');
const slugify = require('./utils/slugify'); const slugify = require('./utils/slugify');
const capitalize = require('./utils/capitalize'); const capitalize = require('./utils/capitalize');
@ -139,53 +140,6 @@ function getBoolean(value) {
return null; return null;
} }
function getMostFrequent(items) {
const { mostFrequent } = items.reduce((acc, item) => {
if (item === undefined || item === null) {
return acc;
}
const slug = slugify(item);
acc.counts[slug] = (acc.counts[slug] || 0) + 1;
if (!acc.mostFrequent || acc.counts[slug] > acc.counts[slugify(acc.mostFrequent)]) {
acc.mostFrequent = item;
}
return acc;
}, {
counts: {},
mostFrequent: null,
});
return mostFrequent;
}
function getMostFrequentDate(dates) {
const year = getMostFrequent(dates.map((dateX) => dateX.getFullYear()));
const month = getMostFrequent(dates.map((dateX) => dateX.getMonth()));
const date = getMostFrequent(dates.map((dateX) => dateX.getDate()));
if (year === null || month === null || date === null) {
return null;
}
return moment({ year, month, date }).toDate();
}
function getHighest(items) {
return items.reduce((prevItem, item) => (item > prevItem ? item : prevItem), null);
}
function getLongest(items) {
return items.sort((itemA, itemB) => itemB.length - itemA.length)[0] || null;
}
function getAverage(items) {
return Math.round(items.reduce((acc, item) => acc + item, 0) / items.length) || null;
}
function toBaseActors(actorsOrNames, release) { function toBaseActors(actorsOrNames, release) {
if (!actorsOrNames) { if (!actorsOrNames) {
return []; return [];
@ -387,6 +341,246 @@ function curateProfileEntry(profile) {
return curatedProfileEntry; return curatedProfileEntry;
} }
async function fetchProfiles(actorIdsOrNames) {
return knex('actors_profiles')
.select(knex.raw('actors_profiles.*, actors.name, row_to_json(media) as avatar'))
.leftJoin('actors', 'actors.id', 'actors_profiles.actor_id')
.modify((query) => {
if (actorIdsOrNames) {
query
.whereIn('actor_id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((builder) => {
builder
.whereIn('actors.name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('actors.entity_id');
});
}
})
.leftJoin('media', 'actors_profiles.avatar_media_id', 'media.id');
}
/*
function getMostFrequent(items) {
const { mostFrequent } = items.reduce((acc, item) => {
if (item === undefined || item === null) {
return acc;
}
const slug = slugify(item);
acc.counts[slug] = (acc.counts[slug] || 0) + 1;
if (!acc.mostFrequent || acc.counts[slug] > acc.counts[slugify(acc.mostFrequent)]) {
acc.mostFrequent = item;
}
return acc;
}, {
counts: {},
mostFrequent: null,
});
return mostFrequent;
}
function getMostFrequentDate(dates) {
const year = getMostFrequent(dates.map((dateX) => dateX.getFullYear()));
const month = getMostFrequent(dates.map((dateX) => dateX.getMonth()));
const date = getMostFrequent(dates.map((dateX) => dateX.getDate()));
if (year === null || month === null || date === null) {
return null;
}
return moment({ year, month, date }).toDate();
}
function getHighest(items) {
return items.reduce((prevItem, item) => (item > prevItem ? item : prevItem), null);
}
function getLongest(items) {
return items.sort((itemA, itemB) => itemB.length - itemA.length)[0] || null;
}
function getAverage(items) {
return Math.round(items.reduce((acc, item) => acc + item, 0) / items.length) || null;
}
function mergeMainProfile(profile, mainProfile) {
const preservedKeys = ['id'];
const mergedProfile = Object.fromEntries(Object.entries(profile).map(([key, value]) => [key, mainProfile[key] === null || preservedKeys.includes(key)
? value
: mainProfile[key]]));
return mergedProfile;
}
*/
async function interpolateProfiles(actorIdsOrNames) {
try {
await interpolateProfilesUtil(actorIdsOrNames, {
knex,
logger,
moment,
slugify,
omit,
});
} catch (error) {
console.log(error);
}
/*
const profiles = await fetchProfiles(actorIdsOrNames);
const profilesByActorId = profiles.reduce((acc, profile) => ({
...acc,
[profile.actor_id]: [
...(acc[profile.actor_id] || []),
profile,
],
}), {});
logger.info(`Interpolating ${profiles.length} profiles from ${Object.keys(profilesByActorId).length} actors`);
const interpolatedProfiles = Object.entries(profilesByActorId).map(([actorId, actorProfiles]) => {
// group values from each profile
const valuesByProperty = actorProfiles
.filter((profile) => profile.entity_id !== null) // main profile is interpolated separately at the end
.reduce((acc, profile) => Object
.entries(profile)
.reduce((profileAcc, [property, value]) => ({
...profileAcc,
[property]: [
...(acc[property] || []),
...(value === null ? [] : Array.from({ length: profile.priority }, () => value)), // multiply by priority, increasing the odds of being the most frequent value
],
}), {
// bundle location values so they can be assessed together, to ensure the most frequent city is in the most frequent state is in most frequent country
origin: [...acc.origin || [], {
...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }),
...(profile.birth_state && { state: profile.birth_state }),
...(profile.birth_city && { city: profile.birth_city }),
}].filter((location) => Object.keys(location).length > 0),
residence: [...acc.residence || [], {
...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }),
...(profile.residence_state && { state: profile.residence_state }),
...(profile.residence_city && { city: profile.residence_city }),
}].filter((location) => Object.keys(location).length > 0),
}), {});
const mostFrequentValues = [
'gender',
'orientation',
'ethnicity',
'cup',
'bust',
'waist',
'hip',
'leg',
'thigh',
'foot',
'shoe_size',
'penis_length',
'penis_girth',
'circumcised',
'hair_color',
'eyes',
'has_tattoos',
'has_piercings',
'blood_type',
].reduce((acc, property) => ({
...acc,
[property]: getMostFrequent(valuesByProperty[property]),
}), {});
const profile = {
id: actorId,
...mostFrequentValues,
};
profile.height = getMostFrequent(valuesByProperty.height.filter((height) => height > 50 && height < 300)); // remove unlikely values
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
profile.age = getHighest(valuesByProperty.age);
profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs);
// ensure most frequent country, city and state match up
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map((location) => location.country));
const remainingOriginCountries = valuesByProperty.origin.filter((location) => location.country === profile.birth_country_alpha2);
profile.birth_state = getMostFrequent(remainingOriginCountries.map((location) => location.state));
const remainingOriginStates = remainingOriginCountries.filter((location) => !profile.birth_state || location.state === profile.birth_state);
profile.birth_city = getMostFrequent(remainingOriginStates.map((location) => location.city));
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map((location) => location.country));
const remainingResidenceCountries = valuesByProperty.residence.filter((location) => location.country === profile.residence_country_alpha2);
profile.residence_state = getMostFrequent(remainingResidenceCountries.map((location) => location.state));
const remainingResidenceStates = remainingResidenceCountries.filter((location) => !profile.residence_state || location.state === profile.residence_state);
profile.residence_city = getMostFrequent(remainingResidenceStates.map((location) => location.city));
profile.weight = getAverage(valuesByProperty.weight);
profile.tattoos = getLongest(valuesByProperty.tattoos);
profile.piercings = getLongest(valuesByProperty.piercings);
profile.avatar_media_id = actorProfiles
.map((actorProfile) => actorProfile.avatar)
.filter((avatar) => avatar && (avatar.entropy === null || avatar.entropy > 5.5))
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
if (!profile.avatar_media_id) {
// try to settle for low quality avatar
profile.avatar_media_id = actorProfiles
.map((actorProfile) => actorProfile.avatar)
.filter((avatar) => avatar)
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
}
const mainProfile = actorProfiles.find((actorProfile) => actorProfile.entity_id === null);
return mergeMainProfile(profile, mainProfile);
});
const transaction = await knex.transaction();
// clear existing interpolated data
const emptyProfile = Object
.keys(omit(curateProfileEntry({ id: 1 }), ['id', 'actor_id', 'entity_id', 'url', 'description_hash']))
.reduce((acc, key) => ({ ...acc, [key]: null }), {});
await knex('actors')
.modify((modifyBuilder) => {
if (actorIdsOrNames) {
modifyBuilder
.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((whereBuilder) => {
whereBuilder
.whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('entity_id');
});
}
})
.update(emptyProfile)
.transacting(transaction);
// insert new interpolated data
const queries = interpolatedProfiles.map((profile) => knex('actors')
.where('id', profile.id)
.update(profile)
.transacting(transaction));
await Promise.all(queries)
.then(transaction.commit)
.catch(transaction.rollback);
*/
}
async function curateProfile(profile, actor) { async function curateProfile(profile, actor) {
if (!profile) { if (!profile) {
return null; return null;
@ -537,170 +731,6 @@ async function curateProfile(profile, actor) {
} }
} }
async function fetchProfiles(actorIdsOrNames) {
return knex('actors_profiles')
.select(knex.raw('actors_profiles.*, row_to_json(actors) as actor, row_to_json(media) as avatar'))
.leftJoin('actors', 'actors.id', 'actors_profiles.actor_id')
.modify((query) => {
if (actorIdsOrNames) {
query
.whereIn('actor_id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((builder) => {
builder
.whereIn('actors.name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('actors.entity_id');
});
}
})
.leftJoin('media', 'actors_profiles.avatar_media_id', 'media.id');
}
async function interpolateProfiles(actorIdsOrNames) {
const profiles = await fetchProfiles(actorIdsOrNames);
const profilesByActorId = profiles.reduce((acc, profile) => ({
...acc,
[profile.actor_id]: [
...(acc[profile.actor_id] || []),
profile,
],
}), {});
logger.info(`Interpolating ${profiles.length} profiles from ${Object.keys(profilesByActorId).length} actors`);
const interpolatedProfiles = Object.entries(profilesByActorId).map(([actorId, actorProfiles]) => {
// group values from each profile
const valuesByProperty = actorProfiles.reduce((acc, profile) => Object
.entries(profile)
.reduce((profileAcc, [property, value]) => ({
...profileAcc,
[property]: [
...(acc[property] || []),
...(value === null ? [] : Array.from({ length: profile.priority }, () => value)), // multiply by priority, increasing the odds of being the most frequent value
],
}), {
// bundle location values so they can be assessed together, to ensure the most frequent city is in the most frequent state is in most frequent country
origin: [...acc.origin || [], {
...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }),
...(profile.birth_state && { state: profile.birth_state }),
...(profile.birth_city && { city: profile.birth_city }),
}].filter((location) => Object.keys(location).length > 0),
residence: [...acc.residence || [], {
...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }),
...(profile.residence_state && { state: profile.residence_state }),
...(profile.residence_city && { city: profile.residence_city }),
}].filter((location) => Object.keys(location).length > 0),
}), {});
const mostFrequentValues = [
'gender',
'orientation',
'ethnicity',
'cup',
'bust',
'waist',
'hip',
'leg',
'thigh',
'foot',
'shoe_size',
'penis_length',
'penis_girth',
'circumcised',
'hair_color',
'eyes',
'has_tattoos',
'has_piercings',
'blood_type',
].reduce((acc, property) => ({
...acc,
[property]: getMostFrequent(valuesByProperty[property]),
}), {});
const profile = {
id: actorId,
...mostFrequentValues,
};
profile.height = getMostFrequent(valuesByProperty.height.filter((height) => height > 50 && height < 300)); // remove unlikely values
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
profile.age = getHighest(valuesByProperty.age);
profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs);
// ensure most frequent country, city and state match up
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map((location) => location.country));
const remainingOriginCountries = valuesByProperty.origin.filter((location) => location.country === profile.birth_country_alpha2);
profile.birth_state = getMostFrequent(remainingOriginCountries.map((location) => location.state));
const remainingOriginStates = remainingOriginCountries.filter((location) => !profile.birth_state || location.state === profile.birth_state);
profile.birth_city = getMostFrequent(remainingOriginStates.map((location) => location.city));
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map((location) => location.country));
const remainingResidenceCountries = valuesByProperty.residence.filter((location) => location.country === profile.residence_country_alpha2);
profile.residence_state = getMostFrequent(remainingResidenceCountries.map((location) => location.state));
const remainingResidenceStates = remainingResidenceCountries.filter((location) => !profile.residence_state || location.state === profile.residence_state);
profile.residence_city = getMostFrequent(remainingResidenceStates.map((location) => location.city));
profile.weight = getAverage(valuesByProperty.weight);
profile.tattoos = getLongest(valuesByProperty.tattoos);
profile.piercings = getLongest(valuesByProperty.piercings);
profile.avatar_media_id = actorProfiles
.map((actorProfile) => actorProfile.avatar)
.filter((avatar) => avatar && (avatar.entropy === null || avatar.entropy > 5.5))
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
if (!profile.avatar_media_id) {
// try to settle for low quality avatar
profile.avatar_media_id = actorProfiles
.map((actorProfile) => actorProfile.avatar)
.filter((avatar) => avatar)
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
}
return profile;
});
const transaction = await knex.transaction();
// clear existing interpolated data
const emptyProfile = Object
.keys(omit(curateProfileEntry({ id: 1 }), ['id', 'actor_id', 'entity_id', 'url', 'description_hash']))
.reduce((acc, key) => ({ ...acc, [key]: null }), {});
await knex('actors')
.modify((modifyBuilder) => {
if (actorIdsOrNames) {
modifyBuilder
.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((whereBuilder) => {
whereBuilder
.whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('entity_id');
});
}
})
.update(emptyProfile)
.transacting(transaction);
// insert new interpolated data
const queries = interpolatedProfiles.map((profile) => knex('actors')
.where('id', profile.id)
.update(profile)
.transacting(transaction));
await Promise.all(queries)
.then(transaction.commit)
.catch(transaction.rollback);
}
async function upsertProfiles(profiles) { async function upsertProfiles(profiles) {
const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean); const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean); const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
@ -1150,7 +1180,7 @@ async function searchActors(query) {
async function flushProfiles(actorIdsOrNames) { async function flushProfiles(actorIdsOrNames) {
const profiles = await fetchProfiles(actorIdsOrNames); const profiles = await fetchProfiles(actorIdsOrNames);
const actorNames = Array.from(new Set(profiles.map((profile) => profile.actor.name))); const actorNames = Array.from(new Set(profiles.map((profile) => profile.name)));
const deleteCount = await knex('actors_profiles') const deleteCount = await knex('actors_profiles')
.whereIn('id', profiles.map((profile) => profile.id)) .whereIn('id', profiles.map((profile) => profile.id))