From eea8d6cc688ec48da013f4e54352d3bae2c2fce6 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Tue, 22 Oct 2024 02:51:00 +0200 Subject: [PATCH] Renamed traxxx utils to common. Added natural lips column. --- .gitmodules | 3 + common | 1 + migrations/20241019013040_actor_revisions.js | 13 +- src/actors.js | 454 ++++++++++--------- 4 files changed, 254 insertions(+), 217 deletions(-) create mode 100644 .gitmodules create mode 160000 common diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..486dabcf --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "common"] + path = common + url = git@unknown.name:DebaucheryLibrarian/traxxx-common.git diff --git a/common b/common new file mode 160000 index 00000000..32f811c2 --- /dev/null +++ b/common @@ -0,0 +1 @@ +Subproject commit 32f811c24d217d15ccb6c65e421231b0a642210b diff --git a/migrations/20241019013040_actor_revisions.js b/migrations/20241019013040_actor_revisions.js index 67deebf1..70077165 100644 --- a/migrations/20241019013040_actor_revisions.js +++ b/migrations/20241019013040_actor_revisions.js @@ -53,12 +53,12 @@ exports.up = async (knex) => { table.enum('boobs_placement', ['over', 'under']); table.string('boobs_surgeon'); + table.boolean('natural_butt'); table.integer('butt_volume'); table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']); + table.boolean('natural_lips'); table.integer('lips_volume'); - - table.boolean('natural_butt'); }); await knex.schema.alterTable('actors_profiles', (table) => { @@ -67,11 +67,12 @@ exports.up = async (knex) => { table.enum('boobs_placement', ['over', 'under']); table.string('boobs_surgeon'); + table.boolean('natural_butt'); table.integer('butt_volume'); table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']); + table.boolean('natural_lips'); table.integer('lips_volume'); - table.boolean('natural_butt'); }); }; @@ -86,11 +87,12 @@ exports.down = async (knex) => { table.dropColumn('boobs_placement'); table.dropColumn('boobs_surgeon'); + table.dropColumn('natural_butt'); table.dropColumn('butt_volume'); table.dropColumn('butt_implant'); + table.dropColumn('natural_lips'); table.dropColumn('lips_volume'); - table.dropColumn('natural_butt'); }); await knex.schema.alterTable('actors_profiles', (table) => { @@ -99,10 +101,11 @@ exports.down = async (knex) => { table.dropColumn('boobs_placement'); table.dropColumn('boobs_surgeon'); + table.dropColumn('natural_butt'); table.dropColumn('butt_volume'); table.dropColumn('butt_implant'); + table.dropColumn('natural_lips'); table.dropColumn('lips_volume'); - table.dropColumn('natural_butt'); }); }; diff --git a/src/actors.js b/src/actors.js index 9171abd6..55e68112 100755 --- a/src/actors.js +++ b/src/actors.js @@ -27,6 +27,7 @@ const { toBaseReleases } = require('./deep'); const { associateAvatars, flushOrphanedMedia } = require('./media'); const { fetchEntitiesBySlug } = require('./entities'); const { deleteScenes } = require('./releases'); +const { interpolateProfiles: interpolateProfilesUtil } = require('../common/actors'); const slugify = require('./utils/slugify'); const capitalize = require('./utils/capitalize'); @@ -139,53 +140,6 @@ function getBoolean(value) { return null; } -function getMostFrequent(items) { - const { mostFrequent } = items.reduce((acc, item) => { - if (item === undefined || item === null) { - return acc; - } - - const slug = slugify(item); - - acc.counts[slug] = (acc.counts[slug] || 0) + 1; - - if (!acc.mostFrequent || acc.counts[slug] > acc.counts[slugify(acc.mostFrequent)]) { - acc.mostFrequent = item; - } - - return acc; - }, { - counts: {}, - mostFrequent: null, - }); - - return mostFrequent; -} - -function getMostFrequentDate(dates) { - const year = getMostFrequent(dates.map((dateX) => dateX.getFullYear())); - const month = getMostFrequent(dates.map((dateX) => dateX.getMonth())); - const date = getMostFrequent(dates.map((dateX) => dateX.getDate())); - - if (year === null || month === null || date === null) { - return null; - } - - return moment({ year, month, date }).toDate(); -} - -function getHighest(items) { - return items.reduce((prevItem, item) => (item > prevItem ? item : prevItem), null); -} - -function getLongest(items) { - return items.sort((itemA, itemB) => itemB.length - itemA.length)[0] || null; -} - -function getAverage(items) { - return Math.round(items.reduce((acc, item) => acc + item, 0) / items.length) || null; -} - function toBaseActors(actorsOrNames, release) { if (!actorsOrNames) { return []; @@ -387,6 +341,246 @@ function curateProfileEntry(profile) { return curatedProfileEntry; } +async function fetchProfiles(actorIdsOrNames) { + return knex('actors_profiles') + .select(knex.raw('actors_profiles.*, actors.name, row_to_json(media) as avatar')) + .leftJoin('actors', 'actors.id', 'actors_profiles.actor_id') + .modify((query) => { + if (actorIdsOrNames) { + query + .whereIn('actor_id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number')) + .orWhere((builder) => { + builder + .whereIn('actors.name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string')) + .whereNull('actors.entity_id'); + }); + } + }) + .leftJoin('media', 'actors_profiles.avatar_media_id', 'media.id'); +} + +/* +function getMostFrequent(items) { + const { mostFrequent } = items.reduce((acc, item) => { + if (item === undefined || item === null) { + return acc; + } + + const slug = slugify(item); + + acc.counts[slug] = (acc.counts[slug] || 0) + 1; + + if (!acc.mostFrequent || acc.counts[slug] > acc.counts[slugify(acc.mostFrequent)]) { + acc.mostFrequent = item; + } + + return acc; + }, { + counts: {}, + mostFrequent: null, + }); + + return mostFrequent; +} + +function getMostFrequentDate(dates) { + const year = getMostFrequent(dates.map((dateX) => dateX.getFullYear())); + const month = getMostFrequent(dates.map((dateX) => dateX.getMonth())); + const date = getMostFrequent(dates.map((dateX) => dateX.getDate())); + + if (year === null || month === null || date === null) { + return null; + } + + return moment({ year, month, date }).toDate(); +} + +function getHighest(items) { + return items.reduce((prevItem, item) => (item > prevItem ? item : prevItem), null); +} + +function getLongest(items) { + return items.sort((itemA, itemB) => itemB.length - itemA.length)[0] || null; +} + +function getAverage(items) { + return Math.round(items.reduce((acc, item) => acc + item, 0) / items.length) || null; +} + +function mergeMainProfile(profile, mainProfile) { + const preservedKeys = ['id']; + + const mergedProfile = Object.fromEntries(Object.entries(profile).map(([key, value]) => [key, mainProfile[key] === null || preservedKeys.includes(key) + ? value + : mainProfile[key]])); + + return mergedProfile; +} +*/ + +async function interpolateProfiles(actorIdsOrNames) { + try { + await interpolateProfilesUtil(actorIdsOrNames, { + knex, + logger, + moment, + slugify, + omit, + }); + } catch (error) { + console.log(error); + } + /* + const profiles = await fetchProfiles(actorIdsOrNames); + + const profilesByActorId = profiles.reduce((acc, profile) => ({ + ...acc, + [profile.actor_id]: [ + ...(acc[profile.actor_id] || []), + profile, + ], + }), {}); + + logger.info(`Interpolating ${profiles.length} profiles from ${Object.keys(profilesByActorId).length} actors`); + + const interpolatedProfiles = Object.entries(profilesByActorId).map(([actorId, actorProfiles]) => { + // group values from each profile + const valuesByProperty = actorProfiles + .filter((profile) => profile.entity_id !== null) // main profile is interpolated separately at the end + .reduce((acc, profile) => Object + .entries(profile) + .reduce((profileAcc, [property, value]) => ({ + ...profileAcc, + [property]: [ + ...(acc[property] || []), + ...(value === null ? [] : Array.from({ length: profile.priority }, () => value)), // multiply by priority, increasing the odds of being the most frequent value + ], + }), { + // bundle location values so they can be assessed together, to ensure the most frequent city is in the most frequent state is in most frequent country + origin: [...acc.origin || [], { + ...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }), + ...(profile.birth_state && { state: profile.birth_state }), + ...(profile.birth_city && { city: profile.birth_city }), + }].filter((location) => Object.keys(location).length > 0), + residence: [...acc.residence || [], { + ...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }), + ...(profile.residence_state && { state: profile.residence_state }), + ...(profile.residence_city && { city: profile.residence_city }), + }].filter((location) => Object.keys(location).length > 0), + }), {}); + + const mostFrequentValues = [ + 'gender', + 'orientation', + 'ethnicity', + 'cup', + 'bust', + 'waist', + 'hip', + 'leg', + 'thigh', + 'foot', + 'shoe_size', + 'penis_length', + 'penis_girth', + 'circumcised', + 'hair_color', + 'eyes', + 'has_tattoos', + 'has_piercings', + 'blood_type', + ].reduce((acc, property) => ({ + ...acc, + [property]: getMostFrequent(valuesByProperty[property]), + }), {}); + + const profile = { + id: actorId, + ...mostFrequentValues, + }; + + profile.height = getMostFrequent(valuesByProperty.height.filter((height) => height > 50 && height < 300)); // remove unlikely values + + profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth); + profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death); + profile.age = getHighest(valuesByProperty.age); + + profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs); + + // ensure most frequent country, city and state match up + profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map((location) => location.country)); + const remainingOriginCountries = valuesByProperty.origin.filter((location) => location.country === profile.birth_country_alpha2); + + profile.birth_state = getMostFrequent(remainingOriginCountries.map((location) => location.state)); + const remainingOriginStates = remainingOriginCountries.filter((location) => !profile.birth_state || location.state === profile.birth_state); + + profile.birth_city = getMostFrequent(remainingOriginStates.map((location) => location.city)); + + profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map((location) => location.country)); + const remainingResidenceCountries = valuesByProperty.residence.filter((location) => location.country === profile.residence_country_alpha2); + + profile.residence_state = getMostFrequent(remainingResidenceCountries.map((location) => location.state)); + const remainingResidenceStates = remainingResidenceCountries.filter((location) => !profile.residence_state || location.state === profile.residence_state); + + profile.residence_city = getMostFrequent(remainingResidenceStates.map((location) => location.city)); + + profile.weight = getAverage(valuesByProperty.weight); + + profile.tattoos = getLongest(valuesByProperty.tattoos); + profile.piercings = getLongest(valuesByProperty.piercings); + + profile.avatar_media_id = actorProfiles + .map((actorProfile) => actorProfile.avatar) + .filter((avatar) => avatar && (avatar.entropy === null || avatar.entropy > 5.5)) + .sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; + + if (!profile.avatar_media_id) { + // try to settle for low quality avatar + profile.avatar_media_id = actorProfiles + .map((actorProfile) => actorProfile.avatar) + .filter((avatar) => avatar) + .sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; + } + + const mainProfile = actorProfiles.find((actorProfile) => actorProfile.entity_id === null); + + return mergeMainProfile(profile, mainProfile); + }); + + const transaction = await knex.transaction(); + + // clear existing interpolated data + const emptyProfile = Object + .keys(omit(curateProfileEntry({ id: 1 }), ['id', 'actor_id', 'entity_id', 'url', 'description_hash'])) + .reduce((acc, key) => ({ ...acc, [key]: null }), {}); + + await knex('actors') + .modify((modifyBuilder) => { + if (actorIdsOrNames) { + modifyBuilder + .whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number')) + .orWhere((whereBuilder) => { + whereBuilder + .whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string')) + .whereNull('entity_id'); + }); + } + }) + .update(emptyProfile) + .transacting(transaction); + + // insert new interpolated data + const queries = interpolatedProfiles.map((profile) => knex('actors') + .where('id', profile.id) + .update(profile) + .transacting(transaction)); + + await Promise.all(queries) + .then(transaction.commit) + .catch(transaction.rollback); + */ +} + async function curateProfile(profile, actor) { if (!profile) { return null; @@ -537,170 +731,6 @@ async function curateProfile(profile, actor) { } } -async function fetchProfiles(actorIdsOrNames) { - return knex('actors_profiles') - .select(knex.raw('actors_profiles.*, row_to_json(actors) as actor, row_to_json(media) as avatar')) - .leftJoin('actors', 'actors.id', 'actors_profiles.actor_id') - .modify((query) => { - if (actorIdsOrNames) { - query - .whereIn('actor_id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number')) - .orWhere((builder) => { - builder - .whereIn('actors.name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string')) - .whereNull('actors.entity_id'); - }); - } - }) - .leftJoin('media', 'actors_profiles.avatar_media_id', 'media.id'); -} - -async function interpolateProfiles(actorIdsOrNames) { - const profiles = await fetchProfiles(actorIdsOrNames); - - const profilesByActorId = profiles.reduce((acc, profile) => ({ - ...acc, - [profile.actor_id]: [ - ...(acc[profile.actor_id] || []), - profile, - ], - }), {}); - - logger.info(`Interpolating ${profiles.length} profiles from ${Object.keys(profilesByActorId).length} actors`); - - const interpolatedProfiles = Object.entries(profilesByActorId).map(([actorId, actorProfiles]) => { - // group values from each profile - const valuesByProperty = actorProfiles.reduce((acc, profile) => Object - .entries(profile) - .reduce((profileAcc, [property, value]) => ({ - ...profileAcc, - [property]: [ - ...(acc[property] || []), - ...(value === null ? [] : Array.from({ length: profile.priority }, () => value)), // multiply by priority, increasing the odds of being the most frequent value - ], - }), { - // bundle location values so they can be assessed together, to ensure the most frequent city is in the most frequent state is in most frequent country - origin: [...acc.origin || [], { - ...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }), - ...(profile.birth_state && { state: profile.birth_state }), - ...(profile.birth_city && { city: profile.birth_city }), - }].filter((location) => Object.keys(location).length > 0), - residence: [...acc.residence || [], { - ...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }), - ...(profile.residence_state && { state: profile.residence_state }), - ...(profile.residence_city && { city: profile.residence_city }), - }].filter((location) => Object.keys(location).length > 0), - }), {}); - - const mostFrequentValues = [ - 'gender', - 'orientation', - 'ethnicity', - 'cup', - 'bust', - 'waist', - 'hip', - 'leg', - 'thigh', - 'foot', - 'shoe_size', - 'penis_length', - 'penis_girth', - 'circumcised', - 'hair_color', - 'eyes', - 'has_tattoos', - 'has_piercings', - 'blood_type', - ].reduce((acc, property) => ({ - ...acc, - [property]: getMostFrequent(valuesByProperty[property]), - }), {}); - - const profile = { - id: actorId, - ...mostFrequentValues, - }; - - profile.height = getMostFrequent(valuesByProperty.height.filter((height) => height > 50 && height < 300)); // remove unlikely values - - profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth); - profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death); - profile.age = getHighest(valuesByProperty.age); - - profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs); - - // ensure most frequent country, city and state match up - profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map((location) => location.country)); - const remainingOriginCountries = valuesByProperty.origin.filter((location) => location.country === profile.birth_country_alpha2); - - profile.birth_state = getMostFrequent(remainingOriginCountries.map((location) => location.state)); - const remainingOriginStates = remainingOriginCountries.filter((location) => !profile.birth_state || location.state === profile.birth_state); - - profile.birth_city = getMostFrequent(remainingOriginStates.map((location) => location.city)); - - profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map((location) => location.country)); - const remainingResidenceCountries = valuesByProperty.residence.filter((location) => location.country === profile.residence_country_alpha2); - - profile.residence_state = getMostFrequent(remainingResidenceCountries.map((location) => location.state)); - const remainingResidenceStates = remainingResidenceCountries.filter((location) => !profile.residence_state || location.state === profile.residence_state); - - profile.residence_city = getMostFrequent(remainingResidenceStates.map((location) => location.city)); - - profile.weight = getAverage(valuesByProperty.weight); - - profile.tattoos = getLongest(valuesByProperty.tattoos); - profile.piercings = getLongest(valuesByProperty.piercings); - - profile.avatar_media_id = actorProfiles - .map((actorProfile) => actorProfile.avatar) - .filter((avatar) => avatar && (avatar.entropy === null || avatar.entropy > 5.5)) - .sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; - - if (!profile.avatar_media_id) { - // try to settle for low quality avatar - profile.avatar_media_id = actorProfiles - .map((actorProfile) => actorProfile.avatar) - .filter((avatar) => avatar) - .sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; - } - - return profile; - }); - - const transaction = await knex.transaction(); - - // clear existing interpolated data - const emptyProfile = Object - .keys(omit(curateProfileEntry({ id: 1 }), ['id', 'actor_id', 'entity_id', 'url', 'description_hash'])) - .reduce((acc, key) => ({ ...acc, [key]: null }), {}); - - await knex('actors') - .modify((modifyBuilder) => { - if (actorIdsOrNames) { - modifyBuilder - .whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number')) - .orWhere((whereBuilder) => { - whereBuilder - .whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string')) - .whereNull('entity_id'); - }); - } - }) - .update(emptyProfile) - .transacting(transaction); - - // insert new interpolated data - const queries = interpolatedProfiles.map((profile) => knex('actors') - .where('id', profile.id) - .update(profile) - .transacting(transaction)); - - await Promise.all(queries) - .then(transaction.commit) - .catch(transaction.rollback); -} - async function upsertProfiles(profiles) { const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean); const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean); @@ -1150,7 +1180,7 @@ async function searchActors(query) { async function flushProfiles(actorIdsOrNames) { const profiles = await fetchProfiles(actorIdsOrNames); - const actorNames = Array.from(new Set(profiles.map((profile) => profile.actor.name))); + const actorNames = Array.from(new Set(profiles.map((profile) => profile.name))); const deleteCount = await knex('actors_profiles') .whereIn('id', profiles.map((profile) => profile.id))