Storing directors.

This commit is contained in:
DebaucheryLibrarian
2021-03-07 00:01:02 +01:00
parent 9ae113ab92
commit 17e6f5a5da
17 changed files with 102 additions and 27 deletions

View File

@@ -760,8 +760,6 @@ async function scrapeActors(argNames) {
const actorNames = await getActorNames(argNames);
const baseActors = toBaseActors(actorNames);
console.log(baseActors);
logger.info(`Scraping profiles for ${actorNames.length} actors`);
const sources = argv.profileSources || config.profiles || Object.keys(scrapers.actors);
@@ -898,13 +896,17 @@ async function getOrCreateActors(baseActors, batchId) {
return existingActors;
}
async function associateActors(releases, batchId) {
async function associatePeople(releases, batchId, type = 'actor') {
try {
const baseActorsByReleaseId = releases.reduce((acc, release) => {
if (release.actors) {
if (type === 'actors' && release.actors) {
acc[release.id] = toBaseActors(release.actors, release);
}
if (type === 'directors' && release.director) {
acc[release.id] = toBaseActors([release.director], release);
}
return acc;
}, {});
@@ -922,6 +924,11 @@ async function associateActors(releases, batchId) {
const uniqueBaseActors = Object.values(baseActorsBySlug);
const actors = await getOrCreateActors(uniqueBaseActors, batchId);
const personKey = ({
actors: 'actor_id',
directors: 'director_id',
})[type];
const actorIdsByEntityIdEntryIdAndSlug = actors.reduce((acc, actor) => ({
...acc,
[actor.entity_id]: {
@@ -929,7 +936,7 @@ async function associateActors(releases, batchId) {
[actor.entry_id]: {
...acc[actor.entity_id]?.[actor.entry_id],
[actor.slug]: {
actor_id: actor.alias_for || actor.id,
[personKey]: actor.alias_for || actor.id,
alias_id: actor.alias_for ? actor.id : null,
},
},
@@ -944,15 +951,15 @@ async function associateActors(releases, batchId) {
})))
.flat();
const validReleaseActorAssociations = releaseActorAssociations.filter(association => association.release_id && association.actor_id);
const validReleaseActorAssociations = releaseActorAssociations.filter(association => association.release_id && association[personKey]);
if (releaseActorAssociations.length > validReleaseActorAssociations.length) {
const invalidReleaseActorAssociations = releaseActorAssociations.filter(association => !association.release_id || !association.actor_id);
const invalidReleaseActorAssociations = releaseActorAssociations.filter(association => !association.release_id || !association[personKey]);
logger.error(invalidReleaseActorAssociations);
}
await bulkInsert('releases_actors', validReleaseActorAssociations, false);
await bulkInsert(`releases_${type}`, validReleaseActorAssociations, false);
logger.verbose(`Associated ${releaseActorAssociations.length} actors to ${releases.length} scenes`);
@@ -964,6 +971,14 @@ async function associateActors(releases, batchId) {
}
}
async function associateActors(releases, batchId) {
return associatePeople(releases, batchId, 'actors');
}
async function associateDirectors(releases, batchId) {
return associatePeople(releases, batchId, 'directors');
}
async function fetchActor(actorId) {
const actor = await knex('actors')
.select(knex.raw(`
@@ -1081,6 +1096,7 @@ async function flushActors() {
module.exports = {
associateActors,
associateDirectors,
deleteActors,
fetchActor,
flushActors,

View File

@@ -9,7 +9,7 @@ const slugify = require('./utils/slugify');
const bulkInsert = require('./utils/bulk-insert');
const resolvePlace = require('./utils/resolve-place');
const { formatDate } = require('./utils/qu');
const { associateActors, scrapeActors, toBaseActors } = require('./actors');
const { associateActors, associateDirectors, scrapeActors, toBaseActors } = require('./actors');
const { associateReleaseTags } = require('./tags');
const { curateEntity } = require('./entities');
const { associateReleaseMedia } = require('./media');
@@ -229,6 +229,7 @@ async function updateReleasesSearch(releaseIds) {
COALESCE(releases.shoot_id, '') || ' ' ||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
) as document
@@ -236,8 +237,10 @@ async function updateReleasesSearch(releaseIds) {
LEFT JOIN entities ON releases.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
LEFT JOIN actors ON local_actors.actor_id = actors.id
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
@@ -319,6 +322,7 @@ async function storeScenes(releases) {
const [actors] = await Promise.all([
associateActors(releasesWithId, batchId),
associateDirectors(releasesWithId, batchId),
associateReleaseTags(releasesWithId),
storeChapters(releasesWithId),
]);