2020-03-16 03:10:52 +00:00
|
|
|
'use strict';
|
|
|
|
|
|
|
|
const config = require('config');
|
2022-01-10 01:17:17 +00:00
|
|
|
const Promise = require('bluebird');
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-05-17 02:59:09 +00:00
|
|
|
const argv = require('./argv');
|
2020-03-21 01:48:24 +00:00
|
|
|
const logger = require('./logger')(__filename);
|
2020-03-16 03:10:52 +00:00
|
|
|
const knex = require('./knex');
|
|
|
|
const slugify = require('./utils/slugify');
|
2020-08-14 21:05:25 +00:00
|
|
|
const bulkInsert = require('./utils/bulk-insert');
|
2020-08-20 02:57:38 +00:00
|
|
|
const resolvePlace = require('./utils/resolve-place');
|
2022-01-10 01:17:17 +00:00
|
|
|
const chunk = require('./utils/chunk');
|
2020-07-08 00:53:46 +00:00
|
|
|
const { formatDate } = require('./utils/qu');
|
2021-03-06 23:01:02 +00:00
|
|
|
const { associateActors, associateDirectors, scrapeActors, toBaseActors } = require('./actors');
|
2020-03-29 02:00:46 +00:00
|
|
|
const { associateReleaseTags } = require('./tags');
|
2020-06-25 00:26:25 +00:00
|
|
|
const { curateEntity } = require('./entities');
|
2020-03-29 02:00:46 +00:00
|
|
|
const { associateReleaseMedia } = require('./media');
|
2021-04-25 01:08:50 +00:00
|
|
|
const { notify } = require('./alerts');
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-08-20 02:57:38 +00:00
|
|
|
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
|
2020-07-03 02:12:56 +00:00
|
|
|
const slugBase = release.title
|
2021-11-20 22:59:15 +00:00
|
|
|
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|
2020-07-08 00:53:46 +00:00
|
|
|
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
|
2020-07-03 02:12:56 +00:00
|
|
|
|| null;
|
|
|
|
|
|
|
|
const slug = slugify(slugBase, '-', {
|
2020-05-14 02:26:05 +00:00
|
|
|
encode: true,
|
|
|
|
limit: config.titleSlugLength,
|
|
|
|
});
|
|
|
|
|
|
|
|
const curatedRelease = {
|
|
|
|
title: release.title,
|
|
|
|
entry_id: release.entryId || null,
|
2020-06-25 00:26:25 +00:00
|
|
|
entity_id: release.entity.id,
|
2020-05-14 02:26:05 +00:00
|
|
|
studio_id: release.studio?.id || null,
|
|
|
|
url: release.url,
|
2020-05-26 02:11:29 +00:00
|
|
|
date: Number(release.date) ? release.date : null,
|
2020-07-03 02:12:56 +00:00
|
|
|
date_precision: release.datePrecision,
|
2020-05-14 02:26:05 +00:00
|
|
|
slug,
|
|
|
|
description: release.description,
|
2022-03-31 12:34:10 +00:00
|
|
|
qualities: release.qualities?.map(Number).filter(Boolean),
|
2020-07-08 00:53:46 +00:00
|
|
|
comment: release.comment,
|
2020-05-14 02:26:05 +00:00
|
|
|
deep: typeof release.deep === 'boolean' ? release.deep : false,
|
|
|
|
deep_url: release.deepUrl,
|
|
|
|
updated_batch_id: batchId,
|
|
|
|
};
|
|
|
|
|
2020-08-01 13:11:07 +00:00
|
|
|
if (type === 'scene') {
|
|
|
|
curatedRelease.shoot_id = release.shootId || null;
|
2020-08-02 01:44:14 +00:00
|
|
|
curatedRelease.production_date = Number(release.productionDate) ? release.productionDate : null;
|
2020-08-01 13:11:07 +00:00
|
|
|
curatedRelease.duration = release.duration;
|
|
|
|
}
|
|
|
|
|
2020-08-20 02:57:38 +00:00
|
|
|
if (release.productionLocation) {
|
|
|
|
curatedRelease.production_location = release.productionLocation;
|
|
|
|
|
|
|
|
if (argv.resolvePlace) {
|
|
|
|
const productionLocation = await resolvePlace(release.productionLocation);
|
|
|
|
|
|
|
|
if (productionLocation) {
|
|
|
|
curatedRelease.production_city = productionLocation.city;
|
|
|
|
curatedRelease.production_state = productionLocation.state;
|
|
|
|
curatedRelease.production_country_alpha2 = productionLocation.country;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
if (!existingRelease && !release.id) {
|
|
|
|
curatedRelease.created_batch_id = batchId;
|
|
|
|
}
|
|
|
|
|
|
|
|
return curatedRelease;
|
2020-03-16 03:10:52 +00:00
|
|
|
}
|
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
async function attachChannelEntities(releases) {
|
2021-11-20 22:59:15 +00:00
|
|
|
const releasesWithoutEntity = releases.filter((release) => release.channel && (!release.entity || release.entity.type === 'network'));
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
const channelEntities = await knex('entities')
|
|
|
|
.select(knex.raw('entities.*, row_to_json(parents) as parent'))
|
2021-11-20 22:59:15 +00:00
|
|
|
.whereIn('entities.slug', releasesWithoutEntity.map((release) => release.channel))
|
2020-06-30 22:25:27 +00:00
|
|
|
.where('entities.type', 'channel')
|
2020-06-25 00:26:25 +00:00
|
|
|
.leftJoin('entities AS parents', 'parents.id', 'entities.parent_id');
|
2020-03-30 01:01:08 +00:00
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
const channelEntitiesBySlug = channelEntities.reduce((acc, entity) => ({ ...acc, [entity.slug]: entity }), {});
|
2020-03-16 23:58:03 +00:00
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
const releasesWithChannelEntity = await Promise.all(releases
|
2020-05-14 02:26:05 +00:00
|
|
|
.map(async (release) => {
|
2020-06-25 00:26:25 +00:00
|
|
|
if (release.channel && channelEntitiesBySlug[release.channel]) {
|
|
|
|
const curatedEntity = await curateEntity(channelEntitiesBySlug[release.channel]);
|
2020-03-30 01:01:08 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return {
|
|
|
|
...release,
|
2020-06-25 00:26:25 +00:00
|
|
|
entity: curatedEntity,
|
2020-05-14 02:26:05 +00:00
|
|
|
};
|
|
|
|
}
|
2020-03-16 23:58:03 +00:00
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
if (release.entity) {
|
2020-05-26 02:11:29 +00:00
|
|
|
return release;
|
|
|
|
}
|
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
logger.error(`Unable to match channel '${release.channel?.slug || release.channel}' from generic URL ${release.url}`);
|
2020-03-16 23:58:03 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return null;
|
|
|
|
}));
|
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
return releasesWithChannelEntity.filter(Boolean);
|
2020-03-16 23:58:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
async function attachStudios(releases) {
|
2021-11-20 22:59:15 +00:00
|
|
|
const studioSlugs = releases.map((release) => release.studio).filter(Boolean);
|
2020-03-16 23:58:03 +00:00
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
const studios = await knex('entities')
|
|
|
|
.whereIn('slug', studioSlugs)
|
2020-06-30 22:25:27 +00:00
|
|
|
.where('type', 'studio');
|
2020-06-25 00:26:25 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
const studioBySlug = studios.reduce((acc, studio) => ({ ...acc, [studio.slug]: studio }), {});
|
2020-03-16 23:58:03 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
const releasesWithStudio = releases.map((release) => {
|
|
|
|
if (release.studio && studioBySlug[release.studio]) {
|
|
|
|
return {
|
|
|
|
...release,
|
|
|
|
studio: studioBySlug[release.studio],
|
|
|
|
};
|
|
|
|
}
|
2020-03-16 23:58:03 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
if (release.studio) {
|
|
|
|
logger.warn(`Unable to match studio '${release.studio}' for ${release.url}`);
|
|
|
|
}
|
2020-03-16 23:58:03 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return release;
|
|
|
|
});
|
2020-03-16 23:58:03 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return releasesWithStudio;
|
2020-03-16 03:10:52 +00:00
|
|
|
}
|
|
|
|
|
2022-01-19 23:54:10 +00:00
|
|
|
function attachReleaseIds(releases, storedReleases, batchId) {
|
2020-06-25 00:26:25 +00:00
|
|
|
const storedReleaseIdsByEntityIdAndEntryId = storedReleases.reduce((acc, release) => {
|
2020-06-17 02:07:24 +00:00
|
|
|
if (!acc[release.entity_id]) acc[release.entity_id] = {};
|
|
|
|
acc[release.entity_id][release.entry_id] = release.id;
|
2020-03-22 02:50:24 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return acc;
|
|
|
|
}, {});
|
2020-03-22 02:50:24 +00:00
|
|
|
|
2020-08-14 21:05:25 +00:00
|
|
|
const releasesWithId = releases.map((release) => {
|
|
|
|
if (!release.entity) {
|
2022-03-26 16:40:20 +00:00
|
|
|
logger.error(`No entity available for ${release.url}`);
|
2020-08-14 21:05:25 +00:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
2021-06-13 14:49:27 +00:00
|
|
|
const id = storedReleaseIdsByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|
|
|
|
|| storedReleaseIdsByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId];
|
2020-08-14 21:05:25 +00:00
|
|
|
|
|
|
|
if (id) {
|
|
|
|
return {
|
|
|
|
...release,
|
|
|
|
id,
|
2022-01-19 23:54:10 +00:00
|
|
|
batchId,
|
2020-08-14 21:05:25 +00:00
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
return null;
|
|
|
|
}).filter(Boolean);
|
2020-03-22 02:50:24 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return releasesWithId;
|
2020-03-22 02:50:24 +00:00
|
|
|
}
|
|
|
|
|
2020-03-28 03:37:04 +00:00
|
|
|
function filterInternalDuplicateReleases(releases) {
|
2020-06-25 00:26:25 +00:00
|
|
|
const releasesByEntityIdAndEntryId = releases.reduce((acc, release) => {
|
|
|
|
if (!release.entity) {
|
2020-05-26 02:11:29 +00:00
|
|
|
return acc;
|
|
|
|
}
|
|
|
|
|
2020-12-30 02:19:09 +00:00
|
|
|
if (!release.entryId) {
|
|
|
|
logger.warn(`No entry ID supplied for "${release.title}" from '${release.entity.name}'`);
|
|
|
|
|
|
|
|
return acc;
|
|
|
|
}
|
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
if (!acc[release.entity.id]) {
|
|
|
|
acc[release.entity.id] = {};
|
2020-05-14 02:26:05 +00:00
|
|
|
}
|
2020-03-28 03:37:04 +00:00
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
acc[release.entity.id][release.entryId] = release;
|
2020-03-28 03:37:04 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return acc;
|
|
|
|
}, {});
|
2020-03-28 03:37:04 +00:00
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
return Object.values(releasesByEntityIdAndEntryId)
|
2021-11-20 22:59:15 +00:00
|
|
|
.map((entityReleases) => Object.values(entityReleases))
|
2020-05-14 02:26:05 +00:00
|
|
|
.flat();
|
2020-03-28 03:37:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
async function filterDuplicateReleases(releases) {
|
2020-05-14 02:26:05 +00:00
|
|
|
const internalUniqueReleases = filterInternalDuplicateReleases(releases);
|
2022-01-10 01:17:17 +00:00
|
|
|
const internalUniqueReleaseChunks = chunk(internalUniqueReleases);
|
2020-03-28 03:37:04 +00:00
|
|
|
|
2022-01-10 01:17:17 +00:00
|
|
|
const duplicateReleaseEntryChunks = await Promise.map(internalUniqueReleaseChunks, async (internalUniqueReleasesChunk) => knex('releases')
|
|
|
|
.whereIn(['entry_id', 'entity_id'], internalUniqueReleasesChunk.map((release) => [release.entryId, release.entity.id]))
|
|
|
|
.orWhereIn(['entry_id', 'entity_id'], internalUniqueReleasesChunk
|
2021-06-13 14:49:27 +00:00
|
|
|
// scene IDs shared across network, mark as duplicate so scene can be updated with channel if only available on release day (i.e. Perv City)
|
2021-11-20 22:59:15 +00:00
|
|
|
.filter((release) => release.entity.parent?.parameters?.networkEntryIds)
|
2022-01-10 01:17:17 +00:00
|
|
|
.map((release) => [release.entryId, release.entity.parent.id])), { concurrency: 10 });
|
|
|
|
|
|
|
|
const duplicateReleaseEntries = duplicateReleaseEntryChunks.flat();
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-06-25 00:26:25 +00:00
|
|
|
const duplicateReleasesByEntityIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
|
2020-06-17 02:07:24 +00:00
|
|
|
if (!acc[release.entity_id]) acc[release.entity_id] = {};
|
|
|
|
acc[release.entity_id][release.entry_id] = true;
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return acc;
|
|
|
|
}, {});
|
2020-03-22 02:50:24 +00:00
|
|
|
|
2021-11-20 22:59:15 +00:00
|
|
|
const duplicateReleases = internalUniqueReleases.filter((release) => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|
2021-06-13 14:49:27 +00:00
|
|
|
|| duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
|
|
|
|
|
2021-11-20 22:59:15 +00:00
|
|
|
const uniqueReleases = internalUniqueReleases.filter((release) => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|
2021-06-13 14:49:27 +00:00
|
|
|
&& !duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
|
2020-03-22 02:50:24 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
return {
|
|
|
|
uniqueReleases,
|
|
|
|
duplicateReleases,
|
|
|
|
duplicateReleaseEntries,
|
|
|
|
};
|
2020-03-16 03:10:52 +00:00
|
|
|
}
|
|
|
|
|
2021-08-22 20:25:20 +00:00
|
|
|
async function updateSceneSearch(releaseIds) {
|
2020-05-14 02:26:05 +00:00
|
|
|
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
|
2020-03-29 02:00:46 +00:00
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
const documents = await knex.raw(`
|
2020-03-29 02:00:46 +00:00
|
|
|
SELECT
|
|
|
|
releases.id AS release_id,
|
|
|
|
TO_TSVECTOR(
|
2021-02-25 01:52:45 +00:00
|
|
|
'english',
|
2020-03-29 21:42:41 +00:00
|
|
|
COALESCE(releases.title, '') || ' ' ||
|
2022-01-12 16:50:23 +00:00
|
|
|
releases.entry_id || ' ' ||
|
2020-06-17 02:07:24 +00:00
|
|
|
entities.name || ' ' ||
|
|
|
|
entities.slug || ' ' ||
|
2020-07-12 20:36:53 +00:00
|
|
|
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
2020-06-29 23:52:17 +00:00
|
|
|
COALESCE(parents.name, '') || ' ' ||
|
|
|
|
COALESCE(parents.slug, '') || ' ' ||
|
2020-07-12 20:36:53 +00:00
|
|
|
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
2020-03-29 02:00:46 +00:00
|
|
|
COALESCE(releases.shoot_id, '') || ' ' ||
|
2021-03-23 14:25:21 +00:00
|
|
|
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
|
2020-03-29 02:00:46 +00:00
|
|
|
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
2021-03-06 23:01:02 +00:00
|
|
|
STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
|
2020-03-29 02:00:46 +00:00
|
|
|
STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
|
|
|
|
STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
|
|
|
|
) as document
|
|
|
|
FROM releases
|
2020-06-17 02:07:24 +00:00
|
|
|
LEFT JOIN entities ON releases.entity_id = entities.id
|
|
|
|
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
2020-03-29 02:00:46 +00:00
|
|
|
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
2021-03-06 23:01:02 +00:00
|
|
|
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
2020-03-29 02:00:46 +00:00
|
|
|
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
|
|
|
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
2021-03-06 23:01:02 +00:00
|
|
|
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
2021-02-25 01:52:45 +00:00
|
|
|
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
|
2020-03-29 02:00:46 +00:00
|
|
|
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
|
|
|
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
|
2020-06-29 23:52:17 +00:00
|
|
|
GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
|
2020-03-29 02:00:46 +00:00
|
|
|
`, releaseIds && [releaseIds]);
|
|
|
|
|
2020-05-14 02:26:05 +00:00
|
|
|
if (documents.rows?.length > 0) {
|
2020-08-14 21:21:53 +00:00
|
|
|
await bulkInsert('releases_search', documents.rows, ['release_id']);
|
2020-05-14 02:26:05 +00:00
|
|
|
}
|
2020-03-29 02:00:46 +00:00
|
|
|
}
|
|
|
|
|
2021-02-26 23:37:22 +00:00
|
|
|
async function storeChapters(releases) {
|
2021-02-27 17:05:06 +00:00
|
|
|
const chapters = releases
|
2021-11-20 22:59:15 +00:00
|
|
|
.map((release) => release.chapters?.map((chapter, index) => ({
|
2021-02-27 17:05:06 +00:00
|
|
|
releaseId: release.id,
|
|
|
|
index: index + 1,
|
|
|
|
time: chapter.time,
|
|
|
|
duration: chapter.duration,
|
|
|
|
title: chapter.title,
|
|
|
|
description: chapter.description,
|
|
|
|
poster: chapter.poster,
|
|
|
|
photos: chapter.photos,
|
|
|
|
tags: chapter.tags,
|
|
|
|
})))
|
|
|
|
.flat()
|
|
|
|
.filter(Boolean)
|
|
|
|
.sort((chapterA, chapterB) => chapterA.time - chapterB.time);
|
2020-08-20 02:57:38 +00:00
|
|
|
|
2021-11-20 22:59:15 +00:00
|
|
|
const curatedChapterEntries = chapters.map((chapter) => ({
|
2021-02-26 23:37:22 +00:00
|
|
|
index: chapter.index,
|
|
|
|
time: chapter.time,
|
|
|
|
duration: chapter.duration,
|
|
|
|
title: chapter.title,
|
|
|
|
description: chapter.description,
|
|
|
|
release_id: chapter.releaseId,
|
2020-08-20 02:57:38 +00:00
|
|
|
}));
|
|
|
|
|
2021-02-26 23:37:22 +00:00
|
|
|
const storedChapters = await bulkInsert('chapters', curatedChapterEntries, ['release_id', 'index']);
|
|
|
|
|
|
|
|
const chapterIdsByReleaseIdAndChapter = storedChapters.reduce((acc, chapter) => ({
|
2020-08-20 02:57:38 +00:00
|
|
|
...acc,
|
2021-02-26 23:37:22 +00:00
|
|
|
[chapter.release_id]: {
|
|
|
|
...acc[chapter.release_id],
|
|
|
|
[chapter.index]: chapter.id,
|
2020-08-20 02:57:38 +00:00
|
|
|
},
|
|
|
|
}), {});
|
|
|
|
|
2021-11-20 22:59:15 +00:00
|
|
|
const chaptersWithId = chapters.map((chapter) => ({
|
2021-02-26 23:37:22 +00:00
|
|
|
...chapter,
|
|
|
|
id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.index],
|
2020-08-20 02:57:38 +00:00
|
|
|
}));
|
|
|
|
|
2021-02-26 23:37:22 +00:00
|
|
|
await associateReleaseTags(chaptersWithId, 'chapter');
|
2020-08-20 02:57:38 +00:00
|
|
|
|
|
|
|
// media is more error-prone, associate separately
|
2021-02-26 23:37:22 +00:00
|
|
|
await associateReleaseMedia(chaptersWithId, 'chapter');
|
2020-08-20 02:57:38 +00:00
|
|
|
}
|
|
|
|
|
2020-08-10 19:39:55 +00:00
|
|
|
async function associateMovieScenes(movies, movieScenes) {
|
2021-01-29 01:38:05 +00:00
|
|
|
const moviesByEntityIdAndEntryId = movies.reduce((acc, movie) => ({
|
2020-08-10 19:39:55 +00:00
|
|
|
...acc,
|
2021-01-29 01:38:05 +00:00
|
|
|
[movie.entity.id]: {
|
|
|
|
...acc[movie.entity.id],
|
|
|
|
[movie.entryId]: movie,
|
2020-08-10 19:39:55 +00:00
|
|
|
},
|
|
|
|
}), {});
|
|
|
|
|
2021-01-29 01:38:05 +00:00
|
|
|
const associations = movieScenes.map((scene) => {
|
|
|
|
if (!scene.movie) {
|
2020-08-10 19:39:55 +00:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
2022-03-04 22:31:59 +00:00
|
|
|
const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]
|
|
|
|
|| moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId];
|
2020-08-10 19:39:55 +00:00
|
|
|
|
2021-01-29 01:38:05 +00:00
|
|
|
if (sceneMovie?.id) {
|
|
|
|
return {
|
|
|
|
movie_id: sceneMovie.id,
|
|
|
|
scene_id: scene.id,
|
|
|
|
};
|
|
|
|
}
|
2020-08-10 19:39:55 +00:00
|
|
|
|
2021-01-29 01:38:05 +00:00
|
|
|
return null;
|
|
|
|
}).filter(Boolean);
|
2020-08-10 19:39:55 +00:00
|
|
|
|
2020-08-14 21:21:53 +00:00
|
|
|
await bulkInsert('movies_scenes', associations, false);
|
2020-08-10 19:39:55 +00:00
|
|
|
}
|
|
|
|
|
2022-03-26 16:40:20 +00:00
|
|
|
async function associateSerieScenes(series, serieScenes) {
|
|
|
|
const seriesByEntityIdAndEntryId = series.reduce((acc, serie) => ({
|
|
|
|
...acc,
|
|
|
|
[serie.entity.id]: {
|
|
|
|
...acc[serie.entity.id],
|
|
|
|
[serie.entryId]: serie,
|
|
|
|
},
|
|
|
|
}), {});
|
|
|
|
|
|
|
|
const associations = serieScenes.map((scene) => {
|
|
|
|
if (!scene.serie) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
const sceneSerie = seriesByEntityIdAndEntryId[scene.entity.id]?.[scene.serie.entryId]
|
|
|
|
|| seriesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.serie.entryId];
|
|
|
|
|
|
|
|
if (sceneSerie?.id) {
|
|
|
|
return {
|
|
|
|
serie_id: sceneSerie.id,
|
|
|
|
scene_id: scene.id,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
return null;
|
|
|
|
}).filter(Boolean);
|
|
|
|
|
|
|
|
await bulkInsert('series_scenes', associations, false);
|
|
|
|
}
|
|
|
|
|
|
|
|
async function updateMovieSearch(movieIds, target = 'movie') {
|
|
|
|
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
|
2021-08-22 20:25:20 +00:00
|
|
|
|
|
|
|
const documents = await knex.raw(`
|
|
|
|
SELECT
|
2022-03-26 16:40:20 +00:00
|
|
|
${target}s.id AS ${target}_id,
|
2021-08-22 20:25:20 +00:00
|
|
|
TO_TSVECTOR(
|
|
|
|
'english',
|
2022-03-26 16:40:20 +00:00
|
|
|
COALESCE(${target}s.title, '') || ' ' ||
|
2021-08-22 20:25:20 +00:00
|
|
|
entities.name || ' ' ||
|
|
|
|
entities.slug || ' ' ||
|
|
|
|
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
|
|
|
COALESCE(parents.name, '') || ' ' ||
|
|
|
|
COALESCE(parents.slug, '') || ' ' ||
|
|
|
|
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
2022-03-26 16:40:20 +00:00
|
|
|
COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
|
2021-08-22 20:25:20 +00:00
|
|
|
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
|
|
|
|
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
|
|
|
STRING_AGG(COALESCE(tags.name, ''), ' ')
|
|
|
|
) as document
|
2022-03-26 16:40:20 +00:00
|
|
|
FROM ${target}s
|
|
|
|
LEFT JOIN entities ON ${target}s.entity_id = entities.id
|
2021-08-22 20:25:20 +00:00
|
|
|
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
2022-03-26 16:40:20 +00:00
|
|
|
LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
|
|
|
|
LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
|
|
|
|
LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
|
2021-08-22 20:25:20 +00:00
|
|
|
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
|
|
|
|
LEFT JOIN actors ON actors.id = releases_actors.actor_id
|
|
|
|
LEFT JOIN tags ON tags.id = releases_tags.tag_id
|
2022-03-26 16:40:20 +00:00
|
|
|
${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
|
|
|
|
GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
|
2021-08-22 20:25:20 +00:00
|
|
|
`, movieIds && [movieIds]);
|
|
|
|
|
|
|
|
if (documents.rows?.length > 0) {
|
2022-03-26 16:40:20 +00:00
|
|
|
await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
|
2021-08-22 20:25:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-01-19 23:54:10 +00:00
|
|
|
async function storeMovies(movies, useBatchId) {
|
2021-01-29 01:38:05 +00:00
|
|
|
if (!movies || movies.length === 0) {
|
2020-08-23 01:41:29 +00:00
|
|
|
return [];
|
|
|
|
}
|
|
|
|
|
2020-08-08 16:10:59 +00:00
|
|
|
const { uniqueReleases } = await filterDuplicateReleases(movies);
|
2022-01-19 23:54:10 +00:00
|
|
|
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
|
2020-08-01 13:11:07 +00:00
|
|
|
|
2021-11-20 22:59:15 +00:00
|
|
|
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
|
2020-08-01 13:11:07 +00:00
|
|
|
|
2020-08-14 21:05:25 +00:00
|
|
|
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
|
2020-08-01 13:11:07 +00:00
|
|
|
const moviesWithId = attachReleaseIds(movies, storedMovies);
|
|
|
|
|
2021-11-20 22:59:15 +00:00
|
|
|
await updateMovieSearch(moviesWithId.map((movie) => movie.id));
|
2020-08-20 02:57:38 +00:00
|
|
|
await associateReleaseMedia(moviesWithId, 'movie');
|
2020-08-01 13:11:07 +00:00
|
|
|
|
2021-01-29 01:38:05 +00:00
|
|
|
return moviesWithId;
|
2020-08-01 13:11:07 +00:00
|
|
|
}
|
|
|
|
|
2022-03-26 16:40:20 +00:00
|
|
|
async function storeSeries(series, useBatchId) {
|
|
|
|
if (!series || series.length === 0) {
|
|
|
|
return [];
|
|
|
|
}
|
|
|
|
|
|
|
|
const { uniqueReleases } = await filterDuplicateReleases(series);
|
|
|
|
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
|
|
|
|
|
|
|
|
const curatedSerieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'serie')));
|
|
|
|
|
|
|
|
const storedSeries = await bulkInsert('series', curatedSerieEntries, ['entity_id', 'entry_id'], true);
|
|
|
|
const seriesWithId = attachReleaseIds(series, storedSeries);
|
|
|
|
|
|
|
|
await updateMovieSearch(seriesWithId.map((serie) => serie.id), 'serie');
|
|
|
|
await associateReleaseMedia(seriesWithId, 'serie');
|
|
|
|
|
|
|
|
return seriesWithId;
|
|
|
|
}
|
|
|
|
|
|
|
|
async function storeScenes(releases, useBatchId) {
|
|
|
|
if (!releases || releases.length === 0) {
|
|
|
|
return [];
|
|
|
|
}
|
|
|
|
|
|
|
|
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
|
|
|
|
|
|
|
|
const releasesWithChannels = await attachChannelEntities(releases);
|
|
|
|
const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) }));
|
|
|
|
const releasesWithStudios = await attachStudios(releasesWithBaseActors);
|
|
|
|
|
|
|
|
// uniqueness is entity ID + entry ID, filter uniques after adding entities
|
|
|
|
const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
|
|
|
|
|
|
|
|
const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId)));
|
|
|
|
const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries);
|
|
|
|
|
|
|
|
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
|
|
|
|
|
|
|
|
const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries, batchId);
|
|
|
|
const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries, batchId);
|
|
|
|
const releasesWithId = uniqueReleasesWithId.concat(duplicateReleasesWithId);
|
|
|
|
|
|
|
|
const updated = await knex.raw(`
|
|
|
|
UPDATE releases
|
|
|
|
SET url = COALESCE(new.url, releases.url),
|
|
|
|
date = COALESCE(new.date, releases.date),
|
|
|
|
entity_id = COALESCE((new.entity->>'id')::integer, releases.entity_id),
|
|
|
|
title = COALESCE(new.title, releases.title),
|
|
|
|
description = COALESCE(new.description, releases.description),
|
|
|
|
duration = COALESCE(new.duration, releases.duration),
|
|
|
|
deep = new.url IS NOT NULL,
|
|
|
|
updated_at = NOW()
|
|
|
|
FROM json_to_recordset(:scenes)
|
|
|
|
AS new(id int, url text, date timestamptz, entity json, title text, description text, duration integer, deep boolean)
|
|
|
|
WHERE releases.id = new.id
|
|
|
|
`, {
|
|
|
|
scenes: JSON.stringify(duplicateReleasesWithId),
|
|
|
|
});
|
|
|
|
|
|
|
|
const [actors, storedSeries] = await Promise.all([
|
|
|
|
associateActors(releasesWithId, batchId),
|
|
|
|
storeSeries(releasesWithId.map((release) => release.serie && { ...release.serie, entity: release.entity }).filter(Boolean), batchId),
|
|
|
|
associateReleaseTags(releasesWithId),
|
|
|
|
storeChapters(releasesWithId),
|
|
|
|
]);
|
|
|
|
|
|
|
|
await associateSerieScenes(storedSeries, releasesWithId);
|
|
|
|
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
|
|
|
|
await updateSceneSearch(releasesWithId.map((release) => release.id));
|
|
|
|
|
|
|
|
// media is more error-prone, associate separately
|
|
|
|
await associateReleaseMedia(releasesWithId);
|
|
|
|
|
|
|
|
if (argv.sceneActors && actors) {
|
|
|
|
await scrapeActors(actors.map((actor) => actor.name));
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info(`Stored ${storedReleaseEntries.length}, updated ${updated.rowCount} releases`);
|
|
|
|
|
|
|
|
await notify(releasesWithId);
|
|
|
|
|
|
|
|
return releasesWithId;
|
|
|
|
}
|
|
|
|
|
2020-03-16 03:10:52 +00:00
|
|
|
module.exports = {
|
2021-01-29 01:38:05 +00:00
|
|
|
associateMovieScenes,
|
2020-08-08 16:10:59 +00:00
|
|
|
storeScenes,
|
2020-08-01 13:11:07 +00:00
|
|
|
storeMovies,
|
2021-08-22 20:25:20 +00:00
|
|
|
updateSceneSearch,
|
|
|
|
updateMovieSearch,
|
2020-03-16 03:10:52 +00:00
|
|
|
};
|