DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Using new bulk insert utility for releases, media and actors.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-08-14 23:21:53 +02:00
parent e996a45bf5
commit 50c5f921f5
11 changed files with 40 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/actors.js
View File

@@ -16,6 +16,7 @@ const scrapers = require('./scrapers/scrapers').actors;
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const bulkInsert = require('./utils/bulk-insert');
const logger = require('./logger')(__filename);
const { toBaseReleases } = require('./deep');
@@ -485,7 +486,7 @@ async function upsertProfiles(profiles) {
const updatingProfileEntries = profiles.filter(profile => profile.update).map(profile => curateProfileEntry(profile));
if (newProfileEntries.length > 0) {
await knex.batchInsert('actors_profiles', newProfileEntries);
await bulkInsert('actors_profiles', newProfileEntries);
logger.info(`Saved ${newProfileEntries.length} actor profiles`);
}
@@ -639,9 +640,7 @@ async function scrapeActors(argNames) {
// TODO: associate entity when entry ID is provided
const newActorEntries = batchId && await knex('actors')
.insert(curatedActorEntries)
.returning(['id', 'name', 'slug', 'entry_id']);
const newActorEntries = batchId && await bulkInsert('actors', curatedActorEntries);
const actors = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []);
@@ -697,7 +696,7 @@ async function getOrCreateActors(baseActors, batchId) {
const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.entity.id]?.[baseActor.slug] && !existingActorSlugs.null?.[baseActor.slug]);
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
const newActors = await knex('actors').insert(curatedActorEntries, ['id', 'alias_for', 'name', 'slug', 'entity_id']);
const newActors = await bulkInsert('actors', curatedActorEntries);
if (Array.isArray(newActors)) {
return newActors.concat(existingActors);
@@ -743,7 +742,7 @@ async function associateActors(releases, batchId) {
})))
.flat();
await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
await bulkInsert('releases_actors', releaseActorAssociations, false);
return actors;
}