DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Using batch insert for profiles to prevent errors on large inserts.

Browse Source
This commit is contained in:
ThePendulum
2020-05-21 03:44:44 +02:00
parent 703b77897d
commit 532a4b679b
1005 changed files with 13340 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/actors.js
View File

@@ -39,6 +39,10 @@ const hairColors = {
raven: 'black',
red: 'red',
redhead: 'red',
blue: 'blue',
green: 'green',
purple: 'purple',
pink: 'pink',
};
const eyeColors = {
@@ -474,9 +478,9 @@ async function upsertProfiles(profiles) {
const updatingProfileEntries = profiles.filter(profile => profile.update).map(profile => curateProfileEntry(profile));
if (newProfileEntries.length > 0) {
await knex('actors_profiles').insert(newProfileEntries);
await knex.batchInsert('actors_profiles', newProfileEntries);
logger.info(`Saved ${newProfileEntries.length} new actor profiles`);
logger.info(`Saved ${newProfileEntries.length} actor profiles`);
}
if (argv.force && updatingProfileEntries.length > 0) {
@@ -583,8 +587,12 @@ async function scrapeActors(actorNames) {
.leftJoin('networks', 'sites.network_id', 'networks.id'),
knex('actors')
.select(['id', 'name', 'slug'])
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('network_id'),
.modify((queryBuilder) => {
if (actorNames.length > 0) {
queryBuilder.whereIn('slug', baseActors.map(baseActor => baseActor.slug));
}
})
.whereNull('alias_for'),
]);
const networksBySlug = networks.reduce((acc, network) => ({ ...acc, [network.slug]: network }), {});

12
src/releases.js
View File

@@ -35,8 +35,11 @@ function curateRelease(release, withMedia = false) {
gender: actor.gender,
networkId: actor.network_id,
aliasFor: actor.alias_for,
dateOfBirth: actor.date_of_birth,
birthCountry: actor.birth_country_alpha2,
})),
tags: (release.tags || []).map(tag => ({
id: tag.id,
name: tag.name,
slug: tag.slug,
})),
...(withMedia && {
poster: release.poster ? {
@@ -67,7 +70,8 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') {
row_to_json(sites) as site,
row_to_json(networks) as network,
row_to_json(site_networks) as site_network,
COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors
COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
`))
.where('type', type)
.leftJoin('sites', 'sites.id', 'releases.site_id')
@@ -75,6 +79,8 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') {
.leftJoin('networks as site_networks', 'site_networks.id', 'sites.network_id')
.leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id')
.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
.leftJoin('releases_tags', 'releases_tags.release_id', 'releases.id')
.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
.groupBy(knex.raw(`
releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
sites.id, networks.id, site_networks.id