2020-03-16 03:10:52 +00:00
|
|
|
'use strict';
|
|
|
|
|
|
|
|
const config = require('config');
|
|
|
|
|
2020-03-21 01:48:24 +00:00
|
|
|
const logger = require('./logger')(__filename);
|
2020-03-16 03:10:52 +00:00
|
|
|
const knex = require('./knex');
|
|
|
|
const slugify = require('./utils/slugify');
|
2020-03-22 02:50:24 +00:00
|
|
|
const { associateTags } = require('./tags');
|
2020-03-24 02:48:24 +00:00
|
|
|
const { associateActors } = require('./actors');
|
2020-03-16 03:10:52 +00:00
|
|
|
|
|
|
|
function curateReleaseEntry(release, batchId, existingRelease) {
|
|
|
|
const slug = slugify(release.title, '-', {
|
|
|
|
encode: true,
|
|
|
|
limit: config.titleSlugLength,
|
|
|
|
});
|
|
|
|
|
|
|
|
const curatedRelease = {
|
|
|
|
title: release.title,
|
|
|
|
entry_id: release.entryId || null,
|
|
|
|
site_id: release.site.id,
|
|
|
|
shoot_id: release.shootId || null,
|
|
|
|
studio_id: release.studio?.id || null,
|
|
|
|
url: release.url,
|
|
|
|
date: release.date,
|
|
|
|
slug,
|
|
|
|
description: release.description,
|
|
|
|
duration: release.duration,
|
|
|
|
type: release.type,
|
|
|
|
// director: release.director,
|
|
|
|
// likes: release.rating && release.rating.likes,
|
|
|
|
// dislikes: release.rating && release.rating.dislikes,
|
|
|
|
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
|
|
|
|
deep: typeof release.deep === 'boolean' ? release.deep : false,
|
|
|
|
deep_url: release.deepUrl,
|
|
|
|
updated_batch_id: batchId,
|
|
|
|
};
|
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
if (!existingRelease && !release.id) {
|
2020-03-16 03:10:52 +00:00
|
|
|
curatedRelease.created_batch_id = batchId;
|
|
|
|
}
|
|
|
|
|
|
|
|
return curatedRelease;
|
|
|
|
}
|
|
|
|
|
2020-03-16 23:58:03 +00:00
|
|
|
async function attachChannelSites(releases) {
|
|
|
|
const releasesWithoutSite = releases.filter(release => release.channel && (!release.site || release.site.isFallback));
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-03-16 23:58:03 +00:00
|
|
|
const channelSites = await knex('sites').whereIn('slug', releasesWithoutSite.map(release => release.channel));
|
|
|
|
const channelSitesBySlug = channelSites.reduce((acc, site) => ({ ...acc, [site.slug]: site }), {});
|
|
|
|
|
|
|
|
const releasesWithChannelSite = releases
|
|
|
|
.map((release) => {
|
|
|
|
if (release.site && !release.site.isFallback) {
|
|
|
|
return release;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (release.channel && channelSitesBySlug[release.channel]) {
|
|
|
|
return {
|
|
|
|
...release,
|
|
|
|
site: channelSitesBySlug[release.channel],
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
logger.error(`Unable to match channel '${release.channel?.slug || release.channel}' from generic URL ${release.url}`);
|
2020-03-16 23:58:03 +00:00
|
|
|
|
|
|
|
return null;
|
|
|
|
})
|
|
|
|
.filter(Boolean);
|
|
|
|
|
|
|
|
return releasesWithChannelSite;
|
|
|
|
}
|
|
|
|
|
|
|
|
async function attachStudios(releases) {
|
|
|
|
const studioSlugs = releases.map(release => release.studio).filter(Boolean);
|
|
|
|
|
|
|
|
const studios = await knex('studios').whereIn('slug', studioSlugs);
|
|
|
|
const studioBySlug = studios.reduce((acc, studio) => ({ ...acc, [studio.slug]: studio }), {});
|
|
|
|
|
|
|
|
const releasesWithStudio = releases.map((release) => {
|
|
|
|
if (release.studio && studioBySlug[release.studio]) {
|
|
|
|
return {
|
|
|
|
...release,
|
|
|
|
studio: release.studio,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
if (release.studio) {
|
|
|
|
logger.warn(`Unable to match studio '${release.studio}' for ${release.url}`);
|
|
|
|
}
|
|
|
|
|
|
|
|
return release;
|
|
|
|
});
|
|
|
|
|
|
|
|
return releasesWithStudio;
|
2020-03-16 03:10:52 +00:00
|
|
|
}
|
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
function attachReleaseIds(releases, storedReleases) {
|
|
|
|
const storedReleaseIdsBySiteIdAndEntryId = storedReleases.reduce((acc, release) => {
|
|
|
|
if (!acc[release.site_id]) acc[release.site_id] = {};
|
|
|
|
acc[release.site_id][release.entry_id] = release.id;
|
|
|
|
|
|
|
|
return acc;
|
|
|
|
}, {});
|
|
|
|
|
|
|
|
const releasesWithId = releases.map(release => ({
|
|
|
|
...release,
|
|
|
|
id: storedReleaseIdsBySiteIdAndEntryId[release.site.id][release.entryId],
|
|
|
|
}));
|
|
|
|
|
|
|
|
return releasesWithId;
|
|
|
|
}
|
|
|
|
|
2020-03-16 03:10:52 +00:00
|
|
|
async function extractUniqueReleases(releases) {
|
|
|
|
const duplicateReleaseEntries = await knex('releases')
|
|
|
|
.whereIn(['entry_id', 'site_id'], releases.map(release => [release.entryId, release.site.id]));
|
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
const duplicateReleasesBySiteIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
|
|
|
|
if (!acc[release.site_id]) acc[release.site_id] = {};
|
|
|
|
acc[release.site_id][release.entry_id] = true;
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
return acc;
|
|
|
|
}, {});
|
|
|
|
|
|
|
|
const duplicateReleases = releases.filter(release => duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
|
|
|
|
const uniqueReleases = releases.filter(release => !duplicateReleasesBySiteIdAndEntryId[release.site.id]?.[release.entryId]);
|
|
|
|
|
|
|
|
return {
|
|
|
|
uniqueReleases,
|
|
|
|
duplicateReleases,
|
|
|
|
duplicateReleaseEntries,
|
|
|
|
};
|
2020-03-16 03:10:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
async function storeReleases(releases) {
|
|
|
|
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
|
|
|
|
|
2020-03-16 23:58:03 +00:00
|
|
|
const releasesWithSites = await attachChannelSites(releases);
|
|
|
|
const releasesWithStudios = await attachStudios(releasesWithSites);
|
|
|
|
|
|
|
|
// uniqueness is site ID + entry ID, filter uniques after adding sites
|
2020-03-22 02:50:24 +00:00
|
|
|
const { uniqueReleases, duplicateReleaseEntries } = await extractUniqueReleases(releasesWithStudios);
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
const curatedNewReleaseEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId));
|
2020-03-21 01:48:24 +00:00
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
const storedReleases = await knex('releases').insert(curatedNewReleaseEntries).returning('*');
|
|
|
|
// TODO: update duplicate releases
|
2020-03-16 03:10:52 +00:00
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
|
|
|
|
const releasesWithId = attachReleaseIds(releases, [].concat(storedReleaseEntries, duplicateReleaseEntries));
|
|
|
|
|
2020-03-24 02:48:24 +00:00
|
|
|
await Promise.all([
|
|
|
|
associateTags(releasesWithId),
|
|
|
|
associateActors(releasesWithId),
|
|
|
|
]);
|
2020-03-21 01:48:24 +00:00
|
|
|
|
2020-03-22 02:50:24 +00:00
|
|
|
return releasesWithId;
|
2020-03-16 03:10:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
module.exports = {
|
|
|
|
storeReleases,
|
|
|
|
};
|