traxxx/src/releases.js

461 lines
14 KiB
JavaScript
Raw Normal View History

'use strict';
const config = require('config');
const Promise = require('bluebird');
2019-12-10 21:35:00 +00:00
const moment = require('moment');
const logger = require('./logger');
const knex = require('./knex');
const argv = require('./argv');
const whereOr = require('./utils/where-or');
const { associateTags } = require('./tags');
2020-01-27 00:41:04 +00:00
const { associateActors, scrapeBasicActors } = require('./actors');
const {
createMediaDirectory,
storePhotos,
// storeReleasePhotos,
storeTrailer,
} = require('./media');
const { fetchSites, findSiteByUrl } = require('./sites');
const slugify = require('./utils/slugify');
function commonQuery(queryBuilder, {
filter = [],
after = new Date(0), // January 1970
before = new Date(2 ** 44), // May 2109
limit = 100,
}) {
const finalFilter = [].concat(filter); // ensure filter is array
queryBuilder
.leftJoin('sites', 'releases.site_id', 'sites.id')
.leftJoin('studios', 'releases.studio_id', 'studios.id')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.select(
'releases.*',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description',
)
.whereNotExists((builder) => {
// apply tag filters
builder
.select('*')
.from('tags_associated')
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
.whereIn('tags.slug', finalFilter)
.where('tags_associated.domain', 'releases')
.whereRaw('tags_associated.target_id = releases.id');
})
.andWhere('releases.date', '>', after)
.andWhere('releases.date', '<=', before)
.orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }])
.limit(limit);
}
2019-12-15 22:46:42 +00:00
async function curateRelease(release) {
const [actors, tags, media] = await Promise.all([
knex('actors_associated')
.select(
'actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'actors.birthdate',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'media.thumbnail as avatar',
)
.where({ release_id: release.id })
.leftJoin('actors', 'actors.id', 'actors_associated.actor_id')
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('media', (builder) => {
builder
.on('media.target_id', 'actors.id')
.andOnVal('media.domain', 'actors')
.andOnVal('media.index', '0');
})
.orderBy('actors.gender'),
knex('tags_associated')
.select('tags.name', 'tags.slug')
.where({
domain: 'releases',
target_id: release.id,
})
.leftJoin('tags', 'tags.id', 'tags_associated.tag_id')
.orderBy('tags.priority', 'desc'),
knex('media')
.where({
target_id: release.id,
domain: 'releases',
})
.orderBy(['role', 'index']),
]);
const curatedRelease = {
id: release.id,
type: release.type,
title: release.title,
date: release.date,
dateAdded: release.created_at,
description: release.description,
url: release.url,
shootId: release.shoot_id,
entryId: release.entry_id,
actors: actors.map(actor => ({
id: actor.id,
slug: actor.slug,
name: actor.name,
gender: actor.gender,
birthdate: actor.birthdate,
age: moment().diff(actor.birthdate, 'years'),
ageThen: moment(release.date).diff(actor.birthdate, 'years'),
avatar: actor.avatar,
origin: actor.birth_country_alpha2
? {
country: {
name: actor.birth_country_alias,
alpha2: actor.birth_country_alpha2,
},
}
: null,
})),
director: release.director,
tags,
duration: release.duration,
photos: media.filter(item => item.role === 'photo'),
poster: media.filter(item => item.role === 'poster')[0],
covers: media.filter(item => item.role === 'cover'),
trailer: media.filter(item => item.role === 'trailer')[0],
site: {
id: release.site_id,
name: release.site_name,
independent: release.site_parameters
? (JSON.parse(release.site_parameters).independent || false)
: false,
slug: release.site_slug,
url: release.site_url,
},
studio: release.studio_id
? {
id: release.studio_id,
name: release.studio_name,
slug: release.studio_slug,
url: release.studio_url,
}
: null,
network: {
id: release.network_id,
name: release.network_name,
description: release.network_description,
slug: release.network_slug,
url: release.network_url,
},
};
return curatedRelease;
}
function curateReleases(releases) {
return Promise.all(releases.map(async release => curateRelease(release)));
}
async function attachChannelSite(release) {
2019-12-15 22:14:43 +00:00
if (!release.site.isFallback) {
2019-12-15 22:46:42 +00:00
return release;
2019-12-15 22:14:43 +00:00
}
if (!release.channel) {
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
}
const [site] = await fetchSites({
name: release.channel,
slug: release.channel,
});
if (site) {
return {
...release,
site,
};
}
try {
const urlSite = await findSiteByUrl(release.channel);
return {
...release,
site: urlSite,
};
} catch (error) {
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
}
}
async function attachStudio(release) {
if (!release.studio) {
return release;
}
const studio = await knex('studios')
.where('name', release.studio)
.orWhere('slug', release.studio)
.orWhere('url', release.studio)
.first();
return {
...release,
studio,
};
}
async function curateReleaseEntry(release) {
const slug = slugify(release.title, {
encode: true,
limit: config.titleSlugLength,
});
const curatedRelease = {
site_id: release.site.id,
studio_id: release.studio ? release.studio.id : null,
shoot_id: release.shootId || null,
entry_id: release.entryId || null,
parent_id: release.parentId,
type: release.type,
url: release.url,
title: release.title,
slug,
date: release.date,
description: release.description,
// director: release.director,
duration: release.duration,
2019-12-11 04:15:23 +00:00
// likes: release.rating && release.rating.likes,
// dislikes: release.rating && release.rating.dislikes,
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
2019-12-09 04:00:49 +00:00
deep: typeof release.deep === 'boolean' ? release.deep : false,
};
return curatedRelease;
}
async function fetchReleases(queryObject = {}, options = {}) {
const releases = await knex('releases')
.modify(commonQuery, options)
.andWhere(builder => whereOr(queryObject, 'releases', builder));
return curateReleases(releases);
}
async function fetchSiteReleases(queryObject, options = {}) {
const releases = await knex('releases')
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'sites', builder));
return curateReleases(releases);
}
async function fetchNetworkReleases(queryObject, options = {}) {
const releases = await knex('releases')
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'networks', builder));
return curateReleases(releases);
}
async function fetchActorReleases(queryObject, options = {}) {
const releases = await knex('actors_associated')
.leftJoin('releases', 'actors_associated.release_id', 'releases.id')
.leftJoin('actors', 'actors_associated.actor_id', 'actors.id')
.select(
'actors.name as actor_name',
)
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'actors', builder));
return curateReleases(releases);
}
async function fetchTagReleases(queryObject, options = {}) {
const releases = await knex('tags_associated')
.leftJoin('releases', 'tags_associated.target_id', 'releases.id')
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
.select(
'tags.name as tag_name',
)
.modify(commonQuery, options)
.where('tags_associated.domain', 'releases')
.where(builder => whereOr(queryObject, 'tags', builder));
return curateReleases(releases);
}
function accumulateActors(releases) {
return releases.reduce((acc, release) => {
if (!Array.isArray(release.actors)) return acc;
release.actors.forEach((actor) => {
const actorName = actor.name ? actor.name.trim() : actor.trim();
if (!acc[actorName]) acc[actorName] = new Set();
acc[actorName].add(release.id);
});
return acc;
}, {});
}
function accumulateMovies(releases) {
return releases.reduce((acc, release) => {
if (release.movie) {
if (acc[release.movie]) {
acc[release.movie] = acc[release.movie].concat(release.id);
return acc;
}
acc[release.movie] = [release.id];
}
return acc;
}, {});
}
async function storeReleaseAssets(releases) {
// await storeReleasePhotos(releases);
await Promise.map(releases, async (release) => {
const subpath = `${release.site.network.slug}/${release.site.slug}/${release.id}/`;
const identifier = `"${release.title}" (${release.id})`;
2020-01-14 20:45:30 +00:00
try {
await createMediaDirectory('releases', subpath);
// don't use Promise.all to prevent concurrency issues with duplicate detection
if (release.poster) {
await storePhotos([release.poster], {
role: 'poster',
targetId: release.id,
subpath,
}, identifier);
}
await storePhotos(release.photos, {
targetId: release.id,
subpath,
primaryRole: release.poster ? null : 'poster',
}, identifier);
await storePhotos(release.covers, {
role: 'cover',
targetId: release.id,
subpath,
}, identifier);
await storeTrailer(release.trailer, {
targetId: release.id,
subpath,
role: 'trailer',
}, identifier);
await storeTrailer(release.teaser, {
targetId: release.id,
subpath,
role: 'teaser',
}, identifier);
2020-01-14 20:45:30 +00:00
} catch (error) {
logger.error(error.message);
}
}, {
concurrency: 10,
});
}
2019-11-17 02:56:45 +00:00
async function storeRelease(release) {
const existingRelease = await knex('releases')
.where({
entry_id: release.entryId,
site_id: release.site.id,
})
.first();
const curatedRelease = await curateReleaseEntry(release);
2019-11-17 02:56:45 +00:00
if (existingRelease && !argv.redownload) {
return existingRelease;
}
2019-11-17 02:56:45 +00:00
if (existingRelease && argv.redownload) {
const [updatedRelease] = await knex('releases')
.where('id', existingRelease.id)
.update({
...existingRelease,
...curatedRelease,
})
.returning('*');
2019-11-17 02:56:45 +00:00
if (updatedRelease) {
await associateTags(release, updatedRelease.id);
logger.info(`Updated release "${release.title}" (${existingRelease.id}, ${release.site.name})`);
}
await associateTags(release, existingRelease.id);
2019-11-17 02:56:45 +00:00
return existingRelease;
2019-11-17 02:56:45 +00:00
}
const [releaseEntry] = await knex('releases')
.insert(curatedRelease)
.returning('*');
await associateTags(release, releaseEntry.id);
logger.info(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`);
2019-11-17 02:56:45 +00:00
return releaseEntry;
2019-11-17 02:56:45 +00:00
}
async function storeReleases(releases) {
const storedReleases = await Promise.map(releases, async (release) => {
2019-11-17 02:56:45 +00:00
try {
const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const { id, slug } = await storeRelease(releaseWithStudio);
2019-11-17 02:56:45 +00:00
return {
id,
slug,
...releaseWithChannelSite,
};
2019-11-17 02:56:45 +00:00
} catch (error) {
2020-01-13 22:45:09 +00:00
logger.error(error.message);
2019-11-17 02:56:45 +00:00
return null;
}
}, {
concurrency: 10,
2019-12-09 04:00:49 +00:00
}).filter(release => release);
const actors = accumulateActors(storedReleases);
const movies = accumulateMovies(storedReleases);
await Promise.all([
associateActors(actors, storedReleases),
storeReleaseAssets(storedReleases),
]);
if (argv.withProfiles && Object.keys(actors).length > 0) {
await scrapeBasicActors();
}
2020-01-27 00:41:04 +00:00
return {
releases: storedReleases,
actors,
movies,
};
2019-11-17 02:56:45 +00:00
}
module.exports = {
fetchReleases,
fetchActorReleases,
fetchSiteReleases,
fetchNetworkReleases,
fetchTagReleases,
storeRelease,
storeReleases,
};