334 lines
11 KiB
JavaScript
334 lines
11 KiB
JavaScript
'use strict';
|
|
|
|
const Promise = require('bluebird');
|
|
const knex = require('./knex');
|
|
const argv = require('./argv');
|
|
const whereOr = require('./utils/where-or');
|
|
const { associateTags } = require('./tags');
|
|
const { associateActors } = require('./actors');
|
|
const {
|
|
createReleaseMediaDirectory,
|
|
storePoster,
|
|
storePhotos,
|
|
storeTrailer,
|
|
} = require('./media');
|
|
const { fetchSites, findSiteByUrl } = require('./sites');
|
|
|
|
async function curateRelease(release) {
|
|
const [actors, tags, media] = await Promise.all([
|
|
knex('actors_associated')
|
|
.select('actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'media.thumbnail as avatar')
|
|
.where({ release_id: release.id })
|
|
.leftJoin('actors', 'actors.id', 'actors_associated.actor_id')
|
|
.leftJoin('media', (builder) => {
|
|
builder
|
|
.on('media.target_id', 'actors.id')
|
|
.andOnVal('media.domain', 'actors')
|
|
.andOnVal('media.index', '0');
|
|
})
|
|
.orderBy('actors.gender'),
|
|
knex('tags_associated')
|
|
.select('tags.name', 'tags.slug')
|
|
.where({
|
|
domain: 'releases',
|
|
target_id: release.id,
|
|
})
|
|
.leftJoin('tags', 'tags.id', 'tags_associated.tag_id')
|
|
.orderBy('tags.priority', 'desc'),
|
|
knex('media')
|
|
.where({
|
|
target_id: release.id,
|
|
domain: 'releases',
|
|
})
|
|
.orderBy('role'),
|
|
]);
|
|
|
|
return {
|
|
id: release.id,
|
|
title: release.title,
|
|
date: release.date,
|
|
dateAdded: release.created_at,
|
|
description: release.description,
|
|
url: release.url,
|
|
shootId: release.shoot_id,
|
|
entryId: release.entry_id,
|
|
actors,
|
|
director: release.director,
|
|
tags,
|
|
duration: release.duration,
|
|
photos: media.filter(item => item.role === 'photo'),
|
|
poster: media.filter(item => item.role === 'poster')[0],
|
|
trailer: media.filter(item => item.role === 'trailer')[0],
|
|
rating: {
|
|
likes: release.likes,
|
|
dislikes: release.dislikes,
|
|
stars: release.stars,
|
|
},
|
|
site: {
|
|
id: release.site_id,
|
|
name: release.site_name,
|
|
independent: release.site_parameters
|
|
? (JSON.parse(release.site_parameters).independent || false)
|
|
: false,
|
|
slug: release.site_slug,
|
|
url: release.site_url,
|
|
},
|
|
studio: release.studio_id
|
|
? {
|
|
id: release.studio_id,
|
|
name: release.studio_name,
|
|
slug: release.studio_slug,
|
|
url: release.studio_url,
|
|
}
|
|
: null,
|
|
network: {
|
|
id: release.network_id,
|
|
name: release.network_name,
|
|
description: release.network_description,
|
|
slug: release.network_slug,
|
|
url: release.network_url,
|
|
},
|
|
};
|
|
}
|
|
|
|
function curateReleases(releases) {
|
|
return Promise.all(releases.map(async release => curateRelease(release)));
|
|
}
|
|
|
|
async function getChannelSite(release) {
|
|
try {
|
|
const site = await findSiteByUrl(release.channel);
|
|
|
|
return site || null;
|
|
} catch (error) {
|
|
const [site] = await fetchSites({
|
|
name: release.channel,
|
|
slug: release.channel,
|
|
});
|
|
|
|
return site || null;
|
|
}
|
|
}
|
|
|
|
async function curateScrapedRelease(release) {
|
|
const curatedRelease = {
|
|
site_id: release.site.id,
|
|
studio_id: release.studio ? release.studio.id : null,
|
|
shoot_id: release.shootId || null,
|
|
entry_id: release.entryId || null,
|
|
url: release.url,
|
|
title: release.title,
|
|
date: release.date,
|
|
description: release.description,
|
|
// director: release.director,
|
|
duration: release.duration,
|
|
likes: release.rating && release.rating.likes,
|
|
dislikes: release.rating && release.rating.dislikes,
|
|
rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
|
|
deep: typeof release.deep === 'boolean' ? release.deep : false,
|
|
};
|
|
|
|
if (release.site.isFallback && release.channel) {
|
|
const site = await getChannelSite(release);
|
|
|
|
if (site) {
|
|
curatedRelease.site_id = site.id;
|
|
return curatedRelease;
|
|
}
|
|
}
|
|
|
|
return curatedRelease;
|
|
}
|
|
|
|
function commonQuery(queryBuilder, {
|
|
filter = [],
|
|
after = new Date(0), // January 1970
|
|
before = new Date(2 ** 44), // May 2109
|
|
limit = 100,
|
|
}) {
|
|
const finalFilter = [].concat(filter); // ensure filter is array
|
|
|
|
queryBuilder
|
|
.leftJoin('sites', 'releases.site_id', 'sites.id')
|
|
.leftJoin('studios', 'releases.studio_id', 'studios.id')
|
|
.leftJoin('networks', 'sites.network_id', 'networks.id')
|
|
.select(
|
|
'releases.*',
|
|
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
|
|
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
|
|
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description',
|
|
)
|
|
.whereNotExists((builder) => {
|
|
// apply tag filters
|
|
builder
|
|
.select('*')
|
|
.from('tags_associated')
|
|
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
|
|
.whereIn('tags.slug', finalFilter)
|
|
.where('tags_associated.domain', 'releases')
|
|
.whereRaw('tags_associated.target_id = releases.id');
|
|
})
|
|
.andWhere('date', '>', after)
|
|
.andWhere('date', '<=', before)
|
|
.orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }])
|
|
.limit(limit);
|
|
}
|
|
|
|
async function fetchReleases(queryObject = {}, options = {}) {
|
|
const releases = await knex('releases')
|
|
.modify(commonQuery, options)
|
|
.andWhere(builder => whereOr(queryObject, 'releases', builder));
|
|
|
|
return curateReleases(releases);
|
|
}
|
|
|
|
async function fetchSiteReleases(queryObject, options = {}) {
|
|
const releases = await knex('releases')
|
|
.modify(commonQuery, options)
|
|
.where(builder => whereOr(queryObject, 'sites', builder));
|
|
|
|
return curateReleases(releases);
|
|
}
|
|
|
|
async function fetchNetworkReleases(queryObject, options = {}) {
|
|
const releases = await knex('releases')
|
|
.modify(commonQuery, options)
|
|
.where(builder => whereOr(queryObject, 'networks', builder));
|
|
|
|
return curateReleases(releases);
|
|
}
|
|
|
|
async function fetchActorReleases(queryObject, options = {}) {
|
|
const releases = await knex('actors_associated')
|
|
.leftJoin('releases', 'actors_associated.release_id', 'releases.id')
|
|
.leftJoin('actors', 'actors_associated.actor_id', 'actors.id')
|
|
.select(
|
|
'actors.name as actor_name',
|
|
)
|
|
.modify(commonQuery, options)
|
|
.where(builder => whereOr(queryObject, 'actors', builder));
|
|
|
|
return curateReleases(releases);
|
|
}
|
|
|
|
async function fetchTagReleases(queryObject, options = {}) {
|
|
const releases = await knex('tags_associated')
|
|
.leftJoin('releases', 'tags_associated.target_id', 'releases.id')
|
|
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
|
|
.select(
|
|
'tags.name as tag_name',
|
|
)
|
|
.modify(commonQuery, options)
|
|
.where('tags_associated.domain', 'releases')
|
|
.where(builder => whereOr(queryObject, 'tags', builder));
|
|
|
|
return curateReleases(releases);
|
|
}
|
|
|
|
async function storeReleaseAssets(release, releaseId) {
|
|
await createReleaseMediaDirectory(release, releaseId);
|
|
|
|
try {
|
|
await Promise.all([
|
|
associateTags(release, releaseId),
|
|
storePhotos(release, releaseId),
|
|
storePoster(release, releaseId),
|
|
storeTrailer(release, releaseId),
|
|
]);
|
|
} catch (error) {
|
|
console.log(release.url, error);
|
|
}
|
|
}
|
|
|
|
async function storeRelease(release) {
|
|
const existingRelease = await knex('releases').where('entry_id', release.entryId).first();
|
|
const curatedRelease = await curateScrapedRelease(release);
|
|
|
|
if (existingRelease && !argv.redownload) {
|
|
return existingRelease.id;
|
|
}
|
|
|
|
if (existingRelease && argv.redownload) {
|
|
const [updatedRelease] = await knex('releases')
|
|
.where('entry_id', existingRelease.id)
|
|
.update({
|
|
...existingRelease,
|
|
...curatedRelease,
|
|
})
|
|
.returning('*');
|
|
|
|
// await storeReleaseAssets(release, existingRelease.id);
|
|
console.log(`Updated release "${release.title}" (${existingRelease.id}, ${release.site.name})`);
|
|
|
|
return updatedRelease ? updatedRelease.id : existingRelease.id;
|
|
}
|
|
|
|
const [releaseEntry] = await knex('releases')
|
|
.insert(curatedRelease)
|
|
.returning('*');
|
|
|
|
// await storeReleaseAssets(release, releaseEntry.id);
|
|
console.log(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`);
|
|
|
|
return releaseEntry.id;
|
|
}
|
|
|
|
async function storeReleases(releases) {
|
|
const storedReleases = await Promise.map(releases, async (release) => {
|
|
if (release.site.isFallback && !release.channel) {
|
|
console.error(`Unable to derive channel site from generic URL: ${release.url}.`);
|
|
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const releaseId = await storeRelease(release);
|
|
|
|
return {
|
|
id: releaseId,
|
|
...release,
|
|
};
|
|
} catch (error) {
|
|
console.error(error);
|
|
|
|
return null;
|
|
}
|
|
}, {
|
|
concurrency: 10,
|
|
}).filter(release => release);
|
|
|
|
const actors = storedReleases.reduce((acc, release) => {
|
|
if (!release.actors) return acc;
|
|
|
|
release.actors.forEach((actor) => {
|
|
const trimmedActor = actor.trim();
|
|
|
|
if (acc[trimmedActor]) {
|
|
acc[trimmedActor] = acc[trimmedActor].concat(release.id);
|
|
return;
|
|
}
|
|
|
|
acc[trimmedActor] = [release.id];
|
|
});
|
|
|
|
return acc;
|
|
}, {});
|
|
|
|
await Promise.all([
|
|
associateActors(actors, storedReleases),
|
|
Promise.all(storedReleases.map(async release => storeReleaseAssets(release, release.id))),
|
|
]);
|
|
|
|
return storedReleases;
|
|
}
|
|
|
|
module.exports = {
|
|
fetchReleases,
|
|
fetchActorReleases,
|
|
fetchSiteReleases,
|
|
fetchNetworkReleases,
|
|
fetchTagReleases,
|
|
storeRelease,
|
|
storeReleases,
|
|
};
|