'use strict'; const Promise = require('bluebird'); const knex = require('./knex'); const argv = require('./argv'); const whereOr = require('./utils/where-or'); const { storeTags } = require('./tags'); const { storeActors } = require('./actors'); const { createMediaDirectory, storePoster, storePhotos, storeTrailer, } = require('./media'); async function curateRelease(release) { const [actors, tags, media] = await Promise.all([ knex('actors_associated') .select('actors.id', 'actors.name', 'actors.gender', 'actors.slug') .where({ release_id: release.id }) .leftJoin('actors', 'actors.id', 'actors_associated.actor_id'), knex('tags_associated') .select('tags.name', 'tags.slug') .where({ release_id: release.id }) .leftJoin('tags', 'tags.id', 'tags_associated.tag_id') .orderBy('tags.priority', 'desc'), knex('media') .where({ target_id: release.id }) .orderBy('role'), ]); return { id: release.id, title: release.title, date: release.date, dateAdded: release.created_at, description: release.description, url: release.url, shootId: release.shoot_id, entryId: release.entry_id, actors, director: release.director, tags, duration: release.duration, photos: media.filter(item => item.role === 'photo'), poster: media.filter(item => item.role === 'poster')[0], trailer: media.filter(item => item.role === 'trailer')[0], rating: { likes: release.likes, dislikes: release.dislikes, stars: release.stars, }, site: { id: release.site_id, name: release.site_name, independent: release.site_parameters ? (JSON.parse(release.site_parameters).independent || false) : false, slug: release.site_slug, url: release.site_url, }, studio: release.studio_id ? { id: release.studio_id, name: release.studio_name, slug: release.studio_slug, url: release.studio_url, } : null, network: { id: release.network_id, name: release.network_name, description: release.network_description, slug: release.network_slug, url: release.network_url, }, }; } function curateReleases(releases) { return Promise.all(releases.map(async release => curateRelease(release))); } function curateScrapedRelease(release) { return { site_id: release.site.id, studio_id: release.studio ? release.studio.id : null, shoot_id: release.shootId || null, entry_id: release.entryId || null, url: release.url, title: release.title, date: release.date, description: release.description, // director: release.director, duration: release.duration, likes: release.rating && release.rating.likes, dislikes: release.rating && release.rating.dislikes, rating: release.rating && release.rating.stars && Math.floor(release.rating.stars), deep: Boolean(argv.deep && release.url && !release.upcoming), }; } async function storeRelease(release) { const curatedRelease = curateScrapedRelease(release); const releaseEntries = await knex('releases') .insert(curatedRelease) .returning('*'); if (releaseEntries.length) { const releaseEntry = releaseEntries[0]; console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`); await createMediaDirectory(release, releaseEntry.id); await Promise.all([ storeActors(release, releaseEntry), storeTags(release, releaseEntry), storePhotos(release, releaseEntry), storePoster(release, releaseEntry), storeTrailer(release, releaseEntry), ]); return releaseEntry.id; } console.error(`Unable to save scene to database, possible collision: "${release.title}" (${release.site.name})`); return null; } async function storeReleases(releases) { return Promise.map(releases, async (release) => { try { const releaseId = await storeRelease(release); return releaseId; } catch (error) { console.error(error); return null; } }, { concurrency: 2, }); } function commonQuery(queryBuilder, { filter = [], after = new Date(0), // January 1970 before = new Date(2 ** 44), // May 2109 limit = 100, }) { const finalFilter = [].concat(filter); // ensure filter is array queryBuilder .leftJoin('sites', 'releases.site_id', 'sites.id') .leftJoin('studios', 'releases.studio_id', 'studios.id') .leftJoin('networks', 'sites.network_id', 'networks.id') .select( 'releases.*', 'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters', 'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', ) .whereNotExists((builder) => { // apply tag filters builder .select('*') .from('tags_associated') .leftJoin('tags', 'tags_associated.tag_id', 'tags.id') .whereIn('tags.slug', finalFilter) .andWhereRaw('tags_associated.release_id = releases.id'); }) .andWhere('date', '>', after) .andWhere('date', '<=', before) .orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }]) .limit(limit); } async function fetchReleases(queryObject = {}, options = {}) { const releases = await knex('releases') .modify(commonQuery, options) .andWhere(builder => whereOr(queryObject, 'releases', builder)); return curateReleases(releases); } async function fetchSiteReleases(queryObject, options = {}) { const releases = await knex('releases') .modify(commonQuery, options) .where(builder => whereOr(queryObject, 'sites', builder)); return curateReleases(releases); } async function fetchNetworkReleases(queryObject, options = {}) { const releases = await knex('releases') .modify(commonQuery, options) .where(builder => whereOr(queryObject, 'networks', builder)); return curateReleases(releases); } async function fetchActorReleases(queryObject, options = {}) { const releases = await knex('actors_associated') .leftJoin('releases', 'actors_associated.release_id', 'releases.id') .leftJoin('actors', 'actors_associated.actor_id', 'actors.id') .select( 'actors.name as actor_name', ) .modify(commonQuery, options) .where(builder => whereOr(queryObject, 'actors', builder)); return curateReleases(releases); } async function fetchTagReleases(queryObject, options = {}) { const releases = await knex('tags_associated') .leftJoin('releases', 'tags_associated.release_id', 'releases.id') .leftJoin('tags', 'tags_associated.tag_id', 'tags.id') .select( 'tags.name as tag_name', ) .modify(commonQuery, options) .where(builder => whereOr(queryObject, 'tags', builder)); return curateReleases(releases); } module.exports = { fetchReleases, fetchActorReleases, fetchSiteReleases, fetchNetworkReleases, fetchTagReleases, storeRelease, storeReleases, };