'use strict'; const config = require('config'); const fs = require('fs'); const path = require('path'); const moment = require('moment'); const Promise = require('bluebird'); const bhttp = require('bhttp'); const { nanoid } = require('nanoid/non-secure'); const AWS = require('aws-sdk'); const { graphql } = require('../web/graphql'); const knex = require('../knex'); const args = require('../argv'); const endpoint = new AWS.Endpoint('s3.eu-central-1.wasabisys.com'); const s3 = new AWS.S3({ // region: 'eu-central-1', endpoint, credentials: { accessKeyId: config.s3.accessKey, secretAccessKey: config.s3.secretKey, }, }); // NOT TRANSFERRED, unutilized on old server: production location, availabile qualities, actor alias for, actor entry id, chapter posters, chapter photos const sceneFields = ` entryId shootId title url date datePrecision productionDate description duration entity { slug type } studio { slug } movies: moviesScenesBySceneId { movie { title entryId entity { slug type } } } actors: releasesActors { actor { name slug entryId entity { slug type } } } directors: releasesDirectors { director { name slug entryId entity { slug type } } } tags: releasesTags { tag { slug } } chapters(orderBy: TIME_ASC) { index time duration title description tags: chaptersTags { tag { slug } } } poster: releasesPoster { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } photos: releasesPhotos { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } covers: releasesCovers { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } trailer: releasesTrailer { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } teaser: releasesTeaser { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } createdAt `; const movieFields = ` entryId title url date datePrecision entity { slug type } poster: moviesPoster { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } covers: moviesCovers { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } createdAt `; async function save() { const limit = args.limit || 1000; const offset = args.start || 0; const { releases } = await graphql(` query SearchScenes( $limit: Int = 20 $offset: Int = 0 ) { releases( first: $limit offset: $offset orderBy: DATE_DESC ) { ${sceneFields} } } `, { limit, offset, }, 'owner'); const { movies } = await graphql(` query SearchScenes( $limit: Int = 20 $offset: Int = 0 ) { movies( first: $limit offset: $offset orderBy: DATE_DESC ) { ${movieFields} } } `, { limit, offset, }, 'owner'); const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`; let savedScenes = 0; let savedMovies = 0; await releases.reduce(async (chain, release) => { await chain; const entry = JSON.stringify({ ...release, type: 'release', actors: release.actors.filter(Boolean).map(({ actor }) => actor), directors: release.directors.filter(Boolean).map(({ director }) => director), studio: release.studio?.slug, tags: release.tags.map(({ tag }) => tag?.slug).filter(Boolean), movies: release.movies?.map(({ movie }) => movie) || [], chapters: release.chapters.filter(Boolean).map((chapter) => ({ ...chapter, tags: chapter.tags.map(({ tag }) => tag?.slug).filter(Boolean), })), poster: release.poster?.media, trailer: release.trailer?.media, teaser: release.teaser?.media, photos: release.photos.filter(Boolean).map(({ media }) => media), covers: release.covers.filter(Boolean).map(({ media }) => media), }); await fs.promises.appendFile(filename, `${entry}\n`); savedScenes += 1; }, Promise.resolve()); await movies.reduce(async (chain, movie) => { await chain; const entry = JSON.stringify({ ...movie, type: 'movie', poster: movie.poster?.media, covers: movie.covers.filter(Boolean).map(({ media }) => media), }); await fs.promises.appendFile(filename, `${entry}\n`); savedMovies += 1; }, Promise.resolve()); console.log(`Saved ${savedScenes} scenes and ${savedMovies} movies to ${filename}`); process.exit(); } async function addReleaseTags(release, context) { if (release.tags.length === 0) { return; } await knex('releases_tags').insert(release.tags.map((tag) => ({ tag_id: context.tagIdsBySlug[tag], release_id: release.id, original_tag: tag, }))); } async function addNewActor(actor, entity, context) { const [actorId] = await knex('actors') .insert({ name: actor.name, slug: actor.slug, entity_id: entity?.id, batch_id: context.batchId, }) .returning('id'); return actorId; } async function addReleaseActors(release, context, target = 'actor') { await release[`${target}s`].reduce(async (chain, actor) => { await chain; const entity = actor.entity ? await knex('entities').where(actor.entity).first() : null; if (actor.entity && !entity) { throw new Error(`Actor ${actor.slug} contains non-existent ${release.entity.type} '${release.entity.slug}'`); } const existingActor = await knex('actors') .where('slug', actor.slug) .where((builder) => { if (entity) { builder.where('entity_id', entity.id); return; } builder.whereNull('entity_id'); }) .first(); const actorId = existingActor?.id || await addNewActor(actor, entity, context); await knex(`releases_${target}s`).insert({ release_id: release.id, [`${target}_id`]: actorId, }); }, Promise.resolve()); } async function addReleaseDirectors(release, context) { return addReleaseActors(release, context, 'director'); } async function addReleaseChapters(release, context) { await release.chapters.reduce(async (chain, chapter) => { await chain; const [chapterId] = await knex('chapters') .insert({ release_id: release.id, index: chapter.index, time: chapter.time, duration: chapter.duration, description: chapter.description, }) .returning('id'); if (chapter.tags.length > 0) { await knex('chapters_tags').insert(chapter.tags.map((tag) => ({ tag_id: context.tagIdsBySlug[tag], chapter_id: chapterId, original_tag: tag, }))); } }, Promise.resolve()); } const dirs = { path: '', thumbnail: 'thumbs', lazy: 'lazy', }; async function transferMedia(media, target) { return ['path', 'thumbnail', 'lazy'].reduce(async (chain, type) => { await chain; const filename = `${media.hash}${path.extname(media[type])}`; const filepath = path.join(target, dirs[type], filename); const temp = path.join('media/temp', filepath); const url = new URL(media[type], `${media.s3 ? config.media.transferSources.s3 : config.media.transferSources.local}/`).href; if (args.logLevel === 'debug') { console.log('Transferring media', url); } const res = await bhttp.get(url, { stream: true }); if (res.statusCode !== 200) { console.warn(`Missing ${target} ${url}`); return; } await fs.promises.mkdir(path.dirname(temp), { recursive: true }); await new Promise((resolve, reject) => { const fileStream = fs.createWriteStream(temp); res.pipe(fileStream); res.on('error', () => { reject(); }); fileStream.on('finish', () => { resolve(); }); fileStream.on('error', () => { reject(); }); }); await s3.upload({ Bucket: config.s3.bucket, Body: fs.createReadStream(temp), Key: filepath, ContentType: media.mime, }).promise(); await fs.promises.unlink(temp); }, Promise.resolve()); } async function addReleaseMedia(medias, release, target) { await medias.filter(Boolean).reduce(async (chain, media) => { await chain; const existingMedia = await knex('media') .where('hash', media.hash) .orWhere('source', media.source) .first(); const id = existingMedia?.id || nanoid(); if (!existingMedia) { await knex('media').insert({ id, hash: media.hash, path: path.join(target, '', `${media.hash}${path.extname(media.path)}`), thumbnail: path.join(target, 'thumbs', `${media.hash}${path.extname(media.thumbnail)}`), lazy: path.join(target, 'lazy', `${media.hash}${path.extname(media.lazy)}`), // is_s3: media.s3, is_s3: true, index: media.index, mime: media.mime, size: media.size, width: media.width, height: media.height, source: media.source, source_page: media.sourcePage, }); await transferMedia(media, target); } await knex(`${release.type}s_${target}`).insert({ [`${release.type}_id`]: release.id, media_id: id, }); }, Promise.resolve()); } async function linkMovieScenes(release, context) { await release.movies.reduce(async (chain, linkedMovie) => { await chain; const movie = context.movies.find((storedMovie) => storedMovie.entryId === linkedMovie.entryId && storedMovie.entity.slug === linkedMovie.entity.slug && storedMovie.entity.type === linkedMovie.entity.type); console.log('movie', linkedMovie, movie); if (!movie) { throw new Error(`Missing ${linkedMovie.entity.slug} movie '${linkedMovie.title}' in '${release.title}'`); } await knex('movies_scenes').insert({ movie_id: movie.id, scene_id: release.id, }); }, Promise.resolve()); } async function addRelease(release, context) { const existingRelease = await knex(`${release.type}s`) .select(`${release.type}s.*`, 'entities.name as entity_name') .leftJoin('entities', 'entities.id', `${release.type}s.entity_id`) .where('entry_id', release.entryId) .where('entities.slug', release.entity.slug) .where('entities.type', release.entity.type) .first(); if (existingRelease) { console.log(`Skipping ${release.entity.slug} release "${release.title}", already in database`); return { ...release, skipped: true, id: existingRelease.id, entityName: existingRelease.entity_name, }; } const [entity] = await Promise.all([ knex('entities').select(['id', 'name']).where(release.entity).first(), ]); if (!entity) { throw new Error(`Release contains non-existent ${release.entity.type} '${release.entity.slug}'`); } const [releaseEntry] = await knex(`${release.type}s`) .insert({ entry_id: release.entryId, entity_id: entity.id, url: release.url, title: release.title, slug: release.slug, date: release.date, date_precision: release.datePrecision, created_batch_id: context.batchId, updated_batch_id: context.batchId, ...(release.type === 'scene' && { shoot_id: release.shootId, studio_id: context.studioIdsBySlug[release.studio], production_date: release.productionDate, description: release.description, duration: release.duration, }), }) .returning(['id', 'entry_id']); const releaseWithId = { ...release, id: releaseEntry.id, entityName: entity.name, }; await Promise.all([ addReleaseMedia([releaseWithId.poster], releaseWithId, 'posters', context), ...(release.type === 'release' ? [ addReleaseTags(releaseWithId, context), addReleaseActors(releaseWithId, context), addReleaseDirectors(releaseWithId, context), addReleaseChapters(releaseWithId, context), addReleaseMedia(releaseWithId.photos, releaseWithId, 'photos', context), linkMovieScenes(releaseWithId, context), ] : []), ...(release.type === 'movie' ? [ addReleaseMedia(releaseWithId.covers, releaseWithId, 'covers', context), ] : []), ]); return releaseWithId; } async function load() { const file = await fs.promises.readFile(args.file, 'utf8'); const start = args.start || 0; const end = args.limit ? start + args.limit : Infinity; const releases = file.split('\n') .filter(Boolean) .map((data) => JSON.parse(data)) .filter((release) => (args.entity ? release.entity.slug === args.entity : true)) .slice(start, end); if (releases.length === 0) { console.log('Nothing to load'); return; } const [batchId] = await knex('batches').insert({ comment: `import ${args.file}` }).returning('id'); const aggTags = Array.from(new Set(releases.filter((release) => release.type === 'release').flatMap((release) => [...release.tags, ...release.chapters.flatMap((chapter) => chapter.tags)]).filter(Boolean))); const aggStudios = Array.from(new Set(releases.map((release) => release.studio).filter(Boolean))); const tags = await knex('tags') .select('id', 'slug') .whereIn('slug', aggTags); const studios = await knex('entities') .select('id', 'slug') .where('type', 'studio') .whereIn('slug', aggStudios); const tagIdsBySlug = Object.fromEntries(tags.map((tag) => [tag.slug, tag.id])); const studioIdsBySlug = Object.fromEntries(studios.map((studio) => [studio.slug, studio.id])); const addedMovies = await releases.filter((release) => release.type === 'movie').reduce(async (chain, release, index, array) => { const acc = await chain; const movie = await addRelease(release, { batchId, tagIdsBySlug, studioIdsBySlug }); if (!movie.skipped) { console.log(`Loaded ${index}/${array.length} '${movie.entityName}' movie "${movie.title}"`); } return acc.concat(movie); }, Promise.resolve([])); const addedScenes = await releases.filter((release) => release.type === 'release').reduce(async (chain, release, index, array) => { const acc = await chain; const scene = await addRelease(release, { batchId, movies: addedMovies, tagIdsBySlug, studioIdsBySlug }); if (!scene.skipped) { console.log(`Loaded ${index}/${array.length} '${scene.entityName}' scene "${scene.title}"`); } return acc.concat((!!scene && !scene.skipped)); }, Promise.resolve([])); console.log(`Loaded ${addedMovies.filter((movie) => movie && !movie.skipped).length}/${releases.filter((release) => release.type === 'movie').length} movies in batch ${batchId}`); console.log(`Loaded ${addedScenes.filter((scene) => scene && !scene.skipped).length}/${releases.filter((release) => release.type === 'release').length} scenes in batch ${batchId}`); process.exit(); } ({ save, load, })[args._]();