traxxx/src/tools/transfer.js

'use strict';

const config = require('config');
const fs = require('fs');
const path = require('path');
const moment = require('moment');
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { nanoid } = require('nanoid/non-secure');
const AWS = require('aws-sdk');

const { graphql } = require('../web/graphql');
const knex = require('../knex');
const args = require('../argv');

const endpoint = new AWS.Endpoint('s3.eu-central-1.wasabisys.com');

const s3 = new AWS.S3({
	// region: 'eu-central-1',
	endpoint,
	credentials: {
		accessKeyId: config.s3.accessKey,
		secretAccessKey: config.s3.secretKey,
	},
});

// NOT TRANSFERRED, unutilized on old server: production location, availabile qualities, actor alias for, actor entry id, chapter posters, chapter photos

const sceneFields = `
	entryId
	shootId
	title
	url
	date
	datePrecision
	productionDate
	description
	duration
	entity {
		slug
		type
	}
	studio {
		slug
	}
	movies: moviesScenesBySceneId {
		movie {
			title
			entryId
			entity {
				slug
				type
			}
		}
	}
	actors: releasesActors {
		actor {
			name
			slug
			entryId
			entity {
				slug
				type
			}
		}
	}
	directors: releasesDirectors {
		director {
			name
			slug
			entryId
			entity {
				slug
				type
			}
		}
	}
	tags: releasesTags {
		tag {
			slug
		}
	}
	chapters(orderBy: TIME_ASC) {
		index
		time
		duration
		title
		description
		tags: chaptersTags {
			tag {
				slug
			}
		}
	}
	poster: releasesPoster {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	photos: releasesPhotos {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	covers: releasesCovers {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	trailer: releasesTrailer {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	teaser: releasesTeaser {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	createdAt
`;

const movieFields = `
	entryId
	title
	url
	date
	datePrecision
	entity {
		slug
		type
	}
	poster: moviesPoster {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	covers: moviesCovers {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	createdAt
`;

async function save() {
	const limit = args.limit || 1000;
	const offset = args.start || 0;

	const { releases } = await graphql(`
		query SearchScenes(
			$limit: Int = 20
			$offset: Int = 0
		) {
			releases(
				first: $limit
				offset: $offset
				orderBy: DATE_DESC
			) {
				${sceneFields}
			}
		}
	`, {
		limit,
		offset,
	}, 'owner');

	const { movies } = await graphql(`
		query SearchScenes(
			$limit: Int = 20
			$offset: Int = 0
		) {
			movies(
				first: $limit
				offset: $offset
				orderBy: DATE_DESC
			) {
				${movieFields}
			}
		}
	`, {
		limit,
		offset,
	}, 'owner');

	const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`;

	let savedScenes = 0;
	let savedMovies = 0;

	await releases.reduce(async (chain, release) => {
		await chain;

		const entry = JSON.stringify({
			...release,
			type: 'release',
			description: release.description.replace(/\n/g, ' '),
			actors: release.actors.filter(Boolean).map(({ actor }) => actor),
			directors: release.directors.filter(Boolean).map(({ director }) => director),
			studio: release.studio?.slug,
			tags: release.tags.map(({ tag }) => tag?.slug).filter(Boolean),
			movies: release.movies?.map(({ movie }) => movie) || [],
			chapters: release.chapters.filter(Boolean).map((chapter) => ({
				...chapter,
				tags: chapter.tags.map(({ tag }) => tag?.slug).filter(Boolean),
			})),
			poster: release.poster?.media,
			trailer: release.trailer?.media,
			teaser: release.teaser?.media,
			photos: release.photos.filter(Boolean).map(({ media }) => media),
			covers: release.covers.filter(Boolean).map(({ media }) => media),
		});

		await fs.promises.appendFile(filename, `${entry}\n`);

		savedScenes += 1;
	}, Promise.resolve());

	await movies.reduce(async (chain, movie) => {
		await chain;

		const entry = JSON.stringify({
			...movie,
			type: 'movie',
			poster: movie.poster?.media,
			covers: movie.covers.filter(Boolean).map(({ media }) => media),
		});

		await fs.promises.appendFile(filename, `${entry}\n`);

		savedMovies += 1;
	}, Promise.resolve());

	console.log(`Saved ${savedScenes} scenes and ${savedMovies} movies to ${filename}`);

	process.exit();
}

async function addReleaseTags(release, context) {
	if (release.tags.length === 0) {
		return;
	}

	await knex('releases_tags').insert(release.tags.map((tag) => ({
		tag_id: context.tagIdsBySlug[tag],
		release_id: release.id,
		original_tag: tag,
	})));
}

async function addNewActor(actor, entity, context) {
	const [actorId] = await knex('actors')
		.insert({
			name: actor.name,
			slug: actor.slug,
			entity_id: entity?.id,
			batch_id: context.batchId,
		})
		.returning('id');

	return actorId;
}

async function addReleaseActors(release, context, target = 'actor') {
	await release[`${target}s`].reduce(async (chain, actor) => {
		await chain;

		const entity = actor.entity
			? await knex('entities').where(actor.entity).first()
			: null;

		if (actor.entity && !entity) {
			throw new Error(`Actor ${actor.slug} contains non-existent ${release.entity.type} '${release.entity.slug}'`);
		}

		const existingActor = await knex('actors')
			.where('slug', actor.slug)
			.where((builder) => {
				if (entity) {
					builder.where('entity_id', entity.id);
					return;
				}

				builder.whereNull('entity_id');
			})
			.first();

		const actorId = existingActor?.id
			|| await addNewActor(actor, entity, context);

		await knex(`releases_${target}s`).insert({
			release_id: release.id,
			[`${target}_id`]: actorId,
		});
	}, Promise.resolve());
}

async function addReleaseDirectors(release, context) {
	return addReleaseActors(release, context, 'director');
}

async function addReleaseChapters(release, context) {
	await release.chapters.reduce(async (chain, chapter) => {
		await chain;

		const [chapterId] = await knex('chapters')
			.insert({
				release_id: release.id,
				index: chapter.index,
				time: chapter.time,
				duration: chapter.duration,
				description: chapter.description,
			})
			.returning('id');

		if (chapter.tags.length > 0) {
			await knex('chapters_tags').insert(chapter.tags.map((tag) => ({
				tag_id: context.tagIdsBySlug[tag],
				chapter_id: chapterId,
				original_tag: tag,
			})));
		}
	}, Promise.resolve());
}

const dirs = {
	path: '',
	thumbnail: 'thumbs',
	lazy: 'lazy',
};

async function transferMedia(media, target) {
	return ['path', 'thumbnail', 'lazy'].reduce(async (chain, type) => {
		await chain;

		const filename = `${media.hash}${path.extname(media[type])}`;
		const filepath = path.join(target, dirs[type], filename);
		const temp = path.join('media/temp', filepath);
		const url = new URL(media[type], `${media.s3 ? config.media.transferSources.s3 : config.media.transferSources.local}/`).href;

		console.log('Transferring media', url);

		const res = await bhttp.get(url, { stream: true });

		if (res.statusCode !== 200) {
			console.warn(`Missing ${target} ${url}`);
			return;
		}

		await fs.promises.mkdir(path.dirname(temp), { recursive: true });

		await new Promise((resolve, reject) => {
			const fileStream = fs.createWriteStream(temp);

			res.pipe(fileStream);

			res.on('error', () => { reject(); });

			fileStream.on('finish', () => { resolve(); });
			fileStream.on('error', () => { reject(); });
		});

		await s3.upload({
			Bucket: config.s3.bucket,
			Body: fs.createReadStream(temp),
			Key: filepath,
			ContentType: media.mime,
		}).promise();

		await fs.promises.unlink(temp);
	}, Promise.resolve());
}

async function addReleaseMedia(medias, release, target) {
	await medias.filter(Boolean).reduce(async (chain, media) => {
		await chain;

		const existingMedia = await knex('media')
			.where('hash', media.hash)
			.first();

		const id = existingMedia?.id || nanoid();

		if (!existingMedia) {
			await knex('media').insert({
				id,
				hash: media.hash,
				path: path.join(target, '', `${media.hash}${path.extname(media.path)}`),
				thumbnail: path.join(target, 'thumbs', `${media.hash}${path.extname(media.thumbnail)}`),
				lazy: path.join(target, 'lazy', `${media.hash}${path.extname(media.lazy)}`),
				// is_s3: media.s3,
				is_s3: true,
				index: media.index,
				mime: media.mime,
				size: media.size,
				width: media.width,
				height: media.height,
				source: media.source,
				source_page: media.sourcePage,
			});

			await transferMedia(media, target);
		}

		await knex(`${release.type}s_${target}`).insert({
			[`${release.type}_id`]: release.id,
			media_id: id,
		});
	}, Promise.resolve());
}

async function linkMovieScenes(release, context) {
	await release.movies.reduce(async (chain, linkedMovie) => {
		await chain;

		const movie = context.movies.find((storedMovie) => storedMovie.entryId === linkedMovie.entryId
			&& storedMovie.entity.slug === linkedMovie.entity.slug
			&& storedMovie.entity.type === linkedMovie.entity.type);

		console.log('movie', linkedMovie, movie);

		if (!movie) {
			throw new Error(`Missing ${linkedMovie.entity.slug} movie '${linkedMovie.title}' in '${release.title}'`);
		}

		await knex('movies_scenes').insert({
			movie_id: movie.id,
			scene_id: release.id,
		});
	}, Promise.resolve());
}

async function addRelease(release, context) {
	const existingRelease = await knex(`${release.type}s`)
		.select(`${release.type}s.*`, 'entities.name as entity_name')
		.leftJoin('entities', 'entities.id', `${release.type}s.entity_id`)
		.where('entry_id', release.entryId)
		.where('entities.slug', release.entity.slug)
		.where('entities.type', release.entity.type)
		.first();

	if (existingRelease) {
		console.log(`Skipping ${release.entity.slug} release "${release.title}", already in database`);

		return {
			...release,
			id: existingRelease.id,
			entityName: existingRelease.entity_name,
		};
	}

	const [entity] = await Promise.all([
		knex('entities').select(['id', 'name']).where(release.entity).first(),
	]);

	if (!entity) {
		throw new Error(`Release contains non-existent ${release.entity.type} '${release.entity.slug}'`);
	}

	const [releaseEntry] = await knex(`${release.type}s`)
		.insert({
			entry_id: release.entryId,
			entity_id: entity.id,
			url: release.url,
			title: release.title,
			slug: release.slug,
			date: release.date,
			date_precision: release.datePrecision,
			created_batch_id: context.batchId,
			updated_batch_id: context.batchId,
			...(release.type === 'scene' && {
				shoot_id: release.shootId,
				studio_id: context.studioIdsBySlug[release.studio],
				production_date: release.productionDate,
				description: release.description,
				duration: release.duration,
			}),
		})
		.returning(['id', 'entry_id']);

	const releaseWithId = {
		...release,
		id: releaseEntry.id,
		entityName: entity.name,
	};

	await Promise.all([
		addReleaseMedia([releaseWithId.poster], releaseWithId, 'posters', context),
		...(release.type === 'release' ? [
			addReleaseTags(releaseWithId, context),
			addReleaseActors(releaseWithId, context),
			addReleaseDirectors(releaseWithId, context),
			addReleaseChapters(releaseWithId, context),
			addReleaseMedia(releaseWithId.photos, releaseWithId, 'photos', context),
			linkMovieScenes(releaseWithId, context),
		] : []),
		...(release.type === 'movie' ? [
			addReleaseMedia(releaseWithId.covers, releaseWithId, 'covers', context),
		] : []),
	]);

	return releaseWithId;
}

async function load() {
	const file = await fs.promises.readFile(args.file, 'utf8');
	const start = args.start || 0;
	const end = args.limit ? start + args.limit : Infinity;

	const releases = file.split('\n')
		.filter(Boolean)
		.map((data) => JSON.parse(data))
		.filter((release) => (args.entity ? release.entity.slug === args.entity : true))
		.slice(start, end);

	if (releases.length === 0) {
		console.log('Nothing to load');
		return;
	}

	const [batchId] = await knex('batches').insert({ comment: `import ${args.file}` }).returning('id');

	const aggTags = Array.from(new Set(releases.filter((release) => release.type === 'release').flatMap((release) => [...release.tags, ...release.chapters.flatMap((chapter) => chapter.tags)]).filter(Boolean)));
	const aggStudios = Array.from(new Set(releases.map((release) => release.studio).filter(Boolean)));

	const tags = await knex('tags')
		.select('id', 'slug')
		.whereIn('slug', aggTags);

	const studios = await knex('entities')
		.select('id', 'slug')
		.where('type', 'studio')
		.whereIn('slug', aggStudios);

	const tagIdsBySlug = Object.fromEntries(tags.map((tag) => [tag.slug, tag.id]));
	const studioIdsBySlug = Object.fromEntries(studios.map((studio) => [studio.slug, studio.id]));

	const addedMovies = await releases.filter((release) => release.type === 'movie').reduce(async (chain, release) => {
		const acc = await chain;
		const movie = await addRelease(release, { batchId, tagIdsBySlug, studioIdsBySlug });

		console.log(`Loaded '${movie.entityName}' movie "${movie.title}"`);

		return acc.concat(movie);
	}, Promise.resolve([]));

	const addedScenes = await releases.filter((release) => release.type === 'release').reduce(async (chain, release) => {
		const acc = await chain;
		const scene = await addRelease(release, { batchId, movies: addedMovies, tagIdsBySlug, studioIdsBySlug });

		console.log(`Loaded '${scene.entityName}' scene "${scene.title}"`);

		return acc.concat(!!scene);
	}, Promise.resolve([]));

	console.log(`Loaded ${addedMovies.filter(Boolean).length}/${releases.filter((release) => release.type === 'movie').length} movies in batch ${batchId}`);
	console.log(`Loaded ${addedScenes.filter(Boolean).length}/${releases.filter((release) => release.type === 'release').length} scenes in batch ${batchId}`);

	process.exit();
}

({
	save,
	load,
})[args._]();