traxxx/src/tools/transfer.js

'use strict';

const config = require('config');
const fs = require('fs').promises;
const moment = require('moment');
const Promise = require('bluebird');
const { nanoid } = require('nanoid/non-secure');
const AWS = require('aws-sdk');

const { graphql } = require('../web/graphql');
const knex = require('../knex');
const args = require('../argv');

const endpoint = new AWS.Endpoint('s3.eu-central-1.wasabisys.com');

const s3 = new AWS.S3({
	// region: 'eu-central-1',
	endpoint,
	credentials: {
		accessKeyId: config.s3.accessKey,
		secretAccessKey: config.s3.secretKey,
	},
});

console.log(Object.keys(s3));

// NOT TRANSFERRED, unutilized on old server: production location, availabile qualities, actor alias for, actor entry id, chapter posters, chapter photos

const releaseFields = `
	entryId
	shootId
	title
	url
	date
	datePrecision
	productionDate
	description
	duration
	entity {
		slug
		type
	}
	studio {
		slug
	}
	actors: releasesActors {
		actor {
			name
			slug
			entryId
			entity {
				slug
				type
			}
		}
	}
	directors: releasesDirectors {
		director {
			slug
			entryId
			entity {
				slug
				type
			}
		}
	}
	tags: releasesTags {
		tag {
			slug
		}
	}
	chapters(orderBy: TIME_ASC) {
		index
		time
		duration
		title
		description
		tags: chaptersTags {
			tag {
				slug
			}
		}
	}
	poster: releasesPoster {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	photos: releasesPhotos {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	covers: releasesCovers {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	trailer: releasesTrailer {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	teaser: releasesTeaser {
		media {
			hash
			path
			thumbnail
			lazy
			s3: isS3
			mime
			index
			width
			height
			size
			source
			sourcePage
		}
	}
	createdAt
`;

async function save() {
	const limit = args.limit || 1000;
	const offset = args.offset || 0;

	const { releases } = await graphql(`
		query SearchReleases(
			$limit: Int = 20
			$offset: Int = 0
		) {
			releases(
				first: $limit
				offset: $offset
				orderBy: DATE_DESC
			) {
				${releaseFields}
			}
		}
	`, {
		limit,
		offset,
	});

	const curatedReleases = releases.map((release) => ({
		...release,
		actors: release.actors.filter(Boolean).map(({ actor }) => actor),
		directors: release.directors.filter(Boolean).map(({ director }) => director),
		studio: release.studio?.slug,
		tags: release.tags.map(({ tag }) => tag?.slug).filter(Boolean),
		chapters: release.chapters.filter(Boolean).map((chapter) => ({
			...chapter,
			tags: chapter.tags.map(({ tag }) => tag?.slug).filter(Boolean),
		})),
		poster: release.poster?.media,
		trailer: release.trailer?.media,
		teaser: release.teaser?.media,
		photos: release.photos.filter(Boolean).map(({ media }) => media),
		covers: release.covers.filter(Boolean).map(({ media }) => media),
	}));

	const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`;
	const serializedData = JSON.stringify(curatedReleases, null, 4);

	await fs.writeFile(filename, serializedData);

	console.log(`Saved ${releases.length} releases to ${filename}`);

	process.exit();
}

async function addReleaseTags(release, context) {
	if (release.tags.length === 0) {
		return;
	}

	await knex('releases_tags').insert(release.tags.map((tag) => ({
		tag_id: context.tagIdsBySlug[tag],
		release_id: release.id,
		original_tag: tag,
	})));
}

async function addNewActor(actor, entity, context) {
	const [actorId] = await knex('actors')
		.insert({
			name: actor.name,
			slug: actor.slug,
			entity_id: entity?.id,
			batch_id: context.batchId,
		})
		.returning('id');

	return actorId;
}

async function addReleaseActors(release, context, target = 'actor') {
	await release[`${target}s`].reduce(async (chain, actor) => {
		await chain;

		const entity = actor.entity
			? await knex('entities').where(actor.entity).first()
			: null;

		if (actor.entity && !entity) {
			throw new Error(`Actor ${actor.slug} contains non-existent ${release.entity.type} '${release.entity.slug}'`);
		}

		const existingActor = await knex('actors')
			.where('slug', actor.slug)
			.where((builder) => {
				if (entity) {
					builder.where('entity_id', entity.id);
					return;
				}

				builder.whereNull('entity_id');
			})
			.first();

		const actorId = existingActor?.id
			|| await addNewActor(actor, entity, context);

		await knex(`releases_${target}s`).insert({
			release_id: release.id,
			[`${target}_id`]: actorId,
		});
	}, Promise.resolve());
}

async function addReleaseDirectors(release, context) {
	return addReleaseActors(release, context, 'director');
}

async function addReleaseChapters(release, context) {
	await release.chapters.reduce(async (chain, chapter) => {
		await chain;

		const [chapterId] = await knex('chapters')
			.insert({
				release_id: release.id,
				index: chapter.index,
				time: chapter.time,
				duration: chapter.duration,
				description: chapter.description,
			})
			.returning('id');

		if (chapter.tags.length > 0) {
			await knex('chapters_tags').insert(chapter.tags.map((tag) => ({
				tag_id: context.tagIdsBySlug[tag],
				chapter_id: chapterId,
				original_tag: tag,
			})));
		}
	}, Promise.resolve());
}

async function addReleaseMedia(medias, release, target) {
	return Promise.all(medias.filter(Boolean).map(async (media) => {
		try {
			const id = nanoid();

			await knex('media').insert({
				id,
				hash: media.hash,
				path: media.path,
				thumbnail: media.thumbnail,
				lazy: media.lazy,
				is_s3: media.s3,
				index: media.index,
				mime: media.mime,
				size: media.size,
				width: media.width,
				height: media.height,
				source: media.source,
				source_page: media.sourcePage,
			});

			await knex(`releases_${target}`).insert({
				release_id: release.id,
				media_id: id,
			});
		} catch (error) {
			console.log(`Skipped existing media ${media.hash} from ${media.url}: ${error.message}`);
		}
	}));
}

async function addRelease(release, context) {
	const existingRelease = await knex('releases')
		.leftJoin('entities', 'entities.id', 'releases.entity_id')
		.where('entry_id', release.entryId)
		.where('entities.slug', release.entity.slug)
		.where('entities.type', release.entity.type)
		.first();

	if (existingRelease) {
		return false;
	}

	const [entity] = await Promise.all([
		knex('entities').select('id').where(release.entity).first(),
	]);

	if (!entity) {
		throw new Error(`Release contains non-existent ${release.entity.type} '${release.entity.slug}'`);
	}

	const [releaseId] = await knex('releases')
		.insert({
			entry_id: release.entryId,
			entity_id: entity.id,
			studio_id: context.studioIdsBySlug[release.studio],
			shoot_id: release.shootId,
			url: release.url,
			title: release.title,
			slug: release.slug,
			date: release.date,
			date_precision: release.datePrecision,
			production_date: release.productionDate,
			description: release.description,
			duration: release.duration,
			created_batch_id: context.batchId,
			updated_batch_id: context.batchId,
		})
		.returning('id');

	const releaseWithId = { ...release, id: releaseId };

	await Promise.all([
		addReleaseTags(releaseWithId, context),
		addReleaseActors(releaseWithId, context),
		addReleaseDirectors(releaseWithId, context),
		addReleaseChapters(releaseWithId, context),
		addReleaseMedia([releaseWithId.poster], releaseWithId, 'posters', context),
		addReleaseMedia(releaseWithId.photos, releaseWithId, 'photos', context),
		// addReleaseMedia(releaseWithId.covers, releaseWithId, 'covers', context),
	]);

	return true;
}

async function load() {
	const file = await fs.readFile(args.file, 'utf8');
	const releases = JSON.parse(file);

	const [batchId] = await knex('batches').insert({ comment: `import ${args.file}` }).returning('id');

	const aggTags = Array.from(new Set(releases.flatMap((release) => [...release.tags, ...release.chapters.flatMap((chapter) => chapter.tags)]).filter(Boolean)));
	const aggStudios = Array.from(new Set(releases.map((release) => release.studio).filter(Boolean)));

	const tags = await knex('tags')
		.select('id', 'slug')
		.whereIn('slug', aggTags);

	const studios = await knex('entities')
		.select('id', 'slug')
		.where('type', 'studio')
		.whereIn('slug', aggStudios);

	const tagIdsBySlug = Object.fromEntries(tags.map((tag) => [tag.slug, tag.id]));
	const studioIdsBySlug = Object.fromEntries(studios.map((studio) => [studio.slug, studio.id]));

	const added = await releases.reduce(async (chain, release) => {
		const acc = await chain;
		const isAdded = await addRelease(release, { batchId, tagIdsBySlug, studioIdsBySlug });

		return acc.concat(isAdded);
	}, Promise.resolve([]));

	console.log(`Loaded ${added.filter(Boolean).length}/${releases.length} scenes in batch ${batchId}`);

	process.exit();
}

({
	save,
	load,
})[args._]();