'use strict'; const config = require('config'); const fs = require('fs').promises; const moment = require('moment'); const Promise = require('bluebird'); const { nanoid } = require('nanoid/non-secure'); const AWS = require('aws-sdk'); const { graphql } = require('../web/graphql'); const knex = require('../knex'); const args = require('../argv'); const endpoint = new AWS.Endpoint('s3.eu-central-1.wasabisys.com'); const s3 = new AWS.S3({ // region: 'eu-central-1', endpoint, credentials: { accessKeyId: config.s3.accessKey, secretAccessKey: config.s3.secretKey, }, }); console.log(Object.keys(s3)); // NOT TRANSFERRED, unutilized on old server: production location, availabile qualities, actor alias for, actor entry id, chapter posters, chapter photos const releaseFields = ` entryId shootId title url date datePrecision productionDate description duration entity { slug type } studio { slug } actors: releasesActors { actor { name slug entryId entity { slug type } } } directors: releasesDirectors { director { slug entryId entity { slug type } } } tags: releasesTags { tag { slug } } chapters(orderBy: TIME_ASC) { index time duration title description tags: chaptersTags { tag { slug } } } poster: releasesPoster { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } photos: releasesPhotos { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } covers: releasesCovers { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } trailer: releasesTrailer { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } teaser: releasesTeaser { media { hash path thumbnail lazy s3: isS3 mime index width height size source sourcePage } } createdAt `; async function save() { const limit = args.limit || 1000; const offset = args.offset || 0; const { releases } = await graphql(` query SearchReleases( $limit: Int = 20 $offset: Int = 0 ) { releases( first: $limit offset: $offset orderBy: DATE_DESC ) { ${releaseFields} } } `, { limit, offset, }); const curatedReleases = releases.map((release) => ({ ...release, actors: release.actors.filter(Boolean).map(({ actor }) => actor), directors: release.directors.filter(Boolean).map(({ director }) => director), studio: release.studio?.slug, tags: release.tags.map(({ tag }) => tag?.slug).filter(Boolean), chapters: release.chapters.filter(Boolean).map((chapter) => ({ ...chapter, tags: chapter.tags.map(({ tag }) => tag?.slug).filter(Boolean), })), poster: release.poster?.media, trailer: release.trailer?.media, teaser: release.teaser?.media, photos: release.photos.filter(Boolean).map(({ media }) => media), covers: release.covers.filter(Boolean).map(({ media }) => media), })); const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`; const serializedData = JSON.stringify(curatedReleases, null, 4); await fs.writeFile(filename, serializedData); console.log(`Saved ${releases.length} releases to ${filename}`); process.exit(); } async function addReleaseTags(release, context) { if (release.tags.length === 0) { return; } await knex('releases_tags').insert(release.tags.map((tag) => ({ tag_id: context.tagIdsBySlug[tag], release_id: release.id, original_tag: tag, }))); } async function addNewActor(actor, entity, context) { const [actorId] = await knex('actors') .insert({ name: actor.name, slug: actor.slug, entity_id: entity?.id, batch_id: context.batchId, }) .returning('id'); return actorId; } async function addReleaseActors(release, context, target = 'actor') { await release[`${target}s`].reduce(async (chain, actor) => { await chain; const entity = actor.entity ? await knex('entities').where(actor.entity).first() : null; if (actor.entity && !entity) { throw new Error(`Actor ${actor.slug} contains non-existent ${release.entity.type} '${release.entity.slug}'`); } const existingActor = await knex('actors') .where('slug', actor.slug) .where((builder) => { if (entity) { builder.where('entity_id', entity.id); return; } builder.whereNull('entity_id'); }) .first(); const actorId = existingActor?.id || await addNewActor(actor, entity, context); await knex(`releases_${target}s`).insert({ release_id: release.id, [`${target}_id`]: actorId, }); }, Promise.resolve()); } async function addReleaseDirectors(release, context) { return addReleaseActors(release, context, 'director'); } async function addReleaseChapters(release, context) { await release.chapters.reduce(async (chain, chapter) => { await chain; const [chapterId] = await knex('chapters') .insert({ release_id: release.id, index: chapter.index, time: chapter.time, duration: chapter.duration, description: chapter.description, }) .returning('id'); if (chapter.tags.length > 0) { await knex('chapters_tags').insert(chapter.tags.map((tag) => ({ tag_id: context.tagIdsBySlug[tag], chapter_id: chapterId, original_tag: tag, }))); } }, Promise.resolve()); } async function addReleaseMedia(medias, release, target) { return Promise.all(medias.filter(Boolean).map(async (media) => { try { const id = nanoid(); await knex('media').insert({ id, hash: media.hash, path: media.path, thumbnail: media.thumbnail, lazy: media.lazy, is_s3: media.s3, index: media.index, mime: media.mime, size: media.size, width: media.width, height: media.height, source: media.source, source_page: media.sourcePage, }); await knex(`releases_${target}`).insert({ release_id: release.id, media_id: id, }); } catch (error) { console.log(`Skipped existing media ${media.hash} from ${media.url}: ${error.message}`); } })); } async function addRelease(release, context) { const existingRelease = await knex('releases') .leftJoin('entities', 'entities.id', 'releases.entity_id') .where('entry_id', release.entryId) .where('entities.slug', release.entity.slug) .where('entities.type', release.entity.type) .first(); if (existingRelease) { return false; } const [entity] = await Promise.all([ knex('entities').select('id').where(release.entity).first(), ]); if (!entity) { throw new Error(`Release contains non-existent ${release.entity.type} '${release.entity.slug}'`); } const [releaseId] = await knex('releases') .insert({ entry_id: release.entryId, entity_id: entity.id, studio_id: context.studioIdsBySlug[release.studio], shoot_id: release.shootId, url: release.url, title: release.title, slug: release.slug, date: release.date, date_precision: release.datePrecision, production_date: release.productionDate, description: release.description, duration: release.duration, created_batch_id: context.batchId, updated_batch_id: context.batchId, }) .returning('id'); const releaseWithId = { ...release, id: releaseId }; await Promise.all([ addReleaseTags(releaseWithId, context), addReleaseActors(releaseWithId, context), addReleaseDirectors(releaseWithId, context), addReleaseChapters(releaseWithId, context), addReleaseMedia([releaseWithId.poster], releaseWithId, 'posters', context), addReleaseMedia(releaseWithId.photos, releaseWithId, 'photos', context), // addReleaseMedia(releaseWithId.covers, releaseWithId, 'covers', context), ]); return true; } async function load() { const file = await fs.readFile(args.file, 'utf8'); const releases = JSON.parse(file); const [batchId] = await knex('batches').insert({ comment: `import ${args.file}` }).returning('id'); const aggTags = Array.from(new Set(releases.flatMap((release) => [...release.tags, ...release.chapters.flatMap((chapter) => chapter.tags)]).filter(Boolean))); const aggStudios = Array.from(new Set(releases.map((release) => release.studio).filter(Boolean))); const tags = await knex('tags') .select('id', 'slug') .whereIn('slug', aggTags); const studios = await knex('entities') .select('id', 'slug') .where('type', 'studio') .whereIn('slug', aggStudios); const tagIdsBySlug = Object.fromEntries(tags.map((tag) => [tag.slug, tag.id])); const studioIdsBySlug = Object.fromEntries(studios.map((studio) => [studio.slug, studio.id])); const added = await releases.reduce(async (chain, release) => { const acc = await chain; const isAdded = await addRelease(release, { batchId, tagIdsBySlug, studioIdsBySlug }); return acc.concat(isAdded); }, Promise.resolve([])); console.log(`Loaded ${added.filter(Boolean).length}/${releases.length} scenes in batch ${batchId}`); process.exit(); } ({ save, load, })[args._]();