Simplified tag media seed file. Renamed tags. Media refactor experiment.

src/argv.js

@@ -70,6 +70,11 @@ const { argv } = yargs
         type: 'boolean',
         default: true,
     })
+    .option('level', {
+        describe: 'Log level',
+        type: 'string',
+        default: process.env.NODE_ENV === 'development' ? 'silly' : 'info',
+    })
     .option('debug', {
         describe: 'Show error stack traces',
         type: 'boolean',

src/logger.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const util = require('util');
+const winston = require('winston');
+const args = require('./argv');
+
+const logger = winston.createLogger({
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.printf(info => `${info.timestamp} ${info.level}: ${util.inspect(info.message)}`),
+    ),
+    transports: [
+        new winston.transports.Console({
+            level: args.level,
+            format: winston.format.combine(
+                winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+                winston.format.colorize(),
+                winston.format.printf(info => `${info.timestamp} ${info.level}: ${info.message}`),
+            ),
+            timestamp: true,
+        }),
+    ],
+});
+
+module.exports = logger;

src/media.js

@@ -9,6 +9,7 @@ const mime = require('mime');
 const sharp = require('sharp');
 const blake2 = require('blake2');
 
+const logger = require('./logger');
 const knex = require('./knex');
 const upsert = require('./utils/upsert');
 
@@ -75,11 +76,11 @@ async function findDuplicates(photos, identifier, prop = null, label) {
         : !duplicateLookup.has(prop ? source[prop] : source)));
 
     if (duplicates.length > 0) {
-        console.log(`${duplicates.length} media items already present by ${identifier} for ${label}`);
+        logger.info(`${duplicates.length} media items already present by ${identifier} for ${label}`);
     }
 
     if (originals.length > 0) {
-        console.log(`Fetching ${originals.length} new media items for ${label}`);
+        logger.info(`Fetching ${originals.length} new media items for ${label}`);
     }
 
     return [duplicates, originals];
@@ -181,8 +182,6 @@ async function storePhotos(photos, {
         concurrency: 10,
     }).filter(photo => photo);
 
-    return metaFiles;
-
     const [hashDuplicates, hashOriginals] = await findDuplicates(metaFiles, 'hash', 'hash', label);
 
     const savedPhotos = await savePhotos(hashOriginals, {
@@ -230,6 +229,39 @@ async function storePhotos(photos, {
     await upsert(`${domain}s_${role}s`, photoAssociations, [`${domain}_id`, 'media_id']);
 }
 
+/*
+async function storeReleasePhotos(releases, label) {
+    const sources = releases.map(release => pluckPhotos(release.photos)).flat();
+    const uniqueSources = Array.from(new Set(sources));
+
+    const [sourceDuplicates, sourceOriginals] = await findDuplicates(uniqueSources, 'source', null, label);
+
+    const metaFiles = await Promise.map(
+        sourceOriginals,
+        async (photoUrl, index) => fetchPhoto(photoUrl, index, label),
+        { concurrency: 10 },
+    )
+        .filter(photo => photo);
+
+    const hashUniques = Object.values(metaFiles.reduce((acc, file) => {
+        if (!acc[file.hash]) acc[file.hash] = file;
+
+        return acc;
+    }, {}));
+
+    const [hashDuplicates, hashOriginals] = await findDuplicates(hashUniques, 'hash', 'hash', label);
+
+    const sourceHashes = metaFiles.concat(sourceDuplicates).reduce((acc, file) => {
+        acc[file.source] = file.hash;
+        return acc;
+    }, {});
+
+    const associations = releases.map(release => release.photos.map(source => [release.id, sourceHashes[source]])).flat();
+
+    console.log(associations);
+}
+*/
+
 async function storeTrailer(trailers, {
     domain = 'releases',
     targetId,
@@ -292,5 +324,6 @@ async function storeTrailer(trailers, {
 module.exports = {
     createMediaDirectory,
     storePhotos,
+    // storeReleasePhotos,
     storeTrailer,
 };

src/releases.js

@@ -11,6 +11,7 @@ const { associateActors } = require('./actors');
 const {
     createMediaDirectory,
     storePhotos,
+    // storeReleasePhotos,
     storeTrailer,
 } = require('./media');
 const { fetchSites, findSiteByUrl } = require('./sites');
@@ -309,41 +310,49 @@ function accumulateMovies(releases) {
     }, {});
 }
 
-async function storeReleaseAssets(release, releaseId) {
-    const subpath = `${release.site.network.slug}/${release.site.slug}/${release.id}/`;
-    const identifier = `"${release.title}" (${releaseId})`;
+async function storeReleaseAssets(releases) {
+    // await storeReleasePhotos(releases);
 
-    await createMediaDirectory('releases', subpath);
+    await Promise.map(releases, async (release) => {
+        const subpath = `${release.site.network.slug}/${release.site.slug}/${release.id}/`;
+        const identifier = `"${release.title}" (${release.id})`;
 
-    try {
-        // don't use Promise.all to prevent concurrency issues with duplicate detection
-        if (release.poster) {
-            await storePhotos([release.poster], {
-                role: 'poster',
-                targetId: releaseId,
+        await createMediaDirectory('releases', subpath);
+
+        console.log(release.id);
+
+        try {
+            // don't use Promise.all to prevent concurrency issues with duplicate detection
+            if (release.poster) {
+                await storePhotos([release.poster], {
+                    role: 'poster',
+                    targetId: release.id,
+                    subpath,
+                }, identifier);
+            }
+
+            await storePhotos(release.photos, {
+                targetId: release.id,
+                subpath,
+                primaryRole: release.poster ? null : 'poster',
+            }, identifier);
+
+            await storePhotos(release.covers, {
+                role: 'cover',
+                targetId: release.id,
                 subpath,
             }, identifier);
+
+            await storeTrailer(release.trailer, {
+                targetId: release.id,
+                subpath,
+            }, identifier);
+        } catch (error) {
+            console.log(release.url, error);
         }
-
-        await storePhotos(release.photos, {
-            targetId: releaseId,
-            subpath,
-            primaryRole: release.poster ? null : 'poster',
-        }, identifier);
-
-        await storePhotos(release.covers, {
-            role: 'cover',
-            targetId: releaseId,
-            subpath,
-        }, identifier);
-
-        await storeTrailer(release.trailer, {
-            targetId: releaseId,
-            subpath,
-        }, identifier);
-    } catch (error) {
-        console.log(release.url, error);
-    }
+    }, {
+        concurrency: 10,
+    });
 }
 
 async function storeRelease(release) {
@@ -414,9 +423,7 @@ async function storeReleases(releases) {
 
     await Promise.all([
         associateActors(actors, storedReleases),
-        Promise.map(storedReleases, async release => storeReleaseAssets(release, release.id), {
-            concurrency: 10,
-        }),
+        storeReleaseAssets(storedReleases),
     ]);
 
     return {