Drastically improved memory performance media module using streams and temp files.

src/utils/http.js

@@ -1,10 +1,13 @@
 'use strict';
 
+const util = require('util');
+const stream = require('stream');
 const config = require('config');
 const tunnel = require('tunnel');
 const bhttp = require('bhttp');
 const taskQueue = require('promise-task-queue');
 
+const pipeline = util.promisify(stream.pipeline);
 const logger = require('../logger')(__filename);
 
 const defaultHeaders = {
@@ -68,6 +71,10 @@ queue.define('http', async ({
         ? await bhttp[method.toLowerCase()](url, body, reqOptions)
         : await bhttp[method.toLowerCase()](url, reqOptions);
 
+    if (options.stream && options.destination) {
+        await pipeline(res, ...(options.transforms || []), options.destination);
+    }
+
     const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
     const json = Buffer.isBuffer(res.body) ? null : res.body;
 

src/utils/media.js

@@ -0,0 +1,110 @@
+'use strict';
+
+const config = require('config');
+const fs = require('fs');
+const fsPromises = require('fs').promises;
+const Promise = require('bluebird');
+const blake2 = require('blake2');
+const sharp = require('sharp');
+const nanoid = require('nanoid');
+const { PassThrough } = require('stream');
+
+const http = require('./http');
+
+function getMemoryUsage() {
+    return process.memoryUsage().rss / (10 ** 6);
+}
+
+let peakMemoryUsage = getMemoryUsage();
+
+async function fetchSource(link) {
+    const id = nanoid();
+
+    const hasher = new blake2.Hash('blake2b');
+    hasher.setEncoding('hex');
+
+    const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
+    const tempFileStream = fs.createWriteStream(tempFilePath);
+    const hashStream = new PassThrough();
+
+    hashStream.on('data', chunk => hasher.write(chunk));
+
+    try {
+        const res = await http.get(link, null, {
+            stream: true,
+            transforms: [hashStream],
+            destination: tempFileStream,
+            timeout: 5000,
+        });
+
+        if (!res.ok) {
+            throw new Error(res.status);
+        }
+
+        hasher.end();
+        const hash = hasher.read();
+
+        const memoryUsage = getMemoryUsage();
+        peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
+
+        console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
+
+        return {
+            id,
+            path: tempFilePath,
+            hash,
+        };
+    } catch (error) {
+        await fsPromises.unlink(tempFilePath);
+
+        throw error;
+    }
+}
+
+async function init() {
+    const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
+    const links = linksFile.split('\n').filter(Boolean);
+
+    await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
+
+    console.time('thumbs');
+
+    const files = await Promise.map(links, async (link) => {
+        try {
+            return await fetchSource(link);
+        } catch (error) {
+            console.log(`Failed to fetch ${link}: ${error.message}`);
+            return null;
+        }
+    });
+
+    await Promise.map(files.filter(Boolean), async (file) => {
+        const image = sharp(file.path).jpeg();
+
+        const [{ width, height }, { size }] = await Promise.all([
+            image.metadata(),
+            fsPromises.stat(file.path),
+        ]);
+
+        await Promise.all([
+            image
+                .toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
+            image
+                .resize({
+                    height: config.media.thumbnailSize,
+                    withoutEnlargement: true,
+                })
+                .toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
+        ]);
+
+        const memoryUsage = getMemoryUsage();
+        peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
+
+        console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
+    }, { concurrency: 10 });
+
+    console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
+    console.timeEnd('thumbs');
+}
+
+init();