111 lines
2.6 KiB
JavaScript
111 lines
2.6 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const fs = require('fs');
|
|
const fsPromises = require('fs').promises;
|
|
const Promise = require('bluebird');
|
|
const blake2 = require('blake2');
|
|
const sharp = require('sharp');
|
|
const nanoid = require('nanoid');
|
|
const { PassThrough } = require('stream');
|
|
|
|
const http = require('./http');
|
|
|
|
function getMemoryUsage() {
|
|
return process.memoryUsage().rss / (10 ** 6);
|
|
}
|
|
|
|
let peakMemoryUsage = getMemoryUsage();
|
|
|
|
async function fetchSource(link) {
|
|
const id = nanoid();
|
|
|
|
const hasher = new blake2.Hash('blake2b');
|
|
hasher.setEncoding('hex');
|
|
|
|
const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
|
|
const tempFileStream = fs.createWriteStream(tempFilePath);
|
|
const hashStream = new PassThrough();
|
|
|
|
hashStream.on('data', (chunk) => hasher.write(chunk));
|
|
|
|
try {
|
|
const res = await http.get(link, null, {
|
|
stream: true,
|
|
transforms: [hashStream],
|
|
destination: tempFileStream,
|
|
timeout: 5000,
|
|
});
|
|
|
|
if (!res.ok) {
|
|
throw new Error(res.status);
|
|
}
|
|
|
|
hasher.end();
|
|
const hash = hasher.read();
|
|
|
|
const memoryUsage = getMemoryUsage();
|
|
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
|
|
|
|
console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
|
|
|
|
return {
|
|
id,
|
|
path: tempFilePath,
|
|
hash,
|
|
};
|
|
} catch (error) {
|
|
await fsPromises.unlink(tempFilePath);
|
|
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function init() {
|
|
const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
|
|
const links = linksFile.split('\n').filter(Boolean);
|
|
|
|
await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
|
|
|
|
console.time('thumbs');
|
|
|
|
const files = await Promise.map(links, async (link) => {
|
|
try {
|
|
return await fetchSource(link);
|
|
} catch (error) {
|
|
console.log(`Failed to fetch ${link}: ${error.message}`);
|
|
return null;
|
|
}
|
|
});
|
|
|
|
await Promise.map(files.filter(Boolean), async (file) => {
|
|
const image = sharp(file.path).jpeg();
|
|
|
|
const [{ width, height }, { size }] = await Promise.all([
|
|
image.metadata(),
|
|
fsPromises.stat(file.path),
|
|
]);
|
|
|
|
await Promise.all([
|
|
image
|
|
.toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
|
|
image
|
|
.resize({
|
|
height: config.media.thumbnailSize,
|
|
withoutEnlargement: true,
|
|
})
|
|
.toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
|
|
]);
|
|
|
|
const memoryUsage = getMemoryUsage();
|
|
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
|
|
|
|
console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
|
|
}, { concurrency: 10 });
|
|
|
|
console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
|
|
console.timeEnd('thumbs');
|
|
}
|
|
|
|
init();
|