DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Hashing media in transform in an attempt to improve reliability. Added seed file for user abilities.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-06-17 00:57:45 +02:00
parent 43d58ff093
commit da6b54079f
1 changed files with 21 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/media.js
View File

@@ -28,7 +28,6 @@ const chunk = require('./utils/chunk');
const { get } = require('./utils/qu'); const { get } = require('./utils/qu');
const { fetchEntityReleaseIds } = require('./entity-releases'); const { fetchEntityReleaseIds } = require('./entity-releases');
// const pipeline = util.promisify(stream.pipeline);
const streamQueue = taskQueue(); const streamQueue = taskQueue();
const s3 = new S3Client({ const s3 = new S3Client({
@@ -516,16 +515,6 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
writeLazy(image, lazypath, info), writeLazy(image, lazypath, info),
]); ]);
/*
if (isProcessed) {
// file already stored, remove temporary file
await fsPromises.unlink(media.file.path);
} else {
// image not processed, simply move temporary file to final location
await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
}
*/
await fsPromises.unlink(media.file.path); await fsPromises.unlink(media.file.path);
if (config.s3.enabled) { if (config.s3.enabled) {
@@ -731,31 +720,31 @@ async function fetchSource(source, baseMedia) {
} }
async function attempt(attempts = 1) { async function attempt(attempts = 1) {
const hasher = new blake2.Hash('blake2b', { digestLength: 24 }); const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
let hasherReady = true; let tempFileTarget;
hasher.setEncoding('hex');
const hasher = blake2.createHash('blake2b', { digestLength: 24 });
let size = 0;
const hashStream = new stream.Transform({
transform(streamChunk, _encoding, callback) {
size += streamChunk.length;
hasher.update(streamChunk);
this.push(streamChunk);
callback();
},
});
try { try {
const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`); tempFileTarget = fs.createWriteStream(tempFilePath);
const tempFileTarget = fs.createWriteStream(tempFilePath);
const hashStream = new stream.PassThrough();
let size = 0;
hashStream.on('data', (streamChunk) => {
size += streamChunk.length;
if (hasherReady) {
hasher.write(streamChunk);
}
});
const { mimetype } = source.stream const { mimetype } = source.stream
? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream }) ? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream })
: await fetchHttpSource(source, tempFileTarget, hashStream); : await fetchHttpSource(source, tempFileTarget, hashStream);
hasher.end(); const hash = hasher.digest('hex');
const hash = hasher.read();
const [type, subtype] = mimetype.split('/'); const [type, subtype] = mimetype.split('/');
const extension = mime.getExtension(mimetype); const extension = mime.getExtension(mimetype);
@@ -778,8 +767,10 @@ async function fetchSource(source, baseMedia) {
}, },
}; };
} catch (error) { } catch (error) {
hasherReady = false; // hasherReady = false;
hasher.end(); // hasher.end();
hashStream.destroy();
tempFileTarget?.destroy();
if (error.code !== 'VERIFY_TYPE') { if (error.code !== 'VERIFY_TYPE') {
logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`); logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);