DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added batch ID and better feedback to StashDB import tool.

This commit is contained in:
DebaucheryLibrarian 2026-01-26 02:14:18 +01:00
parent 40bb73e897
commit 36ba4542a6
2 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
migrations/20260125052234_fingerprints.js
View File

@ -42,6 +42,11 @@ exports.up = async (knex) => {
table.integer('source_submissions'); table.integer('source_submissions');
table.json('source_meta'); table.json('source_meta');
table.integer('batch_id')
.notNullable()
.references('id')
.inTable('batches');
table.datetime('source_created_at'); table.datetime('source_created_at');
table.datetime('created_at') table.datetime('created_at')

12
src/tools/stashdb-hashes.js
View File

@ -1,6 +1,6 @@
'use strict'; 'use strict';
const util = require('util'); // const util = require('util');
const unprint = require('unprint'); const unprint = require('unprint');
const args = require('yargs').argv; const args = require('yargs').argv;
@ -79,7 +79,10 @@ async function curateData(data) {
return release; return release;
}); });
console.log(`Found ${stashScenes.length} scenes in StashDB`);
const stashScenesByIdentifiers = Object.fromEntries(stashScenes.flatMap((scene) => [[scene.entryId || scene.urlId, scene], [scene.url, scene]])); const stashScenesByIdentifiers = Object.fromEntries(stashScenes.flatMap((scene) => [[scene.entryId || scene.urlId, scene], [scene.url, scene]]));
const [{ id: batchId }] = await knex('batches').insert({ showcased: false, comment: 'StashDB fingerprints' }).returning('id');
const sceneEntries = await knex('releases') const sceneEntries = await knex('releases')
.select('releases.*') .select('releases.*')
@ -88,6 +91,8 @@ async function curateData(data) {
.where('entities.slug', 'hardx') .where('entities.slug', 'hardx')
.whereIn('entry_id', stashScenes.map((scene) => scene.entryId || scene.urlId)); .whereIn('entry_id', stashScenes.map((scene) => scene.entryId || scene.urlId));
console.log(`Matched ${sceneEntries.length} scenes`);
const fpEntries = sceneEntries.flatMap((scene) => { const fpEntries = sceneEntries.flatMap((scene) => {
const stashScene = stashScenesByIdentifiers[scene.entry_id] const stashScene = stashScenesByIdentifiers[scene.entry_id]
|| stashScenesByIdentifiers[scene.url]; || stashScenesByIdentifiers[scene.url];
@ -108,15 +113,16 @@ async function curateData(data) {
url: stashScene.url, url: stashScene.url,
date: stashScene.date, date: stashScene.date,
}, },
batch_id: batchId,
})); }));
}); });
console.log(util.inspect(fpEntries, { colors: true, depth: null, maxArrayLength: null })); // console.log(util.inspect(fpEntries, { colors: true, depth: null, maxArrayLength: null }));
if (args.update) { if (args.update) {
await bulkInsert('releases_fingerprints', fpEntries, false); await bulkInsert('releases_fingerprints', fpEntries, false);
console.log(`Inserted ${fpEntries.length} hash entries`); console.log(`Inserted ${fpEntries.length} hash entries in batch ${batchId}`);
} }
} }