DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Re-added image entropy filter.

This commit is contained in:
ThePendulum 2020-02-22 23:50:25 +01:00
parent 06fa428790
commit 74274f879e
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/media.js
View File

@ -231,7 +231,7 @@ function groupItems(items) {
}); });
} }
async function storeMedia(sources, domain, role) { async function storeMedia(sources, domain, role, { entropyFilter = 2.5 } = {}) {
const presentSources = sources.filter(Boolean); const presentSources = sources.filter(Boolean);
if (presentSources.length === 0) { if (presentSources.length === 0) {
@ -250,7 +250,7 @@ async function storeMedia(sources, domain, role) {
const { hash: fetchedItemsByHash } = groupItems(fetchedItems); const { hash: fetchedItemsByHash } = groupItems(fetchedItems);
// find hash duplicates that don't need to be re-saved // find hash duplicates that don't need to be re-saved
const uniqueFetchedItems = Object.values(fetchedItemsByHash); const uniqueFetchedItems = Object.values(fetchedItemsByHash).filter(item => !entropyFilter || item.entropy >= entropyFilter);
const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash)); const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash));
const { hash: existingHashItemsByHash } = groupItems(existingHashItems); const { hash: existingHashItemsByHash } = groupItems(existingHashItems);