DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Compare commits

base: DebaucheryLibrarian:06fa428790fa3a8e34215734d56826920f01142b
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental
...
compare: DebaucheryLibrarian:5afd86a932e3dd9c3cbbd9815a9f96d1f17cd5f4
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental

2 Commits
06fa428790 ... 5afd86a932

Author SHA1 Message Date
ThePendulum 5afd86a932 1.99.1 2020-02-22 23:50:28 +01:00
ThePendulum 74274f879e Re-added image entropy filter. 2020-02-22 23:50:25 +01:00
3 changed files with 4 additions and 4 deletions
Show Stats Expand all files Collapse all files

2
package-lock.json generated
View File

@ -1,6 +1,6 @@
{ {
"name": "traxxx", "name": "traxxx",
"version": "1.99.0", "version": "1.99.1",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "traxxx", "name": "traxxx",
"version": "1.99.0", "version": "1.99.1",
"description": "All the latest porn releases in one place", "description": "All the latest porn releases in one place",
"main": "src/app.js", "main": "src/app.js",
"scripts": { "scripts": {

4
src/media.js
View File

@ -231,7 +231,7 @@ function groupItems(items) {
}); });
} }
async function storeMedia(sources, domain, role) { async function storeMedia(sources, domain, role, { entropyFilter = 2.5 } = {}) {
const presentSources = sources.filter(Boolean); const presentSources = sources.filter(Boolean);
if (presentSources.length === 0) { if (presentSources.length === 0) {
@ -250,7 +250,7 @@ async function storeMedia(sources, domain, role) {
const { hash: fetchedItemsByHash } = groupItems(fetchedItems); const { hash: fetchedItemsByHash } = groupItems(fetchedItems);
// find hash duplicates that don't need to be re-saved // find hash duplicates that don't need to be re-saved
const uniqueFetchedItems = Object.values(fetchedItemsByHash); const uniqueFetchedItems = Object.values(fetchedItemsByHash).filter(item => !entropyFilter || item.entropy >= entropyFilter);
const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash)); const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash));
const { hash: existingHashItemsByHash } = groupItems(existingHashItems); const { hash: existingHashItemsByHash } = groupItems(existingHashItems);