From 5f4039c5d4eabdb085d81b070802249d8913bf9a Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Sun, 20 Dec 2020 04:21:28 +0100 Subject: [PATCH] Added sharpness and re-added entropy to avatars, ignoring low-entropy photos as main avatar and in profile photo list. --- assets/components/actors/photos.vue | 7 +- assets/js/actors/actions.js | 2 + migrations/20190325001339_releases.js | 1 + package-lock.json | 164 +++++++++++++++----------- package.json | 2 +- src/actors.js | 4 +- src/media.js | 35 +++--- 7 files changed, 128 insertions(+), 87 deletions(-) diff --git a/assets/components/actors/photos.vue b/assets/components/actors/photos.vue index 520d7966..461cbbdd 100644 --- a/assets/components/actors/photos.vue +++ b/assets/components/actors/photos.vue @@ -25,7 +25,7 @@ diff --git a/assets/js/actors/actions.js b/assets/js/actors/actions.js index 897a3903..b32a0917 100644 --- a/assets/js/actors/actions.js +++ b/assets/js/actors/actions.js @@ -119,6 +119,8 @@ function initActorActions(store, router) { hash comment credit + entropy + sharpness sfw: sfwMedia { id thumbnail diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 8d1948c1..71b667cb 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -35,6 +35,7 @@ exports.up = knex => Promise.resolve() table.integer('width', 6); table.integer('height', 6); table.float('entropy'); + table.float('sharpness'); table.text('scraper', 32); table.text('credit', 100); diff --git a/package-lock.json b/package-lock.json index 64640ba8..b3c5b1d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2066,8 +2066,7 @@ "base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", - "dev": true + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -2150,13 +2149,24 @@ } }, "bl": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.1.tgz", - "integrity": "sha512-jrCW5ZhfQ/Vt07WX1Ngs+yn9BDqPL/gw28S7s9H6QK/gupnizNzJAss5akW20ISgOrbLTlXOOCTJeNUQqruAWQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", + "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", "requires": { - "readable-stream": "^3.0.1" + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" }, "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -2761,12 +2771,23 @@ } }, "color": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/color/-/color-3.1.2.tgz", - "integrity": "sha512-vXTJhHebByxZn3lDvDJYw4lR5+uB3vuoHsuYA5AKuxRVn5wzzIfQKGLBmgdVRHKTJYeK5rvJcHnrd0Li49CFpg==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/color/-/color-3.1.3.tgz", + "integrity": "sha512-xgXAcTHa2HeFCGLE9Xs/R82hujGtu9Jd9x4NW3T34+OMs7VoPsjwzRczKHvTAHeJwWFwX5j15+MgAppE8ztObQ==", "requires": { "color-convert": "^1.9.1", - "color-string": "^1.5.2" + "color-string": "^1.5.4" + }, + "dependencies": { + "color-string": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", + "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + } } }, "color-convert": { @@ -7720,9 +7741,9 @@ } }, "napi-build-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.1.tgz", - "integrity": "sha512-boQj1WFgQH3v4clhu3mTNfP+vOBxorDlE8EKiMjUlLG3C4qAESnn9AxIOkFgTR2c9LtzNjPrjS60cT27ZKBhaA==" + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" }, "natural-compare": { "version": "1.4.0", @@ -7768,13 +7789,18 @@ "dev": true }, "node-abi": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.15.0.tgz", - "integrity": "sha512-FeLpTS0F39U7hHZU1srAK4Vx+5AHNVOTP+hxBNQknR/54laTHSFIJkDWDqiquY1LeLUgTfPN7sLPhMubx0PLAg==", + "version": "2.19.3", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.19.3.tgz", + "integrity": "sha512-9xZrlyfvKhWme2EXFKQhZRp1yNWT/uI1luYPr3sFl+H4keYY4xR+1jO7mvTTijIsHf1M+QDe9uWuKeEpLInIlg==", "requires": { "semver": "^5.4.1" } }, + "node-addon-api": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz", + "integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw==" + }, "node-fetch": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", @@ -8907,9 +8933,9 @@ } }, "prebuild-install": { - "version": "5.3.6", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.6.tgz", - "integrity": "sha512-s8Aai8++QQGi4sSbs/M1Qku62PFK49Jm1CbgXklGz4nmHveDq0wzJkg7Na5QbnO1uNH8K7iqx2EQ/mV0MZEmOg==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.0.0.tgz", + "integrity": "sha512-h2ZJ1PXHKWZpp1caLw0oX9sagVpL2YTk+ZwInQbQ3QqNd4J03O6MpFNmMTJlkfgPENWqe5kP0WjQLqz5OjLfsw==", "requires": { "detect-libc": "^1.0.3", "expand-template": "^2.0.3", @@ -10071,62 +10097,64 @@ } }, "sharp": { - "version": "0.23.4", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.23.4.tgz", - "integrity": "sha512-fJMagt6cT0UDy9XCsgyLi0eiwWWhQRxbwGmqQT6sY8Av4s0SVsT/deg8fobBQCTDU5iXRgz0rAeXoE2LBZ8g+Q==", + "version": "0.26.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.26.3.tgz", + "integrity": "sha512-NdEJ9S6AMr8Px0zgtFo1TJjMK/ROMU92MkDtYn2BBrDjIx3YfH9TUyGdzPC+I/L619GeYQc690Vbaxc5FPCCWg==", "requires": { - "color": "^3.1.2", + "array-flatten": "^3.0.0", + "color": "^3.1.3", "detect-libc": "^1.0.3", - "nan": "^2.14.0", + "node-addon-api": "^3.0.2", "npmlog": "^4.1.2", - "prebuild-install": "^5.3.3", - "semver": "^6.3.0", - "simple-get": "^3.1.0", - "tar": "^5.0.5", + "prebuild-install": "^6.0.0", + "semver": "^7.3.2", + "simple-get": "^4.0.0", + "tar-fs": "^2.1.1", "tunnel-agent": "^0.6.0" }, "dependencies": { - "fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "array-flatten": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-3.0.0.tgz", + "integrity": "sha512-zPMVc3ZYlGLNk4mpK1NzP2wg0ml9t7fUgDsayR5Y5rSzxQilzR9FGu/EH2jQOcKSAeAfWeylyW8juy3OkWRvNA==" + }, + "decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "requires": { - "minipass": "^3.0.0" + "mimic-response": "^3.1.0" } }, - "minipass": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.1.tgz", - "integrity": "sha512-UFqVihv6PQgwj8/yTGvl9kPz7xIAY+R5z6XYjRInD3Gk3qx6QGSD6zEcpeG4Dy/lQnv1J6zv8ejV90hyYIKf3w==", + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "requires": { "yallist": "^4.0.0" } }, - "minizlib": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.0.tgz", - "integrity": "sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA==", - "requires": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" - } + "mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" - }, - "tar": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/tar/-/tar-5.0.5.tgz", - "integrity": "sha512-MNIgJddrV2TkuwChwcSNds/5E9VijOiw7kAc1y5hTNJoLDSuIyid2QtLYiCYNnICebpuvjhPQZsXwUL0O3l7OQ==", + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==", "requires": { - "chownr": "^1.1.3", - "fs-minipass": "^2.0.0", - "minipass": "^3.0.0", - "minizlib": "^2.1.0", - "mkdirp": "^0.5.0", - "yallist": "^4.0.0" + "lru-cache": "^6.0.0" + } + }, + "simple-get": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz", + "integrity": "sha512-ZalZGexYr3TA0SwySsr5HlgOOinS4Jsa8YB2GJ6lUNAazyAu4KG/VmzMTwAt2YVXzzVj8QmefmAonZIK2BSGcQ==", + "requires": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" } }, "yallist": { @@ -11016,22 +11044,22 @@ } }, "tar-fs": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz", - "integrity": "sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", "requires": { "chownr": "^1.1.1", - "mkdirp": "^0.5.1", + "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", - "tar-stream": "^2.0.0" + "tar-stream": "^2.1.4" } }, "tar-stream": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.0.tgz", - "integrity": "sha512-+DAn4Nb4+gz6WZigRzKEZl1QuJVOLtAwwF+WUxy1fJ6X63CaGaUAxJRD2KEn1OMfcbCjySTYpNC6WmfQoIEOdw==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz", + "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==", "requires": { - "bl": "^3.0.0", + "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", diff --git a/package.json b/package.json index 21309d7a..e8662ece 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,7 @@ "prop-types": "^15.7.2", "react": "^16.13.0", "react-dom": "^16.13.0", - "sharp": "^0.23.4", + "sharp": "^0.26.3", "showdown": "^1.9.1", "source-map-support": "^0.5.16", "template-format": "^1.2.5", diff --git a/src/actors.js b/src/actors.js index e4566b56..fc8ba61c 100644 --- a/src/actors.js +++ b/src/actors.js @@ -526,7 +526,9 @@ async function interpolateProfiles(actorIds) { profile.tattoos = getLongest(valuesByProperty.tattoos); profile.piercings = getLongest(valuesByProperty.piercings); - profile.avatar_media_id = avatars.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; + profile.avatar_media_id = avatars + .filter(avatar => avatar.entropy > 6) + .sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; return profile; }); diff --git a/src/media.js b/src/media.js index 7dc9c030..a4d8287d 100644 --- a/src/media.js +++ b/src/media.js @@ -126,7 +126,7 @@ function toBaseSource(rawSource) { return null; } -function baseSourceToBaseMedia(baseSource, role, metadata, options) { +function baseSourceToBaseMedia(baseSource, role, metadata) { if (Array.isArray(baseSource)) { if (baseSource.length > 0) { return { @@ -134,7 +134,6 @@ function baseSourceToBaseMedia(baseSource, role, metadata, options) { id: nanoid(), role, sources: baseSource, - ...options, }; } @@ -147,7 +146,6 @@ function baseSourceToBaseMedia(baseSource, role, metadata, options) { id: nanoid(), role, sources: [baseSource], - ...options, }; } @@ -281,7 +279,7 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) { throw new Error(`Could not extract source from ${baseSource.url}: ${res.status}`); } -async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath) { +async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options) { logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`); try { @@ -298,20 +296,22 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil ]); const image = sharp(media.file.path); - const info = await image.metadata(); const isProcessed = media.meta.subtype !== 'jpeg' || media.process; - console.log(media); + const [info, stats] = await Promise.all([ + image.metadata(), + options?.stats && image.stats(), + ]); if (media.process) { - Object.entries(media.process).forEach(([operation, options]) => { + Object.entries(media.process).forEach(([operation, processOptions]) => { if (image[operation]) { - image[operation](...(Array.isArray(options) ? options : [options])); + image[operation](...(Array.isArray(processOptions) ? processOptions : [processOptions])); return; } if (operation === 'crop') { - image.extract(...(Array.isArray(options) ? options : [options])); + image.extract(...(Array.isArray(processOptions) ? processOptions : [processOptions])); return; } @@ -365,6 +365,8 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil ...media.meta, width: info.width, height: info.height, + entropy: stats?.entropy || null, + sharpness: stats?.sharpness || null, }, }; } catch (error) { @@ -376,7 +378,7 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil } } -async function storeFile(media) { +async function storeFile(media, options) { try { const hashDir = media.meta.hash.slice(0, 2); const hashSubDir = media.meta.hash.slice(2, 4); @@ -399,7 +401,7 @@ async function storeFile(media) { } if (media.meta.type === 'image') { - return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath); + return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options); } const [stat] = await Promise.all([ @@ -609,6 +611,7 @@ function curateMediaEntry(media, index) { width: media.meta.width, height: media.meta.height, entropy: media.meta.entropy, + sharpness: media.meta.sharpness, source: media.src, source_page: media.url, scraper: media.scraper, @@ -623,7 +626,7 @@ function curateMediaEntry(media, index) { }; } -async function storeMedias(baseMedias) { +async function storeMedias(baseMedias, options) { await fsPromises.mkdir(path.join(config.media.path, 'temp'), { recursive: true }); const [existingSourceMediaByUrl, existingExtractMediaByUrl] = await findSourceDuplicates(baseMedias); @@ -638,7 +641,7 @@ async function storeMedias(baseMedias) { const savedMedias = await Promise.map( uniqueHashMedias, - async baseMedia => storeFile(baseMedia), + async baseMedia => storeFile(baseMedia, options), { concurrency: 100 }, // don't overload disk ); @@ -646,7 +649,7 @@ async function storeMedias(baseMedias) { // overwrite files in case image processing was changed await Promise.map( existingHashMedias, - async baseMedia => storeFile(baseMedia), + async baseMedia => storeFile(baseMedia, options), { concurrency: 100 }, // don't overload disk ); } @@ -733,14 +736,14 @@ async function associateAvatars(profiles) { avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', { credit: profile.credit || profile.entity?.name || null, scraper: profile.scraper || null, - }, { stats: true })[0], + })[0], } : profile )); const baseMedias = profilesWithBaseMedias.map(profile => profile.avatarBaseMedia).filter(Boolean); - const storedMedias = await storeMedias(baseMedias); + const storedMedias = await storeMedias(baseMedias, { stats: true }); const storedMediasById = itemsByKey(storedMedias, 'id'); const profilesWithAvatarIds = profilesWithBaseMedias.map((profile) => {