boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added sharpness and re-added entropy to avatars, ignoring low-entropy photos as main avatar and in profile photo list.

This commit is contained in:
DebaucheryLibrarian 2020-12-20 04:21:28 +01:00
parent cbcac0725d
commit 5f4039c5d4
7 changed files with 128 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
assets/components/actors/photos.vue
View File

@ -25,7 +25,7 @@
</a>
<a
v-for="photo in actor.photos"
v-for="photo in photos"
:key="`photo-${photo.id}`"
:href="`/media/${photo.path}`"
target="_blank"
@ -45,6 +45,10 @@
</template>
<script>
function photos() {
return this.actor.photos.filter(photo => photo.entropy > 6);
}
function sfw() {
return this.$store.state.ui.sfw;
}
@ -58,6 +62,7 @@ export default {
},
computed: {
sfw,
photos,
},
};
</script>

2
assets/js/actors/actions.js
View File

@ -119,6 +119,8 @@ function initActorActions(store, router) {
hash
comment
credit
entropy
sharpness
sfw: sfwMedia {
id
thumbnail

1
migrations/20190325001339_releases.js
View File

@ -35,6 +35,7 @@ exports.up = knex => Promise.resolve()
table.integer('width', 6);
table.integer('height', 6);
table.float('entropy');
table.float('sharpness');
table.text('scraper', 32);
table.text('credit', 100);

164
package-lock.json generated
View File

@ -2066,8 +2066,7 @@
"base64-js": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
"integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==",
"dev": true
"integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
},
"bcrypt-pbkdf": {
"version": "1.0.2",
@ -2150,13 +2149,24 @@
}
},
"bl": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/bl/-/bl-3.0.1.tgz",
"integrity": "sha512-jrCW5ZhfQ/Vt07WX1Ngs+yn9BDqPL/gw28S7s9H6QK/gupnizNzJAss5akW20ISgOrbLTlXOOCTJeNUQqruAWQ==",
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
"integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
"requires": {
"readable-stream": "^3.0.1"
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
},
"dependencies": {
"buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@ -2761,12 +2771,23 @@
}
},
"color": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/color/-/color-3.1.2.tgz",
"integrity": "sha512-vXTJhHebByxZn3lDvDJYw4lR5+uB3vuoHsuYA5AKuxRVn5wzzIfQKGLBmgdVRHKTJYeK5rvJcHnrd0Li49CFpg==",
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/color/-/color-3.1.3.tgz",
"integrity": "sha512-xgXAcTHa2HeFCGLE9Xs/R82hujGtu9Jd9x4NW3T34+OMs7VoPsjwzRczKHvTAHeJwWFwX5j15+MgAppE8ztObQ==",
"requires": {
"color-convert": "^1.9.1",
"color-string": "^1.5.2"
"color-string": "^1.5.4"
},
"dependencies": {
"color-string": {
"version": "1.5.4",
"resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz",
"integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==",
"requires": {
"color-name": "^1.0.0",
"simple-swizzle": "^0.2.2"
}
}
}
},
"color-convert": {
@ -7720,9 +7741,9 @@
}
},
"napi-build-utils": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.1.tgz",
"integrity": "sha512-boQj1WFgQH3v4clhu3mTNfP+vOBxorDlE8EKiMjUlLG3C4qAESnn9AxIOkFgTR2c9LtzNjPrjS60cT27ZKBhaA=="
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
"integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
},
"natural-compare": {
"version": "1.4.0",
@ -7768,13 +7789,18 @@
"dev": true
},
"node-abi": {
"version": "2.15.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.15.0.tgz",
"integrity": "sha512-FeLpTS0F39U7hHZU1srAK4Vx+5AHNVOTP+hxBNQknR/54laTHSFIJkDWDqiquY1LeLUgTfPN7sLPhMubx0PLAg==",
"version": "2.19.3",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.19.3.tgz",
"integrity": "sha512-9xZrlyfvKhWme2EXFKQhZRp1yNWT/uI1luYPr3sFl+H4keYY4xR+1jO7mvTTijIsHf1M+QDe9uWuKeEpLInIlg==",
"requires": {
"semver": "^5.4.1"
}
},
"node-addon-api": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz",
"integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw=="
},
"node-fetch": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz",
@ -8907,9 +8933,9 @@
}
},
"prebuild-install": {
"version": "5.3.6",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.6.tgz",
"integrity": "sha512-s8Aai8++QQGi4sSbs/M1Qku62PFK49Jm1CbgXklGz4nmHveDq0wzJkg7Na5QbnO1uNH8K7iqx2EQ/mV0MZEmOg==",
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.0.0.tgz",
"integrity": "sha512-h2ZJ1PXHKWZpp1caLw0oX9sagVpL2YTk+ZwInQbQ3QqNd4J03O6MpFNmMTJlkfgPENWqe5kP0WjQLqz5OjLfsw==",
"requires": {
"detect-libc": "^1.0.3",
"expand-template": "^2.0.3",
@ -10071,62 +10097,64 @@
}
},
"sharp": {
"version": "0.23.4",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.23.4.tgz",
"integrity": "sha512-fJMagt6cT0UDy9XCsgyLi0eiwWWhQRxbwGmqQT6sY8Av4s0SVsT/deg8fobBQCTDU5iXRgz0rAeXoE2LBZ8g+Q==",
"version": "0.26.3",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.26.3.tgz",
"integrity": "sha512-NdEJ9S6AMr8Px0zgtFo1TJjMK/ROMU92MkDtYn2BBrDjIx3YfH9TUyGdzPC+I/L619GeYQc690Vbaxc5FPCCWg==",
"requires": {
"color": "^3.1.2",
"array-flatten": "^3.0.0",
"color": "^3.1.3",
"detect-libc": "^1.0.3",
"nan": "^2.14.0",
"node-addon-api": "^3.0.2",
"npmlog": "^4.1.2",
"prebuild-install": "^5.3.3",
"semver": "^6.3.0",
"simple-get": "^3.1.0",
"tar": "^5.0.5",
"prebuild-install": "^6.0.0",
"semver": "^7.3.2",
"simple-get": "^4.0.0",
"tar-fs": "^2.1.1",
"tunnel-agent": "^0.6.0"
},
"dependencies": {
"fs-minipass": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
"integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
"array-flatten": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-3.0.0.tgz",
"integrity": "sha512-zPMVc3ZYlGLNk4mpK1NzP2wg0ml9t7fUgDsayR5Y5rSzxQilzR9FGu/EH2jQOcKSAeAfWeylyW8juy3OkWRvNA=="
},
"decompress-response": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
"requires": {
"minipass": "^3.0.0"
"mimic-response": "^3.1.0"
}
},
"minipass": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.1.tgz",
"integrity": "sha512-UFqVihv6PQgwj8/yTGvl9kPz7xIAY+R5z6XYjRInD3Gk3qx6QGSD6zEcpeG4Dy/lQnv1J6zv8ejV90hyYIKf3w==",
"lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"requires": {
"yallist": "^4.0.0"
}
},
"minizlib": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.0.tgz",
"integrity": "sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA==",
"requires": {
"minipass": "^3.0.0",
"yallist": "^4.0.0"
}
"mimic-response": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
},
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
},
"tar": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/tar/-/tar-5.0.5.tgz",
"integrity": "sha512-MNIgJddrV2TkuwChwcSNds/5E9VijOiw7kAc1y5hTNJoLDSuIyid2QtLYiCYNnICebpuvjhPQZsXwUL0O3l7OQ==",
"version": "7.3.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
"integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
"requires": {
"chownr": "^1.1.3",
"fs-minipass": "^2.0.0",
"minipass": "^3.0.0",
"minizlib": "^2.1.0",
"mkdirp": "^0.5.0",
"yallist": "^4.0.0"
"lru-cache": "^6.0.0"
}
},
"simple-get": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz",
"integrity": "sha512-ZalZGexYr3TA0SwySsr5HlgOOinS4Jsa8YB2GJ6lUNAazyAu4KG/VmzMTwAt2YVXzzVj8QmefmAonZIK2BSGcQ==",
"requires": {
"decompress-response": "^6.0.0",
"once": "^1.3.1",
"simple-concat": "^1.0.0"
}
},
"yallist": {
@ -11016,22 +11044,22 @@
}
},
"tar-fs": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz",
"integrity": "sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA==",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
"integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
"requires": {
"chownr": "^1.1.1",
"mkdirp": "^0.5.1",
"mkdirp-classic": "^0.5.2",
"pump": "^3.0.0",
"tar-stream": "^2.0.0"
"tar-stream": "^2.1.4"
}
},
"tar-stream": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.0.tgz",
"integrity": "sha512-+DAn4Nb4+gz6WZigRzKEZl1QuJVOLtAwwF+WUxy1fJ6X63CaGaUAxJRD2KEn1OMfcbCjySTYpNC6WmfQoIEOdw==",
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz",
"integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==",
"requires": {
"bl": "^3.0.0",
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",

2
package.json
View File

@ -117,7 +117,7 @@
"prop-types": "^15.7.2",
"react": "^16.13.0",
"react-dom": "^16.13.0",
"sharp": "^0.23.4",
"sharp": "^0.26.3",
"showdown": "^1.9.1",
"source-map-support": "^0.5.16",
"template-format": "^1.2.5",

4
src/actors.js
View File

@ -526,7 +526,9 @@ async function interpolateProfiles(actorIds) {
profile.tattoos = getLongest(valuesByProperty.tattoos);
profile.piercings = getLongest(valuesByProperty.piercings);
profile.avatar_media_id = avatars.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
profile.avatar_media_id = avatars
.filter(avatar => avatar.entropy > 6)
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
return profile;
});

35
src/media.js
View File

@ -126,7 +126,7 @@ function toBaseSource(rawSource) {
return null;
}
function baseSourceToBaseMedia(baseSource, role, metadata, options) {
function baseSourceToBaseMedia(baseSource, role, metadata) {
if (Array.isArray(baseSource)) {
if (baseSource.length > 0) {
return {
@ -134,7 +134,6 @@ function baseSourceToBaseMedia(baseSource, role, metadata, options) {
id: nanoid(),
role,
sources: baseSource,
...options,
};
}
@ -147,7 +146,6 @@ function baseSourceToBaseMedia(baseSource, role, metadata, options) {
id: nanoid(),
role,
sources: [baseSource],
...options,
};
}
@ -281,7 +279,7 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
throw new Error(`Could not extract source from ${baseSource.url}: ${res.status}`);
}
async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath) {
async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options) {
logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`);
try {
@ -298,20 +296,22 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
]);
const image = sharp(media.file.path);
const info = await image.metadata();
const isProcessed = media.meta.subtype !== 'jpeg' || media.process;
console.log(media);
const [info, stats] = await Promise.all([
image.metadata(),
options?.stats && image.stats(),
]);
if (media.process) {
Object.entries(media.process).forEach(([operation, options]) => {
Object.entries(media.process).forEach(([operation, processOptions]) => {
if (image[operation]) {
image[operation](...(Array.isArray(options) ? options : [options]));
image[operation](...(Array.isArray(processOptions) ? processOptions : [processOptions]));
return;
}
if (operation === 'crop') {
image.extract(...(Array.isArray(options) ? options : [options]));
image.extract(...(Array.isArray(processOptions) ? processOptions : [processOptions]));
return;
}
@ -365,6 +365,8 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
...media.meta,
width: info.width,
height: info.height,
entropy: stats?.entropy || null,
sharpness: stats?.sharpness || null,
},
};
} catch (error) {
@ -376,7 +378,7 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
}
}
async function storeFile(media) {
async function storeFile(media, options) {
try {
const hashDir = media.meta.hash.slice(0, 2);
const hashSubDir = media.meta.hash.slice(2, 4);
@ -399,7 +401,7 @@ async function storeFile(media) {
}
if (media.meta.type === 'image') {
return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath);
return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options);
}
const [stat] = await Promise.all([
@ -609,6 +611,7 @@ function curateMediaEntry(media, index) {
width: media.meta.width,
height: media.meta.height,
entropy: media.meta.entropy,
sharpness: media.meta.sharpness,
source: media.src,
source_page: media.url,
scraper: media.scraper,
@ -623,7 +626,7 @@ function curateMediaEntry(media, index) {
};
}
async function storeMedias(baseMedias) {
async function storeMedias(baseMedias, options) {
await fsPromises.mkdir(path.join(config.media.path, 'temp'), { recursive: true });
const [existingSourceMediaByUrl, existingExtractMediaByUrl] = await findSourceDuplicates(baseMedias);
@ -638,7 +641,7 @@ async function storeMedias(baseMedias) {
const savedMedias = await Promise.map(
uniqueHashMedias,
async baseMedia => storeFile(baseMedia),
async baseMedia => storeFile(baseMedia, options),
{ concurrency: 100 }, // don't overload disk
);
@ -646,7 +649,7 @@ async function storeMedias(baseMedias) {
// overwrite files in case image processing was changed
await Promise.map(
existingHashMedias,
async baseMedia => storeFile(baseMedia),
async baseMedia => storeFile(baseMedia, options),
{ concurrency: 100 }, // don't overload disk
);
}
@ -733,14 +736,14 @@ async function associateAvatars(profiles) {
avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', {
credit: profile.credit || profile.entity?.name || null,
scraper: profile.scraper || null,
}, { stats: true })[0],
})[0],
}
: profile
));
const baseMedias = profilesWithBaseMedias.map(profile => profile.avatarBaseMedia).filter(Boolean);
const storedMedias = await storeMedias(baseMedias);
const storedMedias = await storeMedias(baseMedias, { stats: true });
const storedMediasById = itemsByKey(storedMedias, 'id');
const profilesWithAvatarIds = profilesWithBaseMedias.map((profile) => {