Added primitive thumbnailer.

This commit is contained in:
DebaucheryLibrarian 2023-06-26 00:58:44 +02:00
parent e5b102ce07
commit 83fcdba93a
9 changed files with 2532 additions and 4 deletions

1
.gitignore vendored
View File

@ -2,6 +2,7 @@ node_modules/
dist/ dist/
config/ config/
log/ log/
media/
!config/default.js !config/default.js
assets/js/config/ assets/js/config/
!assets/js/config/default.js !assets/js/config/default.js

View File

@ -27,7 +27,14 @@
class="title-link" class="title-link"
> >
<img <img
v-if="post.hasThumbnail"
class="thumbnail" class="thumbnail"
:src="`/media/thumbnails/${post.id}.jpeg`"
>
<img
v-else
class="thumbnail missing"
:src="blockedIcon" :src="blockedIcon"
> >
</a> </a>
@ -172,11 +179,16 @@ async function submitVote(value) {
width: 7rem; width: 7rem;
height: 4rem; height: 4rem;
box-sizing: border-box; box-sizing: border-box;
padding: 1rem;
border-radius: .25rem; border-radius: .25rem;
margin: .5rem; margin: .5rem;
background: var(--grey-light-10); object-fit: cover;
opacity: .25;
&.missing {
padding: 1rem;
background: var(--grey-light-10);
opacity: .25;
object-fit: contain;
}
} }
.votes { .votes {

View File

@ -115,6 +115,10 @@ export async function up(knex) {
.references('id') .references('id')
.inTable('users'); .inTable('users');
table.text('thumbnail')
.notNullable()
.default(false);
table.datetime('created_at') table.datetime('created_at')
.notNullable() .notNullable()
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
@ -205,6 +209,7 @@ export async function down(knex) {
await knex.schema.dropTableIfExists('posts_votes'); await knex.schema.dropTableIfExists('posts_votes');
await knex.schema.dropTableIfExists('posts'); await knex.schema.dropTableIfExists('posts');
await knex.schema.dropTableIfExists('shelves_settings'); await knex.schema.dropTableIfExists('shelves_settings');
await knex.schema.dropTableIfExists('shelves_subscriptions');
await knex.schema.dropTableIfExists('shelves'); await knex.schema.dropTableIfExists('shelves');
await knex.schema.dropTableIfExists('users'); await knex.schema.dropTableIfExists('users');

2405
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -20,6 +20,7 @@
"build": "vite build", "build": "vite build",
"server": "node --experimental-specifier-resolution=node ./src/web/server", "server": "node --experimental-specifier-resolution=node ./src/web/server",
"server:prod": "cross-env NODE_ENV=production node ./src/web/server", "server:prod": "cross-env NODE_ENV=production node ./src/web/server",
"thumbs": "node --experimental-specifier-resolution=node ./src/cli thumbs",
"migrate-make": "knex migrate:make", "migrate-make": "knex migrate:make",
"migrate": "knex migrate:latest", "migrate": "knex migrate:latest",
"rollback": "knex migrate:rollback" "rollback": "knex migrate:rollback"
@ -55,8 +56,10 @@
"pg": "^8.11.0", "pg": "^8.11.0",
"pinia": "^2.1.3", "pinia": "^2.1.3",
"redis": "^4.6.6", "redis": "^4.6.6",
"sharp": "^0.32.1",
"short-uuid": "^4.2.2", "short-uuid": "^4.2.2",
"sirv": "^2.0.2", "sirv": "^2.0.2",
"unprint": "^0.9.3",
"vite": "^4.0.3", "vite": "^4.0.3",
"vite-plugin-ssr": "^0.4.126", "vite-plugin-ssr": "^0.4.126",
"vite-plugin-vue-markdown": "^0.23.5", "vite-plugin-vue-markdown": "^0.23.5",

View File

@ -1,7 +1,8 @@
// import config from 'config'; // import config from 'config';
import yargs from 'yargs'; import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
const { argv } = yargs const { argv } = yargs(hideBin(process.argv))
.command('npm start') .command('npm start')
.option('log-level', { .option('log-level', {
alias: 'level', alias: 'level',

View File

@ -5,8 +5,13 @@ import { HttpError } from './errors';
import { fetchShelf, fetchShelves } from './shelves'; import { fetchShelf, fetchShelves } from './shelves';
import { fetchUsers } from './users'; import { fetchUsers } from './users';
import { generateThumbnail } from './thumbnails';
import slugify from './utils/slugify'; import slugify from './utils/slugify';
import initLogger from './logger';
const logger = initLogger();
const emptyVote = { const emptyVote = {
tally: 0, tally: 0,
total: 0, total: 0,
@ -25,6 +30,7 @@ function curateDatabasePost(post, {
link: post.link, link: post.link,
shelfId: post.shelf_id, shelfId: post.shelf_id,
createdAt: post.created_at, createdAt: post.created_at,
hasThumbnail: post.thumbnail,
shelf: shelf || shelves?.[post.shelf_id], shelf: shelf || shelves?.[post.shelf_id],
user: users.find((user) => user.id === post.user_id), user: users.find((user) => user.id === post.user_id),
vote: vote || votes?.[post.id] || emptyVote, vote: vote || votes?.[post.id] || emptyVote,
@ -166,6 +172,8 @@ async function votePost(postId, value, user) {
.merge(); .merge();
} }
logger.silly(`User ${user.username} voted ${value} on post ${postId}`);
const votes = await fetchPostVotes([postId], user); const votes = await fetchPostVotes([postId], user);
return votes[postId] || emptyVote; return votes[postId] || emptyVote;
@ -198,6 +206,10 @@ async function createPost(post, shelfId, user) {
votePost(postEntry.id, 1, user), votePost(postEntry.id, 1, user),
]); ]);
logger.verbose(`User ${user.username} created post ${postEntry.id} on s/${shelf.slug}`);
generateThumbnail(postEntry.id);
return curateDatabasePost(postEntry, { shelf, users, vote }); return curateDatabasePost(postEntry, { shelf, users, vote });
} }

80
src/thumbnails.js Normal file
View File

@ -0,0 +1,80 @@
import bhttp from 'bhttp';
import unprint from 'unprint';
import fs from 'fs';
import sharp from 'sharp';
import knex from './knex';
import initLogger from './logger';
const logger = initLogger();
async function createThumbnail(data, post) {
await fs.promises.mkdir('media/thumbnails', { recursive: true });
await sharp(data)
.resize(300, 300)
.jpeg({
quality: 80,
})
.toFile(`media/thumbnails/${post.id}.jpeg`);
await knex('posts')
.where('id', post.id)
.update('thumbnail', true);
logger.debug(`Saved thumbnail for ${post.id}`);
}
async function generateThumbnail(postId) {
const post = await knex('posts').where('id', postId).first();
if (!post) {
logger.warn(`Cannot generate thumbnail for non-existent post ${postId}`);
return;
}
if (!post.link) {
logger.debug(`Skipped thumbnail generation for text-only post ${postId}`);
return;
}
const res = await bhttp.get(post.link);
if (res.statusCode !== 200) {
logger.warn(`Failed thumbnail generation for ${post.link} from ${postId} (${res.statusCode})`);
return;
}
if (res.headers['content-type'].includes('image/')) {
await createThumbnail(res.body, post);
return;
}
const { query } = unprint.init(res.body.toString());
const ogHeader = query.attribute('meta[property="og:image"]', 'content');
if (ogHeader) {
const imgRes = await bhttp.get(ogHeader);
if (imgRes.statusCode === 200 && imgRes.headers['content-type'].includes('image/')) {
await createThumbnail(imgRes.body, post);
}
}
}
async function generateMissingThumbnails(force = false) {
const postsWithoutThumbnail = await knex('posts')
.select('id')
.modify((builder) => {
if (!force) {
builder.where('thumbnail', false);
}
});
await Promise.all(postsWithoutThumbnail.map(async (post) => generateThumbnail(post.id)));
}
export {
generateThumbnail,
generateMissingThumbnails,
};

View File

@ -0,0 +1,9 @@
import { generateMissingThumbnails } from '../thumbnails';
import args from '../cli';
async function init() {
await generateMissingThumbnails(!!args.force);
process.exit();
}
init();