Removed HTML scraper from imgur module, no longer support due to login restriction. Preserving 'preview' property in index file.

This commit is contained in:
DebaucheryLibrarian 2024-09-11 05:16:57 +02:00
parent d5d56b83bd
commit 416a3254fb
7 changed files with 392 additions and 372 deletions

740
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -77,6 +77,7 @@ function fetchSavePosts(userPosts, ep) {
async function initApp() {
try {
const userPosts = await getCompleteUserPosts();
const ep = new exiftool.ExiftoolProcess(exiftoolBin);
await ep.open();

View File

@ -10,7 +10,7 @@ const { isAfter, isBefore, isEqual } = require('date-fns');
function report(curatedPosts, indexed, user, args) {
const {
posts, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount, requestedIgnored,
indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount, requestedIgnored,
} = curatedPosts;
if (indexedUpdated.length > 0) {
@ -68,7 +68,11 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
};
if (indexed.entries.length) {
if (!args.redownload && indexed.entries.find(entry => entry.id === post.id)) {
const indexedPost = indexed.entries.find(entry => entry.id === post.id);
if (indexedPost && !args.redownload) {
curatedPost.previewFallback = indexedPost.preview;
return { ...acc, indexedUpdated: [...acc.indexedUpdated, curatedPost] };
}

View File

@ -19,6 +19,7 @@ const attachContentInfo = users => Promise.reduce(Object.values(users), async (a
return [...accPosts, { ...post, content: await methods[post.host.method](post) }];
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`);
console.log(error);
if (config.fetch.archives.preview && post.preview) {
console.log(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);

View File

@ -44,6 +44,10 @@ async function imgurAlbumApi(post) {
}
async function imgurAlbum(post) {
return imgurAlbumApi(post);
/*
* as of late 2019, imgur requires log in to view albums and gallery images
const res = await fetch(`https://imgur.com/a/${post.host.id}`);
const html = await res.text();
@ -80,6 +84,7 @@ async function imgurAlbum(post) {
datetime: new Date(item.datetime),
})),
};
*/
}
module.exports = imgurAlbum;

View File

@ -29,6 +29,10 @@ async function imgurImageApi(post) {
}
async function imgurImage(post) {
return imgurImageApi(post);
/*
* as of late 2019, imgur requires log in to view albums and gallery images
const res = await fetch(`https://imgur.com/${post.host.id}`);
const html = await res.text();
@ -56,6 +60,7 @@ async function imgurImage(post) {
datetime: new Date(data.timestamp || data.datetime),
}],
};
*/
}
module.exports = imgurImage;

View File

@ -10,6 +10,8 @@ async function getUser(username, reddit) {
try {
const user = await reddit.getUser(username).fetch();
console.log(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
return curateUser(user);
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
@ -28,6 +30,8 @@ async function getPosts(username, reddit, args) {
limit: Infinity,
});
console.log(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);
return submissions;
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);