Fetching posts in series to prevent directory creation race condition. Removed duplicate check for now (will check per post, not per image, in the future).

This commit is contained in:
DebaucheryLibrarian 2024-09-11 05:16:53 +02:00
parent a139ccde34
commit e721c9fc8c
7 changed files with 160 additions and 3 deletions

1
.gitignore vendored
View File

@ -1,5 +1,4 @@
node_modules/
config/*.js
!config/example.js
/*
output/

7
app.js
View File

@ -10,8 +10,9 @@ const dissectLink = require('./dissectLink.js');
const fetchContent = require('./fetchContent.js');
const reddit = new snoowrap(config.api);
const user = yargs.user;
reddit.getUser(yargs.user).getSubmissions({
reddit.getUser(user).getSubmissions({
sort: 'top'
}).then(submissions => {
const curatedPosts = submissions.map(submission => {
@ -20,6 +21,8 @@ reddit.getUser(yargs.user).getSubmissions({
title: submission.title,
permalink: submission.permalink,
url: submission.url,
datetime: submission.created_utc,
subreddit: submission.subreddit.display_name,
host: dissectLink(submission.url)
};
});
@ -37,6 +40,6 @@ reddit.getUser(yargs.user).getSubmissions({
return acc;
}, []));
}).then(fetchContent).catch(error => {
}).then(posts => fetchContent(posts, user)).catch(error => {
note(error);
});

30
dissectLink.js Normal file
View File

@ -0,0 +1,30 @@
'use strict';
const urlPattern = require('url-pattern');
const hosts = [{
method: 'imgurImage',
pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?:num)')
}, {
method: 'imgurAlbum',
pattern: new urlPattern('http(s)\\://imgur.com/:type/:id')
}];
module.exports = function dissectLink(url) {
return hosts.reduce((acc, host) => {
if(acc) {
return acc;
}
const match = host.pattern.match(url);
if(match) {
return Object.assign(match, {
url: url,
method: host.method
});
}
return null;
}, null);
};

47
fetchContent.js Normal file
View File

@ -0,0 +1,47 @@
'use strict';
const fs = require('fs-extra');
const path = require('path');
const fetch = require('node-fetch');
function saveToDisk(buffer, item, index, filename, post, user) {
const stream = fs.createWriteStream(filename);
return new Promise((resolve, reject) => {
buffer.body.pipe(stream).on('error', error => {
reject(error);
}).on('finish', () => {
console.log(`Fetched and saved ${filename}`);
resolve(filename);
});
});
};
function fetchItem(item, index, post, user) {
return fetch(item.url).then(res => {
return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
});
};
module.exports = function(posts, user) {
return Promise.all(posts.map(post => {
const filepath = post.content.album ? `output/${user}/${post.content.album.datetime} - ${post.content.album.id} - ${post.title}/` : `output/${user}/`;
return Promise.resolve().then(() => {
if(post.content.album) {
return fs.ensureDir(filepath);
};
}).then(() => {
return Promise.all(post.content.items.map((item, index) => {
return fetchItem(item, index, post, user).then(buffer => Object.assign(item, {buffer}));
}));
}).then(items => {
return Promise.all(items.map((item, index) => {
const filename = post.content.album ? `${filepath}${index + 1} - ${item.id}.jpg` : `${filepath}${item.datetime} - ${item.id}.jpg`;
return saveToDisk(item.buffer, item, index, filename, post, user)
}));
});
}));
};

38
methods/imgurAlbum.js Normal file
View File

@ -0,0 +1,38 @@
'use strict';
const note = require('note-log');
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
function imgurAlbum(post) {
return fetch(`https://api.imgur.com/3/album/${post.host.id}`, {
headers: {
'Authorization': `Client-ID ${config.methods.imgur.clientId}`
}
}).then(res => res.json()).then(res => {
return {
album: {
id: res.data.id,
url: res.data.link,
title: res.data.title,
description: res.data.description,
datetime: res.data.datetime,
original: res.data
},
items: res.data.images.map(item => ({
id: item.id,
url: item.link,
title: item.title,
description: item.description,
datetime: item.datetime,
type: item.type,
original: item
}))
};
}).catch(error => {
note(error);
});
};
module.exports = imgurAlbum;

31
methods/imgurImage.js Normal file
View File

@ -0,0 +1,31 @@
'use strict';
const note = require('note-log');
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
function imgurImage(post) {
return fetch(`https://api.imgur.com/3/image/${post.host.id}`, {
headers: {
'Authorization': `Client-ID ${config.methods.imgur.clientId}`
}
}).then(res => res.json()).then(res => {
return {
album: null,
items: [{
id: res.data.id,
url: res.data.link,
title: res.data.title,
description: res.data.description,
type: res.data.type,
datetime: res.data.datetime,
original: res.data
}]
};
}).catch(error => {
note(error);
});
};
module.exports = imgurImage;

9
methods/methods.js Normal file
View File

@ -0,0 +1,9 @@
'use strict';
const imgurImage = require('./imgurImage.js');
const imgurAlbum = require('./imgurAlbum.js');
module.exports = {
imgurImage: imgurImage,
imgurAlbum: imgurAlbum
};