DebaucheryLibrarian
/
ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Flow and modularization refactor. Added duplicates option and applying limit after fetch.

This commit is contained in:
DebaucheryLibrarian 2024-09-11 05:16:53 +02:00
parent 1b0bd3a5a4
commit df3421639f
13 changed files with 141 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
app.js
View File

@ -6,55 +6,21 @@ const fs = require('fs-extra');
const yargs = require('yargs').argv;
const snoowrap = require('snoowrap');
const curateSubmissions = require('./curate/submissions.js');
const curateUser = require('./curate/user.js');
const methods = require('./methods/methods.js');
const interpolate = require('./interpolate.js');
const fetchItem = require('./fetchItem.js');
const fetchContent = require('./fetchContent.js');
const save = require('./save.js');
const textToStream = require('./textToStream.js');
const reddit = new snoowrap(config.reddit.api);
function saveProfileDetails(user) {
if(config.library.profile.image) {
// pass profile image as item to interpolate extension variable
const filepath = interpolate(config.library.profile.image, user, null, {
url: user.profile.image
});
const curateSubmissions = require('./curate/submissions.js');
const curateUser = require('./curate/user.js');
fetchItem(user.profile.image).then(stream => save(filepath, stream)).catch(error => {
console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
});
}
const methods = require('./methods/methods.js');
const interpolate = require('./interpolate.js');
if(config.library.profile.description) {
if(user.profile.description) {
const filepath = interpolate(config.library.profile.description, user);
const stream = textToStream(user.profile.description);
const fetchInfo = require('./fetch/info.js');
const fetchContent = require('./fetch/content.js');
save(filepath, stream).catch(error => {
console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
});
} else {
console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
}
}
};
const save = require('./save/save.js');
const saveProfileDetails = require('./save/profileDetails.js');
function getSubmissions(users, sort, limit) {
return users.reduce((chain, user) => {
return chain.then(acc => {
return reddit.getUser(user).getSubmissions({
sort: sort,
limit: limit
}).then(submissions => {
return acc.concat(submissions);
});
});
}, Promise.resolve([]));
};
const limit = yargs.limit || config.fetch.limit;
if(!yargs.user && typeof yargs.users !== 'string') {
return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user=[user], or multiple users with --users=[user1,user2] or --user=[user1] --user=[user2]');
@ -62,50 +28,28 @@ if(!yargs.user && typeof yargs.users !== 'string') {
const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
users.forEach(user => {
users.forEach(username => {
return Promise.resolve().then(() => {
// get reddit profile
return reddit.getUser(user).fetch().then(curateUser);
return reddit.getUser(username).fetch().then(curateUser);
}).then(user => {
return saveProfileDetails(user);
// get submissions
}).catch(error => {
return console.log('\x1b[33m%s\x1b[0m', error);
});
});
/*
Promise.resolve().then(() => {
if(yargs.user || yargs.users) {
const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
return Promise.resolve().then(() => {
if(config.library.profile) {
return getProfiles(users);
}
}).then(() => {
return getSubmissions(users, yargs.sort || config.reddit.sort, yargs.limit === undefined ? config.reddit.limit : yargs.limit);
});
}
return Promise.reject('Please supply at least one user with one or multiple --user, or --users!');
}).then(submissions => {
return Promise.all(curate(submissions).reduce((acc, post) => {
if(post.host && methods[post.host.method]) {
acc = acc.concat(methods[post.host.method](post).then(content => {
post.content = content;
return post;
}).then(user => {
return reddit.getUser(username).getSubmissions({
sort: yargs.sort || config.fetch.sort
}).then(submissions => ({
user,
submissions
}));
} else {
console.log('\x1b[33m%s\x1b[0m', `"${post.title}": '${post.url}' not supported :(`);
}
}).then(({user, submissions}) => {
const posts = curateSubmissions(submissions).slice(0, limit);
return acc;
}, []));
}).then(posts => {
return fetchContent(posts);
}).catch(error => {
return console.log('\x1b[31m%s\x1b[0m', error);
return fetchInfo(posts).then(info => ({
user,
posts
}));
}).then(({user, posts}) => {
return fetchContent(posts, user);
}).catch(error => {
return console.log(error);
});
});
*/

9
config/default.js
View File

@ -10,8 +10,8 @@ module.exports = {
video: '$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext'
},
profile: {
image: 'profile$ext',
description: 'profile ($userVerified$userVerifiedEmail$userGold)'
image: '$userCreated - profile$ext',
description: '$userCreated - profile ($userVerified$userVerifiedEmail$userGold$profileOver18)'
},
booleans: {
extracted: 'extracted-',
@ -25,9 +25,12 @@ module.exports = {
indexOffset: 1,
slashSubstitute: '#',
},
reddit: {
fetch: {
sort: 'top',
limit: 1000,
ignoreDuplicates: true
},
reddit: {
api: {
userAgent: 'wat',
clientId: 'VPquALMpTGl3ag',

19
curate/submissions.js
View File

@ -1,10 +1,19 @@
'use strict';
const config = require('config');
const dissectLink = require('../dissectLink.js');
function curateSubmissions(submissions) {
return submissions.map((submission, index) => {
return {
const processed = new Set();
return submissions.reduce((acc, submission, index) => {
if(config.fetch.ignoreDuplicates && processed.has(submission.url)) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`);
return acc;
}
const curatedSubmission = {
id: submission.id,
index: index,
title: submission.title,
@ -16,7 +25,11 @@ function curateSubmissions(submissions) {
subreddit: submission.subreddit.display_name,
host: dissectLink(submission.url)
};
});
processed.add(submission.url);
return acc.concat(curatedSubmission);
}, []);
};
module.exports = curateSubmissions;

2
curate/user.js
View File

@ -3,8 +3,6 @@
const path = require('path');
function curateUser(user) {
console.log(user);
return {
id: user.id,
name: user.name,

12
fetchContent.js → fetch/content.js
View File

@ -4,12 +4,12 @@ const fs = require('fs-extra');
const path = require('path');
const config = require('config');
const fetchItem = require('./fetchItem');
const save = require('./save.js');
const interpolate = require('./interpolate.js');
const textToStream = require('./textToStream.js');
const fetchItem = require('./item.js');
const interpolate = require('../interpolate.js');
const save = require('../save/save.js');
const textToStream = require('../save/textToStream.js');
module.exports = function(posts) {
module.exports = function(posts, user) {
return Promise.all(posts.map(post => {
return Promise.resolve().then(() => {
return Promise.all(post.content.items.map((item, index) => {
@ -26,7 +26,7 @@ module.exports = function(posts) {
}).then(items => {
return Promise.all(items.map(item => {
const type = item.type.split('/')[0];
const filepath = post.content.album ? interpolate(config.library.album[type], post.user, post, item) : interpolate(config.library[type], post.user, post, item);
const filepath = post.content.album ? interpolate(config.library.album[type], user, post, item) : interpolate(config.library[type], user, post, item);
return Promise.resolve().then(() => {
return fs.ensureDir(path.dirname(filepath));

21
fetch/info.js Normal file
View File

@ -0,0 +1,21 @@
'use strict';
const methods = require('../methods/methods.js');
function fetchInfo(posts) {
return Promise.all(posts.reduce((acc, post) => {
if(post.host && methods[post.host.method]) {
acc = acc.concat(methods[post.host.method](post).then(content => {
post.content = content;
return post;
}));
} else {
console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.title}' - ${post.url}`);
}
return acc;
}, []));
};
module.exports = fetchInfo;

0
fetchItem.js → fetch/item.js
View File

2
interpolate.js
View File

@ -64,7 +64,7 @@ function interpolate(pattern, user, post, item) {
$itemDate: dateFns.format(item.datetime, dateFormat),
$itemIndex: item.index + config.library.indexOffset,
$extracted: item.extracted ? config.library.booleans.extracted : '',
$ext: item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname)
$ext: item.extension || (item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname))
});
}

2
methods/imgurAlbum.js
View File

@ -10,7 +10,7 @@ function imgurAlbum(post) {
'Authorization': `Client-ID ${config.methods.imgur.clientId}`
}
}).then(res => res.json()).then(res => {
const extract = config.patterns.album.extractSingleItem && res.data.images.length === 1;
const extract = config.library.album.extractSingleItem && res.data.images.length === 1;
return {
album: extract ? null : {

19
save.js
View File

@ -1,19 +0,0 @@
'use strict';
const fs = require('fs-extra');
function save(filepath, stream) {
const file = fs.createWriteStream(filepath);
return new Promise((resolve, reject) => {
stream.pipe(file).on('error', error => {
reject(error);
}).on('finish', () => {
console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
resolve(filepath);
});
});
};
module.exports = save;

38
save/profileDetails.js Normal file
View File

@ -0,0 +1,38 @@
'use strict';
const config = require('config');
const interpolate = require('../interpolate.js');
const fetchItem = require('../fetch/item.js');
const textToStream = require('./textToStream.js');
const save = require('./save.js');
function saveProfileDetails(user) {
if(config.library.profile.image) {
const filepath = interpolate(config.library.profile.image, user, null, {
// pass profile image as item to interpolate extension variable
url: user.profile.image
});
fetchItem(user.profile.image).then(stream => save(filepath, stream)).catch(error => {
console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
});
}
if(config.library.profile.description) {
if(user.profile.description) {
const filepath = interpolate(config.library.profile.description, user);
const stream = textToStream(user.profile.description);
save(filepath, stream).catch(error => {
console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
});
} else {
console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
}
}
return user;
};
module.exports = saveProfileDetails;

24
save/save.js Normal file
View File

@ -0,0 +1,24 @@
'use strict';
const fs = require('fs-extra');
const path = require('path');
function save(filepath, stream) {
return Promise.resolve().then(() => {
return fs.ensureDir(path.dirname(filepath));
}).then(() => {
const file = fs.createWriteStream(filepath);
return new Promise((resolve, reject) => {
stream.pipe(file).on('error', error => {
reject(error);
}).on('finish', () => {
console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
resolve(filepath);
});
});
});
};
module.exports = save;

0
textToStream.js → save/textToStream.js
View File