Refactoring flow. Added user and profile saving and variables.
This commit is contained in:
parent
a3f0ad41ea
commit
dc3f3c8440
71
app.js
71
app.js
|
|
@ -2,20 +2,53 @@
|
|||
|
||||
const config = require('config');
|
||||
const util = require('util');
|
||||
const fs = require('fs-extra');
|
||||
const yargs = require('yargs').argv;
|
||||
const snoowrap = require('snoowrap');
|
||||
|
||||
const curateSubmissions = require('./curate/submissions.js');
|
||||
const curateUser = require('./curate/user.js');
|
||||
const methods = require('./methods/methods.js');
|
||||
const curate = require('./curate.js');
|
||||
const interpolate = require('./interpolate.js');
|
||||
const fetchItem = require('./fetchItem.js');
|
||||
const fetchContent = require('./fetchContent.js');
|
||||
const save = require('./save.js');
|
||||
const textToStream = require('./textToStream.js');
|
||||
|
||||
const reddit = new snoowrap(config.reddit.api);
|
||||
|
||||
function getSubmissions(users) {
|
||||
function saveProfileDetails(user) {
|
||||
if(config.library.profile.image) {
|
||||
// pass profile image as item to interpolate extension variable
|
||||
const filepath = interpolate(config.library.profile.image, user, null, {
|
||||
url: user.profile.image
|
||||
});
|
||||
|
||||
fetchItem(user.profile.image).then(stream => save(filepath, stream)).catch(error => {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
|
||||
});
|
||||
}
|
||||
|
||||
if(config.library.profile.description) {
|
||||
if(user.profile.description) {
|
||||
const filepath = interpolate(config.library.profile.description, user);
|
||||
const stream = textToStream(user.profile.description);
|
||||
|
||||
save(filepath, stream).catch(error => {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
|
||||
});
|
||||
} else {
|
||||
console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
function getSubmissions(users, sort, limit) {
|
||||
return users.reduce((chain, user) => {
|
||||
return chain.then(acc => {
|
||||
return reddit.getUser(user).getSubmissions({
|
||||
sort: yargs.sort || config.reddit.sort,
|
||||
limit: yargs.limit || config.reddit.limit
|
||||
sort: sort,
|
||||
limit: limit
|
||||
}).then(submissions => {
|
||||
return acc.concat(submissions);
|
||||
});
|
||||
|
|
@ -23,11 +56,36 @@ function getSubmissions(users) {
|
|||
}, Promise.resolve([]));
|
||||
};
|
||||
|
||||
if(!yargs.user && typeof yargs.users !== 'string') {
|
||||
return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user=[user], or multiple users with --users=[user1,user2] or --user=[user1] --user=[user2]');
|
||||
}
|
||||
|
||||
const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
|
||||
|
||||
users.forEach(user => {
|
||||
return Promise.resolve().then(() => {
|
||||
// get reddit profile
|
||||
return reddit.getUser(user).fetch().then(curateUser);
|
||||
}).then(user => {
|
||||
return saveProfileDetails(user);
|
||||
// get submissions
|
||||
}).catch(error => {
|
||||
return console.log('\x1b[33m%s\x1b[0m', error);
|
||||
});
|
||||
});
|
||||
|
||||
/*
|
||||
Promise.resolve().then(() => {
|
||||
if(yargs.user || yargs.users) {
|
||||
const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
|
||||
|
||||
return getSubmissions(users);
|
||||
return Promise.resolve().then(() => {
|
||||
if(config.library.profile) {
|
||||
return getProfiles(users);
|
||||
}
|
||||
}).then(() => {
|
||||
return getSubmissions(users, yargs.sort || config.reddit.sort, yargs.limit === undefined ? config.reddit.limit : yargs.limit);
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.reject('Please supply at least one user with one or multiple --user, or --users!');
|
||||
|
|
@ -48,5 +106,6 @@ Promise.resolve().then(() => {
|
|||
}).then(posts => {
|
||||
return fetchContent(posts);
|
||||
}).catch(error => {
|
||||
return console.log('\x1b[33m%s\x1b[0m', error);
|
||||
return console.log('\x1b[31m%s\x1b[0m', error);
|
||||
});
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,18 +1,29 @@
|
|||
module.exports = {
|
||||
patterns: {
|
||||
image: 'output/$postUser/$postDate - $itemId - $postTitle$ext',
|
||||
video: 'output/$postUser/$postDate - $itemId - $postTitle$ext',
|
||||
text: 'output/$postUser/$postDate - $postId - $postTitle',
|
||||
library: {
|
||||
base: 'output/$user/',
|
||||
image: '$postDate - $itemId - $postTitle$ext',
|
||||
video: '$postDate - $itemId - $postTitle$ext',
|
||||
text: '$postDate - $postId - $postTitle',
|
||||
album: {
|
||||
extractSingleItem: true,
|
||||
image: 'output/$postUser/$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext',
|
||||
video: 'output/$postUser/$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext'
|
||||
image: '$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext',
|
||||
video: '$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext'
|
||||
},
|
||||
profile: {
|
||||
image: 'profile$ext',
|
||||
description: 'profile ($userVerified$userVerifiedEmail$userGold)'
|
||||
},
|
||||
booleans: {
|
||||
extracted: 'extracted-',
|
||||
verified: '✔',
|
||||
verifiedEmail: '✉',
|
||||
gold: '★',
|
||||
over18: '♥'
|
||||
},
|
||||
dateFormat: 'YYYYMMDD',
|
||||
titleLength: 200,
|
||||
indexOffset: 1,
|
||||
slashSubstitute: '#',
|
||||
extractedLabel: 'extracted-'
|
||||
},
|
||||
reddit: {
|
||||
sort: 'top',
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
'use strict';
|
||||
|
||||
const dissectLink = require('./dissectLink.js');
|
||||
const dissectLink = require('../dissectLink.js');
|
||||
|
||||
function curate(submissions) {
|
||||
function curateSubmissions(submissions) {
|
||||
return submissions.map((submission, index) => {
|
||||
return {
|
||||
id: submission.id,
|
||||
|
|
@ -19,4 +19,4 @@ function curate(submissions) {
|
|||
});
|
||||
};
|
||||
|
||||
module.exports = curate;
|
||||
module.exports = curateSubmissions;
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
'use strict';
|
||||
|
||||
const path = require('path');
|
||||
|
||||
function curateUser(user) {
|
||||
console.log(user);
|
||||
|
||||
return {
|
||||
id: user.id,
|
||||
name: user.name,
|
||||
created: new Date(user.created_utc * 1000),
|
||||
gold: user.is_gold,
|
||||
verified: user.verified,
|
||||
verifiedEmail: user.has_verified_email,
|
||||
profile: {
|
||||
id: user.subreddit.display_name.name,
|
||||
title: user.subreddit.display_name.title,
|
||||
image: user.subreddit.display_name.icon_img,
|
||||
banner: user.subreddit.display_name.banner_img,
|
||||
description: user.subreddit.display_name.public_description,
|
||||
over18: user.subreddit.display_name.over_18
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = curateUser;
|
||||
|
|
@ -1,58 +1,13 @@
|
|||
'use strict';
|
||||
|
||||
const Readable = require('stream').Readable;
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
const config = require('config');
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
const fetchItem = require('./fetchItem');
|
||||
const save = require('./save.js');
|
||||
const interpolate = require('./interpolate.js');
|
||||
|
||||
function saveItemToDisk(stream, filepath) {
|
||||
const file = fs.createWriteStream(filepath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.pipe(file).on('error', error => {
|
||||
reject(error);
|
||||
}).on('finish', () => {
|
||||
console.log(`Saved '${filepath}'`);
|
||||
|
||||
resolve(filepath);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
function textPostToStream(item) {
|
||||
const stream = new Readable();
|
||||
|
||||
stream.push(item.text);
|
||||
stream.push(null);
|
||||
|
||||
return Object.assign(item, {
|
||||
stream: stream
|
||||
});
|
||||
};
|
||||
|
||||
function fetchItem(item, post, attempt) {
|
||||
function retry(error) {
|
||||
console.log(error);
|
||||
|
||||
if(attempt < 3) {
|
||||
console.log('Retrying...');
|
||||
|
||||
return fetchItem(item, post, ++attempt);
|
||||
}
|
||||
};
|
||||
|
||||
return fetch(item.url).then(res => {
|
||||
return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
|
||||
}).then(res => {
|
||||
console.log(`Fetched '${item.url}'`);
|
||||
|
||||
return Object.assign({}, item, {
|
||||
stream: res.body
|
||||
});
|
||||
}).catch(retry);
|
||||
};
|
||||
const textToStream = require('./textToStream.js');
|
||||
|
||||
module.exports = function(posts) {
|
||||
return Promise.all(posts.map(post => {
|
||||
|
|
@ -61,20 +16,22 @@ module.exports = function(posts) {
|
|||
item.index = index;
|
||||
|
||||
if(item.self) {
|
||||
return textPostToStream(item);
|
||||
return Object.assign({}, item, {stream: textToStream(item.text)});
|
||||
}
|
||||
|
||||
return fetchItem(item, post, 0);
|
||||
return fetchItem(item.url, 0).then(stream => {
|
||||
return Object.assign({}, item, {stream});
|
||||
});
|
||||
}));
|
||||
}).then(items => {
|
||||
return Promise.all(items.map(item => {
|
||||
const type = item.type.split('/')[0];
|
||||
const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item) : interpolate(config.patterns[type], post, item);
|
||||
const filepath = post.content.album ? interpolate(config.library.album[type], post.user, post, item) : interpolate(config.library[type], post.user, post, item);
|
||||
|
||||
return Promise.resolve().then(() => {
|
||||
return fs.ensureDir(path.dirname(filepath));
|
||||
}).then(() => {
|
||||
return saveItemToDisk(item.stream, filepath)
|
||||
return save(filepath, item.stream)
|
||||
});
|
||||
}));
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
'use strict';
|
||||
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
function fetchItem(url, attempt) {
|
||||
function retry(error) {
|
||||
console.log(error);
|
||||
|
||||
if(attempt < 3) {
|
||||
console.log('Retrying...');
|
||||
|
||||
return fetchItem(url, ++attempt);
|
||||
}
|
||||
};
|
||||
|
||||
return fetch(url).then(res => {
|
||||
return res.ok ? res : Promise.reject(`Failed to fetch ${url}`);
|
||||
}).then(res => {
|
||||
console.log(`Fetched '${url}'`);
|
||||
|
||||
return res.body;
|
||||
}).catch(retry);
|
||||
};
|
||||
|
||||
module.exports = fetchItem;
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const path = require('path');
|
||||
const url = require('url');
|
||||
const dateFns = require('date-fns');
|
||||
|
||||
const extensions = {
|
||||
|
|
@ -10,45 +12,68 @@ const extensions = {
|
|||
'video/webm': '.webm'
|
||||
};
|
||||
|
||||
function interpolate(path, post, item) {
|
||||
const dateFormat = config.patterns.dateFormat || 'YYYYMMDD';
|
||||
function interpolate(pattern, user, post, item) {
|
||||
const dateFormat = config.library.dateFormat || 'YYYYMMDD';
|
||||
const vars = {};
|
||||
|
||||
const vars = {
|
||||
$postId: post.id,
|
||||
$postTitle: (post.title || '').slice(0, config.patterns.titleLength),
|
||||
$postUser: post.user,
|
||||
$postDate: dateFns.format(post.datetime, dateFormat),
|
||||
$postIndex: post.index + config.patterns.indexOffset,
|
||||
$host: post.host.label
|
||||
};
|
||||
if(config.library.base) {
|
||||
pattern = path.join(config.library.base, pattern);
|
||||
}
|
||||
|
||||
if(post.content.album) {
|
||||
if(user) {
|
||||
Object.assign(vars, {
|
||||
$albumId: post.content.album.id,
|
||||
$albumTitle: (post.content.album.title || '').slice(0, config.patterns.titleLength),
|
||||
$albumDescription: post.content.album.description,
|
||||
$albumDate: dateFns.format(post.content.album.datetime, dateFormat)
|
||||
$user: user.name,
|
||||
$username: user.name,
|
||||
$userId: user.id,
|
||||
$userCreated: dateFns.format(user.created, dateFormat),
|
||||
$userVerified: user.verified ? config.library.booleans.verified : '',
|
||||
$userVerifiedEmail: user.verifiedEmail ? config.library.booleans.verifiedEmail : '',
|
||||
$userGold: user.gold ? config.library.booleans.gold : '',
|
||||
$profileId: user.profile.id,
|
||||
$profileTitle: user.profile.title,
|
||||
$profileDescription: user.profile.description,
|
||||
$profileOver18: user.profile.over18 ? config.library.booleans.over18 : ''
|
||||
});
|
||||
}
|
||||
|
||||
if(post) {
|
||||
Object.assign(vars, {
|
||||
$postId: post.id,
|
||||
$postTitle: (post.title || '').slice(0, config.library.titleLength),
|
||||
$postUser: post.user || user.user,
|
||||
$postDate: dateFns.format(post.datetime, dateFormat),
|
||||
$postIndex: post.index + config.library.indexOffset,
|
||||
$host: post.host.label
|
||||
});
|
||||
|
||||
if(post.content.album) {
|
||||
Object.assign(vars, {
|
||||
$albumId: post.content.album.id,
|
||||
$albumTitle: (post.content.album.title || '').slice(0, config.library.titleLength),
|
||||
$albumDescription: post.content.album.description,
|
||||
$albumDate: dateFns.format(post.content.album.datetime, dateFormat)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if(item) {
|
||||
Object.assign(vars, {
|
||||
$itemId: item.id,
|
||||
$itemTitle: (item.title || '').slice(0, config.patterns.titleLength),
|
||||
$itemTitle: (item.title || '').slice(0, config.library.titleLength),
|
||||
$itemDescription: item.description,
|
||||
$itemDate: dateFns.format(item.datetime, dateFormat),
|
||||
$itemIndex: item.index + config.patterns.indexOffset,
|
||||
$extracted: item.extracted ? config.patterns.extractedLabel : '',
|
||||
$ext: extensions[item.type]
|
||||
$itemIndex: item.index + config.library.indexOffset,
|
||||
$extracted: item.extracted ? config.library.booleans.extracted : '',
|
||||
$ext: item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname)
|
||||
});
|
||||
}
|
||||
|
||||
return Object.entries(vars).reduce((acc, [key, value], index) => {
|
||||
// strip slashes for filesystem compatability
|
||||
value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
|
||||
// substitute slashes for filesystem compatability
|
||||
value = (value || '').toString().replace(/\//g, config.library.slashSubstitute);
|
||||
|
||||
return acc.replace(key, value);
|
||||
}, path);
|
||||
}, pattern);
|
||||
};
|
||||
|
||||
module.exports = interpolate;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
'use strict';
|
||||
|
||||
const fs = require('fs-extra');
|
||||
|
||||
function save(filepath, stream) {
|
||||
const file = fs.createWriteStream(filepath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.pipe(file).on('error', error => {
|
||||
reject(error);
|
||||
}).on('finish', () => {
|
||||
console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
|
||||
|
||||
resolve(filepath);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
module.exports = save;
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
'use strict';
|
||||
|
||||
const Readable = require('stream').Readable;
|
||||
|
||||
function textToStream(text) {
|
||||
const stream = new Readable();
|
||||
|
||||
stream.push(text);
|
||||
stream.push(null);
|
||||
|
||||
return stream;
|
||||
};
|
||||
|
||||
module.exports = textToStream;
|
||||
Loading…
Reference in New Issue