2018-04-09 22:26:30 +00:00
|
|
|
'use strict';
|
|
|
|
|
|
|
|
const config = require('config');
|
2018-06-16 23:11:10 +00:00
|
|
|
const Snoowrap = require('snoowrap');
|
2018-07-01 22:25:48 +00:00
|
|
|
const fs = require('fs-extra');
|
2018-07-02 01:45:20 +00:00
|
|
|
const Promise = require('bluebird');
|
2018-06-12 23:51:45 +00:00
|
|
|
const exiftool = require('node-exiftool');
|
|
|
|
const exiftoolBin = require('dist-exiftool');
|
2018-07-01 21:16:03 +00:00
|
|
|
const cron = require('node-cron');
|
2018-05-04 22:51:58 +00:00
|
|
|
|
|
|
|
require('array.prototype.flatten').shim();
|
2018-04-22 21:46:14 +00:00
|
|
|
|
2018-06-16 23:11:10 +00:00
|
|
|
const reddit = new Snoowrap(config.reddit.api);
|
2021-12-17 01:04:25 +00:00
|
|
|
const args = require('./cli')();
|
|
|
|
const logger = require('./logger')(__filename);
|
2018-04-22 23:50:07 +00:00
|
|
|
|
2021-12-17 01:04:25 +00:00
|
|
|
const dissectLink = require('./dissectLink');
|
|
|
|
const curatePosts = require('./curate/posts');
|
2018-04-22 23:50:07 +00:00
|
|
|
|
2021-12-17 01:04:25 +00:00
|
|
|
const { attachContentInfo, getInfo } = require('./fetch/info');
|
|
|
|
const { fetchSaveUserContent, fetchSaveDirectContent } = require('./fetch/content');
|
2018-04-22 21:46:14 +00:00
|
|
|
|
2021-12-17 01:04:25 +00:00
|
|
|
const getPosts = require('./sources/getPosts')(reddit, args);
|
|
|
|
const getUserPosts = require('./sources/getUserPosts')(reddit, args);
|
2018-04-18 02:04:39 +00:00
|
|
|
|
2018-07-01 22:25:48 +00:00
|
|
|
async function getFileContents(location, label) {
|
|
|
|
try {
|
|
|
|
const fileContents = await fs.readFile(location, 'utf8');
|
|
|
|
|
2019-11-12 02:38:26 +00:00
|
|
|
return fileContents.split('\n').filter(entry => entry && entry.slice(0, 1) !== '#');
|
2018-07-01 22:25:48 +00:00
|
|
|
} catch (error) {
|
2021-12-17 01:04:25 +00:00
|
|
|
logger.error(`Could not read ${label} file '${location}': ${error}.`);
|
2018-07-01 22:25:48 +00:00
|
|
|
|
|
|
|
return [];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-01 03:22:36 +00:00
|
|
|
async function getCompletePosts() {
|
2018-06-30 23:07:32 +00:00
|
|
|
let userPosts = {};
|
2018-07-02 00:33:34 +00:00
|
|
|
let ignoreIds = [];
|
2018-07-01 22:25:48 +00:00
|
|
|
let usernames = args.users || [];
|
|
|
|
let postIds = args.posts || [];
|
|
|
|
|
|
|
|
if (args.fileUsers) {
|
|
|
|
usernames = usernames.concat(await getFileContents(args.fileUsers, 'username'));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (args.filePosts) {
|
|
|
|
postIds = postIds.concat(await getFileContents(args.filePosts, 'post ID'));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!usernames.length && !postIds.length) {
|
2019-11-02 01:19:23 +00:00
|
|
|
return null;
|
2018-07-01 22:25:48 +00:00
|
|
|
}
|
2018-06-30 23:07:32 +00:00
|
|
|
|
2018-07-01 22:25:48 +00:00
|
|
|
if (usernames.length) {
|
|
|
|
userPosts = await getUserPosts(usernames);
|
2018-06-30 23:07:32 +00:00
|
|
|
}
|
2018-06-16 23:11:10 +00:00
|
|
|
|
2018-07-01 22:25:48 +00:00
|
|
|
if (postIds.length) {
|
|
|
|
userPosts = await getPosts(postIds, userPosts);
|
2018-06-16 23:11:10 +00:00
|
|
|
}
|
|
|
|
|
2018-07-02 00:33:34 +00:00
|
|
|
if (args.fileIgnore) {
|
|
|
|
ignoreIds = await getFileContents(args.fileIgnore, 'ignore');
|
|
|
|
}
|
|
|
|
|
|
|
|
const curatedUserPosts = curatePosts(userPosts, ignoreIds, args);
|
2018-06-16 23:11:10 +00:00
|
|
|
|
2019-11-05 00:58:00 +00:00
|
|
|
return attachContentInfo(curatedUserPosts, reddit);
|
2018-04-22 21:46:14 +00:00
|
|
|
}
|
|
|
|
|
2019-11-01 04:55:55 +00:00
|
|
|
async function getDirectContent(links, ep) {
|
|
|
|
return Promise.map(links, async (link) => {
|
|
|
|
const host = dissectLink(link);
|
2019-11-12 02:38:26 +00:00
|
|
|
const info = await getInfo(host, reddit, link);
|
2019-11-01 04:55:55 +00:00
|
|
|
|
2019-11-12 02:38:26 +00:00
|
|
|
if (info) {
|
|
|
|
return fetchSaveDirectContent(info, host, ep);
|
|
|
|
}
|
|
|
|
|
|
|
|
return null;
|
2019-11-01 04:55:55 +00:00
|
|
|
}, {
|
|
|
|
concurrency: 5,
|
|
|
|
});
|
|
|
|
}
|
2019-11-01 03:22:36 +00:00
|
|
|
|
2019-11-01 04:55:55 +00:00
|
|
|
async function getCompleteContents(ep) {
|
|
|
|
if (args.fetch) {
|
2019-11-12 02:38:26 +00:00
|
|
|
return getDirectContent(args.fetch, ep);
|
2019-11-01 04:55:55 +00:00
|
|
|
}
|
2019-11-01 03:22:36 +00:00
|
|
|
|
2019-11-01 04:55:55 +00:00
|
|
|
if (args.fileDirect) {
|
|
|
|
return getDirectContent(await getFileContents(args.fileDirect, 'direct'), ep);
|
|
|
|
}
|
2019-11-01 03:22:36 +00:00
|
|
|
|
2019-11-01 04:55:55 +00:00
|
|
|
return null;
|
2019-11-01 03:22:36 +00:00
|
|
|
}
|
|
|
|
|
2018-06-16 23:11:10 +00:00
|
|
|
function fetchSavePosts(userPosts, ep) {
|
2018-07-02 01:45:20 +00:00
|
|
|
// don't map to apply concurrency limit and reduce network stress
|
2019-11-01 03:22:36 +00:00
|
|
|
return Promise.reduce(Object.values(userPosts), (acc, user) => fetchSaveUserContent(user, ep, args), null);
|
2018-06-16 23:11:10 +00:00
|
|
|
}
|
2018-06-12 23:51:45 +00:00
|
|
|
|
2018-06-16 23:11:10 +00:00
|
|
|
async function initApp() {
|
|
|
|
try {
|
|
|
|
const ep = new exiftool.ExiftoolProcess(exiftoolBin);
|
2019-11-01 04:55:55 +00:00
|
|
|
await ep.open();
|
2018-06-16 23:11:10 +00:00
|
|
|
|
2019-11-01 04:55:55 +00:00
|
|
|
if (args.fetch || args.fileDirect) {
|
|
|
|
await getCompleteContents(ep);
|
2019-11-01 03:22:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
const userPosts = await getCompletePosts();
|
|
|
|
|
2019-11-02 01:19:23 +00:00
|
|
|
if (userPosts) {
|
|
|
|
await fetchSavePosts(userPosts, ep);
|
|
|
|
}
|
|
|
|
|
2018-06-17 01:39:12 +00:00
|
|
|
await ep.close();
|
2018-07-01 01:06:57 +00:00
|
|
|
|
|
|
|
if (args.watch) {
|
2021-12-17 01:04:25 +00:00
|
|
|
logger.info(`Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`);
|
2018-07-01 01:06:57 +00:00
|
|
|
}
|
2018-06-16 23:11:10 +00:00
|
|
|
} catch (error) {
|
2018-07-05 21:27:11 +00:00
|
|
|
if (args.debug) {
|
2021-12-17 01:04:25 +00:00
|
|
|
logger.error(error.stack);
|
2018-07-05 21:27:11 +00:00
|
|
|
} else {
|
2021-12-17 01:04:25 +00:00
|
|
|
logger.error(error.message);
|
2018-07-05 21:27:11 +00:00
|
|
|
}
|
2018-06-16 23:11:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
initApp();
|
2018-07-01 21:16:03 +00:00
|
|
|
|
|
|
|
if (args.watch) {
|
|
|
|
cron.schedule(config.fetch.watch.schedule, initApp);
|
|
|
|
}
|