139 lines
3.6 KiB
JavaScript
139 lines
3.6 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const Snoowrap = require('snoowrap');
|
|
const fs = require('fs-extra');
|
|
const Promise = require('bluebird');
|
|
const exiftool = require('node-exiftool');
|
|
const exiftoolBin = require('dist-exiftool');
|
|
const cron = require('node-cron');
|
|
|
|
require('array.prototype.flatten').shim();
|
|
|
|
const reddit = new Snoowrap(config.reddit.api);
|
|
const args = require('./cli')();
|
|
const logger = require('./logger')(__filename);
|
|
|
|
const dissectLink = require('./dissectLink');
|
|
const curatePosts = require('./curate/posts');
|
|
|
|
const { attachContentInfo, getInfo } = require('./fetch/info');
|
|
const { fetchSaveUserContent, fetchSaveDirectContent } = require('./fetch/content');
|
|
|
|
const getPosts = require('./sources/getPosts')(reddit, args);
|
|
const getUserPosts = require('./sources/getUserPosts')(reddit, args);
|
|
|
|
async function getFileContents(location, label) {
|
|
try {
|
|
const fileContents = await fs.readFile(location, 'utf8');
|
|
|
|
return fileContents.split('\n').filter(entry => entry && entry.slice(0, 1) !== '#');
|
|
} catch (error) {
|
|
logger.error(`Could not read ${label} file '${location}': ${error}.`);
|
|
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async function getCompletePosts() {
|
|
let userPosts = {};
|
|
let ignoreIds = [];
|
|
let usernames = args.users || [];
|
|
let postIds = args.posts || [];
|
|
|
|
if (args.fileUsers) {
|
|
usernames = usernames.concat(await getFileContents(args.fileUsers, 'username'));
|
|
}
|
|
|
|
if (args.filePosts) {
|
|
postIds = postIds.concat(await getFileContents(args.filePosts, 'post ID'));
|
|
}
|
|
|
|
if (!usernames.length && !postIds.length) {
|
|
return null;
|
|
}
|
|
|
|
if (usernames.length) {
|
|
userPosts = await getUserPosts(usernames);
|
|
}
|
|
|
|
if (postIds.length) {
|
|
userPosts = await getPosts(postIds, userPosts);
|
|
}
|
|
|
|
if (args.fileIgnore) {
|
|
ignoreIds = await getFileContents(args.fileIgnore, 'ignore');
|
|
}
|
|
|
|
const curatedUserPosts = curatePosts(userPosts, ignoreIds, args);
|
|
|
|
return attachContentInfo(curatedUserPosts, reddit);
|
|
}
|
|
|
|
async function getDirectContent(links, ep) {
|
|
return Promise.map(links, async (link) => {
|
|
const host = dissectLink(link);
|
|
const info = await getInfo(host, reddit, link);
|
|
|
|
if (info) {
|
|
return fetchSaveDirectContent(info, host, ep);
|
|
}
|
|
|
|
return null;
|
|
}, {
|
|
concurrency: 5,
|
|
});
|
|
}
|
|
|
|
async function getCompleteContents(ep) {
|
|
if (args.fetch) {
|
|
return getDirectContent(args.fetch, ep);
|
|
}
|
|
|
|
if (args.fileDirect) {
|
|
return getDirectContent(await getFileContents(args.fileDirect, 'direct'), ep);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function fetchSavePosts(userPosts, ep) {
|
|
// don't map to apply concurrency limit and reduce network stress
|
|
return Promise.reduce(Object.values(userPosts), (acc, user) => fetchSaveUserContent(user, ep, args), null);
|
|
}
|
|
|
|
async function initApp() {
|
|
try {
|
|
const ep = new exiftool.ExiftoolProcess(exiftoolBin);
|
|
await ep.open();
|
|
|
|
if (args.fetch || args.fileDirect) {
|
|
await getCompleteContents(ep);
|
|
}
|
|
|
|
const userPosts = await getCompletePosts();
|
|
|
|
if (userPosts) {
|
|
await fetchSavePosts(userPosts, ep);
|
|
}
|
|
|
|
await ep.close();
|
|
|
|
if (args.watch) {
|
|
logger.info(`Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`);
|
|
}
|
|
} catch (error) {
|
|
if (args.debug) {
|
|
logger.error(error.stack);
|
|
} else {
|
|
logger.error(error.message);
|
|
}
|
|
}
|
|
}
|
|
|
|
initApp();
|
|
|
|
if (args.watch) {
|
|
cron.schedule(config.fetch.watch.schedule, initApp);
|
|
}
|