'use strict'; const config = require('config'); const Snoowrap = require('snoowrap'); const fs = require('fs-extra'); const Promise = require('bluebird'); const exiftool = require('node-exiftool'); const exiftoolBin = require('dist-exiftool'); const cron = require('node-cron'); const { format } = require('date-fns'); require('array.prototype.flatten').shim(); const reddit = new Snoowrap(config.reddit.api); const args = require('./cli.js')(); const dissectLink = require('./dissectLink.js'); const curatePosts = require('./curate/posts.js'); const { attachContentInfo, getInfo } = require('./fetch/info.js'); const { fetchSaveUserContent, fetchSaveDirectContent } = require('./fetch/content.js'); const getPosts = require('./sources/getPosts.js')(reddit, args); const getUserPosts = require('./sources/getUserPosts.js')(reddit, args); async function getFileContents(location, label) { try { const fileContents = await fs.readFile(location, 'utf8'); return fileContents.split('\n').filter(entry => entry); } catch (error) { console.log('\x1b[31m%s\x1b[0m', `Could not read ${label} file '${location}': ${error}.`); return []; } } async function getCompletePosts() { let userPosts = {}; let ignoreIds = []; let usernames = args.users || []; let postIds = args.posts || []; if (args.fileUsers) { usernames = usernames.concat(await getFileContents(args.fileUsers, 'username')); } if (args.filePosts) { postIds = postIds.concat(await getFileContents(args.filePosts, 'post ID')); } if (!usernames.length && !postIds.length) { throw new Error('Could not retrieve any posts. Did you supply --users, --posts, --file-users or --file-posts?'); } if (usernames.length) { userPosts = await getUserPosts(usernames); } if (postIds.length) { userPosts = await getPosts(postIds, userPosts); } if (args.fileIgnore) { ignoreIds = await getFileContents(args.fileIgnore, 'ignore'); } const curatedUserPosts = curatePosts(userPosts, ignoreIds, args); return attachContentInfo(curatedUserPosts); } async function getDirectContent(link, ep) { const host = dissectLink(link); const info = await getInfo(host); console.log(info); return fetchSaveDirectContent(info, host, ep); /* console.log(info); await Promise.all(info.items.map(async (item, index) => { const stream = await fetchItem(item.url); if (info.album) { const filepath = `./output/${host.label}/${host.id} - ${info.album.title}/${index + 1} - ${item.id}${path.extname(url.parse(item.url).pathname)}`; console.log(filepath); return saveItem(filepath, stream, item); } const filepath = `./output/${host.label}/${item.id}${path.extname(url.parse(item.url).pathname)}`; console.log(filepath); return saveItem(filepath, stream, item); })); */ } function fetchSavePosts(userPosts, ep) { // don't map to apply concurrency limit and reduce network stress return Promise.reduce(Object.values(userPosts), (acc, user) => fetchSaveUserContent(user, ep, args), null); } async function initApp() { try { const ep = new exiftool.ExiftoolProcess(exiftoolBin); if (args.fetch) { await getDirectContent(args.fetch, ep); return; } const userPosts = await getCompletePosts(); await ep.open(); await fetchSavePosts(userPosts, ep); await ep.close(); if (args.watch) { console.log(`[${format(new Date(), 'YYYY-MM-DD HH:mm:ss')}] Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`); } } catch (error) { if (args.debug) { console.log('\x1b[31m%s\x1b[0m', error.stack); } else { console.log('\x1b[31m%s\x1b[0m', error.message); } } } initApp(); if (args.watch) { cron.schedule(config.fetch.watch.schedule, initApp); }