Writing rudimentary index file to feed path.

.nvmrc

@@ -1 +1 @@
-14.13.0
+14.17.0

src/app.js

@@ -1,15 +1,14 @@
 'use strict';
 
-const util = require('util');
-
 const { fetchFeeds } = require('./feeds/feeds');
 const { fetchFeedsContent } = require('./content/fetch');
+const { saveIndex } = require('./feeds/saveIndex');
 
 async function init() {
 	const feeds = await fetchFeeds();
 	const feedsWithContent = await fetchFeedsContent(feeds);
 
-	// console.log(util.inspect(feedsWithContent, null, null));
+	await saveIndex(feedsWithContent);
 }
 
 init();

src/content/store.js

@@ -9,13 +9,7 @@ const moment = require('moment');
 
 const logger = require('../logger')(__filename);
 
-function interpolate(item, post, feed) {
-	const type = item.type.split('/')[0];
-
-	const pattern = feed.type === 'direct'
-		? config.library.direct[type]
-		: config.library.feed[type];
-
+function interpolate(pattern, item, post, feed) {
 	const data = {};
 
 	if (feed) {
@@ -28,6 +22,7 @@ function interpolate(item, post, feed) {
 	if (post) {
 		data.post = {
 			title: post.title,
+			date: post.date && moment(post.date).format(config.library.dateFormat),
 		};
 
 		data.host = {
@@ -84,7 +79,13 @@ async function storeItem(item, post, feed) {
 		return item;
 	}
 
-	const destination = interpolate(item, post, feed);
+	const type = item.type.split('/')[0];
+
+	const pattern = feed.type === 'direct'
+		? config.library.direct[type]
+		: config.library.feed[type];
+
+	const destination = interpolate(pattern, item, post, feed);
 
 	logger.info(`Storing '${item.title || post.title}' from '${feed.name}' at ${destination}`);
 
@@ -116,4 +117,5 @@ async function storeContent(post, feed) {
 
 module.exports = {
 	storeContent,
+	interpolate,
 };

src/feeds/feeds.js

@@ -25,6 +25,8 @@ async function fetchFeeds() {
 	};
 
 	const postsPerFeed = (await Promise.all(Object.keys(feeds).map(async (feedType) => {
+		console.log(feedType, argv);
+
 		// iterate through feed types and check if an argument is provided for it
 		if (argv[feedType]) {
 			return Promise.all(argv[feedType].map(async (channelName) => {

src/feeds/reddit.js

@@ -9,7 +9,7 @@ function curatePost(rawPost) {
 		id: rawPost.id,
 		title: rawPost.title,
 		url: `https://reddit.com${rawPost.permalink}`,
-		subreddit: rawPost.subreddit,
+		channel: rawPost.subreddit,
 		username: rawPost.author,
 		isPinned: rawPost.pinned,
 		isStickied: rawPost.stickied,

src/feeds/saveIndex.js

@@ -0,0 +1,57 @@
+'use strict';
+
+const config = require('config');
+const util = require('util');
+const yaml = require('js-yaml');
+const fsPromises = require('fs').promises;
+
+const logger = require('../logger')(__filename);
+const { interpolate } = require('../content/store');
+
+function curatePost(post, feed) {
+	return {
+		id: post.id,
+		channel: post.channel,
+		title: post.title,
+		date: post.date,
+		hash: post.hash,
+		indexed: new Date(),
+	};
+}
+
+function curateIndex(feed) {
+	return JSON.parse(JSON.stringify({ // removes undefined values that cannot be saved as YAML
+		posts: feed.posts.map((post) => curatePost(post, feed)),
+	}));
+}
+
+async function saveIndex(feeds) {
+	console.log(util.inspect(feeds, null, null));
+
+	await Promise.all(feeds.map(async (feed) => {
+		try {
+			if (feed.posts.length === 0) {
+				return;
+			}
+
+			const basepath = interpolate(feed.type === 'direct' ? config.library.base.direct : config.library.base.feed, null, null, feed);
+			const filepath = interpolate(config.library.index.file, null, null, feed);
+
+			const index = yaml.dump(curateIndex(feed), {
+				skipInvalid: true,
+			});
+
+			await fsPromises.mkdir(basepath, {
+				recursive: true,
+			});
+
+			await fsPromises.writeFile(filepath, index);
+		} catch (error) {
+			logger.error(`Failed to save index for ${feed.type} '${feed.name}': ${error.message}`);
+		}
+	}));
+}
+
+module.exports = {
+	saveIndex,
+};

src/utils/http.js

@@ -67,7 +67,9 @@ async function request(method = 'get', url, body, requestOptions, session) {
 
 	if (Buffer.isBuffer(res.body)) {
 		const html = res.body.toString();
-		const window = new JSDOM(html).window;
+		const window = new JSDOM(html, {
+			runScripts: 'dangerously',
+		}).window;
 
 		return {
 			...res,