Using YAML rather than TSV for index files. Improves both readability and reindexability.

This commit is contained in:
2018-06-30 03:33:30 +02:00
parent 1c4ec06f68
commit 74e36a6826
9 changed files with 73 additions and 91 deletions

View File

@@ -2,19 +2,31 @@
const config = require('config');
const fs = require('fs-extra');
const Promise = require('bluebird');
const csvStringify = Promise.promisify(require('csv').stringify);
const yaml = require('js-yaml');
const interpolate = require('../interpolate.js');
async function writeToIndex(posts, user) {
const filename = interpolate(config.library.index.file, user, null, false);
const newEntries = posts.map(post => config.library.index.keys.map(key => interpolate(`$${key}`, user, post, null, false, 'YYYY-MM-DDTHH:mm:ssZ')));
const fullEntries = newEntries.concat(user.indexed.map(entry => Object.values(entry)));
const now = new Date();
const tsvString = await csvStringify(fullEntries, { delimiter: '\t', header: true, columns: config.library.index.keys });
// Individual posts are wrapped in [] to get a YAML array value for each individual item, allowing them to be joined manually with a newline
// between each entry to improve human readability of the index while maintaining a valid YAML list
const oldEntries = user.indexed.map(entry => yaml.safeDump([entry]));
const newEntries = posts.map(post => yaml.safeDump([{
id: post.id,
subreddit: post.subreddit,
permalink: post.permalink,
url: post.url,
hostId: post.host.id,
date: post.datetime,
indexed: now,
title: post.title,
}]));
return fs.writeFile(filename, tsvString);
const entries = newEntries.concat(oldEntries).join('\n');
return fs.writeFile(filename, entries);
}
module.exports = writeToIndex;