Browse Source

Writing rudimentary index file to feed path.

master
ThePendulum 1 year ago
parent
commit
194e4e46d2
  1. 2
      .nvmrc
  2. 5
      src/app.js
  3. 18
      src/content/store.js
  4. 2
      src/feeds/feeds.js
  5. 2
      src/feeds/reddit.js
  6. 57
      src/feeds/saveIndex.js
  7. 4
      src/utils/http.js

2
.nvmrc

@ -1 +1 @@
14.13.0
14.17.0

5
src/app.js

@ -1,15 +1,14 @@
'use strict';
const util = require('util');
const { fetchFeeds } = require('./feeds/feeds');
const { fetchFeedsContent } = require('./content/fetch');
const { saveIndex } = require('./feeds/saveIndex');
async function init() {
const feeds = await fetchFeeds();
const feedsWithContent = await fetchFeedsContent(feeds);
// console.log(util.inspect(feedsWithContent, null, null));
await saveIndex(feedsWithContent);
}
init();

18
src/content/store.js

@ -9,13 +9,7 @@ const moment = require('moment');
const logger = require('../logger')(__filename);
function interpolate(item, post, feed) {
const type = item.type.split('/')[0];
const pattern = feed.type === 'direct'
? config.library.direct[type]
: config.library.feed[type];
function interpolate(pattern, item, post, feed) {
const data = {};
if (feed) {
@ -28,6 +22,7 @@ function interpolate(item, post, feed) {
if (post) {
data.post = {
title: post.title,
date: post.date && moment(post.date).format(config.library.dateFormat),
};
data.host = {
@ -84,7 +79,13 @@ async function storeItem(item, post, feed) {
return item;
}
const destination = interpolate(item, post, feed);
const type = item.type.split('/')[0];
const pattern = feed.type === 'direct'
? config.library.direct[type]
: config.library.feed[type];
const destination = interpolate(pattern, item, post, feed);
logger.info(`Storing '${item.title || post.title}' from '${feed.name}' at ${destination}`);
@ -116,4 +117,5 @@ async function storeContent(post, feed) {
module.exports = {
storeContent,
interpolate,
};

2
src/feeds/feeds.js

@ -25,6 +25,8 @@ async function fetchFeeds() {
};
const postsPerFeed = (await Promise.all(Object.keys(feeds).map(async (feedType) => {
console.log(feedType, argv);
// iterate through feed types and check if an argument is provided for it
if (argv[feedType]) {
return Promise.all(argv[feedType].map(async (channelName) => {

2
src/feeds/reddit.js

@ -9,7 +9,7 @@ function curatePost(rawPost) {
id: rawPost.id,
title: rawPost.title,
url: `https://reddit.com${rawPost.permalink}`,
subreddit: rawPost.subreddit,
channel: rawPost.subreddit,
username: rawPost.author,
isPinned: rawPost.pinned,
isStickied: rawPost.stickied,

57
src/feeds/saveIndex.js

@ -0,0 +1,57 @@
'use strict';
const config = require('config');
const util = require('util');
const yaml = require('js-yaml');
const fsPromises = require('fs').promises;
const logger = require('../logger')(__filename);
const { interpolate } = require('../content/store');
function curatePost(post, feed) {
return {
id: post.id,
channel: post.channel,
title: post.title,
date: post.date,
hash: post.hash,
indexed: new Date(),
};
}
function curateIndex(feed) {
return JSON.parse(JSON.stringify({ // removes undefined values that cannot be saved as YAML
posts: feed.posts.map((post) => curatePost(post, feed)),
}));
}
async function saveIndex(feeds) {
console.log(util.inspect(feeds, null, null));
await Promise.all(feeds.map(async (feed) => {
try {
if (feed.posts.length === 0) {
return;
}
const basepath = interpolate(feed.type === 'direct' ? config.library.base.direct : config.library.base.feed, null, null, feed);
const filepath = interpolate(config.library.index.file, null, null, feed);
const index = yaml.dump(curateIndex(feed), {
skipInvalid: true,
});
await fsPromises.mkdir(basepath, {
recursive: true,
});
await fsPromises.writeFile(filepath, index);
} catch (error) {
logger.error(`Failed to save index for ${feed.type} '${feed.name}': ${error.message}`);
}
}));
}
module.exports = {
saveIndex,
};

4
src/utils/http.js

@ -67,7 +67,9 @@ async function request(method = 'get', url, body, requestOptions, session) {
if (Buffer.isBuffer(res.body)) {
const html = res.body.toString();
const window = new JSDOM(html).window;
const window = new JSDOM(html, {
runScripts: 'dangerously',
}).window;
return {
...res,

Loading…
Cancel
Save