83 lines
2.2 KiB
JavaScript
83 lines
2.2 KiB
JavaScript
'use strict';
|
|
|
|
const Readable = require('stream').Readable;
|
|
const fs = require('fs-extra');
|
|
const path = require('path');
|
|
const config = require('config');
|
|
const fetch = require('node-fetch');
|
|
const interpolate = require('./interpolate.js');
|
|
|
|
function saveItemToDisk(stream, filepath) {
|
|
const file = fs.createWriteStream(filepath);
|
|
|
|
return new Promise((resolve, reject) => {
|
|
stream.pipe(file).on('error', error => {
|
|
reject(error);
|
|
}).on('finish', () => {
|
|
console.log(`Saved '${filepath}'`);
|
|
|
|
resolve(filepath);
|
|
});
|
|
});
|
|
};
|
|
|
|
function textPostToStream(item) {
|
|
const stream = new Readable();
|
|
|
|
stream.push(item.text);
|
|
stream.push(null);
|
|
|
|
return Object.assign(item, {
|
|
stream: stream
|
|
});
|
|
};
|
|
|
|
function fetchItem(item, post, attempt) {
|
|
function retry(error) {
|
|
console.log(error);
|
|
|
|
if(attempt < 3) {
|
|
console.log('Retrying...');
|
|
|
|
return fetchItem(item, post, ++attempt);
|
|
}
|
|
};
|
|
|
|
return fetch(item.url).then(res => {
|
|
return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
|
|
}).then(res => {
|
|
console.log(`Fetched '${item.url}'`);
|
|
|
|
return Object.assign({}, item, {
|
|
stream: res.body
|
|
});
|
|
}).catch(retry);
|
|
};
|
|
|
|
module.exports = function(posts) {
|
|
return Promise.all(posts.map(post => {
|
|
return Promise.resolve().then(() => {
|
|
return Promise.all(post.content.items.map((item, index) => {
|
|
item.index = index;
|
|
|
|
if(item.self) {
|
|
return textPostToStream(item);
|
|
}
|
|
|
|
return fetchItem(item, post, 0);
|
|
}));
|
|
}).then(items => {
|
|
return Promise.all(items.map(item => {
|
|
const type = item.type.split('/')[0];
|
|
const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item) : interpolate(config.patterns[type], post, item);
|
|
|
|
return Promise.resolve().then(() => {
|
|
return fs.ensureDir(path.dirname(filepath));
|
|
}).then(() => {
|
|
return saveItemToDisk(item.stream, filepath)
|
|
});
|
|
}));
|
|
});
|
|
}));
|
|
};
|