ripunzel/fetchContent.js

108 lines
3.3 KiB
JavaScript
Raw Normal View History

'use strict';
const fs = require('fs-extra');
const path = require('path');
const config = require('config');
const fetch = require('node-fetch');
const dateFns = require('date-fns');
const extensions = {
'image/jpeg': '.jpg',
'image/gif': '.gif',
'video/mp4': '.mp4',
'video/webm': '.webm'
};
function interpolate(path, post, item, index) {
2018-04-16 22:14:23 +00:00
const dateFormat = config.patterns.dateformat || 'YYYYMMDD';
const vars = {
$postId: post.id,
$postTitle: post.title,
$postUser: post.user,
$postDate: dateFns.format(post.datetime, dateFormat)
};
if(post.content.album) {
Object.assign(vars, {
$albumId: post.content.album.id,
$albumTitle: post.content.album.title,
$albumDescription: post.content.album.description,
$albumDate: dateFns.format(post.content.album.datetime, dateFormat)
});
}
if(item) {
Object.assign(vars, {
$itemId: item.id,
$itemTitle: item.title,
$itemDescription: item.description,
$itemDate: dateFns.format(item.datetime, dateFormat),
$itemIndex: index + config.patterns.indexOffset,
$ext: extensions[item.type]
});
}
return Object.entries(vars).reduce((acc, [key, value], index) => {
// strip slashes for filesystem compatability
value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
return acc.replace(key, value);
}, path);
};
function saveItemToDisk(buffer, item, index, filename, post) {
const stream = fs.createWriteStream(filename);
return new Promise((resolve, reject) => {
buffer.body.pipe(stream).on('error', error => {
reject(error);
}).on('finish', () => {
console.log(`Saved '${filename}'`);
resolve(filename);
});
});
};
function fetchItem(item, index, post, attempt) {
function retry(error) {
console.log(error);
if(attempt < 3) {
console.log('Retrying...');
return fetchItem(item, index, post, ++attempt);
}
};
return fetch(item.url).then(res => {
return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
}).then(res => {
console.log(`Fetched '${item.url}'`);
return res;
}).catch(retry);
};
module.exports = function(posts) {
return Promise.all(posts.map(post => {
return Promise.resolve().then(() => {
return Promise.all(post.content.items.map((item, index) => {
return fetchItem(item, index, post, 0).then(buffer => Object.assign(item, {buffer}));
}));
}).then(items => {
return Promise.all(items.map((item, index) => {
const type = item.type.split('/')[0];
const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item, index) : interpolate(config.patterns[type], post, item, index);
return Promise.resolve().then(() => {
return fs.ensureDir(path.dirname(filepath));
}).then(() => {
return saveItemToDisk(item.buffer, item, index, filepath, post);
});
}));
});
}));
};