ripunzel/fetchContent.js

'use strict';

const Readable = require('stream').Readable;
const fs = require('fs-extra');
const path = require('path');
const config = require('config');
const fetch = require('node-fetch');
const interpolate = require('./interpolate.js');

function saveItemToDisk(stream, filepath) {
    const file = fs.createWriteStream(filepath);

    return new Promise((resolve, reject) => {
        stream.pipe(file).on('error', error => {
            reject(error);
        }).on('finish', () => {
            console.log(`Saved '${filepath}'`);

            resolve(filepath);
        });
    });
};

function textPostToStream(item) {
    const stream = new Readable();

    stream.push(item.text);
    stream.push(null);

    return Object.assign(item, {
        stream: stream
    });
};

function fetchItem(item, post, attempt) {
    function retry(error) {
        console.log(error);

        if(attempt < 3) {
            console.log('Retrying...');

            return fetchItem(item, post, ++attempt);
        }
    };

    return fetch(item.url).then(res => {
        return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
    }).then(res => {
        console.log(`Fetched '${item.url}'`);

        return Object.assign({}, item, {
            stream: res.body
        });
    }).catch(retry);
};

module.exports = function(posts) {
    return Promise.all(posts.map(post => {
        return Promise.resolve().then(() => {
            return Promise.all(post.content.items.map((item, index) => {
                item.index = index;

                if(item.self) {
                    return textPostToStream(item);
                }

                return fetchItem(item, post, 0);
            }));
        }).then(items => {
            return Promise.all(items.map(item => {
                const type = item.type.split('/')[0];
                const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item) : interpolate(config.patterns[type], post, item);

                return Promise.resolve().then(() => {
                    return fs.ensureDir(path.dirname(filepath));
                }).then(() => {
                    return saveItemToDisk(item.stream, filepath)
                });
            }));
        });
    }));
};