ripunzel/fetchContent.js

'use strict';

const fs = require('fs-extra');
const path = require('path');
const config = require('config');
const fetch = require('node-fetch');
const dateFns = require('date-fns');

const extensions = {
    'image/jpeg': '.jpg',
    'image/gif': '.gif',
    'video/mp4': '.mp4',
    'video/webm': '.webm'
};

function interpolate(path, post, item, index) {
    const dateFormat = config.patterns.dateformat || 'YYYYMMDD';

    const vars = {
        $postId: post.id,
        $postTitle: post.title,
        $postUser: post.user,
        $postDate: dateFns.format(post.datetime, dateFormat)
    };

    if(post.content.album) {
        Object.assign(vars, {
            $albumId: post.content.album.id,
            $albumTitle: post.content.album.title,
            $albumDescription: post.content.album.description,
            $albumDate: dateFns.format(post.content.album.datetime, dateFormat)
        });
    }

    if(item) {
        Object.assign(vars, {
            $itemId: item.id,
            $itemTitle: item.title,
            $itemDescription: item.description,
            $itemDate: dateFns.format(item.datetime, dateFormat),
            $itemIndex: index + config.patterns.indexOffset,
            $ext: extensions[item.type]
        });
    }

    return Object.entries(vars).reduce((acc, [key, value], index) =>  {
        // strip slashes for filesystem compatability
        value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);

        return acc.replace(key, value);
    }, path);
};

function saveItemToDisk(buffer, item, index, filename, post) {
    const stream = fs.createWriteStream(filename);

    return new Promise((resolve, reject) => {
        buffer.body.pipe(stream).on('error', error => {
            reject(error);
        }).on('finish', () => {
            console.log(`Saved '${filename}'`);

            resolve(filename);
        });
    });
};

function fetchItem(item, index, post, attempt) {
    function retry(error) {
        console.log(error);

        if(attempt < 3) {
            console.log('Retrying...');

            return fetchItem(item, index, post, ++attempt);
        }
    };

    return fetch(item.url).then(res => {
        return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
    }).then(res => {
        console.log(`Fetched '${item.url}'`);

        return res;
    }).catch(retry);
};

module.exports = function(posts) {
    return Promise.all(posts.map(post => {
        return Promise.resolve().then(() => {
            return Promise.all(post.content.items.map((item, index) => {
                return fetchItem(item, index, post, 0).then(buffer => Object.assign(item, {buffer}));
            }));
        }).then(items => {
            return Promise.all(items.map((item, index) => {
                const type = item.type.split('/')[0];
                const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item, index) : interpolate(config.patterns[type], post, item, index);

                return Promise.resolve().then(() => {
                    return fs.ensureDir(path.dirname(filepath));
                }).then(() => {
                    return saveItemToDisk(item.buffer, item, index, filepath, post);
                });
            }));
        });
    }));
};