Refactored info content fetching and saving to handle users object and utilize async/await.

This commit is contained in:
DebaucheryLibrarian 2024-09-11 05:16:56 +02:00
parent bba0f8f1d7
commit 6696438da0
6 changed files with 1306 additions and 70 deletions

8
.eslintrc Normal file
View File

@ -0,0 +1,8 @@
{
"extends": "airbnb-base",
"rules": {
"no-console": 0,
"indent": ["error", 4],
"max-len": [2, {"code": 125, "tabWidth": 4, "ignoreUrls": true}]
}
}

1237
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -44,5 +44,10 @@
"snoowrap": "^1.15.2",
"url-pattern": "^1.0.3",
"yargs": "^11.0.0"
},
"devDependencies": {
"eslint": "^4.19.1",
"eslint-config-airbnb-base": "^12.1.0",
"eslint-plugin-import": "^2.12.0"
}
}

View File

@ -5,6 +5,8 @@ const util = require('util');
const fs = require('fs-extra');
const snoowrap = require('snoowrap');
const omit = require('object.omit');
const exiftool = require('node-exiftool');
const exiftoolBin = require('dist-exiftool');
require('promise.prototype.finally').shim();
require('array.prototype.flatten').shim();
@ -26,6 +28,8 @@ if(!(args.users && args.users.length) && !(args.posts && args.posts.length)) {
return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user <user> or one post with --post <post-id>. See --help for more options.');
}
const ep = new exiftool.ExiftoolProcess(exiftoolBin);
Promise.resolve().then(async () => {
const initUsers = args.users ? args.users.reduce((acc, username) => ({...acc, [username]: {name: username, posts: []}}), {}) : {};
let userPosts = await getUserPosts(initUsers);
@ -37,15 +41,9 @@ Promise.resolve().then(async () => {
const curatedUserPosts = curatePosts(userPosts, args);
const infoUserPosts = await attachContentInfo(curatedUserPosts);
console.log(util.inspect(infoUserPosts, {depth: 10}));
/*
for(const user of args.users) {
console.log(user);
}
*/
// return fetchContent(posts, userPosts);
await ep.open();
await Promise.all(Object.values(infoUserPosts).map(user => fetchContent(user, ep)));
await ep.close();
}).catch(error => {
return console.error(error);
});

View File

@ -1,8 +1,6 @@
'use strict';
const fs = require('fs-extra');
const path = require('path');
const config = require('config');
const Promise = require('bluebird');
const fetchItem = require('./item.js');
const interpolate = require('../interpolate.js');
@ -11,72 +9,62 @@ const textToStream = require('../save/textToStream.js');
const saveMeta = require('../save/meta.js');
const mux = require('../save/mux.js');
const exiftool = require('node-exiftool');
const exiftoolBin = require('dist-exiftool');
async function getStreams(item, post) {
if(item.self) {
return [textToStream(item.text)];
}
const ep = new exiftool.ExiftoolProcess(exiftoolBin);
const sources = item.mux ? [item.url].concat(item.mux) : [item.url];
const streams = await Promise.map(sources, source => fetchItem(source, 0, post));
module.exports = function(posts, users) {
return Promise.resolve().then(() => {
return ep.open();
}).then(() => {
// console.log(users);
}).then(() => {
return Promise.all(posts.map(post => {
return Promise.all(post.content.items.map((item, index) => {
item.index = index;
if (streams.filter(stream => stream).length > 0) {
return streams;
}
if(item.self) {
return Object.assign({}, item, {stream: textToStream(item.text)});
}
return null;
}
// some videos are delivered with separate audio and are fetched separately to be muxed later
const sources = item.mux ? [item.url].concat(item.mux) : [item.url];
async function fetchContent(user, ep) {
await Promise.map(user.posts, async (post) => {
const items = await Promise.reduce(post.content.items, async (accItems, originalItem, index) => {
const item = { ...originalItem, index };
const streams = await getStreams(item, post);
return Promise.all(sources.map(source => {
return fetchItem(source, 0, post);
})).then(streams => {
if(streams.filter(stream => stream).length > 0) {
return Object.assign({}, item, {streams})
}
// no streams, ignore item
if (streams.length <= 0) {
return accItems;
}
return null;
});
})).then(items => items.filter(item => item)).then(items => {
return Promise.all(items.map(item => {
const type = item.type.split('/')[0];
const filepath = post.content.album ? interpolate(config.library.album[type], post.user, post, item) : interpolate(config.library[type], post.user, post, item);
const type = item.type.split('/')[0];
const filepath = post.content.album
? interpolate(config.library.album[type], user, post, item)
: interpolate(config.library[type], user, post, item);
return Promise.resolve().then(() => {
return save(filepath, item.streams || item.stream, item, post);
}).then(sourcePaths => {
if(item.mux) {
return mux(filepath, sourcePaths, item);
}
}).then(() => {
const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
const interpolatedValue = interpolate(value, post.user, post, item);
const sourcePaths = await save(filepath, streams, item, post);
if(interpolatedValue) {
acc[key] = interpolatedValue;
}
if (item.mux) {
await mux(filepath, sourcePaths, item);
}
return acc;
}, {});
const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
const interpolatedValue = interpolate(value, user, post, item);
if(Object.keys(meta).length > 0) {
return saveMeta(filepath, meta, ep);
}
});
})).then(result => {
const filename = interpolate(config.library.index.file, post.user, post);
const entry = interpolate(config.library.index.entry, post.user, post, null, false) + '\n';
return interpolatedValue ? { ...acc, [key]: interpolatedValue } : acc;
}, {});
return fs.appendFile(filename, config.library.index.unique ? `${post.hash} ${entry}` : entry);
});
});
}));
}).finally(() => {
return ep.close();
if (Object.keys(meta).length > 0) {
await saveMeta(filepath, meta, ep);
}
return sourcePaths;
}, []);
console.log(items);
const filename = interpolate(config.library.index.file, user, post);
const entry = `${interpolate(config.library.index.entry, user, post, null, false)}\n`;
await fs.appendFile(filename, config.library.index.unique ? `${post.hash} ${entry}` : entry);
});
};
}
module.exports = fetchContent;

View File

@ -14,7 +14,7 @@ const attachContentInfo = users => {
}
try {
return [...accPosts, await methods[post.host.method](post)];
return [...accPosts, {...post, content: await methods[post.host.method](post)}];
} catch(error) {
console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`);