DebaucheryLibrarian
/
ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added support for fetching content directly from host. Improved pattern interpolation. Refactored content modules.

This commit is contained in:
ThePendulum 2019-11-01 04:22:36 +01:00
parent d773920da8
commit eee3ba0abc
26 changed files with 572 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
config/default.js
View File

@ -2,26 +2,41 @@
module.exports = {
library: {
base: 'output/$user/',
image: '$base$postDate - $preview$itemId - $postTitle$ext',
video: '$base$postDate - $preview$itemId - $postTitle$ext',
text: '$base$postDate - $preview$postId - $postTitle',
album: {
image: '$base$postDate - $preview$albumId - $postTitle/$itemIndex - $itemId$ext',
video: '$base$postDate - $preview$albumId - $postTitle/$itemIndex - $itemId$ext',
extractSingleItem: true,
base: {
posts: 'output/{user.name}/',
direct: 'output/{host.name}/',
},
posts: {
image: '{base.posts}{post.date} - {tags.preview}{item.id} - {post.title}{ext}',
video: '{base.posts}{post.date} - {tags.preview}{item.id} - {post.title}{ext}',
text: '{base.posts}{post.date} - {tags.preview}{post.id} - {post.title}',
album: {
image: '{base.posts}{post.date} - {tags.preview}{album.id} - {post.title}/{item.index} - {item.id}{ext}',
video: '{base.posts}{post.date} - {tags.preview}{album.id} - {post.title}/{item.index} - {item.id}{ext}',
},
},
direct: {
image: '{base.direct}{item.date} - {tags.preview}{item.id} - {item.title}{ext}',
video: '{base.direct}{item.date} - {tags.preview}{item.id} - {item.title}{ext}',
text: '{base.direct}{item.date} - {tags.preview}{item.id} - {item.title}',
album: {
image: '{base.direct}{album.date} - {tags.preview}{album.id} - {album.title}/{item.index} - {item.id}{ext}',
video: '{base.direct}{album.date} - {tags.preview}{album.id} - {album.title}/{item.index} - {item.id}{ext}',
},
},
extractSingleAlbumItem: true,
profile: {
image: '$base$userCreated - profile$ext',
description: '$base$userCreated - profile ($userVerified$userVerifiedEmail$userGold$profileOver18)',
image: '{base.posts}{user.created} - profile{ext}',
description: '{base.posts}{user.created} - profile ({tags.verified}{tags.verifiedEmail}{tags.gold}{tags.over18})',
avoidAvatar: true,
},
index: {
file: '$base/index',
format: 'tsv',
keys: ['postId', 'subreddit', 'postDate', 'url', 'hostId', 'postTitle'],
file: '{base.posts}index',
},
booleans: {
meta: {
comment: '{item.description}',
},
tags: {
extracted: 'extracted-',
preview: 'preview-',
verified: '✔',
@ -29,9 +44,6 @@ module.exports = {
gold: '★',
over18: '♥',
},
meta: {
comment: '$itemDescription',
},
dateFormat: 'YYYYMMDD',
truncate: {
limit: 250,

7
package-lock.json generated
View File

@ -528,7 +528,7 @@
"integrity": "sha1-i9F7rB5D3qzYHZrazQ8eJMaQEA8=",
"requires": {
"exiftool.exe": "10.53",
"exiftool.pl": "10.53",
"exiftool.pl": "^10.53.0",
"platform-dependent-modules": "0.0.14"
}
},
@ -2204,6 +2204,11 @@
"string-width": "^2.1.1"
}
},
"template-format": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/template-format/-/template-format-1.2.4.tgz",
"integrity": "sha512-+8ItNMtMTBbsEHyPR1l7Ke1WZfl91PAcoTvwAcx5U28CRLd7ylzDLazv0kuDTzNmdq/RAOnsxFVWzr4QwVIFVg=="
},
"text-table": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",

1
package.json
View File

@ -44,6 +44,7 @@
"object.omit": "^3.0.0",
"object.pick": "^1.3.0",
"snoowrap": "^1.15.2",
"template-format": "^1.2.4",
"url-pattern": "^1.0.3",
"yargs": "^11.0.0"
},

49
src/app.js
View File

@ -14,10 +14,11 @@ require('array.prototype.flatten').shim();
const reddit = new Snoowrap(config.reddit.api);
const args = require('./cli.js')();
const dissectLink = require('./dissectLink.js');
const curatePosts = require('./curate/posts.js');
const attachContentInfo = require('./fetch/info.js');
const fetchSaveContent = require('./fetch/content.js');
const { attachContentInfo, getInfo } = require('./fetch/info.js');
const { fetchSaveUserContent, fetchSaveDirectContent } = require('./fetch/content.js');
const getPosts = require('./sources/getPosts.js')(reddit, args);
const getUserPosts = require('./sources/getUserPosts.js')(reddit, args);
@ -34,7 +35,7 @@ async function getFileContents(location, label) {
}
}
async function getCompleteUserPosts() {
async function getCompletePosts() {
let userPosts = {};
let ignoreIds = [];
let usernames = args.users || [];
@ -69,17 +70,51 @@ async function getCompleteUserPosts() {
return attachContentInfo(curatedUserPosts);
}
async function getDirectContent(link, ep) {
const host = dissectLink(link);
const info = await getInfo(host);
console.log(info);
return fetchSaveDirectContent(info, host, ep);
/*
console.log(info);
await Promise.all(info.items.map(async (item, index) => {
const stream = await fetchItem(item.url);
if (info.album) {
const filepath = `./output/${host.label}/${host.id} - ${info.album.title}/${index + 1} - ${item.id}${path.extname(url.parse(item.url).pathname)}`;
console.log(filepath);
return saveItem(filepath, stream, item);
}
const filepath = `./output/${host.label}/${item.id}${path.extname(url.parse(item.url).pathname)}`;
console.log(filepath);
return saveItem(filepath, stream, item);
}));
*/
}
function fetchSavePosts(userPosts, ep) {
// don't map to apply concurrency limit and reduce network stress
return Promise.reduce(Object.values(userPosts), (acc, user) => fetchSaveContent(user, ep, args), null);
return Promise.reduce(Object.values(userPosts), (acc, user) => fetchSaveUserContent(user, ep, args), null);
}
async function initApp() {
try {
const userPosts = await getCompleteUserPosts();
const ep = new exiftool.ExiftoolProcess(exiftoolBin);
if (args.fetch) {
await getDirectContent(args.fetch, ep);
return;
}
const userPosts = await getCompletePosts();
await ep.open();
await fetchSavePosts(userPosts, ep);
await ep.close();
@ -94,8 +129,6 @@ async function initApp() {
console.log('\x1b[31m%s\x1b[0m', error.message);
}
}
return true;
}
initApp();

5
src/cli.js
View File

@ -24,6 +24,11 @@ function getArgs() {
describe: 'Load reddit post IDs from file',
type: 'string',
})
.option('direct', {
describe: 'Get content directly from imgur and other hosts',
type: 'string',
alias: 'fetch',
})
.option('limit', {
describe: 'Maximum amount of posts to fetch per supplied user (!), after filtering out ignored, cross- and reposts',
type: 'number',

54
src/fetch/content.js
View File

@ -14,7 +14,7 @@ const writeToIndex = require('../save/writeToIndex.js');
async function getStreams(item, post) {
if (item.self) {
return [textToStream(item.text)];
return [textToStream(post.text)];
}
const sources = item.mux ? [item.url].concat(item.mux) : [item.url];
@ -33,7 +33,7 @@ async function addMeta(filepath, item, post, user, ep) {
}
const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
const interpolatedValue = interpolate(value, user, post, item);
const interpolatedValue = interpolate(value, item, null, null, post, user);
return interpolatedValue ? { ...acc, [key]: interpolatedValue } : acc;
}, {});
@ -45,15 +45,25 @@ async function addMeta(filepath, item, post, user, ep) {
return false;
}
function getFilepath(item, post, user) {
function getFilepath(item, content, host, post, user) {
const type = item.type.split('/')[0];
return post.content.album
? interpolate(config.library.album[type], user, post, item)
: interpolate(config.library[type], user, post, item);
if (post && content.album) {
return interpolate(config.library.posts.album[type], item, content, host, post, user);
}
if (post) {
return interpolate(config.library.posts[type], item, content, host, post, user);
}
if (content.album) {
return interpolate(config.library.direct.album[type], item, content, host);
}
return interpolate(config.library.direct[type], item, content, host);
}
async function fetchSaveContent(user, ep, args) {
async function fetchSaveUserContent(user, ep, args) {
const profilePaths = await saveProfileDetails(user, args);
const posts = await Promise.map(user.posts, async (post) => {
@ -66,7 +76,7 @@ async function fetchSaveContent(user, ep, args) {
return accItems;
}
const filepath = getFilepath(item, post, user);
const filepath = getFilepath(item, post.content, post.host, post, user);
const sourcePaths = await save(filepath, streams, item, post);
if (item.mux) {
@ -86,4 +96,30 @@ async function fetchSaveContent(user, ep, args) {
return writeToIndex(posts, profilePaths, user, args);
}
module.exports = fetchSaveContent;
async function fetchSaveDirectContent(content, host, ep) {
const data = await Promise.reduce(content.items, async (accItems, originalItem, index) => {
const item = { ...originalItem, index };
const streams = await getStreams(item, null);
// no streams, ignore item
if (!streams || streams.length === 0) {
return accItems;
}
const filepath = getFilepath(item, content, host, null, null);
const sourcePaths = await save(filepath, streams, item, null);
if (item.mux) {
await mux(filepath, sourcePaths, item);
}
await addMeta(filepath, item, null, null, ep);
return sourcePaths;
}, []);
}
module.exports = {
fetchSaveUserContent,
fetchSaveDirectContent,
};

14
src/fetch/info.js
View File

@ -2,6 +2,7 @@
const config = require('config');
const Promise = require('bluebird');
const methods = require('../methods/methods.js');
const attachContentInfo = users => Promise.reduce(Object.values(users), async (accUsers, user) => ({
@ -16,14 +17,14 @@ const attachContentInfo = users => Promise.reduce(Object.values(users), async (a
}
try {
return [...accPosts, { ...post, content: await methods[post.host.method](post) }];
return [...accPosts, { ...post, content: await methods[post.host.method](post.host, post) }];
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`);
if (config.fetch.archives.preview && post.preview) {
console.log(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);
return [...accPosts, { ...post, previewFallback: true, content: await methods.redditPreview(post) }];
return [...accPosts, { ...post, previewFallback: true, content: await methods.redditPreview(post.host, post) }];
}
return accPosts;
@ -32,4 +33,11 @@ const attachContentInfo = users => Promise.reduce(Object.values(users), async (a
},
}), {});
module.exports = attachContentInfo;
async function getInfo(host) {
return methods[host.method](host);
}
module.exports = {
attachContentInfo,
getInfo,
};

28
src/fetch/item.js
View File

@ -3,30 +3,32 @@
const config = require('config');
const fetch = require('node-fetch');
function fetchItem(url, attempt, post) {
function retry(error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch '${url}': ${error.message} (${post.permalink})`);
async function fetchItem(url, attempt, post) {
async function retry(error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch '${url}': ${error.message} (${post ? post.permalink : 'no post'})`);
if(attempt < config.fetch.retries) {
if (attempt < config.fetch.retries) {
console.log('Retrying...');
return fetchItem(url, ++attempt, post);
return fetchItem(url, attempt + 1, post);
}
return null;
};
}
return fetch(url).then(res => {
if(!res.ok) {
try {
const res = await fetch(url);
if (!res.ok) {
throw new Error(`Response not OK for '${url}', HTTP code '${res.status}'`);
}
return res;
}).then(res => {
console.log(`Fetched '${url}' (${post.permalink})`);
console.log(`Fetched '${url}' (${post ? post.permalink : 'no post'})`);
return res.body;
}).catch(retry);
};
} catch (error) {
return retry(error);
}
}
module.exports = fetchItem;

165
src/interpolate.js
View File

@ -5,79 +5,130 @@ const path = require('path');
const url = require('url');
const dateFns = require('date-fns');
const mime = require('mime-types');
const format = require('template-format');
function interpolate(pattern, user, post, item, strip = true, dateFormat = config.library.dateFormat) {
const vars = {
$base: config.library.base,
function interpolate(pattern, item = null, content = null, host = null, post = null, user = null, strip = true, dateFormat = config.library.dateFormat) {
const data = {
tags: {},
};
if (user) {
Object.assign(vars, {
$user: user.name,
$username: user.name,
$userId: user.id,
$userCreated: dateFns.format(user.created, dateFormat),
$userVerified: user.verified ? config.library.booleans.verified : '',
$userVerifiedEmail: user.verifiedEmail ? config.library.booleans.verifiedEmail : '',
$userGold: user.gold ? config.library.booleans.gold : '',
if (item) {
Object.assign(data, {
item: {
id: item.id,
title: item.title && item.title.slice(0, config.library.titleLength),
description: item.description,
date: dateFns.format(item.datetime, dateFormat),
index: item.index + config.library.indexOffset,
},
ext: item.type
? `.${mime.extension(item.type)}`
: path.extname(url.parse(item.url).pathname),
tags: {
...data.tags,
extracted: item.extracted && config.library.tags.extracted,
},
});
}
if (user.profile) {
Object.assign(vars, {
$profileId: user.profile.id,
$profileTitle: user.profile.title,
$profileDescription: user.profile.description,
$profileOver18: user.profile.over18 ? config.library.booleans.over18 : '',
});
}
if (host) {
Object.assign(data, {
host: {
id: host.id,
label: host.label,
name: host.label,
},
});
}
if (content && content.album) {
Object.assign(data, {
album: {
id: content.album.id,
title: content.album.title && content.album.title.slice(0, config.library.titleLength),
description: content.album.description,
date: dateFns.format(content.album.datetime, dateFormat),
},
});
}
if (post) {
Object.assign(vars, {
$postId: post.id,
$postTitle: (post.title || '').slice(0, config.library.titleLength),
$postUser: post.user,
$postDate: dateFns.format(post.datetime, dateFormat),
$postIndex: post.index + config.library.indexOffset,
$postHash: post.hash,
$url: post.url,
$score: post.score,
$subreddit: post.subreddit,
$hostLabel: post.host.label,
$hostId: post.host.id,
Object.assign(data, {
post: {
id: post.id,
title: post.title && post.title.slice(0, config.library.titleLength),
url: post.url,
user: post.user,
date: dateFns.format(post.datetime, dateFormat),
index: post.index + config.library.indexOffset,
hash: post.hash,
score: post.score,
subreddit: post.subreddit,
},
});
}
if (user) {
Object.assign(data, {
user: {
name: user.name,
username: user.name,
id: user.id,
created: dateFns.format(user.created, dateFormat),
},
tags: {
...data.tags,
verified: user.verified && config.library.tags.verified,
verifiedEmail: user.verifiedEmail && config.library.tags.verifiedEmail,
gold: user.gold && config.library.tags.gold,
},
});
if (post.content.album) {
Object.assign(vars, {
$albumId: post.content.album.id,
$albumTitle: (post.content.album.title || '').slice(0, config.library.titleLength),
$albumDescription: post.content.album.description,
$albumDate: dateFns.format(post.content.album.datetime, dateFormat),
if (user.profile) {
Object.assign(data, {
profile: {
id: user.profile.id,
title: user.profile.title,
description: user.profile.description,
},
tags: {
...data.tags,
over18: user.profile.over18 && config.library.tags.over18,
},
});
}
}
if (item) {
Object.assign(vars, {
$itemId: item.id,
$itemTitle: (item.title || '').slice(0, config.library.titleLength),
$itemDescription: item.description,
$itemDate: dateFns.format(item.datetime, dateFormat),
$itemIndex: item.index + config.library.indexOffset,
$extracted: item.extracted ? config.library.booleans.extracted : '',
$preview: item.preview ? config.library.booleans.preview : '',
$ext: item.type ? `.${mime.extension(item.type)}` : path.extname(url.parse(item.url).pathname),
});
}
const strippedData = strip
? Object.entries(data).reduce((acc, [key, value]) => {
if (typeof value === 'string') {
return {
...acc,
[key]: value && value.toString().replace(/\//g, config.library.slashSubstitute),
};
}
return Object.entries(vars).reduce((acc, [key, value]) => {
// substitute slashes for filesystem compatability
if (key !== '$base' && strip) {
value = (value || '').toString().replace(/\//g, config.library.slashSubstitute);
}
return {
...acc,
[key]: Object.entries(value).reduce((subacc, [subkey, subvalue]) => ({
...subacc,
[subkey]: subvalue && subvalue.toString().replace(/\//g, config.library.slashSubstitute),
}), {}),
};
}, {})
: data;
return acc.replace(key, value);
}, pattern);
const base = {
posts: format(config.library.base.posts, strippedData),
direct: format(config.library.base.direct, strippedData),
};
const interpolated = format(pattern, {
base,
...strippedData,
});
return interpolated;
}
module.exports = interpolate;

36
src/methods/erome.js
View File

@ -1,32 +1,30 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const cheerio = require('cheerio');
const base = 'https://www.erome.com/';
function erome(post) {
return fetch(`${base}a/${post.host.id}`).then(res => {
if(res.ok) {
return res.text();
}
async function erome(host) {
const res = await fetch(`${base}a/${host.id}`);
throw new Error(`Unable to retrieve info for Erome album '${post.host.id}' :(`);
}).then(res => {
const $ = cheerio.load(res);
const videoUrls = $('video').toArray().map(videoEl => {
const sourceHd = $(videoEl).find('source[label="HD"]');
const sourceSd = $(videoEl).find('source[label="SD"]');
if (res.ok) {
throw new Error(`Unable to retrieve info for Erome album '${host.id}' :(`);
}
console.log(sourceHd.attr('src'));
const html = await res.text();
return sourceHd ? base + sourceHd.attr('src') : base + sourceSd.attr('src');
});
const $ = cheerio.load(html);
const videoUrls = $('video').toArray().map((videoEl) => {
const sourceHd = $(videoEl).find('source[label="HD"]');
const sourceSd = $(videoEl).find('source[label="SD"]');
console.log(videoUrls);
console.log(sourceHd.attr('src'));
return sourceHd ? base + sourceHd.attr('src') : base + sourceSd.attr('src');
});
};
console.log(videoUrls);
}
module.exports = erome;

64
src/methods/eroshareAlbum.js
View File

@ -1,42 +1,38 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
function eroshareAlbum(post) {
return fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${post.host.id}`).then(res => {
if(res.ok) {
return res.text();
}
async function eroshareAlbum(host) {
const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);
return Promise.reject(`Unable to recover Eroshare album or item '${post.host.id}' :(`);
}).then(res => {
const data = JSON.parse(res.match(/var album = .*/)[0].slice(12, -1));
const extract = config.library.album.extractSingleItem && data.items.length === 1;
if (!res.ok) {
throw new Error(`Unable to recover Eroshare album or item '${host.id}' :(`);
}
return {
album: extract ? null : {
id: data.slug,
title: data.title,
datetime: new Date(data.created_at)
},
items: data.items.map(item => {
return {
extracted: extract,
id: item.slug,
url: item.type === 'Image' ? item.url_full_protocol : item.url_mp4,
title: data.title,
description: item.description,
type: item.type === 'Image' ? 'image/jpeg' : 'video/mp4',
datetime: new Date(data.created_at),
width: data.width,
height: data.height,
original: item
};
})
};
});
};
const html = await res.text();
const data = JSON.parse(html.match(/var album = .*/)[0].slice(12, -1));
const extract = config.library.extractSingleAlbumItem && data.items.length === 1;
return {
album: extract ? null : {
id: data.slug,
title: data.title,
datetime: new Date(data.created_at),
},
items: data.items.map(item => ({
extracted: extract,
id: item.slug,
url: item.type === 'Image' ? item.url_full_protocol : item.url_mp4,
title: data.title,
description: item.description,
type: item.type === 'Image' ? 'image/jpeg' : 'video/mp4',
datetime: new Date(data.created_at),
width: data.width,
height: data.height,
original: item,
})),
};
}
module.exports = eroshareAlbum;

48
src/methods/eroshareItem.js
View File

@ -1,34 +1,30 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const cheerio = require('cheerio');
const mime = require('mime-types');
function eroshareItem(post) {
return fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${post.host.id}`).then(res => {
if(res.ok) {
return res.text();
}
async function eroshareItem(host, post) {
const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${host.id}`);
if (!res.ok) {
throw new Error(`Unable to recover Eroshare item '${host.id}' :(`);
}
return Promise.reject(`Unable to recover Eroshare item '${post.host.id}' :(`);
}).then(res => {
const $ = cheerio.load(res);
const videoElement = $('source[data-default="true"]');
const html = await res.text();
return {
album: null,
items: [{
id: post.host.id,
url: videoElement.attr('src'),
title: post.title,
type: videoElement.attr('type'),
datetime: post.datetime,
original: post
}]
};
});
};
const $ = cheerio.load(html);
const videoElement = $('source[data-default="true"]');
return {
album: null,
items: [{
id: host.id,
url: videoElement.attr('src'),
title: post ? post.title : null,
type: videoElement.attr('type'),
datetime: post ? post.datetime : null,
original: post || null,
}],
};
}
module.exports = eroshareItem;

4
src/methods/gfycat.js
View File

@ -2,8 +2,8 @@
const fetch = require('node-fetch');
async function gfycat(post) {
const res = await fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`);
async function gfycat(host) {
const res = await fetch(`https://api.gfycat.com/v1/gfycats/${host.id}`);
const data = await res.json();
if (data.error) {

81
src/methods/imgurAlbum.js
View File

@ -2,52 +2,51 @@
const config = require('config');
const fetch = require('node-fetch');
const mime = require('mime-types');
// const mime = require('mime-types');
async function imgurAlbumApi(post) {
return fetch(`https://api.imgur.com/3/album/${post.host.id}`, {
async function imgurAlbumApi(host, post) {
const res = await fetch(`https://api.imgur.com/3/album/${host.id}`, {
headers: {
Authorization: `Client-ID ${config.methods.imgur.clientId}`,
},
}).then(res => res.json()).then((res) => {
if (res.status !== 200) {
throw new Error(`Could not fetch info for imgur album '${post.host.id}': ${res.data.error}`);
}
const extract = config.library.album.extractSingleItem && res.data.images.length === 1;
if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.data.link}`);
}
return {
album: extract ? null : {
id: res.data.id,
url: res.data.link,
title: res.data.title,
description: res.data.description,
datetime: new Date(res.data.datetime * 1000),
original: res.data,
},
items: res.data.images.map(item => ({
extracted: extract,
id: item.id,
url: item.animated ? item.mp4 : item.link,
title: item.title || (extract ? res.data.title : null),
description: item.description || (extract ? res.data.description : null),
type: item.animated ? 'video/mp4' : item.type,
datetime: item.datetime * 1000,
original: item,
})),
};
});
if (res.status !== 200) {
throw new Error(`Could not fetch info for imgur album '${host.id}': ${res.data.error}`);
}
const { data } = await res.json();
const extract = config.library.extractSingleAlbumItem && data.images.length === 1;
if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${data.link}' (${post ? post.url : 'no post'})`);
}
return {
album: extract ? null : {
id: data.id,
url: data.link,
title: data.title,
description: data.description,
datetime: new Date(data.datetime * 1000),
original: data,
},
items: data.images.map(item => ({
extracted: extract,
id: item.id,
url: item.animated ? item.mp4 : item.link,
title: item.title || data.title || null,
description: item.description || data.description || null,
type: item.animated ? 'video/mp4' : item.type,
datetime: item.datetime * 1000,
original: item,
})),
};
}
async function imgurAlbum(post) {
return imgurAlbumApi(post);
/*
* as of late 2019, imgur requires log in to view albums and gallery images
/*
* as of late 2019, imgur requires log in to view albums and gallery images
async function imgurAlbum(host, post) {
const res = await fetch(`https://imgur.com/a/${post.host.id}`);
const html = await res.text();
@ -84,7 +83,7 @@ async function imgurAlbum(post) {
datetime: new Date(item.datetime),
})),
};
*/
}
*/
module.exports = imgurAlbum;
module.exports = imgurAlbumApi;

44
src/methods/imgurImage.js
View File

@ -3,33 +3,35 @@
const config = require('config');
const fetch = require('node-fetch');
async function imgurImageApi(post) {
return fetch(`https://api.imgur.com/3/image/${post.host.id}`, {
async function imgurImageApi(host) {
const res = await fetch(`https://api.imgur.com/3/image/${host.id}`, {
headers: {
Authorization: `Client-ID ${config.methods.imgur.clientId}`,
},
}).then(res => res.json()).then((res) => {
if (res.status !== 200) {
throw new Error(`Could not fetch info for imgur image '${post.host.id}': ${res.data.error}`);
}
return {
album: null,
items: [{
id: res.data.id,
url: res.data.animated ? res.data.mp4 : res.data.link,
title: res.data.title,
description: res.data.description,
type: res.data.animated ? 'video/mp4' : res.data.type,
datetime: new Date(res.data.datetime * 1000),
original: res.data,
}],
};
});
const { data } = await res.json();
if (res.status !== 200) {
throw new Error(`Could not fetch info for imgur image '${host.id}': ${res.data.error}`);
}
return {
album: null,
items: [{
id: data.id,
url: data.animated ? data.mp4 : data.link,
title: data.title,
description: data.description,
type: data.animated ? 'video/mp4' : data.type,
datetime: new Date(data.datetime * 1000),
original: data,
}],
};
}
async function imgurImage(post) {
return imgurImageApi(post);
async function imgurImage(host, post) {
return imgurImageApi(host, post);
/*
* as of late 2019, imgur requires log in to view albums and gallery images

42
src/methods/methods.js
View File

@ -1,31 +1,33 @@
'use strict';
const self = require('./self');
const redditImage = require('./redditImage');
const redditVideo = require('./redditVideo');
const redditPreview = require('./redditPreview');
const imgurImage = require('./imgurImage');
const imgurAlbum = require('./imgurAlbum');
const vidbleImage = require('./vidbleImage');
const vidbleVideo = require('./vidbleVideo');
const vidbleAlbum = require('./vidbleAlbum');
const gfycat = require('./gfycat');
const erome = require('./erome');
const eroshareAlbum = require('./eroshareAlbum');
const eroshareItem = require('./eroshareItem');
const gfycat = require('./gfycat');
const imgurAlbum = require('./imgurAlbum');
const imgurImage = require('./imgurImage');
const pornhub = require('./pornhub');
const redditImage = require('./redditImage');
const redditPreview = require('./redditPreview');
const redditVideo = require('./redditVideo');
const self = require('./self');
const vidbleAlbum = require('./vidbleAlbum');
const vidbleImage = require('./vidbleImage');
const vidbleVideo = require('./vidbleVideo');
module.exports = {
self,
redditImage,
redditVideo,
redditPreview,
imgurImage,
imgurAlbum,
vidbleImage,
vidbleVideo,
vidbleAlbum,
gfycat,
erome,
eroshareAlbum,
eroshareItem,
gfycat,
imgurAlbum,
imgurImage,
pornhub,
redditImage,
redditPreview,
redditVideo,
self,
vidbleAlbum,
vidbleImage,
vidbleVideo,
};

12
src/methods/pornhub.js
View File

@ -2,11 +2,11 @@
const fetch = require('node-fetch');
async function pornhub(post) {
const res = await fetch(`https://www.pornhub.com/view_video.php?viewkey=${post.host.id}`);
async function pornhub(host, post) {
const res = await fetch(`https://www.pornhub.com/view_video.php?viewkey=${host.id}`);
if (res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${post.host.id}': '${res.error}'`);
throw new Error(`Could not fetch info PornHub video '${host.id}': '${res.error}'`);
}
const html = await res.text();
@ -27,11 +27,11 @@ async function pornhub(post) {
return {
album: null,
items: [{
id: post.host.id,
id: host.id,
url,
title: post.title,
title: post ? post.title : null,
type: 'video/mp4',
datetime: post.datetime,
datetime: post ? post.datetime : null,
}],
};
}

18
src/methods/redditImage.js
View File

@ -1,21 +1,19 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const mime = require('mime-types');
function redditImage(post) {
return Promise.resolve({
async function redditImage(host, post) {
return {
album: null,
items: [{
id: post.host.id || post.id,
id: host.id || post.id,
url: post.url,
title: post.title,
datetime: post.datetime,
type: mime.lookup(post.url.split('/.')[0]) || 'image/jpeg',
original: post
}]
});
};
original: post,
}],
};
}
module.exports = redditImage;

37
src/methods/redditPreview.js
View File

@ -1,33 +1,26 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const path = require('path');
const fetch = require('node-fetch');
const mime = require('mime-types');
const urlPattern = require('url-pattern');
function redditPreview(post) {
return Promise.resolve({
async function redditPreview(host, post) {
return {
album: post.preview.length > 1 ? {
id: post.host.id || post.id,
url: post.url,
title: post.title,
datetime: post.datetime,
original: post
original: post,
} : null,
items: post.preview.map(image => {
return {
id: post.host.id || post.id,
url: image.url,
title: post.title,
datetime: post.datetime,
type: mime.lookup(image.url.split('?')[0]),
preview: true,
original: post
};
})
});
};
items: post.preview.map(image => ({
id: post.host.id || post.id,
url: image.url,
title: post.title,
datetime: post.datetime,
type: mime.lookup(image.url.split('?')[0]),
preview: true,
original: post,
})),
};
}
module.exports = redditPreview;

57
src/methods/redditVideo.js
View File

@ -1,38 +1,35 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const fs = require('fs-extra');
function redditVideo(post) {
return fetch(`${post.permalink}.json`).then(res => res.json()).then(res => {
return res[0].data.children[0].data.media.reddit_video.fallback_url;
}).then(videoUrl => {
const audioUrl = videoUrl.split('/').slice(0, -1).join('/') + '/audio';
async function redditVideo(host, post) {
const res = await fetch(`${post.permalink}.json`);
const [{ data }] = await res.json();
return fetch(audioUrl, {
method: 'HEAD'
}).then(res => {
const item = {
album: null,
items: [{
id: post.host.id || post.id,
url: videoUrl,
title: post.title,
datetime: post.datetime,
type: 'video/mp4',
original: post
}]
};
const videoUrl = data.children[0].data.media.reddit_video.fallback_url;
const audioUrl = `${videoUrl.split('/').slice(0, -1).join('/')}/audio`;
if(res.status === 200) {
item.items[0].mux = [audioUrl];
}
return item;
});
const audioRes = await fetch(audioUrl, {
method: 'HEAD',
});
};
const item = {
album: null,
items: [{
id: post.host.id || post.id,
url: videoUrl,
title: post.title,
datetime: post.datetime,
type: 'video/mp4',
original: post,
}],
};
if (audioRes.status === 200) {
item.items[0].mux = [audioUrl];
}
return item;
}
module.exports = redditVideo;

77
src/methods/vidbleAlbum.js
View File

@ -1,55 +1,52 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const urlPattern = require('url-pattern');
const UrlPattern = require('url-pattern');
const cheerio = require('cheerio');
const mime = require('mime-types');
const pattern = new urlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');
const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');
function vidbleAlbum(post) {
return fetch(`https://www.vidble.com/album/${post.host.id}`).then(res => {
if(res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${post.host.id}': '${res.error}'`);
}
async function vidbleAlbum(host, post) {
const res = await fetch(`https://www.vidble.com/album/${host.id}`);
return res.text();
}).then(res => {
const $ = cheerio.load(res);
if (res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${host.id}': '${res.error}'`);
}
const title = $('h2').text();
const imgUrls = $('img.img2').toArray().map(img => `https://vidble.com${img.attribs.src || img.attribs['data-original']}`);
const html = await res.text();
const $ = cheerio.load(html);
const extract = config.library.album.extractSingleItem && imgUrls.length === 1;
const title = $('h2').text();
const imgUrls = $('img.img2').toArray().map(img => `https://vidble.com${img.attribs.src || img.attribs['data-original']}`);
const extract = config.library.extractSingleAlbumItem && imgUrls.length === 1;
if(extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.link}`);
}
if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.link}`);
}
return {
album: extract ? null : {
id: post.host.id,
url: post.url,
title: post.title,
datetime: post.datetime
},
items: imgUrls.map(url => {
const components = pattern.match(url);
const id = components.id.replace('_med', '');
const mimetype = mime.lookup(components.ext);
return {
album: extract ? null : {
id: host.id,
url: post ? post.url : null,
title: post ? post.title : title,
datetime: post ? post.datetime : null,
},
items: imgUrls.map((url) => {
const components = pattern.match(url);
const id = components.id.replace('_med', '');
const mimetype = mime.lookup(components.ext);
return {
extracted: extract,
id: id,
url: `https://vidble.com/${id}.${components.ext}`,
type: mimetype,
datetime: post.datetime
};
})
};
});
};
return {
extracted: extract,
id,
url: `https://vidble.com/${id}.${components.ext}`,
type: mimetype,
datetime: post.datetime,
};
}),
};
}
module.exports = vidbleAlbum;

46
src/methods/vidbleImage.js
View File

@ -1,34 +1,30 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const path = require('path');
const fetch = require('node-fetch');
const $ = require('cheerio');
const mime = require('mime-types');
function vidbleImage(post) {
return fetch(`https://vidble.com/${post.host.id}`).then(res => {
if(res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${post.host.id}': '${res.error}'`);
}
async function vidbleImage(host, post) {
const res = await fetch(`https://vidble.com/${host.id}`);
return res.text();
}).then(res => {
const resource = $('img', res).attr('src');
if (res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${host.id}': '${res.error}'`);
}
return {
album: null,
items: [{
id: post.host.id,
url: `https://vidble.com/${resource}`,
title: post.title,
datetime: post.datetime,
type: mime.lookup(resource),
original: post
}]
};
});
};
const html = await res.text();
const resource = $('img', html).attr('src');
return {
album: null,
items: [{
id: host.id,
url: `https://vidble.com/${resource}`,
title: post ? post.title : null,
datetime: post ? post.datetime : null,
type: mime.lookup(resource),
original: post || null,
}],
};
}
module.exports = vidbleImage;

58
src/methods/vidbleVideo.js
View File

@ -1,39 +1,37 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const cheerio = require('cheerio');
function vidbleVideo(post) {
return fetch(`https://www.vidble.com/watch?v=${post.host.id}`).then(res => {
if(res.status !== 200) {
throw new Error(`Could not fetch info for vidble video '${post.host.id}': '${res.error}'`);
}
async function vidbleVideo(host, post) {
const res = await fetch(`https://www.vidble.com/watch?v=${host.id}`);
return res.text();
}).then(res => {
const $ = cheerio.load(res);
const resource = $('video source');
const source = resource.attr('src');
const type = resource.attr('type');
if (res.status !== 200) {
throw new Error(`Could not fetch info for vidble video '${host.id}': '${res.error}'`);
}
if(!source || !type) {
throw new Error(`Failed to retrieve (likely removed) vidble video '${post.host.id}'`);
}
const html = await res.text();
return {
album: null,
items: [{
id: post.host.id,
url: `https://vidble.com/${source}`,
title: post.title,
datetime: post.datetime,
type: type,
original: post
}]
};
});
};
const $ = cheerio.load(html);
const resource = $('video source');
const source = resource.attr('src');
const type = resource.attr('type');
if (!source || !type) {
throw new Error(`Failed to retrieve (likely removed) vidble video '${host.id}'`);
}
return {
album: null,
items: [{
id: host.id,
url: `https://vidble.com/${source}`,
title: post ? post.title : null,
datetime: post ? post.datetime : null,
type,
original: post || null,
}],
};
}
module.exports = vidbleVideo;

15
src/save/profileDetails.js
View File

@ -25,10 +25,17 @@ async function saveProfileImage(user, args) {
return null;
}
const filepath = interpolate(config.library.profile.image, user, null, {
const filepath = interpolate(
config.library.profile.image,
{
// pass profile image as item to interpolate extension variable
url: image,
});
url: image,
},
null,
null,
null,
user,
);
try {
const stream = await fetchItem(image, 0, { permalink: `https://reddit.com/user/${user.name}` });
@ -54,7 +61,7 @@ async function saveProfileDescription(user, args) {
if (config.library.profile.description && !user.fallback && !user.deleted) {
if (user.profile && user.profile.description) {
const filepath = interpolate(config.library.profile.description, user);
const filepath = interpolate(config.library.profile.description, null, null, null, null, user);
const stream = textToStream(user.profile.description);
try {

2
src/save/writeToIndex.js
View File

@ -8,7 +8,7 @@ const textToStream = require('./textToStream');
const save = require('./save');
async function writeToIndex(posts, profilePaths, user, args) {
const filepath = interpolate(config.library.index.file, user, null, false);
const filepath = interpolate(config.library.index.file, null, null, null, null, user, false);
const now = new Date();
const newAndUpdatedEntries = posts.concat(user.indexed.updated, args.indexIgnored ? user.indexed.ignored : []).map((post) => {

2
src/sources/getIndex.js
View File

@ -7,7 +7,7 @@ const yaml = require('js-yaml');
const interpolate = require('../interpolate.js');
async function getIndex(user) {
const indexFilePath = interpolate(config.library.index.file, user, null, null, false);
const indexFilePath = interpolate(config.library.index.file, null, null, null, null, user, false);
try {
const indexFile = await fs.readFile(indexFilePath, 'utf8');