DebaucheryLibrarian
/
ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added support for i.redd.it and self posts. Moved pattern interpolation out of fetch module.

This commit is contained in:
ThePendulum 2018-04-18 00:18:04 +02:00
parent ad1796bc13
commit 821e7fff82
13 changed files with 144 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -28,7 +28,7 @@ Path patterns dictate where and how a file will be saved. Various variables and
* `$itemDescription`: The description of the individual image or video
* `$itemDate`: The submission date of the individual image or video, formatted by the `dateformat` configuration described below
* `$itemIndex`: The index of the individual image or video in an album, offset by the `indexOffset` configuration described below
* `$ext`: The extension of the file. Must practically always be included.
* `$ext`: The extension of the file. Must typically be included, but may be omitted for self (text) posts on Unix systems
### Date format
Affects the representation of `$postDate`, `$albumDate` and `$itemDate` and defaults to `YYYYMMDD`. See [this documentation](https://date-fns.org/v1.29.0/docs/format) for an overview of all available tokens.
@ -37,4 +37,4 @@ Affects the representation of `$postDate`, `$albumDate` and `$itemDate` and defa
Arrays start at 0, but as to not tire myself out debating the matter, you may offset it my any numerical value you like. Affects the `$itemIndex` variable for album items.
### Slash substitute
The patterns are Unix file paths, and a `/` therefore indicates a new directory. You may freely use directories in your paths, but titles or descriptions may contain a `/` that is not supposed to create a new directory. All instances of `/` in a variable value will be replaced with the configured slash substitute.
The patterns represent Unix file paths, and a `/` therefore indicates a new directory. You may freely use directories in your paths, but titles or descriptions may contain a `/` that is not supposed to create a new directory. All instances of `/` in a variable value will be replaced with the configured slash substitute.

7
app.js
View File

@ -2,7 +2,6 @@
const config = require('config');
const util = require('util');
const note = require('note-log');
const yargs = require('yargs').argv;
const snoowrap = require('snoowrap');
const methods = require('./methods/methods.js');
@ -12,13 +11,13 @@ const fetchContent = require('./fetchContent.js');
const reddit = new snoowrap(config.reddit);
reddit.getUser(yargs.user).getSubmissions({
sort: 'top',
limit: Infinity
}).then(submissions => {
const curatedPosts = submissions.map(submission => {
return {
id: submission.id,
title: submission.title,
text: submission.selftext,
user: submission.author.name,
permalink: submission.permalink,
url: submission.url,
@ -36,11 +35,11 @@ reddit.getUser(yargs.user).getSubmissions({
return post;
}));
} else {
note('fetch', 1, `"${post.title}": '${post.url}' not supported :(`);
console.log('\x1b[33m%s\x1b[0m', `"${post.title}": '${post.url}' not supported :(`);
}
return acc;
}, []));
}).then(posts => fetchContent(posts)).catch(error => {
note(error);
console.error(error);
});

8
dissectLink.js
View File

@ -3,11 +3,17 @@
const urlPattern = require('url-pattern');
const hosts = [{
method: 'self',
pattern: new urlPattern('http(s)\\://(www.)reddit.com/r/:subreddit/comments/:id/:uri/')
}, {
method: 'reddit',
pattern: new urlPattern('http(s)\\://i.redd.it/:id.:ext')
}, {
method: 'imgurImage',
pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?:num)')
}, {
method: 'imgurAlbum',
pattern: new urlPattern('http(s)\\://imgur.com/:type/:id')
pattern: new urlPattern('http(s)\\://(m.)imgur.com/:type/:id')
}, {
method: 'gfycat',
pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)')

81
fetchContent.js
View File

@ -1,70 +1,37 @@
'use strict';
const Readable = require('stream').Readable;
const fs = require('fs-extra');
const path = require('path');
const config = require('config');
const fetch = require('node-fetch');
const dateFns = require('date-fns');
const interpolate = require('./interpolate.js');
const extensions = {
'image/jpeg': '.jpg',
'image/gif': '.gif',
'video/mp4': '.mp4',
'video/webm': '.webm'
};
function interpolate(path, post, item, index) {
const dateFormat = config.patterns.dateformat || 'YYYYMMDD';
const vars = {
$postId: post.id,
$postTitle: post.title,
$postUser: post.user,
$postDate: dateFns.format(post.datetime, dateFormat)
};
if(post.content.album) {
Object.assign(vars, {
$albumId: post.content.album.id,
$albumTitle: post.content.album.title,
$albumDescription: post.content.album.description,
$albumDate: dateFns.format(post.content.album.datetime, dateFormat)
});
}
if(item) {
Object.assign(vars, {
$itemId: item.id,
$itemTitle: item.title,
$itemDescription: item.description,
$itemDate: dateFns.format(item.datetime, dateFormat),
$itemIndex: index + config.patterns.indexOffset,
$ext: extensions[item.type]
});
}
return Object.entries(vars).reduce((acc, [key, value], index) => {
// strip slashes for filesystem compatability
value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
return acc.replace(key, value);
}, path);
};
function saveItemToDisk(buffer, item, index, filename, post) {
const stream = fs.createWriteStream(filename);
function saveItemToDisk(stream, filepath) {
const file = fs.createWriteStream(filepath);
return new Promise((resolve, reject) => {
buffer.body.pipe(stream).on('error', error => {
stream.pipe(file).on('error', error => {
reject(error);
}).on('finish', () => {
console.log(`Saved '${filename}'`);
console.log(`Saved '${filepath}'`);
resolve(filename);
resolve(filepath);
});
});
};
function textPostToStream(item) {
const stream = new Readable();
stream.push(item.text);
stream.push(null);
return Object.assign(item, {
stream: stream
});
};
function fetchItem(item, index, post, attempt) {
function retry(error) {
console.log(error);
@ -81,7 +48,9 @@ function fetchItem(item, index, post, attempt) {
}).then(res => {
console.log(`Fetched '${item.url}'`);
return res;
return Object.assign({}, item, {
stream: res.body
});
}).catch(retry);
};
@ -89,7 +58,11 @@ module.exports = function(posts) {
return Promise.all(posts.map(post => {
return Promise.resolve().then(() => {
return Promise.all(post.content.items.map((item, index) => {
return fetchItem(item, index, post, 0).then(buffer => Object.assign(item, {buffer}));
if(item.self) {
return textPostToStream(item);
}
return fetchItem(item, index, post, 0);
}));
}).then(items => {
return Promise.all(items.map((item, index) => {
@ -99,7 +72,7 @@ module.exports = function(posts) {
return Promise.resolve().then(() => {
return fs.ensureDir(path.dirname(filepath));
}).then(() => {
return saveItemToDisk(item.buffer, item, index, filepath, post);
return saveItemToDisk(item.stream, filepath)
});
}));
});

51
interpolate.js Normal file
View File

@ -0,0 +1,51 @@
'use strict';
const config = require('config');
const dateFns = require('date-fns');
const extensions = {
'image/jpeg': '.jpg',
'image/gif': '.gif',
'video/mp4': '.mp4',
'video/webm': '.webm'
};
function interpolate(path, post, item, index) {
const dateFormat = config.patterns.dateformat || 'YYYYMMDD';
const vars = {
$postId: post.id,
$postTitle: post.title,
$postUser: post.user,
$postDate: dateFns.format(post.datetime, dateFormat)
};
if(post.content.album) {
Object.assign(vars, {
$albumId: post.content.album.id,
$albumTitle: post.content.album.title,
$albumDescription: post.content.album.description,
$albumDate: dateFns.format(post.content.album.datetime, dateFormat)
});
}
if(item) {
Object.assign(vars, {
$itemId: item.id,
$itemTitle: item.title,
$itemDescription: item.description,
$itemDate: dateFns.format(item.datetime, dateFormat),
$itemIndex: index + config.patterns.indexOffset,
$ext: extensions[item.type]
});
}
return Object.entries(vars).reduce((acc, [key, value], index) => {
// strip slashes for filesystem compatability
value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
return acc.replace(key, value);
}, path);
};
module.exports = interpolate;

7
methods/gfycat.js
View File

@ -1,11 +1,10 @@
'use strict';
const note = require('note-log');
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
function imgurImage(post) {
function gfycat(post) {
return fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`, {
headers: {
'Authorization': `Bearer ${config.methods.gfycat.key}`
@ -24,8 +23,8 @@ function imgurImage(post) {
}]
};
}).catch(error => {
note(error);
console.error(error);
});
};
module.exports = imgurImage;
module.exports = gfycat;

3
methods/imgurAlbum.js
View File

@ -1,6 +1,5 @@
'use strict';
const note = require('note-log');
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
@ -31,7 +30,7 @@ function imgurAlbum(post) {
}))
};
}).catch(error => {
note(error);
console.error(error);
});
};

3
methods/imgurImage.js
View File

@ -1,6 +1,5 @@
'use strict';
const note = require('note-log');
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
@ -24,7 +23,7 @@ function imgurImage(post) {
}]
};
}).catch(error => {
note(error);
console.error(error);
});
};

6
methods/methods.js
View File

@ -1,10 +1,14 @@
'use strict';
const self = require('./self.js');
const reddit = require('./reddit.js');
const imgurImage = require('./imgurImage.js');
const imgurAlbum = require('./imgurAlbum.js');
const gfycat= require('./gfycat.js');
const gfycat = require('./gfycat.js');
module.exports = {
self: self,
reddit: reddit,
imgurImage: imgurImage,
imgurAlbum: imgurAlbum,
gfycat: gfycat

21
methods/reddit.js Normal file
View File

@ -0,0 +1,21 @@
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
function reddit(post) {
return Promise.resolve({
album: null,
items: [{
id: post.id,
url: post.url,
title: post.title,
datetime: post.datetime,
type: 'image/jpeg',
original: post
}]
});
};
module.exports = reddit;

23
methods/self.js Normal file
View File

@ -0,0 +1,23 @@
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
function self(post) {
return Promise.resolve({
album: null,
items: [{
id: post.id,
url: post.url,
title: post.title,
text: post.text,
datetime: post.datetime,
type: 'text/plain',
self: true,
original: post
}]
});
};
module.exports = self;

133
package-lock.json generated
View File

@ -77,26 +77,6 @@
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"cli-color": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/cli-color/-/cli-color-1.2.0.tgz",
"integrity": "sha1-OlrnT9drYmevZm5p4q+70B3vNNE=",
"requires": {
"ansi-regex": "2.1.1",
"d": "1.0.0",
"es5-ext": "0.10.42",
"es6-iterator": "2.0.3",
"memoizee": "0.4.12",
"timers-ext": "0.1.5"
},
"dependencies": {
"ansi-regex": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
}
}
},
"cliui": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-4.0.0.tgz",
@ -167,14 +147,6 @@
}
}
},
"d": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/d/-/d-1.0.0.tgz",
"integrity": "sha1-dUu1v+VUUdpppYuU1F9MWwRi1Y8=",
"requires": {
"es5-ext": "0.10.42"
}
},
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@ -207,55 +179,6 @@
"jsbn": "0.1.1"
}
},
"es5-ext": {
"version": "0.10.42",
"resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.42.tgz",
"integrity": "sha512-AJxO1rmPe1bDEfSR6TJ/FgMFYuTBhR5R57KW58iCkYACMyFbrkqVyzXSurYoScDGvgyMpk7uRF/lPUPPTmsRSA==",
"requires": {
"es6-iterator": "2.0.3",
"es6-symbol": "3.1.1",
"next-tick": "1.0.0"
}
},
"es6-iterator": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
"integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=",
"requires": {
"d": "1.0.0",
"es5-ext": "0.10.42",
"es6-symbol": "3.1.1"
}
},
"es6-symbol": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz",
"integrity": "sha1-vwDvT9q2uhtG7Le2KbTH7VcVzHc=",
"requires": {
"d": "1.0.0",
"es5-ext": "0.10.42"
}
},
"es6-weak-map": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.2.tgz",
"integrity": "sha1-XjqzIlH/0VOKH45f+hNXdy+S2W8=",
"requires": {
"d": "1.0.0",
"es5-ext": "0.10.42",
"es6-iterator": "2.0.3",
"es6-symbol": "3.1.1"
}
},
"event-emitter": {
"version": "0.3.5",
"resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
"integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=",
"requires": {
"d": "1.0.0",
"es5-ext": "0.10.42"
}
},
"execa": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
@ -401,11 +324,6 @@
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
"integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="
},
"is-promise": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
"integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o="
},
"is-stream": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
@ -502,14 +420,6 @@
"yallist": "2.1.2"
}
},
"lru-queue": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz",
"integrity": "sha1-Jzi9nw089PhEkMVzbEhpmsYyzaM=",
"requires": {
"es5-ext": "0.10.42"
}
},
"mem": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz",
@ -518,21 +428,6 @@
"mimic-fn": "1.2.0"
}
},
"memoizee": {
"version": "0.4.12",
"resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.12.tgz",
"integrity": "sha512-sprBu6nwxBWBvBOh5v2jcsGqiGLlL2xr2dLub3vR8dnE8YB17omwtm/0NSHl8jjNbcsJd5GMWJAnTSVe/O0Wfg==",
"requires": {
"d": "1.0.0",
"es5-ext": "0.10.42",
"es6-weak-map": "2.0.2",
"event-emitter": "0.3.5",
"is-promise": "2.1.0",
"lru-queue": "0.1.0",
"next-tick": "1.0.0",
"timers-ext": "0.1.5"
}
},
"mime-db": {
"version": "1.33.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz",
@ -551,30 +446,11 @@
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
"integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="
},
"moment": {
"version": "2.22.0",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.22.0.tgz",
"integrity": "sha512-1muXCh8jb1N/gHRbn9VDUBr0GYb8A/aVcHlII9QSB68a50spqEVLIGN6KVmCOnSvJrUhC0edGgKU5ofnGXdYdg=="
},
"next-tick": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz",
"integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw="
},
"node-fetch": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz",
"integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U="
},
"note-log": {
"version": "2.1.11",
"resolved": "https://registry.npmjs.org/note-log/-/note-log-2.1.11.tgz",
"integrity": "sha1-DvEbJ2llJ2EfG5NHjaXCjyse7SQ=",
"requires": {
"cli-color": "1.2.0",
"moment": "2.22.0"
}
},
"npm-run-path": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
@ -816,15 +692,6 @@
"resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
"integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="
},
"timers-ext": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.5.tgz",
"integrity": "sha512-tsEStd7kmACHENhsUPaxb8Jf8/+GZZxyNFQbZD07HQOyooOa6At1rQqjffgvg7n+dxscQa9cjjMdWhJtsP2sxg==",
"requires": {
"es5-ext": "0.10.42",
"next-tick": "1.0.0"
}
},
"tough-cookie": {
"version": "2.3.4",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz",

1
package.json
View File

@ -26,7 +26,6 @@
"date-fns": "^1.29.0",
"fs-extra": "^5.0.0",
"node-fetch": "^2.1.2",
"note-log": "^2.1.11",
"snoowrap": "^1.15.2",
"url-pattern": "^1.0.3",
"yargs": "^11.0.0"