From adbbc79b16d1ba216be5cec9ca1f33b22c477b05 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Fri, 11 May 2018 01:46:08 +0200 Subject: [PATCH] Added support for Vidble (image) albums. Replaced extensions.json and reverseLookup with mime-types lib. --- config/default.js | 3 +- package.json | 1 + src/dissectLink.js | 8 ++++++ src/extensions.json | 7 ----- src/interpolate.js | 5 ++-- src/methods/methods.js | 4 +++ src/methods/redditImage.js | 11 ++------ src/methods/redditPreview.js | 11 ++------ src/methods/vidbleAlbum.js | 55 ++++++++++++++++++++++++++++++++++++ src/methods/vidbleImage.js | 26 +++++++++++++++++ 10 files changed, 103 insertions(+), 28 deletions(-) delete mode 100644 src/extensions.json create mode 100644 src/methods/vidbleAlbum.js create mode 100644 src/methods/vidbleImage.js diff --git a/config/default.js b/config/default.js index b1f6458..d85acd9 100644 --- a/config/default.js +++ b/config/default.js @@ -26,7 +26,8 @@ module.exports = { comment: '$itemDescription' }, dateFormat: 'YYYYMMDD', - titleLength: 200, + filenameLimit: 250, + truncator: '...', indexOffset: 1, slashSubstitute: '#', }, diff --git a/package.json b/package.json index 1494394..db8a469 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "dist-exiftool": "^10.53.0", "fluent-ffmpeg": "^2.1.2", "fs-extra": "^5.0.0", + "mime-types": "^2.1.18", "node-exiftool": "^2.3.0", "node-fetch": "^2.1.2", "promise.prototype.finally": "^3.1.0", diff --git a/src/dissectLink.js b/src/dissectLink.js index e0a318f..9926599 100644 --- a/src/dissectLink.js +++ b/src/dissectLink.js @@ -26,6 +26,14 @@ const hosts = [{ method: 'imgurAlbum', label: 'imgur', pattern: new urlPattern('http(s)\\://(:subdomain.)imgur.com/:type/:id') +}, { + method: 'vidbleImage', + label: 'vidble', + pattern: new urlPattern('http(s)\\://(www.)vidble.com/(show/):id(.:ext)') +}, { + method: 'vidbleAlbum', + label: 'vidble', + pattern: new urlPattern('http(s)\\://(www.)vidble.com/album/:id') }, { method: 'gfycat', label: 'gfycat', diff --git a/src/extensions.json b/src/extensions.json deleted file mode 100644 index 995904f..0000000 --- a/src/extensions.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "image/jpeg": ".jpg", - "image/png": ".png", - "image/gif": ".gif", - "video/mp4": ".mp4", - "video/webm": ".webm" -} diff --git a/src/interpolate.js b/src/interpolate.js index 1d81bb8..8ef0dd1 100644 --- a/src/interpolate.js +++ b/src/interpolate.js @@ -4,8 +4,7 @@ const config = require('config'); const path = require('path'); const url = require('url'); const dateFns = require('date-fns'); - -const extensions = require('./extensions.json'); +const mime = require('mime-types'); function interpolate(pattern, user, post, item) { const dateFormat = config.library.dateFormat || 'YYYYMMDD'; @@ -65,7 +64,7 @@ function interpolate(pattern, user, post, item) { $itemIndex: item.index + config.library.indexOffset, $extracted: item.extracted ? config.library.booleans.extracted : '', $preview: item.preview ? config.library.booleans.preview : '', - $ext: item.extension || (item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname)) + $ext: `.${mime.extension(item.type).replace('jpeg', 'jpg') || path.extname(url.parse(item.url).pathname)}` }); } diff --git a/src/methods/methods.js b/src/methods/methods.js index 9a6bbd0..3ec5033 100644 --- a/src/methods/methods.js +++ b/src/methods/methods.js @@ -6,6 +6,8 @@ const redditVideo = require('./redditVideo.js'); const redditPreview = require('./redditPreview.js'); const imgurImage = require('./imgurImage.js'); const imgurAlbum = require('./imgurAlbum.js'); +const vidbleImage = require('./vidbleImage.js'); +const vidbleAlbum = require('./vidbleAlbum.js'); const gfycat = require('./gfycat.js'); const eroshare = require('./eroshare.js'); @@ -16,6 +18,8 @@ module.exports = { redditPreview, imgurImage, imgurAlbum, + vidbleImage, + vidbleAlbum, gfycat, eroshare }; diff --git a/src/methods/redditImage.js b/src/methods/redditImage.js index 83690f4..2642914 100644 --- a/src/methods/redditImage.js +++ b/src/methods/redditImage.js @@ -2,14 +2,7 @@ const util = require('util'); const config = require('config'); -const path = require('path'); -const fetch = require('node-fetch'); - -const extensions = require('../extensions.json'); - -function reverseLookup(extension) { - return Object.keys(extensions).find(mime => extensions[mime] === extension); -}; +const mime = require('mime-types'); function redditImage(post) { return Promise.resolve({ @@ -19,7 +12,7 @@ function redditImage(post) { url: post.url, title: post.title, datetime: post.datetime, - type: reverseLookup(path.extname(post.url)), + type: mime.lookup(post.url), original: post }] }); diff --git a/src/methods/redditPreview.js b/src/methods/redditPreview.js index 44a55fb..811e1eb 100644 --- a/src/methods/redditPreview.js +++ b/src/methods/redditPreview.js @@ -4,14 +4,9 @@ const util = require('util'); const config = require('config'); const path = require('path'); const fetch = require('node-fetch'); +const mime = require('mime-types'); const urlPattern = require('url-pattern'); -const extensions = require('../extensions.json'); - -function reverseLookup(extension) { - return Object.keys(extensions).find(mime => extensions[mime] === extension); -}; - function redditPreview(post) { return Promise.resolve({ album: post.preview.length > 1 ? { @@ -22,14 +17,14 @@ function redditPreview(post) { original: post } : null, items: post.preview.map(image => { - const urlComponents = new urlPattern('http(s)\\://i.redditmedia.com/:id(.:ext)(?*)').match(image.url); + console.log(mime.lookup(image.url)); return { id: post.host.id || post.id, url: image.url, title: post.title, datetime: post.datetime, - type: reverseLookup(`.${urlComponents.ext}`), + type: mime.lookup(image.url), preview: true, original: post }; diff --git a/src/methods/vidbleAlbum.js b/src/methods/vidbleAlbum.js new file mode 100644 index 0000000..b70ebaf --- /dev/null +++ b/src/methods/vidbleAlbum.js @@ -0,0 +1,55 @@ + 'use strict'; + +const util = require('util'); +const config = require('config'); +const fetch = require('node-fetch'); +const urlPattern = require('url-pattern'); +const cheerio = require('cheerio'); +const mime = require('mime-types'); + +const pattern = new urlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)'); + +function vidbleAlbum(post) { + return fetch(`https://www.vidble.com/album/${post.host.id}`).then(res => { + if(res.status !== 200) { + throw new Error(`Could not fetch info for vidble album '${post.host.id}': '${res.error}'`); + } + + return res.text(); + }).then(res => { + const $ = cheerio.load(res); + + const title = $('h2').text(); + const imgUrls = $('img.img2').toArray().map(img => `https://vidble.com${img.attribs.src || img.attribs['data-original']}`); + + const extract = config.library.album.extractSingleItem && imgUrls.length === 1; + + if(extract) { + console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.link}`); + } + + return { + album: extract ? null : { + id: post.host.id, + url: post.url, + title: post.title, + datetime: post.datetime + }, + items: imgUrls.map(url => { + const components = pattern.match(url); + const id = components.id.replace('_med', ''); + const mimetype = mime.lookup(components.ext); + + return { + extracted: extract, + id: id, + url: `https://vidble.com/${id}.${components.ext}`, + type: mimetype, + datetime: post.datetime + }; + }) + }; + }); +}; + +module.exports = vidbleAlbum; diff --git a/src/methods/vidbleImage.js b/src/methods/vidbleImage.js new file mode 100644 index 0000000..8d66b8f --- /dev/null +++ b/src/methods/vidbleImage.js @@ -0,0 +1,26 @@ + 'use strict'; + +const util = require('util'); +const config = require('config'); +const path = require('path'); +const fetch = require('node-fetch'); +const $ = require('cheerio'); + +function vidbleImage(post) { + return fetch(`https://vidble.com/${post.host.id}`).then(res => res.text()).then(res => { + console.log(res, $('img')); + + if(res.status !== 200) { + throw new Error(`Could not fetch info for vidble album '${post.host.id}': '${res.data.error}'`); + } + + const extract = config.library.album.extractSingleItem && res.data.images.length === 1; + + if(extract) { + console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.data.link}`); + } + + }); +}; + +module.exports = vidbleImage;