From 81f8f4be0df8eac9c8cb83cfbeb6a18d41844307 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Thu, 19 Apr 2018 01:47:09 +0200 Subject: [PATCH] Added groundwork for Eroshare support. Removed gfycat API requirement, using JSON output. --- config/default.js | 3 - dissectLink.js | 3 + methods/eroshare.js | 41 ++++++++++++ methods/gfycat.js | 6 +- methods/methods.js | 4 +- package-lock.json | 153 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 7 files changed, 202 insertions(+), 9 deletions(-) create mode 100644 methods/eroshare.js diff --git a/config/default.js b/config/default.js index 1587229..bc7d036 100644 --- a/config/default.js +++ b/config/default.js @@ -28,9 +28,6 @@ module.exports = { methods: { imgur: { clientId: '455ceaa737dbac0' - }, - gfycat: { - key: '2_oOveHD' } } }; diff --git a/dissectLink.js b/dissectLink.js index 322dbbf..46bfcc9 100644 --- a/dissectLink.js +++ b/dissectLink.js @@ -17,6 +17,9 @@ const hosts = [{ }, { method: 'gfycat', pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)') +}, { + method: 'eroshare', + pattern: new urlPattern('http(s)\\://eroshare.com/:id') }]; module.exports = function dissectLink(url) { diff --git a/methods/eroshare.js b/methods/eroshare.js new file mode 100644 index 0000000..dee277e --- /dev/null +++ b/methods/eroshare.js @@ -0,0 +1,41 @@ + 'use strict'; + +const util = require('util'); +const config = require('config'); +const fetch = require('node-fetch'); +const cheerio = require('cheerio'); + +function eroshare(post) { + return fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${post.host.id}`).then(res => { + if(res.ok) { + return res.text() + } + + return Promise.reject(`Unable to recover Eroshare video '${post.host.id}' :(`); + }).then(res => { + const $ = cheerio.load(res); + + $('video').forEach(video => { + console.log(video); + }); + + /* + return { + album: null, + items: [{ + id: res.gfyItem.gfyName, + url: res.gfyItem.webmUrl, + title: res.gfyItem.title, + description: res.gfyItem.description, + type: 'video/webm', + datetime: res.gfyItem.createDate * 1000, + original: res.gfyItem + }] + }; + */ + }).catch(error => { + console.log('\x1b[33m%s\x1b[0m', error); + }); +}; + +module.exports = eroshare; diff --git a/methods/gfycat.js b/methods/gfycat.js index 208235c..84696cf 100644 --- a/methods/gfycat.js +++ b/methods/gfycat.js @@ -5,11 +5,7 @@ const config = require('config'); const fetch = require('node-fetch'); function gfycat(post) { - return fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`, { - headers: { - 'Authorization': `Bearer ${config.methods.gfycat.key}` - } - }).then(res => res.json()).then(res => { + return fetch(`https://gfycat.com/cajax/get/${post.host.id}`).then(res => res.json()).then(res => { return { album: null, items: [{ diff --git a/methods/methods.js b/methods/methods.js index ab5aead..aad635a 100644 --- a/methods/methods.js +++ b/methods/methods.js @@ -5,11 +5,13 @@ const reddit = require('./reddit.js'); const imgurImage = require('./imgurImage.js'); const imgurAlbum = require('./imgurAlbum.js'); const gfycat = require('./gfycat.js'); +const eroshare = require('./eroshare.js'); module.exports = { self: self, reddit: reddit, imgurImage: imgurImage, imgurAlbum: imgurAlbum, - gfycat: gfycat + gfycat: gfycat, + eroshare: eroshare }; diff --git a/package-lock.json b/package-lock.json index 1a39260..1cd5c54 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,11 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/node": { + "version": "9.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.5.tgz", + "integrity": "sha512-NOLEgsT6UiDTjnWG5Hd2Mg25LRyz/oe8ql3wbjzgSFeRzRROhPmtlsvIrei4B46UjERF0td9SZ1ZXPLOdcrBHg==" + }, "ajv": { "version": "5.5.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", @@ -59,6 +64,11 @@ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, "boom": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", @@ -77,6 +87,19 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "cheerio": { + "version": "1.0.0-rc.2", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", + "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=", + "requires": { + "css-select": "1.2.0", + "dom-serializer": "0.1.0", + "entities": "1.1.1", + "htmlparser2": "3.9.2", + "lodash": "4.17.5", + "parse5": "3.0.3" + } + }, "cliui": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.0.0.tgz", @@ -147,6 +170,22 @@ } } }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "1.0.0", + "css-what": "2.1.0", + "domutils": "1.5.1", + "nth-check": "1.0.1" + } + }, + "css-what": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.0.tgz", + "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0=" + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -170,6 +209,44 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "dom-serializer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", + "requires": { + "domelementtype": "1.1.3", + "entities": "1.1.1" + }, + "dependencies": { + "domelementtype": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=" + } + } + }, + "domelementtype": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz", + "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=" + }, + "domhandler": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.1.tgz", + "integrity": "sha1-iS5HAAqZvlW783dP/qBWHYh5wlk=", + "requires": { + "domelementtype": "1.3.0" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0.1.0", + "domelementtype": "1.3.0" + } + }, "ecc-jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", @@ -179,6 +256,11 @@ "jsbn": "0.1.1" } }, + "entities": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", + "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" + }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -304,6 +386,19 @@ "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" }, + "htmlparser2": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz", + "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=", + "requires": { + "domelementtype": "1.3.0", + "domhandler": "2.4.1", + "domutils": "1.5.1", + "entities": "1.1.1", + "inherits": "2.0.3", + "readable-stream": "2.3.6" + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -314,6 +409,11 @@ "sshpk": "1.14.1" } }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, "invert-kv": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", @@ -334,6 +434,11 @@ "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -459,6 +564,14 @@ "path-key": "2.0.1" } }, + "nth-check": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz", + "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=", + "requires": { + "boolbase": "1.0.0" + } + }, "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", @@ -515,6 +628,14 @@ "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=" }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "9.6.5" + } + }, "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", @@ -530,6 +651,11 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, "promise-chains": { "version": "0.3.12", "resolved": "https://registry.npmjs.org/promise-chains/-/promise-chains-0.3.12.tgz", @@ -553,6 +679,20 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, "request": { "version": "2.85.0", "resolved": "https://registry.npmjs.org/request/-/request-2.85.0.tgz", @@ -674,6 +814,14 @@ "strip-ansi": "4.0.0" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "5.1.1" + } + }, "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", @@ -729,6 +877,11 @@ "resolved": "https://registry.npmjs.org/url-pattern/-/url-pattern-1.0.3.tgz", "integrity": "sha1-BAkpJHGyTyPFDWWkeTF5PStaz8E=" }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, "uuid": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", diff --git a/package.json b/package.json index 4742684..02b74d8 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "author": "Niels Simenon", "license": "ISC", "dependencies": { + "cheerio": "^1.0.0-rc.2", "config": "^1.30.0", "date-fns": "^1.29.0", "fs-extra": "^5.0.0",