DebaucheryLibrarian/ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added support for fetching content directly from host. Improved pattern interpolation. Refactored content modules.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2024-09-11 05:16:57 +02:00
parent 20cb522689
commit b9a7e4b83a
26 changed files with 572 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/methods/erome.js
View File

@@ -1,32 +1,30 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const cheerio = require('cheerio');
const base = 'https://www.erome.com/';
function erome(post) {
return fetch(`${base}a/${post.host.id}`).then(res => {
if(res.ok) {
return res.text();
}
async function erome(host) {
const res = await fetch(`${base}a/${host.id}`);
throw new Error(`Unable to retrieve info for Erome album '${post.host.id}' :(`);
}).then(res => {
const $ = cheerio.load(res);
const videoUrls = $('video').toArray().map(videoEl => {
const sourceHd = $(videoEl).find('source[label="HD"]');
const sourceSd = $(videoEl).find('source[label="SD"]');
if (res.ok) {
throw new Error(`Unable to retrieve info for Erome album '${host.id}' :(`);
}
console.log(sourceHd.attr('src'));
const html = await res.text();
return sourceHd ? base + sourceHd.attr('src') : base + sourceSd.attr('src');
});
const $ = cheerio.load(html);
const videoUrls = $('video').toArray().map((videoEl) => {
const sourceHd = $(videoEl).find('source[label="HD"]');
const sourceSd = $(videoEl).find('source[label="SD"]');
console.log(videoUrls);
console.log(sourceHd.attr('src'));
return sourceHd ? base + sourceHd.attr('src') : base + sourceSd.attr('src');
});
};
console.log(videoUrls);
}
module.exports = erome;

64
src/methods/eroshareAlbum.js
View File

@@ -1,42 +1,38 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
function eroshareAlbum(post) {
return fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${post.host.id}`).then(res => {
if(res.ok) {
return res.text();
}
async function eroshareAlbum(host) {
const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);
return Promise.reject(`Unable to recover Eroshare album or item '${post.host.id}' :(`);
}).then(res => {
const data = JSON.parse(res.match(/var album = .*/)[0].slice(12, -1));
const extract = config.library.album.extractSingleItem && data.items.length === 1;
if (!res.ok) {
throw new Error(`Unable to recover Eroshare album or item '${host.id}' :(`);
}
return {
album: extract ? null : {
id: data.slug,
title: data.title,
datetime: new Date(data.created_at)
},
items: data.items.map(item => {
return {
extracted: extract,
id: item.slug,
url: item.type === 'Image' ? item.url_full_protocol : item.url_mp4,
title: data.title,
description: item.description,
type: item.type === 'Image' ? 'image/jpeg' : 'video/mp4',
datetime: new Date(data.created_at),
width: data.width,
height: data.height,
original: item
};
})
};
});
};
const html = await res.text();
const data = JSON.parse(html.match(/var album = .*/)[0].slice(12, -1));
const extract = config.library.extractSingleAlbumItem && data.items.length === 1;
return {
album: extract ? null : {
id: data.slug,
title: data.title,
datetime: new Date(data.created_at),
},
items: data.items.map(item => ({
extracted: extract,
id: item.slug,
url: item.type === 'Image' ? item.url_full_protocol : item.url_mp4,
title: data.title,
description: item.description,
type: item.type === 'Image' ? 'image/jpeg' : 'video/mp4',
datetime: new Date(data.created_at),
width: data.width,
height: data.height,
original: item,
})),
};
}
module.exports = eroshareAlbum;

48
src/methods/eroshareItem.js
View File

@@ -1,34 +1,30 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const cheerio = require('cheerio');
const mime = require('mime-types');
function eroshareItem(post) {
return fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${post.host.id}`).then(res => {
if(res.ok) {
return res.text();
}
async function eroshareItem(host, post) {
const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/i/${host.id}`);
if (!res.ok) {
throw new Error(`Unable to recover Eroshare item '${host.id}' :(`);
}
return Promise.reject(`Unable to recover Eroshare item '${post.host.id}' :(`);
}).then(res => {
const $ = cheerio.load(res);
const videoElement = $('source[data-default="true"]');
const html = await res.text();
return {
album: null,
items: [{
id: post.host.id,
url: videoElement.attr('src'),
title: post.title,
type: videoElement.attr('type'),
datetime: post.datetime,
original: post
}]
};
});
};
const $ = cheerio.load(html);
const videoElement = $('source[data-default="true"]');
return {
album: null,
items: [{
id: host.id,
url: videoElement.attr('src'),
title: post ? post.title : null,
type: videoElement.attr('type'),
datetime: post ? post.datetime : null,
original: post || null,
}],
};
}
module.exports = eroshareItem;

4
src/methods/gfycat.js
View File

@@ -2,8 +2,8 @@
const fetch = require('node-fetch');
async function gfycat(post) {
const res = await fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`);
async function gfycat(host) {
const res = await fetch(`https://api.gfycat.com/v1/gfycats/${host.id}`);
const data = await res.json();
if (data.error) {

81
src/methods/imgurAlbum.js
View File

@@ -2,52 +2,51 @@
const config = require('config');
const fetch = require('node-fetch');
const mime = require('mime-types');
// const mime = require('mime-types');
async function imgurAlbumApi(post) {
return fetch(`https://api.imgur.com/3/album/${post.host.id}`, {
async function imgurAlbumApi(host, post) {
const res = await fetch(`https://api.imgur.com/3/album/${host.id}`, {
headers: {
Authorization: `Client-ID ${config.methods.imgur.clientId}`,
},
}).then(res => res.json()).then((res) => {
if (res.status !== 200) {
throw new Error(`Could not fetch info for imgur album '${post.host.id}': ${res.data.error}`);
}
const extract = config.library.album.extractSingleItem && res.data.images.length === 1;
if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.data.link}`);
}
return {
album: extract ? null : {
id: res.data.id,
url: res.data.link,
title: res.data.title,
description: res.data.description,
datetime: new Date(res.data.datetime * 1000),
original: res.data,
},
items: res.data.images.map(item => ({
extracted: extract,
id: item.id,
url: item.animated ? item.mp4 : item.link,
title: item.title || (extract ? res.data.title : null),
description: item.description || (extract ? res.data.description : null),
type: item.animated ? 'video/mp4' : item.type,
datetime: item.datetime * 1000,
original: item,
})),
};
});
if (res.status !== 200) {
throw new Error(`Could not fetch info for imgur album '${host.id}': ${res.data.error}`);
}
const { data } = await res.json();
const extract = config.library.extractSingleAlbumItem && data.images.length === 1;
if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${data.link}' (${post ? post.url : 'no post'})`);
}
return {
album: extract ? null : {
id: data.id,
url: data.link,
title: data.title,
description: data.description,
datetime: new Date(data.datetime * 1000),
original: data,
},
items: data.images.map(item => ({
extracted: extract,
id: item.id,
url: item.animated ? item.mp4 : item.link,
title: item.title || data.title || null,
description: item.description || data.description || null,
type: item.animated ? 'video/mp4' : item.type,
datetime: item.datetime * 1000,
original: item,
})),
};
}
async function imgurAlbum(post) {
return imgurAlbumApi(post);
/*
* as of late 2019, imgur requires log in to view albums and gallery images
/*
* as of late 2019, imgur requires log in to view albums and gallery images
async function imgurAlbum(host, post) {
const res = await fetch(`https://imgur.com/a/${post.host.id}`);
const html = await res.text();
@@ -84,7 +83,7 @@ async function imgurAlbum(post) {
datetime: new Date(item.datetime),
})),
};
*/
}
*/
module.exports = imgurAlbum;
module.exports = imgurAlbumApi;

44
src/methods/imgurImage.js
View File

@@ -3,33 +3,35 @@
const config = require('config');
const fetch = require('node-fetch');
async function imgurImageApi(post) {
return fetch(`https://api.imgur.com/3/image/${post.host.id}`, {
async function imgurImageApi(host) {
const res = await fetch(`https://api.imgur.com/3/image/${host.id}`, {
headers: {
Authorization: `Client-ID ${config.methods.imgur.clientId}`,
},
}).then(res => res.json()).then((res) => {
if (res.status !== 200) {
throw new Error(`Could not fetch info for imgur image '${post.host.id}': ${res.data.error}`);
}
return {
album: null,
items: [{
id: res.data.id,
url: res.data.animated ? res.data.mp4 : res.data.link,
title: res.data.title,
description: res.data.description,
type: res.data.animated ? 'video/mp4' : res.data.type,
datetime: new Date(res.data.datetime * 1000),
original: res.data,
}],
};
});
const { data } = await res.json();
if (res.status !== 200) {
throw new Error(`Could not fetch info for imgur image '${host.id}': ${res.data.error}`);
}
return {
album: null,
items: [{
id: data.id,
url: data.animated ? data.mp4 : data.link,
title: data.title,
description: data.description,
type: data.animated ? 'video/mp4' : data.type,
datetime: new Date(data.datetime * 1000),
original: data,
}],
};
}
async function imgurImage(post) {
return imgurImageApi(post);
async function imgurImage(host, post) {
return imgurImageApi(host, post);
/*
* as of late 2019, imgur requires log in to view albums and gallery images

42
src/methods/methods.js
View File

@@ -1,31 +1,33 @@
'use strict';
const self = require('./self');
const redditImage = require('./redditImage');
const redditVideo = require('./redditVideo');
const redditPreview = require('./redditPreview');
const imgurImage = require('./imgurImage');
const imgurAlbum = require('./imgurAlbum');
const vidbleImage = require('./vidbleImage');
const vidbleVideo = require('./vidbleVideo');
const vidbleAlbum = require('./vidbleAlbum');
const gfycat = require('./gfycat');
const erome = require('./erome');
const eroshareAlbum = require('./eroshareAlbum');
const eroshareItem = require('./eroshareItem');
const gfycat = require('./gfycat');
const imgurAlbum = require('./imgurAlbum');
const imgurImage = require('./imgurImage');
const pornhub = require('./pornhub');
const redditImage = require('./redditImage');
const redditPreview = require('./redditPreview');
const redditVideo = require('./redditVideo');
const self = require('./self');
const vidbleAlbum = require('./vidbleAlbum');
const vidbleImage = require('./vidbleImage');
const vidbleVideo = require('./vidbleVideo');
module.exports = {
self,
redditImage,
redditVideo,
redditPreview,
imgurImage,
imgurAlbum,
vidbleImage,
vidbleVideo,
vidbleAlbum,
gfycat,
erome,
eroshareAlbum,
eroshareItem,
gfycat,
imgurAlbum,
imgurImage,
pornhub,
redditImage,
redditPreview,
redditVideo,
self,
vidbleAlbum,
vidbleImage,
vidbleVideo,
};

12
src/methods/pornhub.js
View File

@@ -2,11 +2,11 @@
const fetch = require('node-fetch');
async function pornhub(post) {
const res = await fetch(`https://www.pornhub.com/view_video.php?viewkey=${post.host.id}`);
async function pornhub(host, post) {
const res = await fetch(`https://www.pornhub.com/view_video.php?viewkey=${host.id}`);
if (res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${post.host.id}': '${res.error}'`);
throw new Error(`Could not fetch info PornHub video '${host.id}': '${res.error}'`);
}
const html = await res.text();
@@ -27,11 +27,11 @@ async function pornhub(post) {
return {
album: null,
items: [{
id: post.host.id,
id: host.id,
url,
title: post.title,
title: post ? post.title : null,
type: 'video/mp4',
datetime: post.datetime,
datetime: post ? post.datetime : null,
}],
};
}

18
src/methods/redditImage.js
View File

@@ -1,21 +1,19 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const mime = require('mime-types');
function redditImage(post) {
return Promise.resolve({
async function redditImage(host, post) {
return {
album: null,
items: [{
id: post.host.id || post.id,
id: host.id || post.id,
url: post.url,
title: post.title,
datetime: post.datetime,
type: mime.lookup(post.url.split('/.')[0]) || 'image/jpeg',
original: post
}]
});
};
original: post,
}],
};
}
module.exports = redditImage;

37
src/methods/redditPreview.js
View File

@@ -1,33 +1,26 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const path = require('path');
const fetch = require('node-fetch');
const mime = require('mime-types');
const urlPattern = require('url-pattern');
function redditPreview(post) {
return Promise.resolve({
async function redditPreview(host, post) {
return {
album: post.preview.length > 1 ? {
id: post.host.id || post.id,
url: post.url,
title: post.title,
datetime: post.datetime,
original: post
original: post,
} : null,
items: post.preview.map(image => {
return {
id: post.host.id || post.id,
url: image.url,
title: post.title,
datetime: post.datetime,
type: mime.lookup(image.url.split('?')[0]),
preview: true,
original: post
};
})
});
};
items: post.preview.map(image => ({
id: post.host.id || post.id,
url: image.url,
title: post.title,
datetime: post.datetime,
type: mime.lookup(image.url.split('?')[0]),
preview: true,
original: post,
})),
};
}
module.exports = redditPreview;

57
src/methods/redditVideo.js
View File

@@ -1,38 +1,35 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const fs = require('fs-extra');
function redditVideo(post) {
return fetch(`${post.permalink}.json`).then(res => res.json()).then(res => {
return res[0].data.children[0].data.media.reddit_video.fallback_url;
}).then(videoUrl => {
const audioUrl = videoUrl.split('/').slice(0, -1).join('/') + '/audio';
async function redditVideo(host, post) {
const res = await fetch(`${post.permalink}.json`);
const [{ data }] = await res.json();
return fetch(audioUrl, {
method: 'HEAD'
}).then(res => {
const item = {
album: null,
items: [{
id: post.host.id || post.id,
url: videoUrl,
title: post.title,
datetime: post.datetime,
type: 'video/mp4',
original: post
}]
};
const videoUrl = data.children[0].data.media.reddit_video.fallback_url;
const audioUrl = `${videoUrl.split('/').slice(0, -1).join('/')}/audio`;
if(res.status === 200) {
item.items[0].mux = [audioUrl];
}
return item;
});
const audioRes = await fetch(audioUrl, {
method: 'HEAD',
});
};
const item = {
album: null,
items: [{
id: post.host.id || post.id,
url: videoUrl,
title: post.title,
datetime: post.datetime,
type: 'video/mp4',
original: post,
}],
};
if (audioRes.status === 200) {
item.items[0].mux = [audioUrl];
}
return item;
}
module.exports = redditVideo;

77
src/methods/vidbleAlbum.js
View File

@@ -1,55 +1,52 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const urlPattern = require('url-pattern');
const UrlPattern = require('url-pattern');
const cheerio = require('cheerio');
const mime = require('mime-types');
const pattern = new urlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');
const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');
function vidbleAlbum(post) {
return fetch(`https://www.vidble.com/album/${post.host.id}`).then(res => {
if(res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${post.host.id}': '${res.error}'`);
}
async function vidbleAlbum(host, post) {
const res = await fetch(`https://www.vidble.com/album/${host.id}`);
return res.text();
}).then(res => {
const $ = cheerio.load(res);
if (res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${host.id}': '${res.error}'`);
}
const title = $('h2').text();
const imgUrls = $('img.img2').toArray().map(img => `https://vidble.com${img.attribs.src || img.attribs['data-original']}`);
const html = await res.text();
const $ = cheerio.load(html);
const extract = config.library.album.extractSingleItem && imgUrls.length === 1;
const title = $('h2').text();
const imgUrls = $('img.img2').toArray().map(img => `https://vidble.com${img.attribs.src || img.attribs['data-original']}`);
const extract = config.library.extractSingleAlbumItem && imgUrls.length === 1;
if(extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.link}`);
}
if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.link}`);
}
return {
album: extract ? null : {
id: post.host.id,
url: post.url,
title: post.title,
datetime: post.datetime
},
items: imgUrls.map(url => {
const components = pattern.match(url);
const id = components.id.replace('_med', '');
const mimetype = mime.lookup(components.ext);
return {
album: extract ? null : {
id: host.id,
url: post ? post.url : null,
title: post ? post.title : title,
datetime: post ? post.datetime : null,
},
items: imgUrls.map((url) => {
const components = pattern.match(url);
const id = components.id.replace('_med', '');
const mimetype = mime.lookup(components.ext);
return {
extracted: extract,
id: id,
url: `https://vidble.com/${id}.${components.ext}`,
type: mimetype,
datetime: post.datetime
};
})
};
});
};
return {
extracted: extract,
id,
url: `https://vidble.com/${id}.${components.ext}`,
type: mimetype,
datetime: post.datetime,
};
}),
};
}
module.exports = vidbleAlbum;

46
src/methods/vidbleImage.js
View File

@@ -1,34 +1,30 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const path = require('path');
const fetch = require('node-fetch');
const $ = require('cheerio');
const mime = require('mime-types');
function vidbleImage(post) {
return fetch(`https://vidble.com/${post.host.id}`).then(res => {
if(res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${post.host.id}': '${res.error}'`);
}
async function vidbleImage(host, post) {
const res = await fetch(`https://vidble.com/${host.id}`);
return res.text();
}).then(res => {
const resource = $('img', res).attr('src');
if (res.status !== 200) {
throw new Error(`Could not fetch info for vidble album '${host.id}': '${res.error}'`);
}
return {
album: null,
items: [{
id: post.host.id,
url: `https://vidble.com/${resource}`,
title: post.title,
datetime: post.datetime,
type: mime.lookup(resource),
original: post
}]
};
});
};
const html = await res.text();
const resource = $('img', html).attr('src');
return {
album: null,
items: [{
id: host.id,
url: `https://vidble.com/${resource}`,
title: post ? post.title : null,
datetime: post ? post.datetime : null,
type: mime.lookup(resource),
original: post || null,
}],
};
}
module.exports = vidbleImage;

58
src/methods/vidbleVideo.js
View File

@@ -1,39 +1,37 @@
'use strict';
'use strict';
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
const cheerio = require('cheerio');
function vidbleVideo(post) {
return fetch(`https://www.vidble.com/watch?v=${post.host.id}`).then(res => {
if(res.status !== 200) {
throw new Error(`Could not fetch info for vidble video '${post.host.id}': '${res.error}'`);
}
async function vidbleVideo(host, post) {
const res = await fetch(`https://www.vidble.com/watch?v=${host.id}`);
return res.text();
}).then(res => {
const $ = cheerio.load(res);
const resource = $('video source');
const source = resource.attr('src');
const type = resource.attr('type');
if (res.status !== 200) {
throw new Error(`Could not fetch info for vidble video '${host.id}': '${res.error}'`);
}
if(!source || !type) {
throw new Error(`Failed to retrieve (likely removed) vidble video '${post.host.id}'`);
}
const html = await res.text();
return {
album: null,
items: [{
id: post.host.id,
url: `https://vidble.com/${source}`,
title: post.title,
datetime: post.datetime,
type: type,
original: post
}]
};
});
};
const $ = cheerio.load(html);
const resource = $('video source');
const source = resource.attr('src');
const type = resource.attr('type');
if (!source || !type) {
throw new Error(`Failed to retrieve (likely removed) vidble video '${host.id}'`);
}
return {
album: null,
items: [{
id: host.id,
url: `https://vidble.com/${source}`,
title: post ? post.title : null,
datetime: post ? post.datetime : null,
type,
original: post || null,
}],
};
}
module.exports = vidbleVideo;