Using Imgur API rate limit feedback to prevent exceeding it.
This commit is contained in:
parent
7ec4143972
commit
7d633c31b4
|
@ -106,10 +106,12 @@ async function getDirectContent(links, ep) {
|
|||
};
|
||||
});
|
||||
|
||||
// const predata = await fetchPredata(hosts.map(({ host }) => host));
|
||||
const predata = await fetchPredata(hosts.map(({ host }) => host));
|
||||
|
||||
console.log('app predata', predata);
|
||||
|
||||
return Promise.map(hosts, async ({ link, host }) => {
|
||||
const info = await getInfo(host, reddit, link);
|
||||
const info = await getInfo(host, { reddit, link, predata });
|
||||
|
||||
if (info) {
|
||||
return fetchSaveDirectContent(info, host, ep);
|
||||
|
|
|
@ -102,10 +102,12 @@ function getArgs() {
|
|||
choices: ['oldest', 'latest'],
|
||||
})
|
||||
.option('redownload', {
|
||||
alias: 'force',
|
||||
describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed',
|
||||
type: 'boolean',
|
||||
})
|
||||
.option('redownload-profile', {
|
||||
alias: 'force-profile',
|
||||
describe: 'Ignore index file and force a redownload of the profile image and description',
|
||||
type: 'boolean',
|
||||
})
|
||||
|
|
|
@ -6,54 +6,58 @@ const Promise = require('bluebird');
|
|||
const logger = require('../logger')(__filename);
|
||||
const methods = require('../methods/methods');
|
||||
|
||||
const attachContentInfo = (users, { reddit, predata }) => Promise.reduce(Object.values(users), async (accUsers, user) => ({
|
||||
...accUsers,
|
||||
[user.name]: {
|
||||
...user,
|
||||
posts: await Promise.reduce(user.posts, async (accPosts, post) => {
|
||||
if (!post.host || !methods[post.host.method]) {
|
||||
logger.warn(`Ignoring unsupported content '${post.url}' (${post.permalink})`);
|
||||
async function attachContentInfo(users, { reddit, predata }) {
|
||||
return Promise.reduce(Object.values(users), async (accUsers, user) => ({
|
||||
...accUsers,
|
||||
[user.name]: {
|
||||
...user,
|
||||
posts: await Promise.reduce(user.posts, async (accPosts, post) => {
|
||||
if (!post.host || !methods[post.host.method]) {
|
||||
logger.warn(`Ignoring unsupported content '${post.url}' (${post.permalink})`);
|
||||
|
||||
return accPosts;
|
||||
}
|
||||
return accPosts;
|
||||
}
|
||||
|
||||
try {
|
||||
return [
|
||||
...accPosts,
|
||||
{
|
||||
...post,
|
||||
content: await (methods[post.host.method].fetchInfo || methods[post.host.method])(post.host, post, {
|
||||
predata: predata[post.host.method],
|
||||
reddit,
|
||||
}),
|
||||
},
|
||||
];
|
||||
} catch (error) {
|
||||
logger.warn(`${error.message} (${post.permalink})`);
|
||||
|
||||
if (config.fetch.archives.preview && post.preview) {
|
||||
logger.info(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);
|
||||
console.log('attach predata', predata[post.host.method]);
|
||||
|
||||
try {
|
||||
return [
|
||||
...accPosts,
|
||||
{
|
||||
...post,
|
||||
previewFallback: true,
|
||||
content: await methods.redditPreview(post.host, post, {
|
||||
predata: predata.redditPreview,
|
||||
content: await (methods[post.host.method].fetchInfo || methods[post.host.method])(post.host, post, {
|
||||
predata: predata[post.host.method],
|
||||
reddit,
|
||||
}),
|
||||
},
|
||||
];
|
||||
} catch (error) {
|
||||
logger.warn(`${error.message} (${post.permalink})`);
|
||||
|
||||
if (config.fetch.archives.preview && post.preview) {
|
||||
logger.info(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);
|
||||
|
||||
return [
|
||||
...accPosts,
|
||||
{
|
||||
...post,
|
||||
previewFallback: true,
|
||||
content: await methods.redditPreview(post.host, post, {
|
||||
predata: predata.redditPreview,
|
||||
reddit,
|
||||
}),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
return accPosts;
|
||||
}
|
||||
}, []),
|
||||
},
|
||||
}), {});
|
||||
}
|
||||
|
||||
return accPosts;
|
||||
}
|
||||
}, []),
|
||||
},
|
||||
}), {});
|
||||
|
||||
async function getInfo(host, reddit, url) {
|
||||
async function getInfo(host, { reddit, url, predata }) {
|
||||
if (host === null) {
|
||||
try {
|
||||
const info = await methods.tube(host, null, reddit);
|
||||
|
@ -66,7 +70,7 @@ async function getInfo(host, reddit, url) {
|
|||
}
|
||||
}
|
||||
|
||||
return (methods[host.method].fetchInfo || methods[host.method])(host, null, reddit);
|
||||
return (methods[host.method].fetchInfo || methods[host.method])(host, null, { reddit, predata });
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
|
|
@ -4,14 +4,25 @@ const config = require('config');
|
|||
const fetch = require('node-fetch');
|
||||
|
||||
const logger = require('../logger')(__filename);
|
||||
const { fetchPredata } = require('./imgurImage');
|
||||
|
||||
async function imgurAlbumApi(host, post, { predata }) {
|
||||
if (predata.remaining === 10) { // keep a buffer
|
||||
throw new Error(`Reached Imgur API rate limit with source '${host.url}'`);
|
||||
}
|
||||
|
||||
async function imgurAlbumApi(host, post) {
|
||||
const res = await fetch(`https://api.imgur.com/3/album/${host.id}`, {
|
||||
headers: {
|
||||
Authorization: `Client-ID ${config.methods.imgur.clientId}`,
|
||||
},
|
||||
});
|
||||
|
||||
const rateRemaining = Number(res.headers.get('x-ratelimit-userremaining'));
|
||||
|
||||
if (rateRemaining) {
|
||||
predata.setRemaining(rateRemaining);
|
||||
}
|
||||
|
||||
const { data } = await res.json();
|
||||
|
||||
if (res.status !== 200) {
|
||||
|
@ -33,7 +44,7 @@ async function imgurAlbumApi(host, post) {
|
|||
datetime: new Date(data.datetime * 1000),
|
||||
original: data,
|
||||
},
|
||||
items: data.images.map(item => ({
|
||||
items: data.images.map((item) => ({
|
||||
extracted: extract,
|
||||
id: item.id,
|
||||
url: item.animated ? item.mp4 : item.link,
|
||||
|
@ -46,4 +57,7 @@ async function imgurAlbumApi(host, post) {
|
|||
};
|
||||
}
|
||||
|
||||
module.exports = imgurAlbumApi;
|
||||
module.exports = {
|
||||
fetchInfo: imgurAlbumApi,
|
||||
fetchPredata,
|
||||
};
|
||||
|
|
|
@ -3,16 +3,52 @@
|
|||
const config = require('config');
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
async function imgurImageApi(host) {
|
||||
async function fetchPredata() {
|
||||
const data = {
|
||||
limit: 0,
|
||||
remaining: 0,
|
||||
};
|
||||
|
||||
data.setRemaining = (remaining) => {
|
||||
data.remaining = remaining;
|
||||
};
|
||||
|
||||
const res = await fetch('https://api.imgur.com/3/credits', {
|
||||
headers: {
|
||||
Authorization: `Client-ID ${config.methods.imgur.clientId}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
const body = await res.json();
|
||||
|
||||
if (body.success) {
|
||||
data.limit = body.data.UserLimit;
|
||||
data.remaining = body.data.UserRemaining;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
async function imgurImageApi(host, post, { predata } = {}) {
|
||||
if (predata.remaining === 10) { // keep a buffer
|
||||
throw new Error(`Reached Imgur API rate limit with source '${host.url}'`);
|
||||
}
|
||||
|
||||
const res = await fetch(`https://api.imgur.com/3/image/${host.id}`, {
|
||||
headers: {
|
||||
Authorization: `Client-ID ${config.methods.imgur.clientId}`,
|
||||
},
|
||||
});
|
||||
|
||||
console.log('imgur headers', res.headers);
|
||||
const rateRemaining = Number(res.headers.get('x-ratelimit-userremaining'));
|
||||
|
||||
if (res.status !== 200) {
|
||||
if (rateRemaining) {
|
||||
predata.setRemaining(rateRemaining);
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Imgur API returned HTTP ${res.status} for source '${host.url}'`);
|
||||
}
|
||||
|
||||
|
@ -36,8 +72,11 @@ async function imgurImageApi(host) {
|
|||
};
|
||||
}
|
||||
|
||||
async function imgurImage(host, post) {
|
||||
return imgurImageApi(host, post);
|
||||
async function imgurImage(host, post, context) {
|
||||
return imgurImageApi(host, post, context);
|
||||
}
|
||||
|
||||
module.exports = imgurImage;
|
||||
module.exports = {
|
||||
fetchInfo: imgurImage,
|
||||
fetchPredata,
|
||||
};
|
||||
|
|
|
@ -6,6 +6,28 @@ const mime = require('mime');
|
|||
|
||||
const { version } = require('../../package.json');
|
||||
|
||||
async function fetchPredata() {
|
||||
const userAgent = `ripunzel/${version}`;
|
||||
const res = await fetch('https://api.redgifs.com/v2/auth/temporary', {
|
||||
headers: {
|
||||
'user-agent': userAgent,
|
||||
},
|
||||
});
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
if (res.ok) {
|
||||
return {
|
||||
address: data.addr,
|
||||
agent: data.agent,
|
||||
token: data.token,
|
||||
userAgent,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function scrapeGallery(data) {
|
||||
const oldestDate = Math.min(...data.gifs.map((gif) => gif.createDate));
|
||||
|
||||
|
@ -125,28 +147,6 @@ async function redgifs(host, post, { predata }) {
|
|||
*/
|
||||
}
|
||||
|
||||
async function fetchPredata() {
|
||||
const userAgent = `ripunzel/${version}`;
|
||||
const res = await fetch('https://api.redgifs.com/v2/auth/temporary', {
|
||||
headers: {
|
||||
'user-agent': userAgent,
|
||||
},
|
||||
});
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
if (res.ok) {
|
||||
return {
|
||||
address: data.addr,
|
||||
agent: data.agent,
|
||||
token: data.token,
|
||||
userAgent,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchInfo: redgifs,
|
||||
fetchPredata,
|
||||
|
|
Loading…
Reference in New Issue