Using Imgur API rate limit feedback to prevent exceeding it.

This commit is contained in:
DebaucheryLibrarian 2024-09-11 05:16:58 +02:00
parent 7ec4143972
commit 7d633c31b4
6 changed files with 130 additions and 69 deletions

View File

@ -106,10 +106,12 @@ async function getDirectContent(links, ep) {
}; };
}); });
// const predata = await fetchPredata(hosts.map(({ host }) => host)); const predata = await fetchPredata(hosts.map(({ host }) => host));
console.log('app predata', predata);
return Promise.map(hosts, async ({ link, host }) => { return Promise.map(hosts, async ({ link, host }) => {
const info = await getInfo(host, reddit, link); const info = await getInfo(host, { reddit, link, predata });
if (info) { if (info) {
return fetchSaveDirectContent(info, host, ep); return fetchSaveDirectContent(info, host, ep);

View File

@ -102,10 +102,12 @@ function getArgs() {
choices: ['oldest', 'latest'], choices: ['oldest', 'latest'],
}) })
.option('redownload', { .option('redownload', {
alias: 'force',
describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed', describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed',
type: 'boolean', type: 'boolean',
}) })
.option('redownload-profile', { .option('redownload-profile', {
alias: 'force-profile',
describe: 'Ignore index file and force a redownload of the profile image and description', describe: 'Ignore index file and force a redownload of the profile image and description',
type: 'boolean', type: 'boolean',
}) })

View File

@ -6,7 +6,8 @@ const Promise = require('bluebird');
const logger = require('../logger')(__filename); const logger = require('../logger')(__filename);
const methods = require('../methods/methods'); const methods = require('../methods/methods');
const attachContentInfo = (users, { reddit, predata }) => Promise.reduce(Object.values(users), async (accUsers, user) => ({ async function attachContentInfo(users, { reddit, predata }) {
return Promise.reduce(Object.values(users), async (accUsers, user) => ({
...accUsers, ...accUsers,
[user.name]: { [user.name]: {
...user, ...user,
@ -17,6 +18,8 @@ const attachContentInfo = (users, { reddit, predata }) => Promise.reduce(Object.
return accPosts; return accPosts;
} }
console.log('attach predata', predata[post.host.method]);
try { try {
return [ return [
...accPosts, ...accPosts,
@ -51,9 +54,10 @@ const attachContentInfo = (users, { reddit, predata }) => Promise.reduce(Object.
} }
}, []), }, []),
}, },
}), {}); }), {});
}
async function getInfo(host, reddit, url) { async function getInfo(host, { reddit, url, predata }) {
if (host === null) { if (host === null) {
try { try {
const info = await methods.tube(host, null, reddit); const info = await methods.tube(host, null, reddit);
@ -66,7 +70,7 @@ async function getInfo(host, reddit, url) {
} }
} }
return (methods[host.method].fetchInfo || methods[host.method])(host, null, reddit); return (methods[host.method].fetchInfo || methods[host.method])(host, null, { reddit, predata });
} }
module.exports = { module.exports = {

View File

@ -4,14 +4,25 @@ const config = require('config');
const fetch = require('node-fetch'); const fetch = require('node-fetch');
const logger = require('../logger')(__filename); const logger = require('../logger')(__filename);
const { fetchPredata } = require('./imgurImage');
async function imgurAlbumApi(host, post, { predata }) {
if (predata.remaining === 10) { // keep a buffer
throw new Error(`Reached Imgur API rate limit with source '${host.url}'`);
}
async function imgurAlbumApi(host, post) {
const res = await fetch(`https://api.imgur.com/3/album/${host.id}`, { const res = await fetch(`https://api.imgur.com/3/album/${host.id}`, {
headers: { headers: {
Authorization: `Client-ID ${config.methods.imgur.clientId}`, Authorization: `Client-ID ${config.methods.imgur.clientId}`,
}, },
}); });
const rateRemaining = Number(res.headers.get('x-ratelimit-userremaining'));
if (rateRemaining) {
predata.setRemaining(rateRemaining);
}
const { data } = await res.json(); const { data } = await res.json();
if (res.status !== 200) { if (res.status !== 200) {
@ -33,7 +44,7 @@ async function imgurAlbumApi(host, post) {
datetime: new Date(data.datetime * 1000), datetime: new Date(data.datetime * 1000),
original: data, original: data,
}, },
items: data.images.map(item => ({ items: data.images.map((item) => ({
extracted: extract, extracted: extract,
id: item.id, id: item.id,
url: item.animated ? item.mp4 : item.link, url: item.animated ? item.mp4 : item.link,
@ -46,4 +57,7 @@ async function imgurAlbumApi(host, post) {
}; };
} }
module.exports = imgurAlbumApi; module.exports = {
fetchInfo: imgurAlbumApi,
fetchPredata,
};

View File

@ -3,16 +3,52 @@
const config = require('config'); const config = require('config');
const fetch = require('node-fetch'); const fetch = require('node-fetch');
async function imgurImageApi(host) { async function fetchPredata() {
const data = {
limit: 0,
remaining: 0,
};
data.setRemaining = (remaining) => {
data.remaining = remaining;
};
const res = await fetch('https://api.imgur.com/3/credits', {
headers: {
Authorization: `Client-ID ${config.methods.imgur.clientId}`,
},
});
if (res.ok) {
const body = await res.json();
if (body.success) {
data.limit = body.data.UserLimit;
data.remaining = body.data.UserRemaining;
}
}
return data;
}
async function imgurImageApi(host, post, { predata } = {}) {
if (predata.remaining === 10) { // keep a buffer
throw new Error(`Reached Imgur API rate limit with source '${host.url}'`);
}
const res = await fetch(`https://api.imgur.com/3/image/${host.id}`, { const res = await fetch(`https://api.imgur.com/3/image/${host.id}`, {
headers: { headers: {
Authorization: `Client-ID ${config.methods.imgur.clientId}`, Authorization: `Client-ID ${config.methods.imgur.clientId}`,
}, },
}); });
console.log('imgur headers', res.headers); const rateRemaining = Number(res.headers.get('x-ratelimit-userremaining'));
if (res.status !== 200) { if (rateRemaining) {
predata.setRemaining(rateRemaining);
}
if (!res.ok) {
throw new Error(`Imgur API returned HTTP ${res.status} for source '${host.url}'`); throw new Error(`Imgur API returned HTTP ${res.status} for source '${host.url}'`);
} }
@ -36,8 +72,11 @@ async function imgurImageApi(host) {
}; };
} }
async function imgurImage(host, post) { async function imgurImage(host, post, context) {
return imgurImageApi(host, post); return imgurImageApi(host, post, context);
} }
module.exports = imgurImage; module.exports = {
fetchInfo: imgurImage,
fetchPredata,
};

View File

@ -6,6 +6,28 @@ const mime = require('mime');
const { version } = require('../../package.json'); const { version } = require('../../package.json');
async function fetchPredata() {
const userAgent = `ripunzel/${version}`;
const res = await fetch('https://api.redgifs.com/v2/auth/temporary', {
headers: {
'user-agent': userAgent,
},
});
const data = await res.json();
if (res.ok) {
return {
address: data.addr,
agent: data.agent,
token: data.token,
userAgent,
};
}
return null;
}
function scrapeGallery(data) { function scrapeGallery(data) {
const oldestDate = Math.min(...data.gifs.map((gif) => gif.createDate)); const oldestDate = Math.min(...data.gifs.map((gif) => gif.createDate));
@ -125,28 +147,6 @@ async function redgifs(host, post, { predata }) {
*/ */
} }
async function fetchPredata() {
const userAgent = `ripunzel/${version}`;
const res = await fetch('https://api.redgifs.com/v2/auth/temporary', {
headers: {
'user-agent': userAgent,
},
});
const data = await res.json();
if (res.ok) {
return {
address: data.addr,
agent: data.agent,
token: data.token,
userAgent,
};
}
return null;
}
module.exports = { module.exports = {
fetchInfo: redgifs, fetchInfo: redgifs,
fetchPredata, fetchPredata,