Added MOFOS to default rate limit config, added light logos.

This commit is contained in:
DebaucheryLibrarian
2022-02-20 16:52:29 +01:00
parent 1dd935e1e9
commit 690d2bb3ed
43 changed files with 57 additions and 9 deletions

View File

@@ -4,6 +4,7 @@
const Promise = require('bluebird');
const moment = require('moment');
const logger = require('../logger')(__filename);
const http = require('../utils/http');
const slugify = require('../utils/slugify');
@@ -141,24 +142,67 @@ async function getTrailer(scene, channel, url) {
return null;
}
async function getPhotos(url) {
async function getPhotosLegacy(url) {
const htmlRes = await http.get(url, {
extract: {
runScripts: 'dangerously',
},
});
const state = htmlRes?.window.__APOLLO_STATE__;
const key = Object.values(state.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
const data = state[key];
try {
const state = htmlRes?.window?.__APOLLO_STATE__;
console.log(data);
if (!state) {
return [];
}
if (!data) {
const key = Object.values(state?.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
const data = state[key];
if (!data) {
return [];
}
return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
} catch (error) {
logger.warn(`Failed to retrieve Vixen images: ${error.message}`);
return [];
}
}
return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
async function getPhotos(url) {
const htmlRes = await http.get(url, {
parse: true,
extract: {
runScripts: 'dangerously',
},
});
try {
const state = htmlRes?.window?.__APOLLO_STATE__;
console.log('state', state);
if (!state) {
return [];
}
const key = Object.values(state?.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
const data = state[key];
console.log('data', data);
if (!data) {
return [];
}
console.log(data.carousel);
return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
} catch (error) {
logger.warn(`Failed to retrieve Vixen images: ${error.message}`);
return [];
}
}
function scrapeAll(scenes, site, origin) {