Refactored Bang! scraper to match new website, first use of unprint.

This commit is contained in:
DebaucheryLibrarian
2022-11-27 04:22:58 +01:00
parent 3cf8776ca5
commit 6edd62c337
61 changed files with 2371 additions and 1233 deletions

View File

@@ -2,6 +2,7 @@
const util = require('util');
const Promise = require('bluebird');
const unprint = require('unprint');
const { mergeAdvanced: merge } = require('object-merge-advanced');
const argv = require('./argv');
@@ -54,12 +55,33 @@ function toBaseReleases(baseReleasesOrUrls, entity = null) {
.filter(Boolean);
}
async function fetchUnprintScene(scraper, url, entity, baseRelease, options, type) {
const res = await unprint.get(url, {
rejectUnauthorized: false,
});
if (res.ok) {
return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.context, {
url,
entity,
baseRelease,
headers: res.headers,
}, options);
}
return res.status;
}
async function fetchScene(scraper, url, entity, baseRelease, options, type = 'scene') {
if ((type === 'scene' && scraper.fetchScene) || (type === 'movie' && scraper.fetchMovie)) {
return scraper[type === 'movie' ? 'fetchMovie' : 'fetchScene'](baseRelease.url, entity, baseRelease, options, null);
}
if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
if (scraper.useUnprint) {
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
}
const session = qu.session();
const res = await qu.get(url, null, null, {