DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Preventing Bang! scraper from redundant deep scraping when base release is available.

This commit is contained in:
DebaucheryLibrarian 2022-02-12 16:22:00 +01:00
parent 19c7f958e1
commit cde760c1ea
2 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/scrapers/bang.js
View File

@ -102,6 +102,8 @@ async function scrapeScene(scene, entity, options) {
.replace(/[! .]/g, '') .replace(/[! .]/g, '')
.replace('&', 'and'); .replace('&', 'and');
console.log(release);
return release; return release;
} }
@ -357,6 +359,11 @@ async function fetchUpcoming(site, page = 1) {
} }
async function fetchScene(url, entity, baseRelease, options) { async function fetchScene(url, entity, baseRelease, options) {
if (baseRelease?.entryId) {
// overview and deep data is the same, don't hit server unnecessarily
return baseRelease;
}
const encodedId = new URL(url).pathname.split('/')[2]; const encodedId = new URL(url).pathname.split('/')[2];
const entryId = decodeId(encodedId); const entryId = decodeId(encodedId);

3
src/tools/realitykings.js
View File

@ -1,9 +1,6 @@
'use strict'; 'use strict';
const https = require('https');
const bhttp = require('bhttp');
const fetch = require('node-fetch'); const fetch = require('node-fetch');
const { request } = require('undici');
const express = require('express'); const express = require('express');
async function init() { async function init() {