Added conditions to Tokyo Hot scraper to prevent total failure.

This commit is contained in:
DebaucheryLibrarian
2023-07-31 23:41:32 +02:00
parent d2f81d446b
commit ae64c5225f
13 changed files with 170 additions and 95 deletions

View File

@@ -279,7 +279,6 @@ async function findSourceDuplicates(baseMedias) {
const [existingSourceMedia, existingExtractMedia] = await Promise.all([
// may try to check thousands of URLs at once, don't pass all of them to a single query
chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => {
console.log(sourceUrlsChunk);
const accUrls = await chain;
const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk);