Added conditions to Tokyo Hot scraper to prevent total failure.
This commit is contained in:
@@ -279,7 +279,6 @@ async function findSourceDuplicates(baseMedias) {
|
||||
const [existingSourceMedia, existingExtractMedia] = await Promise.all([
|
||||
// may try to check thousands of URLs at once, don't pass all of them to a single query
|
||||
chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => {
|
||||
console.log(sourceUrlsChunk);
|
||||
const accUrls = await chain;
|
||||
const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user