From 3789ef51f2203faf210dee4a680f7b2ecd1ca556 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Fri, 18 Sep 2020 22:43:45 +0200 Subject: [PATCH] (Temporarily) removed studio filter from entity query for performance reasons. --- assets/js/entities/actions.js | 45 ++++++++++++++--------------------- src/scrapers/legalporno.js | 12 +++++----- src/store-releases.js | 1 - src/utils/bulk-insert.js | 6 ++++- 4 files changed, 29 insertions(+), 35 deletions(-) diff --git a/assets/js/entities/actions.js b/assets/js/entities/actions.js index 7abcdbfb..eb48ba3b 100644 --- a/assets/js/entities/actions.js +++ b/assets/js/entities/actions.js @@ -80,36 +80,27 @@ function initEntitiesActions(store, router) { filter: { and: [ { - or: [ - { - entity: { - or: [ - { + entity: { + or: [ + { + slug: { equalTo: $entitySlug } + }, + { + parent: { + slug: { equalTo: $entitySlug } + type: { equalTo: $entityType } + } + }, + { + parent: { + parent: { slug: { equalTo: $entitySlug } - }, - { - parent: { - slug: { equalTo: $entitySlug } - type: { equalTo: $entityType } - } - }, - { - parent: { - parent: { - slug: { equalTo: $entitySlug } - type: { equalTo: $entityType } - } - } + type: { equalTo: $entityType } } - ] + } } - } - { - studio: { - slug: { equalTo: $entitySlug }, - } - } - ] + ] + } } { or: [ diff --git a/src/scrapers/legalporno.js b/src/scrapers/legalporno.js index ffa982fb..7f71afde 100644 --- a/src/scrapers/legalporno.js +++ b/src/scrapers/legalporno.js @@ -1,10 +1,10 @@ 'use strict'; -const bhttp = require('bhttp'); const { JSDOM } = require('jsdom'); const cheerio = require('cheerio'); const moment = require('moment'); +const http = require('../utils/http'); const slugify = require('../utils/slugify'); function extractTitle(originalTitle) { @@ -164,7 +164,7 @@ async function scrapeProfile(html, _url, actorName) { } async function fetchLatest(site, page = 1) { - const res = await bhttp.get(`${site.url}/new-videos/${page}`); + const res = await http.get(`${site.url}/new-videos/${page}`); return scrapeAll(res.body.toString(), site); } @@ -174,20 +174,20 @@ async function fetchScene(url, site) { // TODO: fall back on screenshots when gallery is not available const res = useGallery - ? await bhttp.get(`${url}/gallery#gallery`) - : await bhttp.get(`${url}/screenshots#screenshots`); + ? await http.get(`${url}/gallery#gallery`) + : await http.get(`${url}/screenshots#screenshots`); return scrapeScene(res.body.toString(), url, site, useGallery); } async function fetchProfile({ name: actorName }) { - const res = await bhttp.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`); + const res = await http.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`); const data = res.body; const result = data.terms.find(item => item.type === 'model'); if (result) { - const bioRes = await bhttp.get(result.url); + const bioRes = await http.get(result.url); const html = bioRes.body.toString(); return scrapeProfile(html, result.url, actorName); diff --git a/src/store-releases.js b/src/store-releases.js index 8475431a..34a97c69 100644 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -299,7 +299,6 @@ async function storeScenes(releases) { const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId))); const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries); - // TODO: update duplicate releases const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : []; const releasesWithId = attachReleaseIds([].concat(uniqueReleases, duplicateReleases), [].concat(storedReleaseEntries, duplicateReleaseEntries)); diff --git a/src/utils/bulk-insert.js b/src/utils/bulk-insert.js index b2af09d7..81bb11e3 100644 --- a/src/utils/bulk-insert.js +++ b/src/utils/bulk-insert.js @@ -19,7 +19,11 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) { `); return knex.transaction(async (transaction) => { - const queries = chunk(items, chunkSize) + const chunked = chunk(items, chunkSize); + + // console.log(items.length, chunkSize, chunked.length, chunked[0]?.length); + + const queries = chunked .map(chunkItems => knex.raw(updated || ':query RETURNING *;', { query: knex(table).insert(chunkItems).transacting(transaction), }));