DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Compare commits

base: DebaucheryLibrarian:64a52fbb1e31027d8e3458469a9ba33a46bccf1b
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental
...
compare: DebaucheryLibrarian:f4b1fb4831129a7adc6fda90ba9da25c0bad1d51
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental

4 Commits
64a52fbb1e ... f4b1fb4831

Author SHA1 Message Date
DebaucheryLibrarian f4b1fb4831 1.138.7 2020-10-28 15:28:43 +01:00
DebaucheryLibrarian 8c553d5b3d Added traxxx dummy network to default excludes. 2020-10-28 15:28:39 +01:00
DebaucheryLibrarian e40d7ba181 1.138.6 2020-10-28 03:51:15 +01:00
DebaucheryLibrarian 4469376dd2 Using temporary table instead of WHERE IN to stack depth error when finding duplicate actors. 2020-10-28 03:50:52 +01:00
4 changed files with 14 additions and 7 deletions
Show Stats Expand all files Collapse all files

4
config/default.js
View File

@ -61,6 +61,10 @@ module.exports = {
// mindgeek
'pornhub',
],
networks: [
// dummy network for testing
'traxxx',
],
},
profiles: [
[

2
package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "traxxx",
"version": "1.138.5",
"version": "1.138.7",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "traxxx",
"version": "1.138.5",
"version": "1.138.7",
"description": "All the latest porn releases in one place",
"main": "src/app.js",
"scripts": {

13
src/actors.js
View File

@ -696,11 +696,14 @@ async function scrapeActors(argNames) {
}
async function getOrCreateActors(baseActors, batchId) {
const existingActors = await knex('actors')
.select('id', 'alias_for', 'name', 'slug', 'entity_id')
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('entity_id')
.orWhereIn(['slug', 'entity_id'], baseActors.map(baseActor => [baseActor.slug, baseActor.entity.id]));
// WHERE IN causes stack depth error and performance issues with a large amount of values, no knex VALUES helper available
const actorValues = baseActors.map(actor => knex.raw('(:slug, :entityId)', { slug: actor.slug, entityId: actor.entity.id })).join(', ');
const existingActors = await knex
.select('actors.*')
.from(knex.raw(`actors, (VALUES ${actorValues}) AS base_actors (slug, entity_id)`))
.whereRaw('actors.slug = base_actors.slug AND actors.entity_id IS NULL')
.orWhereRaw('actors.slug = base_actors.slug AND actors.entity_id = base_actors.entity_id');
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
const existingActorSlugs = existingActors.reduce((acc, actor) => ({