diff --git a/assets/components/tags/tags.vue b/assets/components/tags/tags.vue index fd89bc72..b19e621c 100644 --- a/assets/components/tags/tags.vue +++ b/assets/components/tags/tags.vue @@ -103,6 +103,7 @@ async function mounted() { 'toy-dp', 'double-dildo', 'double-dildo-blowjob', + 'double-dildo-kiss', 'double-dildo-anal', ], roleplay: [ diff --git a/package-lock.json b/package-lock.json index 14d9dc8a..d816da0b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1066,6 +1066,56 @@ "tar": "^4.4.6" } }, + "@thependulum/bhttp": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@thependulum/bhttp/-/bhttp-1.2.6.tgz", + "integrity": "sha512-jqYVj99upU9vfMq4cjMEi87lbmf381e2A8dM91IPgU7TVF9ryN0rYHDK/IcIWbPBJwa8q9l0GzazN2xC2R9TgA==", + "requires": { + "bluebird": "^2.8.2", + "concat-stream": "^1.4.7", + "debug": "^2.1.1", + "dev-null": "^0.1.1", + "errors": "^0.2.0", + "extend": "^2.0.0", + "form-data2": "^1.0.0", + "form-fix-array": "^1.0.0", + "lodash.clonedeep": "^4.5.0", + "lodash.merge": "^4.6.2", + "stream-length": "^1.0.2", + "through2-sink": "^1.0.0", + "through2-spy": "^1.2.0", + "tough-cookie": "^2.3.1" + }, + "dependencies": { + "bluebird": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", + "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=" + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + } + } + }, "@tokenizer/token": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.1.1.tgz", diff --git a/package.json b/package.json index c2493304..0b55b7f5 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6", "@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1", "@tensorflow/tfjs-node": "^1.5.2", + "@thependulum/bhttp": "^1.2.6", "babel-polyfill": "^6.26.0", "bhttp": "^1.2.6", "blake2": "^4.0.0", diff --git a/public/img/tags/double-dildo-anal/3.jpeg b/public/img/tags/double-dildo-anal/3.jpeg new file mode 100644 index 00000000..caf3d40b Binary files /dev/null and b/public/img/tags/double-dildo-anal/3.jpeg differ diff --git a/public/img/tags/double-dildo-anal/lazy/3.jpeg b/public/img/tags/double-dildo-anal/lazy/3.jpeg new file mode 100644 index 00000000..062e0095 Binary files /dev/null and b/public/img/tags/double-dildo-anal/lazy/3.jpeg differ diff --git a/public/img/tags/double-dildo-anal/thumbs/3.jpeg b/public/img/tags/double-dildo-anal/thumbs/3.jpeg new file mode 100644 index 00000000..080e39c5 Binary files /dev/null and b/public/img/tags/double-dildo-anal/thumbs/3.jpeg differ diff --git a/public/img/tags/double-dildo-blowjob/2.jpeg b/public/img/tags/double-dildo-blowjob/2.jpeg new file mode 100644 index 00000000..a098bd9e Binary files /dev/null and b/public/img/tags/double-dildo-blowjob/2.jpeg differ diff --git a/public/img/tags/double-dildo-blowjob/3.jpeg b/public/img/tags/double-dildo-blowjob/3.jpeg new file mode 100644 index 00000000..52bce648 Binary files /dev/null and b/public/img/tags/double-dildo-blowjob/3.jpeg differ diff --git a/public/img/tags/double-dildo-blowjob/lazy/2.jpeg b/public/img/tags/double-dildo-blowjob/lazy/2.jpeg new file mode 100644 index 00000000..80795ca9 Binary files /dev/null and b/public/img/tags/double-dildo-blowjob/lazy/2.jpeg differ diff --git a/public/img/tags/double-dildo-blowjob/lazy/3.jpeg b/public/img/tags/double-dildo-blowjob/lazy/3.jpeg new file mode 100644 index 00000000..606956f3 Binary files /dev/null and b/public/img/tags/double-dildo-blowjob/lazy/3.jpeg differ diff --git a/public/img/tags/double-dildo-blowjob/thumbs/0.jpeg b/public/img/tags/double-dildo-blowjob/thumbs/0.jpeg index aa5275f2..0b0c1131 100644 Binary files a/public/img/tags/double-dildo-blowjob/thumbs/0.jpeg and b/public/img/tags/double-dildo-blowjob/thumbs/0.jpeg differ diff --git a/public/img/tags/double-dildo-blowjob/thumbs/1.jpeg b/public/img/tags/double-dildo-blowjob/thumbs/1.jpeg index 1fcfd03b..2af3b482 100644 Binary files a/public/img/tags/double-dildo-blowjob/thumbs/1.jpeg and b/public/img/tags/double-dildo-blowjob/thumbs/1.jpeg differ diff --git a/public/img/tags/double-dildo-blowjob/thumbs/2.jpeg b/public/img/tags/double-dildo-blowjob/thumbs/2.jpeg new file mode 100644 index 00000000..63c94121 Binary files /dev/null and b/public/img/tags/double-dildo-blowjob/thumbs/2.jpeg differ diff --git a/public/img/tags/double-dildo-blowjob/thumbs/3.jpeg b/public/img/tags/double-dildo-blowjob/thumbs/3.jpeg new file mode 100644 index 00000000..57c6830c Binary files /dev/null and b/public/img/tags/double-dildo-blowjob/thumbs/3.jpeg differ diff --git a/public/img/tags/double-dildo-kiss/0.jpeg b/public/img/tags/double-dildo-kiss/0.jpeg new file mode 100644 index 00000000..f044c12c Binary files /dev/null and b/public/img/tags/double-dildo-kiss/0.jpeg differ diff --git a/public/img/tags/double-dildo-kiss/lazy/0.jpeg b/public/img/tags/double-dildo-kiss/lazy/0.jpeg new file mode 100644 index 00000000..9a01681b Binary files /dev/null and b/public/img/tags/double-dildo-kiss/lazy/0.jpeg differ diff --git a/public/img/tags/double-dildo-kiss/thumbs/0.jpeg b/public/img/tags/double-dildo-kiss/thumbs/0.jpeg new file mode 100644 index 00000000..54051a1c Binary files /dev/null and b/public/img/tags/double-dildo-kiss/thumbs/0.jpeg differ diff --git a/public/img/tags/fake-boobs/14.jpeg b/public/img/tags/fake-boobs/14.jpeg new file mode 100644 index 00000000..e6633b5f Binary files /dev/null and b/public/img/tags/fake-boobs/14.jpeg differ diff --git a/public/img/tags/fake-boobs/lazy/14.jpeg b/public/img/tags/fake-boobs/lazy/14.jpeg new file mode 100644 index 00000000..267748f2 Binary files /dev/null and b/public/img/tags/fake-boobs/lazy/14.jpeg differ diff --git a/public/img/tags/fake-boobs/thumbs/14.jpeg b/public/img/tags/fake-boobs/thumbs/14.jpeg new file mode 100644 index 00000000..7f0df8a8 Binary files /dev/null and b/public/img/tags/fake-boobs/thumbs/14.jpeg differ diff --git a/public/img/tags/toy-anal/2.jpeg b/public/img/tags/toy-anal/2.jpeg new file mode 100644 index 00000000..19e9fdbb Binary files /dev/null and b/public/img/tags/toy-anal/2.jpeg differ diff --git a/public/img/tags/toy-anal/lazy/2.jpeg b/public/img/tags/toy-anal/lazy/2.jpeg new file mode 100644 index 00000000..e4627205 Binary files /dev/null and b/public/img/tags/toy-anal/lazy/2.jpeg differ diff --git a/public/img/tags/toy-anal/thumbs/2.jpeg b/public/img/tags/toy-anal/thumbs/2.jpeg new file mode 100644 index 00000000..be1d8aef Binary files /dev/null and b/public/img/tags/toy-anal/thumbs/2.jpeg differ diff --git a/seeds/00_tags.js b/seeds/00_tags.js index 78c5e414..649314f8 100644 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -328,7 +328,7 @@ const tags = [ { name: 'double dildo', slug: 'double-dildo', - description: 'Two girls fucking eachother using either end of a double-sided dildo.', + description: 'Two girls fucking eachother using either end of a double-sided dildo. They can suck it for a [double dildo blowjob](/tag/double-dildo-blowjob), deepthroat it for a [double dildo kiss](/tag/double-dildo-kiss), or put it up their ass for [double dildo anal](/tag/double-dildo-anal).', priority: 4, }, { @@ -1372,6 +1372,10 @@ const aliases = [ name: 'dp', for: 'dp', }, + { + name: 'double dildo deepthroat', + for: 'double-dildo-kiss', + }, { name: 'double penetration (dp)', for: 'dp', diff --git a/seeds/04_media.js b/seeds/04_media.js index 083c7a77..09103b0a 100644 --- a/seeds/04_media.js +++ b/seeds/04_media.js @@ -617,6 +617,7 @@ const tagPosters = [ ['double-dildo', 0, 'Kali Roses in "Double Dildo Party" for KaliRoses.com'], ['double-dildo-anal', 0, 'Vina Sky and Kenzie Reeves in "Vina Sky\'s 1st Lesbian Anal" for HardX'], ['double-dildo-blowjob', 0, 'Adriana Chechik and Vicki Chase in "Anal Savages 1" for Jules Jordan'], + ['double-dildo-kiss', 0, 'Giselle Palmer and Romi Rain in "Punishable Behavior" for Brazzers'], ['dp', 3, 'Hime Marie in LegalPorno AA047'], ['dvp', 'poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'], ['dv-tp', 'poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'], @@ -721,8 +722,11 @@ const tagPhotos = [ ['deepthroat', 1, 'Jynx Maze in "Slutty and Sluttier 13" for Evil Angel'], ['deepthroat', 0, 'Chanel Grey in "Deepthroating Is Fun" for Throated'], ['double-blowjob', 0, 'Kira Noir and Kali Roses for Brazzers'], - ['double-dildo-anal', 2, 'Adria Rae and Megan Rain in "Best Friends Anal" for Holed'], ['double-dildo-anal', 1, 'Sammie Rhodes and Ainsley Addision in "Tickle Me Pink" for We Live Together (Reality Kings)'], + ['double-dildo-anal', 3, 'Amber Rayne, Phoenix Marie and Roxy Raye in "Deep Anal Abyss 4" for Evil Angel'], + ['double-dildo-anal', 2, 'Adria Rae and Megan Rain in "Best Friends Anal" for Holed'], + ['double-dildo-blowjob', 3, 'Angela White and Madison Ivy in "Sunbathing Babes" for Brazzers'], + ['double-dildo-blowjob', 2, 'Giselle Palmer and Romi Rain in "Punishable Behavior" for Brazzers'], ['double-dildo-blowjob', 1, 'Aidra Fox and Reena Sky in "Reena\'s Got A Staring Problem" for Brazzers'], ['double-dildo-dp', 0, 'u/LacyCrow "Sometimes you have to do it yourself"'], ['dp', 5, 'Lana Rhoades in "Gangbang Me 3" for HardX'], @@ -741,6 +745,7 @@ const tagPhotos = [ ['facefucking', 2, 'Jynx Maze for Throated'], ['facefucking', 4, 'Brooklyn Gray in "Throats Fucks 6" for Evil Angel'], ['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'], + ['fake-boobs', 14, 'Rikki Six for Dream Dolls'], ['fake-boobs', 13, 'Kitana Lure for Asshole Fever'], ['fake-boobs', 11, 'Jessa Rhodes and Cali Carter in "Busty Anal Workout" for LesbianX'], ['fake-boobs', 10, 'Tia Cyrus in "Titty-Fucked Yoga Goddess" for Latina Sex Tapes'], @@ -788,6 +793,7 @@ const tagPhotos = [ ['trainbang', 0, 'Nicole Black in GIO971 for LegalPorno'], ['tap', 1, 'Natasha Teen in SZ2098 for LegalPorno'], ['tap', 2, 'Kira Thorn in GIO1018 for LegalPorno'], + ['toy-anal', 2, 'Denise, Irina and Laki in "Sexy Slumber" for Lez Cuties'], ['toy-anal', 0, 'Kira Noir in 1225 for InTheCrack'], ['toy-dp', 0, 'Marley Brinx, Ivy Lebelle and Lyra Law in "Marley Brinx First GGDP" for LesbianX'], ] diff --git a/src/actors.js b/src/actors.js index 27e469f5..335905f8 100644 --- a/src/actors.js +++ b/src/actors.js @@ -792,6 +792,8 @@ async function associateActors(releases, batchId) { await bulkInsert('releases_actors', releaseActorAssociations, false); + logger.verbose(`Associated ${releaseActorAssociations.length} actors to ${releases.length} scenes`); + return actors; } diff --git a/src/entities.js b/src/entities.js index 40379771..9c3fb174 100644 --- a/src/entities.js +++ b/src/entities.js @@ -262,11 +262,13 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) { return; } - await Promise.all([ + const [deletedScenesCount, deletedMoviesCount] = await Promise.all([ deleteScenes(sceneIds), deleteMovies(movieIds), ]); + logger.info(`Removed ${deletedScenesCount} scenes and ${deletedMoviesCount} movies for ${entitySlugs}`); + await flushOrphanedMedia(); } diff --git a/src/media.js b/src/media.js index 9ee9d10f..93a329d5 100644 --- a/src/media.js +++ b/src/media.js @@ -279,6 +279,8 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) { } async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath) { + logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`); + try { const thumbdir = path.join(media.role, 'thumbs', hashDir, hashSubDir); const thumbpath = path.join(thumbdir, filename); @@ -620,6 +622,7 @@ async function storeMedias(baseMedias) { const fetchedMedias = await Promise.map( baseMedias, async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }), + { concurrency: 100 }, // don't overload disk (or network, although this has its own throttling) ); const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(fetchedMedias); @@ -627,6 +630,7 @@ async function storeMedias(baseMedias) { const savedMedias = await Promise.map( uniqueHashMedias, async baseMedia => storeFile(baseMedia), + { concurrency: 100 }, // don't overload disk ); if (argv.force) { @@ -634,6 +638,7 @@ async function storeMedias(baseMedias) { await Promise.map( existingHashMedias, async baseMedia => storeFile(baseMedia), + { concurrency: 100 }, // don't overload disk ); } @@ -784,7 +789,7 @@ async function flushOrphanedMedia() { await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true }); - logger.info('Removed temporary media directory'); + logger.info('Cleared temporary media directory'); } module.exports = { diff --git a/src/releases.js b/src/releases.js index 2fb04114..6e60eeef 100644 --- a/src/releases.js +++ b/src/releases.js @@ -126,15 +126,31 @@ async function searchReleases(query, limit = 100) { } async function deleteScenes(sceneIds) { - await knex('releases') + if (sceneIds.length === 0) { + return 0; + } + + const deleteCount = await knex('releases') .whereIn('id', sceneIds) .delete(); + + logger.info(`Removed ${deleteCount}/${sceneIds.length} scenes`); + + return deleteCount; } async function deleteMovies(movieIds) { - await knex('movies') + if (movieIds.length === 0) { + return 0; + } + + const deleteCount = await knex('movies') .whereIn('id', movieIds) .delete(); + + logger.info(`Removed ${deleteCount}/${movieIds.length} movies`); + + return deleteCount; } async function flushBatches(batchIds) { @@ -161,11 +177,13 @@ async function flushBatches(batchIds) { return; } - await Promise.all([ + const [deletedScenesCount, deletedMoviesCount] = await Promise.all([ deleteScenes(sceneIds), deleteMovies(movieIds), ]); + logger.info(`Removed ${deletedScenesCount} scenes and ${deletedMoviesCount} movies for batches ${batchIds}`); + await flushOrphanedMedia(); } diff --git a/src/scrapers/bang.js b/src/scrapers/bang.js index 50d09417..033cc0e3 100644 --- a/src/scrapers/bang.js +++ b/src/scrapers/bang.js @@ -1,6 +1,6 @@ 'use strict'; -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const { post } = require('../utils/http'); const { extractDate } = require('../utils/qu'); diff --git a/src/scrapers/bangbros.js b/src/scrapers/bangbros.js index b0beb2f2..6a1e186a 100644 --- a/src/scrapers/bangbros.js +++ b/src/scrapers/bangbros.js @@ -1,7 +1,7 @@ 'use strict'; /* eslint-disable newline-per-chained-call */ -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); diff --git a/src/scrapers/dogfart.js b/src/scrapers/dogfart.js index 603ec36b..0334d104 100644 --- a/src/scrapers/dogfart.js +++ b/src/scrapers/dogfart.js @@ -2,7 +2,7 @@ /* eslint-disable newline-per-chained-call */ // const Promise = require('bluebird'); -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const { JSDOM } = require('jsdom'); const moment = require('moment'); diff --git a/src/scrapers/jesseloadsmonsterfacials.js b/src/scrapers/jesseloadsmonsterfacials.js index 29050d2b..5b7242ad 100644 --- a/src/scrapers/jesseloadsmonsterfacials.js +++ b/src/scrapers/jesseloadsmonsterfacials.js @@ -1,6 +1,6 @@ 'use strict'; -const { get, initAll, formatDate } = require('../utils/qu'); +const { get, initAll } = require('../utils/qu'); function scrapeLatest(scenes, dates, site) { return scenes.map(({ qu }, index) => { @@ -8,21 +8,23 @@ function scrapeLatest(scenes, dates, site) { const path = qu.url('a[href*="videos/"]'); if (path) { - release.url = `${site.url}/visitors/${path}`; + if (/\.wmv$/.test(path)) { + release.trailer = `${site.url}/visitors/${path}`; + } else { + release.url = `${site.url}/visitors/${path}`; + } } + console.log(dates, dates[index], path); + if (dates && dates[index]) { release.date = dates[index].qu.date(null, 'MM/DD/YYYY'); } - const entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1] + // release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId; + release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1] || qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1]; - if (!entryId) { - return null; - } - - release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId; release.description = qu.q('tbody tr:nth-child(3) font', true); const infoLine = qu.q('font[color="#663366"]', true); @@ -43,7 +45,14 @@ function scrapeScene({ qu }, url, site) { const release = { url }; const { pathname } = new URL(url); - release.entryId = pathname.match(/videos\/(\w+)_hd_trailer/)[1]; + + release.entryId = pathname?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]; + + if (/\.wmv$/.test(pathname)) { + release.trailer = url; + + return release; + } const actor = qu.q('font[color="#990033"] strong', true); release.actors = [actor]; diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index ed5b1136..7a90dd8e 100644 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -2,7 +2,7 @@ const util = require('util'); const Promise = require('bluebird'); -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const cheerio = require('cheerio'); const { JSDOM } = require('jsdom'); const moment = require('moment'); diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index 44680fdf..10ea4857 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -2,7 +2,7 @@ /* eslint-disable newline-per-chained-call */ const Promise = require('bluebird'); -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const { CookieJar } = Promise.promisifyAll(require('tough-cookie')); const moment = require('moment'); diff --git a/src/scrapers/pornhub.js b/src/scrapers/pornhub.js index c50f746d..1367220c 100644 --- a/src/scrapers/pornhub.js +++ b/src/scrapers/pornhub.js @@ -1,6 +1,6 @@ 'use strict'; -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const { JSDOM } = require('jsdom'); const moment = require('moment'); diff --git a/src/scrapers/private.js b/src/scrapers/private.js index f862002f..9254d22c 100644 --- a/src/scrapers/private.js +++ b/src/scrapers/private.js @@ -1,7 +1,7 @@ 'use strict'; /* eslint-disable newline-per-chained-call */ -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); diff --git a/src/scrapers/realitykings.js b/src/scrapers/realitykings.js index 720ce156..db4bfeed 100644 --- a/src/scrapers/realitykings.js +++ b/src/scrapers/realitykings.js @@ -1,6 +1,6 @@ 'use strict'; -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const cheerio = require('cheerio'); const { diff --git a/src/scrapers/traxxx.js b/src/scrapers/traxxx.js index 41b824cc..76d51060 100644 --- a/src/scrapers/traxxx.js +++ b/src/scrapers/traxxx.js @@ -258,8 +258,7 @@ async function fetchLatest(entity, page, options) { .limit(faker.random.number({ min: 2, max: 15 })) .pluck('name'); - // release.actors = actors(release); - release.actors = [null, 'Charles Darwin']; + release.actors = [...actors(release), null]; // include empty actor to ensure proper handling release.title = title(release); return release; diff --git a/src/scrapers/vivid.js b/src/scrapers/vivid.js index 275cc59b..427a5b4f 100644 --- a/src/scrapers/vivid.js +++ b/src/scrapers/vivid.js @@ -1,7 +1,7 @@ 'use strict'; /* eslint-disable no-unused-vars */ -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const { get, ed } = require('../utils/q'); const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); diff --git a/src/scrapers/xempire.js b/src/scrapers/xempire.js index 1995bc37..75df6e72 100644 --- a/src/scrapers/xempire.js +++ b/src/scrapers/xempire.js @@ -1,6 +1,6 @@ 'use strict'; -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const { fetchLatest, fetchUpcoming, scrapeScene, fetchProfile } = require('./gamma'); diff --git a/src/utils/bulk-insert.js b/src/utils/bulk-insert.js index 81bb11e3..b756191e 100644 --- a/src/utils/bulk-insert.js +++ b/src/utils/bulk-insert.js @@ -21,12 +21,10 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) { return knex.transaction(async (transaction) => { const chunked = chunk(items, chunkSize); - // console.log(items.length, chunkSize, chunked.length, chunked[0]?.length); - const queries = chunked .map(chunkItems => knex.raw(updated || ':query RETURNING *;', { - query: knex(table).insert(chunkItems).transacting(transaction), - })); + query: knex(table).insert(chunkItems), + }).transacting(transaction)); const responses = await Promise.all(queries); diff --git a/src/utils/http.js b/src/utils/http.js index e59b80fa..45925baa 100644 --- a/src/utils/http.js +++ b/src/utils/http.js @@ -4,7 +4,7 @@ const util = require('util'); const stream = require('stream'); const config = require('config'); const tunnel = require('tunnel'); -const bhttp = require('bhttp'); +const bhttp = require('@thependulum/bhttp'); const taskQueue = require('promise-task-queue'); const pipeline = util.promisify(stream.pipeline);