Replaced bhttp with patched fork. Improved Jesse Loads Monster Facials scraper reliability (WIP). Added various tag photos.

assets/components/tags/tags.vue

@@ -103,6 +103,7 @@ async function mounted() {
 			'toy-dp',
 			'double-dildo',
 			'double-dildo-blowjob',
+			'double-dildo-kiss',
 			'double-dildo-anal',
 		],
 		roleplay: [

package-lock.json

@@ -1066,6 +1066,56 @@
                 "tar": "^4.4.6"
             }
         },
+        "@thependulum/bhttp": {
+            "version": "1.2.6",
+            "resolved": "https://registry.npmjs.org/@thependulum/bhttp/-/bhttp-1.2.6.tgz",
+            "integrity": "sha512-jqYVj99upU9vfMq4cjMEi87lbmf381e2A8dM91IPgU7TVF9ryN0rYHDK/IcIWbPBJwa8q9l0GzazN2xC2R9TgA==",
+            "requires": {
+                "bluebird": "^2.8.2",
+                "concat-stream": "^1.4.7",
+                "debug": "^2.1.1",
+                "dev-null": "^0.1.1",
+                "errors": "^0.2.0",
+                "extend": "^2.0.0",
+                "form-data2": "^1.0.0",
+                "form-fix-array": "^1.0.0",
+                "lodash.clonedeep": "^4.5.0",
+                "lodash.merge": "^4.6.2",
+                "stream-length": "^1.0.2",
+                "through2-sink": "^1.0.0",
+                "through2-spy": "^1.2.0",
+                "tough-cookie": "^2.3.1"
+            },
+            "dependencies": {
+                "bluebird": {
+                    "version": "2.11.0",
+                    "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz",
+                    "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE="
+                },
+                "debug": {
+                    "version": "2.6.9",
+                    "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+                    "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+                    "requires": {
+                        "ms": "2.0.0"
+                    }
+                },
+                "ms": {
+                    "version": "2.0.0",
+                    "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+                    "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+                },
+                "tough-cookie": {
+                    "version": "2.5.0",
+                    "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
+                    "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
+                    "requires": {
+                        "psl": "^1.1.28",
+                        "punycode": "^2.1.1"
+                    }
+                }
+            }
+        },
         "@tokenizer/token": {
             "version": "0.1.1",
             "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.1.1.tgz",

package.json

@@ -72,6 +72,7 @@
         "@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
         "@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
         "@tensorflow/tfjs-node": "^1.5.2",
+        "@thependulum/bhttp": "^1.2.6",
         "babel-polyfill": "^6.26.0",
         "bhttp": "^1.2.6",
         "blake2": "^4.0.0",

seeds/00_tags.js

@@ -328,7 +328,7 @@ const tags = [
 	{
 		name: 'double dildo',
 		slug: 'double-dildo',
-		description: 'Two girls fucking eachother using either end of a double-sided dildo.',
+		description: 'Two girls fucking eachother using either end of a double-sided dildo. They can suck it for a [double dildo blowjob](/tag/double-dildo-blowjob), deepthroat it for a [double dildo kiss](/tag/double-dildo-kiss), or put it up their ass for [double dildo anal](/tag/double-dildo-anal).',
 		priority: 4,
 	},
 	{
@@ -1372,6 +1372,10 @@ const aliases = [
 		name: 'dp',
 		for: 'dp',
 	},
+	{
+		name: 'double dildo deepthroat',
+		for: 'double-dildo-kiss',
+	},
 	{
 		name: 'double penetration (dp)',
 		for: 'dp',

seeds/04_media.js

@@ -617,6 +617,7 @@ const tagPosters = [
 	['double-dildo', 0, 'Kali Roses in "Double Dildo Party" for KaliRoses.com'],
 	['double-dildo-anal', 0, 'Vina Sky and Kenzie Reeves in "Vina Sky\'s 1st Lesbian Anal" for HardX'],
 	['double-dildo-blowjob', 0, 'Adriana Chechik and Vicki Chase in "Anal Savages 1" for Jules Jordan'],
+	['double-dildo-kiss', 0, 'Giselle Palmer and Romi Rain in "Punishable Behavior" for Brazzers'],
 	['dp', 3, 'Hime Marie in LegalPorno AA047'],
 	['dvp', 'poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
 	['dv-tp', 'poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'],
@@ -721,8 +722,11 @@ const tagPhotos = [
 	['deepthroat', 1, 'Jynx Maze in "Slutty and Sluttier 13" for Evil Angel'],
 	['deepthroat', 0, 'Chanel Grey in "Deepthroating Is Fun" for Throated'],
 	['double-blowjob', 0, 'Kira Noir and Kali Roses for Brazzers'],
-	['double-dildo-anal', 2, 'Adria Rae and Megan Rain in "Best Friends Anal" for Holed'],
 	['double-dildo-anal', 1, 'Sammie Rhodes and Ainsley Addision in "Tickle Me Pink" for We Live Together (Reality Kings)'],
+	['double-dildo-anal', 3, 'Amber Rayne, Phoenix Marie and Roxy Raye in "Deep Anal Abyss 4" for Evil Angel'],
+	['double-dildo-anal', 2, 'Adria Rae and Megan Rain in "Best Friends Anal" for Holed'],
+	['double-dildo-blowjob', 3, 'Angela White and Madison Ivy in "Sunbathing Babes" for Brazzers'],
+	['double-dildo-blowjob', 2, 'Giselle Palmer and Romi Rain in "Punishable Behavior" for Brazzers'],
 	['double-dildo-blowjob', 1, 'Aidra Fox and Reena Sky in "Reena\'s Got A Staring Problem" for Brazzers'],
 	['double-dildo-dp', 0, 'u/LacyCrow "Sometimes you have to do it yourself"'],
 	['dp', 5, 'Lana Rhoades in "Gangbang Me 3" for HardX'],
@@ -741,6 +745,7 @@ const tagPhotos = [
 	['facefucking', 2, 'Jynx Maze for Throated'],
 	['facefucking', 4, 'Brooklyn Gray in "Throats Fucks 6" for Evil Angel'],
 	['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'],
+	['fake-boobs', 14, 'Rikki Six for Dream Dolls'],
 	['fake-boobs', 13, 'Kitana Lure for Asshole Fever'],
 	['fake-boobs', 11, 'Jessa Rhodes and Cali Carter in "Busty Anal Workout" for LesbianX'],
 	['fake-boobs', 10, 'Tia Cyrus in "Titty-Fucked Yoga Goddess" for Latina Sex Tapes'],
@@ -788,6 +793,7 @@ const tagPhotos = [
 	['trainbang', 0, 'Nicole Black in GIO971 for LegalPorno'],
 	['tap', 1, 'Natasha Teen in SZ2098 for LegalPorno'],
 	['tap', 2, 'Kira Thorn in GIO1018 for LegalPorno'],
+	['toy-anal', 2, 'Denise, Irina and Laki in "Sexy Slumber" for Lez Cuties'],
 	['toy-anal', 0, 'Kira Noir in 1225 for InTheCrack'],
 	['toy-dp', 0, 'Marley Brinx, Ivy Lebelle and Lyra Law in "Marley Brinx First GGDP" for LesbianX'],
 ]

src/actors.js

@@ -792,6 +792,8 @@ async function associateActors(releases, batchId) {
 
 	await bulkInsert('releases_actors', releaseActorAssociations, false);
 
+	logger.verbose(`Associated ${releaseActorAssociations.length} actors to ${releases.length} scenes`);
+
 	return actors;
 }
 

src/entities.js

@@ -262,11 +262,13 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) {
 		return;
 	}
 
-	await Promise.all([
+	const [deletedScenesCount, deletedMoviesCount] = await Promise.all([
 		deleteScenes(sceneIds),
 		deleteMovies(movieIds),
 	]);
 
+	logger.info(`Removed ${deletedScenesCount} scenes and ${deletedMoviesCount} movies for ${entitySlugs}`);
+
 	await flushOrphanedMedia();
 }
 

src/media.js

@@ -279,6 +279,8 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
 }
 
 async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath) {
+	logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`);
+
 	try {
 		const thumbdir = path.join(media.role, 'thumbs', hashDir, hashSubDir);
 		const thumbpath = path.join(thumbdir, filename);
@@ -620,6 +622,7 @@ async function storeMedias(baseMedias) {
 	const fetchedMedias = await Promise.map(
 		baseMedias,
 		async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
+		{ concurrency: 100 }, // don't overload disk (or network, although this has its own throttling)
 	);
 
 	const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(fetchedMedias);
@@ -627,6 +630,7 @@ async function storeMedias(baseMedias) {
 	const savedMedias = await Promise.map(
 		uniqueHashMedias,
 		async baseMedia => storeFile(baseMedia),
+		{ concurrency: 100 }, // don't overload disk
 	);
 
 	if (argv.force) {
@@ -634,6 +638,7 @@ async function storeMedias(baseMedias) {
 		await Promise.map(
 			existingHashMedias,
 			async baseMedia => storeFile(baseMedia),
+			{ concurrency: 100 }, // don't overload disk
 		);
 	}
 
@@ -784,7 +789,7 @@ async function flushOrphanedMedia() {
 
 	await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });
 
-	logger.info('Removed temporary media directory');
+	logger.info('Cleared temporary media directory');
 }
 
 module.exports = {

src/releases.js

@@ -126,15 +126,31 @@ async function searchReleases(query, limit = 100) {
 }
 
 async function deleteScenes(sceneIds) {
-	await knex('releases')
+	if (sceneIds.length === 0) {
+		return 0;
+	}
+
+	const deleteCount = await knex('releases')
 		.whereIn('id', sceneIds)
 		.delete();
+
+	logger.info(`Removed ${deleteCount}/${sceneIds.length} scenes`);
+
+	return deleteCount;
 }
 
 async function deleteMovies(movieIds) {
-	await knex('movies')
+	if (movieIds.length === 0) {
+		return 0;
+	}
+
+	const deleteCount = await knex('movies')
 		.whereIn('id', movieIds)
 		.delete();
+
+	logger.info(`Removed ${deleteCount}/${movieIds.length} movies`);
+
+	return deleteCount;
 }
 
 async function flushBatches(batchIds) {
@@ -161,11 +177,13 @@ async function flushBatches(batchIds) {
 		return;
 	}
 
-	await Promise.all([
+	const [deletedScenesCount, deletedMoviesCount] = await Promise.all([
 		deleteScenes(sceneIds),
 		deleteMovies(movieIds),
 	]);
 
+	logger.info(`Removed ${deletedScenesCount} scenes and ${deletedMoviesCount} movies for batches ${batchIds}`);
+
 	await flushOrphanedMedia();
 }
 

src/scrapers/bang.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 
 const { post } = require('../utils/http');
 const { extractDate } = require('../utils/qu');

src/scrapers/bangbros.js

@@ -1,7 +1,7 @@
 'use strict';
 
 /* eslint-disable newline-per-chained-call */
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 

src/scrapers/dogfart.js

@@ -2,7 +2,7 @@
 
 /* eslint-disable newline-per-chained-call */
 // const Promise = require('bluebird');
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 const { JSDOM } = require('jsdom');
 const moment = require('moment');
 

src/scrapers/jesseloadsmonsterfacials.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { get, initAll, formatDate } = require('../utils/qu');
+const { get, initAll } = require('../utils/qu');
 
 function scrapeLatest(scenes, dates, site) {
 	return scenes.map(({ qu }, index) => {
@@ -8,21 +8,23 @@ function scrapeLatest(scenes, dates, site) {
 		const path = qu.url('a[href*="videos/"]');
 
 		if (path) {
-			release.url = `${site.url}/visitors/${path}`;
+			if (/\.wmv$/.test(path)) {
+				release.trailer = `${site.url}/visitors/${path}`;
+			} else {
+				release.url = `${site.url}/visitors/${path}`;
+			}
 		}
 
+		console.log(dates, dates[index], path);
+
 		if (dates && dates[index]) {
 			release.date = dates[index].qu.date(null, 'MM/DD/YYYY');
 		}
 
-		const entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
+		// release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
+		release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
 			|| qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1];
 
-		if (!entryId) {
-			return null;
-		}
-
-		release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
 		release.description = qu.q('tbody tr:nth-child(3) font', true);
 
 		const infoLine = qu.q('font[color="#663366"]', true);
@@ -43,7 +45,14 @@ function scrapeScene({ qu }, url, site) {
 	const release = { url };
 
 	const { pathname } = new URL(url);
-	release.entryId = pathname.match(/videos\/(\w+)_hd_trailer/)[1];
+
+	release.entryId = pathname?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1];
+
+	if (/\.wmv$/.test(pathname)) {
+		release.trailer = url;
+
+		return release;
+	}
 
 	const actor = qu.q('font[color="#990033"] strong', true);
 	release.actors = [actor];

src/scrapers/julesjordan.js

@@ -2,7 +2,7 @@
 
 const util = require('util');
 const Promise = require('bluebird');
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 const cheerio = require('cheerio');
 const { JSDOM } = require('jsdom');
 const moment = require('moment');

src/scrapers/mindgeek.js

@@ -2,7 +2,7 @@
 
 /* eslint-disable newline-per-chained-call */
 const Promise = require('bluebird');
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
 const moment = require('moment');
 

src/scrapers/pornhub.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 const { JSDOM } = require('jsdom');
 const moment = require('moment');
 

src/scrapers/private.js

@@ -1,7 +1,7 @@
 'use strict';
 
 /* eslint-disable newline-per-chained-call */
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 

src/scrapers/realitykings.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 const cheerio = require('cheerio');
 
 const {

src/scrapers/traxxx.js

@@ -258,8 +258,7 @@ async function fetchLatest(entity, page, options) {
 			.limit(faker.random.number({ min: 2, max: 15 }))
 			.pluck('name');
 
-		// release.actors = actors(release);
+		release.actors = [...actors(release), null]; // include empty actor to ensure proper handling
-		release.actors = [null, 'Charles Darwin'];
 		release.title = title(release);
 
 		return release;

src/scrapers/vivid.js

@@ -1,7 +1,7 @@
 'use strict';
 
 /* eslint-disable no-unused-vars */
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 
 const { get, ed } = require('../utils/q');
 const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');

src/scrapers/xempire.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 
 const { fetchLatest, fetchUpcoming, scrapeScene, fetchProfile } = require('./gamma');
 

src/utils/bulk-insert.js

@@ -21,12 +21,10 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
 	return knex.transaction(async (transaction) => {
 		const chunked = chunk(items, chunkSize);
 
-		// console.log(items.length, chunkSize, chunked.length, chunked[0]?.length);
-
 		const queries = chunked
 			.map(chunkItems => knex.raw(updated || ':query RETURNING *;', {
-				query: knex(table).insert(chunkItems).transacting(transaction),
+				query: knex(table).insert(chunkItems),
-			}));
+			}).transacting(transaction));
 
 		const responses = await Promise.all(queries);
 

src/utils/http.js

@@ -4,7 +4,7 @@ const util = require('util');
 const stream = require('stream');
 const config = require('config');
 const tunnel = require('tunnel');
-const bhttp = require('bhttp');
+const bhttp = require('@thependulum/bhttp');
 const taskQueue = require('promise-task-queue');
 
 const pipeline = util.promisify(stream.pipeline);