Adjusting tooltip arrow position, added open and close events. Fixed search tooltip layout.

src/scrapers/mikeadriano.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const qu = require('../utils/qu');
+const http = require('../utils/http');
 
 function scrapeAll(scenes) {
 	return scenes.map(({ query }) => {
@@ -33,13 +34,11 @@ function scrapeAll(scenes) {
 async function scrapeScene({ query }, url) {
 	const release = { director: 'Mike Adriano' };
 
-	if (query.exists('a[href*="stackpath.com"]')) {
-		throw new Error('URL blocked by StackPath');
-	}
-
 	const pathname = new URL(url).pathname;
 	release.entryId = pathname.match(/\/view\/(\d+)/)?.[1] || pathname.match(/\/view\/([\w-]+)/)?.[1];
 
+	console.log(release);
+
 	release.title = query.cnt('.content-page-info .title');
 	release.description = query.cnt('.content-page-info .desc');
 	release.date = query.date('.content-page-info .date, .content-page-info .hide, .post-date', 'Do MMM YYYY');
@@ -71,10 +70,40 @@ async function fetchLatest(channel, page = 1) {
 }
 
 async function fetchScene(url, channel) {
-	const res = await qu.get(url);
+	const cookieJar = http.cookieJar();
+	const session = http.session({ cookieJar });
+
+	console.log(cookieJar);
+
+	const resA = await http.get(url, {
+		session,
+		extract: {
+			cookieJar,
+			// runScripts: 'dangerously',
+		},
+	});
+
+	console.log(resA.headers, cookieJar.getCookiesSync(url));
+	const cookie = cookieJar.getCookieStringSync(url);
+
+	console.log(cookie);
+
+	const res = await http.get(url, {
+		headers: {
+			cookie,
+		},
+	});
+
+	// console.log(res.req);
 
 	if (res.ok) {
-		return scrapeScene(res.item, url, channel);
+		const item = qu.init(res.document);
+
+		if (item.query.exists('a[href*="stackpath.com"]')) {
+			throw new Error('URL blocked by StackPath');
+		}
+
+		return scrapeScene(item, url, channel);
 	}
 
 	return res.status;

src/utils/http.js

@@ -6,7 +6,7 @@ const util = require('util');
 const stream = require('stream');
 const tunnel = require('tunnel');
 const Bottleneck = require('bottleneck');
-const { JSDOM } = require('jsdom');
+const { JSDOM, CookieJar } = require('jsdom');
 
 const logger = require('../logger')(__filename);
 const virtualConsole = require('./virtual-console')(__filename);
@@ -110,7 +110,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 
 	if (Buffer.isBuffer(res.body)) {
 		const html = res.body.toString();
-		const window = new JSDOM(html, { virtualConsole }).window;
+		const window = new JSDOM(html, { virtualConsole, ...options.extract }).window;
 
 		return {
 			...res,
@@ -165,6 +165,10 @@ function getSession(options) {
 	return bhttp.session({ ...defaultOptions, ...options });
 }
 
+function getCookieJar(store, options) {
+	return new CookieJar(store, options);
+}
+
 module.exports = {
 	get,
 	head,
@@ -173,5 +177,7 @@ module.exports = {
 	put,
 	patch,
 	session: getSession,
+	cookieJar: getCookieJar,
 	getSession,
+	getCookieJar,
 };

src/utils/qu.js

@@ -118,6 +118,12 @@ function html(context, selector) {
 	return el && el.innerHTML;
 }
 
+function htmls(context, selector) {
+	const els = all(context, selector, null, true);
+
+	return els.map(el => el.innerHTML);
+}
+
 function texts(context, selector, applyTrim = true, filter = true) {
 	const el = q(context, selector, null, applyTrim);
 	if (!el) return null;
@@ -363,27 +369,28 @@ const legacyFuncs = {
 
 const quFuncs = {
 	all,
-	html,
-	content,
-	contents,
 	cnt: content,
 	cnts: contents,
+	content,
+	contents,
 	count,
 	date,
 	dateAgo,
 	dur: duration,
 	duration,
-	element: q,
 	el: q,
+	element: q,
 	exists,
+	html,
+	htmls,
 	image,
 	images,
 	img: image,
 	imgs: images,
 	length: duration,
 	meta,
-	number,
 	num: number,
+	number,
 	poster,
 	q,
 	sourceSet,
@@ -444,8 +451,8 @@ function initAll(context, selector, window) {
 		.map(element => init(element, window));
 }
 
-function extract(htmlValue, selector) {
-	const { window } = new JSDOM(htmlValue, { virtualConsole });
+function extract(htmlValue, selector, options) {
+	const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
 
 	if (selector) {
 		return init(window.document.querySelector(selector), window);
@@ -454,8 +461,8 @@ function extract(htmlValue, selector) {
 	return init(window.document, window);
 }
 
-function extractAll(htmlValue, selector) {
-	const { window } = new JSDOM(htmlValue, { virtualConsole });
+function extractAll(htmlValue, selector, options) {
+	const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
 
 	return initAll(window.document, selector, window);
 }
@@ -467,8 +474,8 @@ async function request(method = 'get', urlValue, body, selector, headers, option
 
 	if (res.ok) {
 		const item = queryAll
-			? extractAll(res.body.toString(), selector)
-			: extract(res.body.toString(), selector);
+			? initAll(res.document, selector, res.window)
+			: init(res.document, selector, res.window);
 
 		return {
 			item,
@@ -506,6 +513,10 @@ async function postAll(urlValue, body, selector, headers, options) {
 	return request('post', urlValue, body, selector, headers, options, true);
 }
 
+function session(headers, options) {
+	return http.session(headers, options);
+}
+
 module.exports = {
 	extractDate,
 	extract,
@@ -532,5 +543,6 @@ module.exports = {
 	post,
 	postAll,
 	prefixUrl,
+	session,
 	...legacyFuncs,
 };