Added showcased migration. Added Love Her Films scraper.

src/utils/convert.js

@@ -86,6 +86,18 @@ function convertApi(input, fromOrTo, to) {
 	}
 }
 
+function maleFeetUsToEu(input) {
+	const size = Number(input.toString().match(/\d+(\.\d+)?/)?.[0]);
+
+	return Math.round((1.27 * size + 29.94) / 0.5) * 0.5; // round to nearest half
+}
+
+function femaleFeetUsToEu(input) {
+	const size = Number(input.toString().match(/\d+(\.\d+)?/)?.[0]);
+
+	return Math.round((1.27 * size + 28.67) / 0.5) * 0.5; // round to nearest half
+}
+
 module.exports = {
 	cmToFeetInches,
 	cmToInches,
@@ -94,5 +106,7 @@ module.exports = {
 	inchesToCm,
 	lbsToKg,
 	kgToLbs,
+	maleFeetUsToEu,
+	femaleFeetUsToEu,
 	convert: convertApi,
 };

src/utils/http.js

@@ -9,6 +9,7 @@ const stream = require('stream');
 const tunnel = require('tunnel');
 const Bottleneck = require('bottleneck');
 const { JSDOM, toughCookie } = require('jsdom');
+const puppeteer = require('puppeteer');
 
 const windows = require('./http-windows');
 
@@ -28,6 +29,9 @@ const limiters = {
 
 const bypassSessions = new Map();
 
+let browser = null;
+const browserSessions = new Map();
+
 Promise.config({
 	cancellation: true,
 });
@@ -58,6 +62,24 @@ function useProxy(url) {
 	return config.proxy.hostnames.includes(hostname);
 }
 
+function useBrowserBypass(url, options) {
+	if (!config.bypass.browser.enable) {
+		return null;
+	}
+
+	const { hostname } = new URL(url);
+
+	if (options.bypassBrowser === 'shared') {
+		return true;
+	}
+
+	if (config.bypass.browser.hostnames.includes(hostname)) {
+		return true;
+	}
+
+	return false;
+}
+
 function useCloudflareBypass(url, options) {
 	if (!config.bypass.cloudflare.enable) {
 		return null;
@@ -134,6 +156,42 @@ function extractJson(solution) {
 	return solution.response;
 }
 
+async function getBrowserSession(hostname) {
+	console.log(browserSessions);
+
+	if (browserSessions.has(hostname)) {
+		return browserSessions.get(hostname);
+	}
+
+	if (!browser) {
+		browser = await puppeteer.launch({ headless: false });
+	}
+
+	const page = await browser.newPage();
+
+	browserSessions.set(hostname, page);
+
+	return page;
+}
+
+async function bypassBrowserRequest(url, _options) {
+	const page = await limiters.bypass.schedule(async () => getBrowserSession(new URL(url).hostname));
+
+	const res = await page.goto(url);
+	const body = await page.content();
+
+	console.log(res);
+	console.log(res.status());
+
+	return {
+		body,
+		/*
+		statusCode: res.body.solution.status,
+		headers: res.body.solution.headers,
+		*/
+	};
+}
+
 async function getBypassSession(url, hostname) {
 	if (bypassSessions.has(hostname)) {
 		return bypassSessions.get(hostname);
@@ -236,13 +294,22 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 	};
 
 	const withProxy = useProxy(url);
+	const withBrowserBypass = useBrowserBypass(url, options);
 	const withCloudflareBypass = useCloudflareBypass(url, options);
 
 	if (withProxy) {
 		options.agent = proxyAgent;
 	}
 
-	logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`);
+	logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withBrowserBypass || withCloudflareBypass ? ' bypass' : ''}) ${url}`);
+
+	if (withBrowserBypass) {
+		if (method !== 'get') {
+			throw new Error('Browser bypass only supports GET');
+		}
+
+		return bypassBrowserRequest(url, options);
+	}
 
 	if (withCloudflareBypass) {
 		return bypassCloudflareRequest(url, method, body, withCloudflareBypass, options);
@@ -385,6 +452,8 @@ module.exports = {
 	patch,
 	session: getSession,
 	cookieJar: getCookieJar,
+	getBrowserSession,
+	getBypassSession,
 	getSession,
 	getCookieJar,
 	destroyBypassSessions,