DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Removed direct bhttp usage from scrapers in favor of local http module. Deleted legacy scrapers, as old code is available via git repo history.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-11-23 00:05:02 +01:00
parent 3d427f7e1d
commit 0633197793
22 changed files with 77 additions and 537 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/scrapers/mindgeek.js
View File

@@ -2,12 +2,12 @@
/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('@thependulum/bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const moment = require('moment');
const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
const http = require('../utils/http');
const { inchesToCm, lbsToKg } = require('../utils/convert');
const { cookieToData } = require('../utils/cookies');
@@ -145,14 +145,14 @@ function getUrl(site) {
async function getSession(site) {
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
const session = http.session({ cookieJar });
// const res = await session.get(url);
const sessionUrl = site.parameters?.siteId && !(site.parameters?.childSession || site.parent?.parameters?.childSession)
? site.parent.url
: site.url;
const res = await session.get(sessionUrl);
const res = await http.get(sessionUrl, { session });
if (res.statusCode === 200) {
const cookieString = await cookieJar.getCookieStringAsync(sessionUrl);
@@ -215,7 +215,8 @@ async function fetchLatest(site, page = 1) {
? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
const res = await session.get(apiUrl, {
const res = await http.get(apiUrl, {
session,
headers: {
Instance: instanceToken,
Origin: site.url,
@@ -236,7 +237,8 @@ async function fetchUpcoming(site) {
const apiUrl = 'https://site-api.project1service.com/v2/upcoming-releases';
const res = await session.get(apiUrl, {
const res = await http.get(apiUrl, {
session,
headers: {
Instance: instanceToken,
Origin: site.url,
@@ -260,7 +262,8 @@ async function fetchScene(url, site, baseScene) {
const entryId = url.match(/\d+/)[0];
const { session, instanceToken } = await getSession(site);
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
session,
headers: {
Instance: instanceToken,
},
@@ -277,7 +280,8 @@ async function fetchProfile({ name: actorName }, networkOrNetworkSlug, actorPath
// const url = `https://www.${networkOrNetworkSlug.slug || networkOrNetworkSlug}.com`;
const { session, instanceToken } = await getSession(networkOrNetworkSlug);
const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
const res = await http.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
session,
headers: {
Instance: instanceToken,
},
@@ -291,8 +295,9 @@ async function fetchProfile({ name: actorName }, networkOrNetworkSlug, actorPath
const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`;
const [actorRes, actorReleasesRes] = await Promise.all([
bhttp.get(actorUrl),
session.get(actorReleasesUrl, {
http.get(actorUrl),
http.get(actorReleasesUrl, {
session,
headers: {
Instance: instanceToken,
},