boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Replaced bhttp with patched fork. Improved Jesse Loads Monster Facials scraper reliability (WIP). Added various tag photos.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-10-30 17:37:10 +01:00
parent 4af7597441
commit 39f8c037a5
43 changed files with 128 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/scrapers/bang.js
View File

@@ -1,6 +1,6 @@
'use strict';
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const { post } = require('../utils/http');
const { extractDate } = require('../utils/qu');

2
src/scrapers/bangbros.js
View File

@@ -1,7 +1,7 @@
'use strict';
/* eslint-disable newline-per-chained-call */
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const cheerio = require('cheerio');
const moment = require('moment');

2
src/scrapers/dogfart.js
View File

@@ -2,7 +2,7 @@
/* eslint-disable newline-per-chained-call */
// const Promise = require('bluebird');
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const { JSDOM } = require('jsdom');
const moment = require('moment');

27
src/scrapers/jesseloadsmonsterfacials.js
View File

@@ -1,6 +1,6 @@
'use strict';
const { get, initAll, formatDate } = require('../utils/qu');
const { get, initAll } = require('../utils/qu');
function scrapeLatest(scenes, dates, site) {
return scenes.map(({ qu }, index) => {
@@ -8,21 +8,23 @@ function scrapeLatest(scenes, dates, site) {
const path = qu.url('a[href*="videos/"]');
if (path) {
release.url = `${site.url}/visitors/${path}`;
if (/\.wmv$/.test(path)) {
release.trailer = `${site.url}/visitors/${path}`;
} else {
release.url = `${site.url}/visitors/${path}`;
}
}
console.log(dates, dates[index], path);
if (dates && dates[index]) {
release.date = dates[index].qu.date(null, 'MM/DD/YYYY');
}
const entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
// release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
|| qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1];
if (!entryId) {
return null;
}
release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
release.description = qu.q('tbody tr:nth-child(3) font', true);
const infoLine = qu.q('font[color="#663366"]', true);
@@ -43,7 +45,14 @@ function scrapeScene({ qu }, url, site) {
const release = { url };
const { pathname } = new URL(url);
release.entryId = pathname.match(/videos\/(\w+)_hd_trailer/)[1];
release.entryId = pathname?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1];
if (/\.wmv$/.test(pathname)) {
release.trailer = url;
return release;
}
const actor = qu.q('font[color="#990033"] strong', true);
release.actors = [actor];

2
src/scrapers/julesjordan.js
View File

@@ -2,7 +2,7 @@
const util = require('util');
const Promise = require('bluebird');
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const cheerio = require('cheerio');
const { JSDOM } = require('jsdom');
const moment = require('moment');

2
src/scrapers/mindgeek.js
View File

@@ -2,7 +2,7 @@
/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const moment = require('moment');

2
src/scrapers/pornhub.js
View File

@@ -1,6 +1,6 @@
'use strict';
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const { JSDOM } = require('jsdom');
const moment = require('moment');

2
src/scrapers/private.js
View File

@@ -1,7 +1,7 @@
'use strict';
/* eslint-disable newline-per-chained-call */
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const cheerio = require('cheerio');
const moment = require('moment');

2
src/scrapers/realitykings.js
View File

@@ -1,6 +1,6 @@
'use strict';
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const cheerio = require('cheerio');
const {

3
src/scrapers/traxxx.js
View File

@@ -258,8 +258,7 @@ async function fetchLatest(entity, page, options) {
.limit(faker.random.number({ min: 2, max: 15 }))
.pluck('name');
// release.actors = actors(release);
release.actors = [null, 'Charles Darwin'];
release.actors = [...actors(release), null]; // include empty actor to ensure proper handling
release.title = title(release);
return release;

2
src/scrapers/vivid.js
View File

@@ -1,7 +1,7 @@
'use strict';
/* eslint-disable no-unused-vars */
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const { get, ed } = require('../utils/q');
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');

2
src/scrapers/xempire.js
View File

@@ -1,6 +1,6 @@
'use strict';
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const { fetchLatest, fetchUpcoming, scrapeScene, fetchProfile } = require('./gamma');