DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Hush scraper uses children from entity argument for filter regexp, instead of making its own database request.

This commit is contained in:
DebaucheryLibrarian 2020-08-23 03:31:37 +02:00
parent 278b74e78c
commit c3d771c8fc
2 changed files with 11 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
seeds/01_networks.js
View File

@ -230,24 +230,26 @@ const networks = [
{
slug: 'hussiepass',
name: 'Hussie Pass',
url: 'http://www.hussiepass.com',
url: 'https://www.hussiepass.com',
parent: 'hush',
},
{
slug: 'hushpass',
name: 'Hush Pass',
url: 'http://www.hushpass.com',
url: 'https://www.hushpass.com',
parent: 'hush',
parameters: {
t1: true,
sequential: true,
},
},
{
slug: 'interracialpass',
name: 'Interracial Pass',
url: 'http://www.interracialpass.com',
url: 'https://www.interracialpass.com',
parent: 'hush',
parameters: {
t1: true,
sequential: true,
},
},

27
src/scrapers/hush.js
View File

@ -2,19 +2,10 @@
const util = require('util');
const knex = require('../knex');
const { get, geta, ed, formatDate, ctxa } = require('../utils/q');
const slugify = require('../utils/slugify');
const { feetInchesToCm } = require('../utils/convert');
async function getChannelRegExp(site) {
if (!['hushpass', 'interracialpass'].includes(site.parent.slug)) return null;
const sites = await knex('entities').where('parent_id', site.parent.id);
return new RegExp(sites.map(channel => channel.parameters?.match || channel.name).join('|'), 'i');
}
function deriveEntryId(release) {
if (release.date && release.title) {
return `${slugify(formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
@ -160,7 +151,7 @@ function scrapeScene({ html, qu }, site, url, baseRelease) {
return release;
}
function scrapeSceneT1({ html, qu }, site, url, baseRelease, channelRegExp) {
function scrapeSceneT1({ html, qu }, site, url, baseRelease) {
const release = { url };
release.title = qu.q('.trailer-section-head .section-title', true);
@ -187,16 +178,12 @@ function scrapeSceneT1({ html, qu }, site, url, baseRelease, channelRegExp) {
const stars = qu.q('.update-rating', true).match(/\d.\d/)?.[0];
if (stars) release.stars = Number(stars);
console.log(channelRegExp, site);
if (channelRegExp) {
if (site.type === 'network') {
const channelRegExp = new RegExp(site.children.map(channel => channel.parameters?.match || channel.name).join('|'), 'i');
const channel = release.tags.find(tag => channelRegExp.test(tag));
if (channel) {
release.channel = {
force: true,
slug: slugify(channel, ''),
};
release.channel = slugify(channel, '');
}
}
@ -372,12 +359,11 @@ async function fetchLatest(site, page = 1, include, { uniqueReleases, duplicateR
return scrapeAll(res.items, site, uniqueReleases);
}
async function fetchScene(url, site, baseRelease, include, beforeFetchLatest) {
const channelRegExp = beforeFetchLatest || await getChannelRegExp(site);
async function fetchScene(url, site, baseRelease) {
const res = await get(url);
if (!res.ok) return res.status;
if (site.parameters?.t1) return scrapeSceneT1(res.item, site, url, baseRelease, channelRegExp);
if (site.parameters?.t1) return scrapeSceneT1(res.item, site, url, baseRelease);
if (site.parameters?.tour) return scrapeSceneTour(res.item, site, url, baseRelease);
return scrapeScene(res.item, site, url, baseRelease);
@ -405,7 +391,6 @@ async function fetchProfile({ name: actorName }, { site }) {
}
module.exports = {
beforeFetchLatest: getChannelRegExp,
fetchLatest,
fetchScene,
fetchProfile,