DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added tour layout scraper to Hush, enabling Interracial POVs, POV Pornstars and See Him Fuck.

This commit is contained in:
ThePendulum 2020-03-05 20:31:11 +01:00
parent 0feac66e94
commit fd6e90e74c
4 changed files with 95 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
seeds/00_tags.js
View File

@ -850,10 +850,22 @@ const aliases = [
name: 'anilingus',
for: 'ass-eating',
},
{
name: 'analingus',
for: 'ass-eating',
},
{
name: 'analingus male',
for: 'ass-eating',
},
{
name: 'asians',
for: 'asian',
},
{
name: 'asian female',
for: 'asian',
},
{
name: 'anal fingering',
for: 'ass-fingering',

12
seeds/02_sites.js
View File

@ -2075,6 +2075,9 @@ const sites = [
url: 'https://seehimfuck.com',
tags: ['male-focus'],
network: 'hussiepass',
parameters: {
tour: true,
},
},
{
slug: 'interracialpovs',
@ -2082,13 +2085,20 @@ const sites = [
url: 'https://www.interracialpovs.com',
tags: ['interracial', 'pov'],
network: 'hussiepass',
parameters: {
tour: true,
},
},
{
slug: 'povpornstars',
name: 'POV Pornstars',
url: 'https://www.povpornstars.com',
url: 'http://www.povpornstars.com',
tags: ['pov'],
network: 'hussiepass',
parameters: {
latest: 'http://www.povpornstars.com/tour/categories/movies_%d_d.html',
tour: true,
},
},
// HUSH PASS
{

15
src/releases.js
View File

@ -423,6 +423,11 @@ async function updateReleasesSearch(releaseIds) {
}
async function storeRelease(release, batchId) {
if (!release.entryId) {
logger.warn(`Missing entry ID, unable to store ${release.url}`);
return null;
}
const existingRelease = await knex('releases')
.where({
entry_id: release.entryId,
@ -473,11 +478,11 @@ async function storeReleases(releases) {
try {
const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const { id, slug } = await storeRelease(releaseWithStudio, batchId);
const storedRelease = await storeRelease(releaseWithStudio, batchId);
return {
id,
slug,
return storedRelease && {
id: storedRelease.id,
slug: storedRelease.slug,
...releaseWithChannelSite,
};
} catch (error) {
@ -487,7 +492,7 @@ async function storeReleases(releases) {
}
}, {
concurrency: 10,
}).filter(release => release);
}).filter(Boolean);
logger.info(`Stored ${storedReleases.length} new releases`);

71
src/scrapers/hush.js
View File

@ -6,6 +6,10 @@ const knex = require('../knex');
const { get, geta, fd } = require('../utils/q');
const slugify = require('../utils/slugify');
function deriveEntryId(release) {
return `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
}
function extractPoster(posterPath, site, baseRelease) {
if (posterPath && !/400.jpg/.test(posterPath)) {
const poster = `${site.parameters?.media || site.url}${posterPath}`;
@ -60,9 +64,6 @@ function scrapeLatestT1(scenes, site) {
release.date = qd('.more-info-div', 'MMM D, YYYY');
release.duration = ql('.more-info-div');
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
const posterPath = q('.img-div img', 'src0_1x') || qi('img.video_placeholder');
if (posterPath) {
@ -75,6 +76,27 @@ function scrapeLatestT1(scenes, site) {
];
}
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
return release;
});
}
function scrapeLatestTour(scenes) {
return scenes.map(({ q, qa, qu, qd, qi }) => {
const release = {};
release.title = q('h4 a', true);
release.url = qu('a');
release.date = qd('.tour_update_models + span', 'YYYY-MM-DD');
release.actors = qa('.tour_update_models a', true);
release.poster = qi('a img');
release.entryId = deriveEntryId(release);
return release;
});
}
@ -112,9 +134,6 @@ function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease, cha
release.date = qd('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = ql('.update-info-row:nth-child(2)');
// release.entryId = q('.player-thumb img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
release.actors = qa('.models-list-thumbs a').map(el => ({
name: q(el, 'span', true),
avatar: [
@ -148,6 +167,35 @@ function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease, cha
}
}
// release.entryId = q('.player-thumb img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
return release;
}
function scrapeSceneTour({ html, q, qd, qa, qis }, site, url) {
const release = { url };
release.title = q('.update_title, .video-title', true);
release.description = q('.latest_update_description, .video-summary', true);
const date = qd('.availdate', 'YYYY-MM-DD');
if (date) release.date = date;
release.actors = qa('.update_block_info .tour_update_models a, .video-model .tour_update_models a', true);
release.tags = qa('.update_tags a, .tour_update_tags a', true);
const [photo, poster, ...photos] = qis('.update_image img:not(.play_icon_overlay)');
if (poster || photo) release.poster = poster || photo;
if ((photo && poster) || photos) release.photos = poster ? [photo, ...photos] : photos; // don't use first photo when already used as fallback poster
if (release.date) release.entryId = deriveEntryId(release);
const trailerCode = q('.update_image a', 'onclick');
const trailerPath = trailerCode?.match(/tload\('(.*)'\)/)?.[1] || html.match(/\/trailer\/.*\.mp4/)?.[0];
if (trailerPath && /^http/.test(trailerPath)) release.trailer = { src: trailerPath };
else if (trailerPath) release.trailer = { src: `${site.parameters?.media || site.url}${trailerPath}` };
return release;
}
@ -164,11 +212,13 @@ async function fetchLatest(site, page = 1) {
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|| `${site.url}/categories/movies_${page}_d.html`;
const qLatest = await geta(url, '.modelfeature, .item-video');
const qLatest = await geta(url, '.modelfeature, .item-video, .updateItem');
if (!qLatest) return null;
if (site.parameters?.t1) return scrapeLatestT1(qLatest, site);
if (site.parameters?.tour) return scrapeLatestTour(qLatest, site);
return site.parameters?.t1 ? scrapeLatestT1(qLatest, site) : scrapeLatest(qLatest, site);
return scrapeLatest(qLatest, site);
}
async function fetchScene(url, site, baseRelease, beforeFetchLatest) {
@ -177,7 +227,10 @@ async function fetchScene(url, site, baseRelease, beforeFetchLatest) {
if (!qScene) return null;
return site.parameters?.t1 ? scrapeSceneT1(qScene, site, url, baseRelease, channelRegExp) : scrapeScene(qScene, site, url, baseRelease);
if (site.parameters?.t1) return scrapeSceneT1(qScene, site, url, baseRelease, channelRegExp);
if (site.parameters?.tour) return scrapeSceneTour(qScene, site, url, baseRelease);
return scrapeScene(qScene, site, url, baseRelease);
}
module.exports = {