DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Scraping actor scenes from Hussie Pass. Adding entity to actor base releases.

This commit is contained in:
DebaucheryLibrarian 2020-11-15 23:50:04 +01:00
parent b952b758d7
commit d14ef90136
5 changed files with 75 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
seeds/02_sites.js
View File

@ -3070,7 +3070,7 @@ const sites = [
{
slug: 'interracialpovs',
name: 'Interracial POVs',
url: 'https://www.interracialpovs.com',
url: 'https://interracialpovs.com',
tags: ['interracial', 'pov'],
parent: 'hussiepass',
},

2
src/actors.js
View File

@ -396,7 +396,7 @@ async function curateProfile(profile) {
}).filter(Boolean)
: [];
curatedProfile.releases = toBaseReleases(profile.releases);
curatedProfile.releases = toBaseReleases(profile.releases, profile.entity);
if (profile.ethnicity && !curatedProfile.ethnicity) logger.warn(`Unrecognized ethnicity returned by '${profile.entity.name}' scraper: ${profile.ethnicity}`);
if ((profile.hairColor || profile.hair) && !curatedProfile.hairColor) logger.warn(`Unrecognized hair color returned by '${profile.entity.name}' scraper: ${profile.hairColor || profile.hair}`);

5
src/deep.js
View File

@ -46,7 +46,7 @@ async function findEntities(baseReleases) {
return entitiesBySlug;
}
function toBaseReleases(baseReleasesOrUrls) {
function toBaseReleases(baseReleasesOrUrls, entity = null) {
if (!baseReleasesOrUrls) {
return [];
}
@ -57,6 +57,7 @@ function toBaseReleases(baseReleasesOrUrls) {
// base release with URL
return {
...baseReleaseOrUrl,
entity,
deep: false,
};
}
@ -65,6 +66,7 @@ function toBaseReleases(baseReleasesOrUrls) {
// URL
return {
url: baseReleaseOrUrl,
entity,
deep: false,
};
}
@ -73,6 +75,7 @@ function toBaseReleases(baseReleasesOrUrls) {
// base release without URL, prepare for passthrough
return {
...baseReleaseOrUrl,
entity,
deep: false,
};
}

96
src/scrapers/hush.js
View File

@ -2,7 +2,7 @@
const util = require('util');
const { get, getAll, ed, formatDate, prefixUrl, ctxa } = require('../utils/q');
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
const { feetInchesToCm, inchesToCm } = require('../utils/convert');
@ -10,11 +10,11 @@ function deriveEntryId(release) {
if (release.date && release.url) {
const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
return `${slugify(formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
}
if (release.date && release.title) {
return `${slugify(formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
}
return null;
@ -82,18 +82,18 @@ function scrapeAll(scenes, channel) {
}
function scrapeAllT1(scenes, site, accNetworkReleases) {
return scenes.map(({ qu }) => {
return scenes.map(({ query }) => {
const release = {};
release.title = qu.q('h4 a', 'title') || qu.q('h4 a', true);
release.url = qu.url('h4 a');
release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
release.url = query.url('h4 a');
release.date = qu.date('.more-info-div', 'MMM D, YYYY');
release.duration = qu.dur('.more-info-div');
release.date = query.date('.more-info-div', 'MMM D, YYYY');
release.duration = query.dur('.more-info-div');
if (/bts|behind the scenes/i.test(release.title)) release.tags = ['behind the scenes'];
const posterPath = qu.q('.img-div img', 'src0_1x') || qu.img('img.video_placeholder');
const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${site.parameters?.media || site.url}${posterPath}`;
@ -109,7 +109,7 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
release.entryId = deriveEntryId(release);
if (site.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site, requires sequential site scraping
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
return null;
}
@ -129,10 +129,10 @@ function scrapeScene({ html, query }, channel, url) {
release.actors = query.cnts('.update_models a');
const posterPath = html.match(/poster="([\w-/.]+)"/)?.[1];
const poster = prefixUrl(posterPath, channel.url);
const poster = qu.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
[release.poster, ...release.photos] = [poster, ...query.imgs('.item-thumb img', 'src0_1x', { origin: channel.url })]
.map(src => [
.map(src => src && [
src.replace('-1x', '-3x'),
src.replace('-1x', '-2x'),
src,
@ -141,8 +141,7 @@ function scrapeScene({ html, query }, channel, url) {
const trailerPath = html.match(/\/trailers\/.*.mp4/);
if (trailerPath) {
// release.trailer = { src: `${channel.parameters?.media || channel.url}${trailerPath}` };
release.trailer = prefixUrl(trailerPath, channel.parameters?.media || channel.url);
release.trailer = qu.prefixUrl(trailerPath, channel.parameters?.media || channel.url);
}
release.tags = query.cnts('.featuring a[href*="categories/"]');
@ -153,31 +152,31 @@ function scrapeScene({ html, query }, channel, url) {
return release;
}
function scrapeSceneT1({ html, qu }, site, url, baseRelease) {
function scrapeSceneT1({ html, query }, site, url, baseRelease) {
const release = { url };
release.title = qu.q('.trailer-section-head .section-title', true);
release.description = qu.text('.row .update-info-block');
release.title = query.q('.trailer-section-head .section-title', true);
release.description = query.text('.row .update-info-block');
release.date = qu.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = qu.dur('.update-info-row:nth-child(2)');
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)');
release.actors = qu.all('.models-list-thumbs a').map(el => ({
name: qu.q(el, 'span', true),
avatar: getImageWithFallbacks(qu.q, 'img', site, el),
release.actors = query.all('.models-list-thumbs a').map(el => ({
name: query.q(el, 'span', true),
avatar: getImageWithFallbacks(query.q, 'img', site, el),
}));
release.tags = qu.all('.tags a', true);
release.tags = query.all('.tags a', true);
// const posterPath = html.match(/poster="(.*\.jpg)/)?.[1];
const posterPath = qu.q('.player-thumb img', 'src0_1x');
const posterPath = query.q('.player-thumb img', 'src0_1x');
[release.poster, release.photos] = extractPoster(posterPath, site, baseRelease);
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
if (trailer && /^http/.test(trailer)) release.trailer = { src: trailer, referer: url };
else if (trailer) release.trailer = { src: `${site.parameters?.media || site.url}${trailer}`, referer: url };
const stars = qu.q('.update-rating', true).match(/\d.\d/)?.[0];
const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
if (stars) release.stars = Number(stars);
if (site.type === 'network') {
@ -195,10 +194,10 @@ function scrapeSceneT1({ html, qu }, site, url, baseRelease) {
return release;
}
function scrapeProfileT1({ el, qu }, site) {
function scrapeProfileT1({ el, query }, site) {
const profile = {};
const bio = qu.all('.detail-div + .detail-div p, .detail-div p', true).reduce((acc, info) => {
const bio = query.all('.detail-div + .detail-div p, .detail-div p', true).reduce((acc, info) => {
const [key, value] = info.split(':');
if (!value) return acc;
@ -225,15 +224,30 @@ function scrapeProfileT1({ el, qu }, site) {
if (heightMetric) profile.height = Number(heightMetric[1]);
if (heightImperial) profile.height = feetInchesToCm(Number(heightImperial[0]), Number(heightImperial[1]));
profile.avatar = getImageWithFallbacks(qu.q, '.img-div img', site);
profile.avatar = getImageWithFallbacks(query.q, '.img-div img', site);
const qReleases = ctxa(el, '.item-video');
const qReleases = qu.initAll(el, '.item-video');
profile.releases = scrapeAllT1(qReleases, site);
return profile;
}
function scrapeProfile({ query }, channel) {
async function fetchActorScenes({ query, el }, channel, accScenes = []) {
const scenes = scrapeAll(qu.initAll(el, '.item-video'), channel);
const nextPage = query.url('.next a');
if (nextPage) {
const res = await qu.get(nextPage);
if (res.ok) {
return fetchActorScenes(res.item, channel, scenes.concat(accScenes));
}
}
return accScenes.concat(scenes);
}
async function scrapeProfile({ query, el }, channel, options) {
const profile = {};
const bio = query.all('.stats li').reduce((acc, bioEl) => {
@ -246,7 +260,7 @@ function scrapeProfile({ query }, channel) {
};
}, {});
if (bio.date_of_birth) profile.birthdate = ed(bio.date_of_birth, 'MMMM D, YYYY');
if (bio.date_of_birth) profile.birthdate = qu.extractDate(bio.date_of_birth, 'MMMM D, YYYY');
if (bio.birthplace) profile.birthPlace = bio.birthplace;
if (bio.fun_fact) profile.description = bio.fun_fact;
@ -286,6 +300,10 @@ function scrapeProfile({ query }, channel) {
query.img('.profile-pic img', 'src0_1x', { origin: channel.url }),
];
if (options.includeActorScenes) {
profile.releases = await fetchActorScenes({ query, el }, channel);
}
return profile;
}
@ -294,7 +312,7 @@ async function fetchLatest(site, page = 1, include, { uniqueReleases = [], dupli
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|| `${site.url}/categories/movies_${page}_d.html`;
const res = await getAll(url, '.modelfeature, .item-video, .updateItem');
const res = await qu.getAll(url, '.modelfeature, .item-video, .updateItem');
if (!res.ok) {
return res.status;
@ -308,7 +326,7 @@ async function fetchLatest(site, page = 1, include, { uniqueReleases = [], dupli
}
async function fetchScene(url, site, baseRelease) {
const res = await get(url);
const res = await qu.get(url);
if (!res.ok) {
return res.status;
@ -321,19 +339,19 @@ async function fetchScene(url, site, baseRelease) {
return scrapeScene(res.item, site, url, baseRelease);
}
async function fetchProfile({ name: actorName }, { site }) {
async function fetchProfile({ name: actorName }, { site }, options) {
const actorSlugA = slugify(actorName, '');
const actorSlugB = slugify(actorName);
const t1 = site.parameters?.t1 ? 't1/' : '';
const res1 = site.parameters?.profile
? await get(util.format(site.parameters.profile, actorSlugA))
: await get(`${site.url}/${t1}models/${actorSlugA}.html`, null, null, { followRedirects: false });
? await qu.get(util.format(site.parameters.profile, actorSlugA))
: await qu.get(`${site.url}/${t1}models/${actorSlugA}.html`, null, null, { followRedirects: false });
const res = (res1.ok && res1)
|| (site.parameters?.profile && await get(util.format(site.parameters.profile, actorSlugB)))
|| await get(`${site.url}/${t1}models/${actorSlugB}.html`, null, null, { followRedirects: false });
|| (site.parameters?.profile && await qu.get(util.format(site.parameters.profile, actorSlugB)))
|| await qu.get(`${site.url}/${t1}models/${actorSlugB}.html`, null, null, { followRedirects: false });
if (!res.ok) {
return res.status;
@ -343,7 +361,7 @@ async function fetchProfile({ name: actorName }, { site }) {
return scrapeProfileT1(res.item, site);
}
return scrapeProfile(res.item, site);
return scrapeProfile(res.item, site, options);
}
module.exports = {

24
src/utils/argv-include.js
View File

@ -15,18 +15,18 @@ function include(argv) {
return {
...options,
// legacy
covers: include.includeCovers,
media: include.includeMedia,
photos: include.includePhotos,
videos: include.includeVideos,
poster: include.includePosters,
posters: include.includePosters,
teaser: include.includeTeasers,
teasers: include.includeTeasers,
trailer: include.includeTrailers,
trailers: include.includeTrailers,
releases: include.includeActorScenes,
scenes: include.includeActorScenes,
covers: options.includeCovers,
media: options.includeMedia,
photos: options.includePhotos,
videos: options.includeVideos,
poster: options.includePosters,
posters: options.includePosters,
teaser: options.includeTeasers,
teasers: options.includeTeasers,
trailer: options.includeTrailers,
trailers: options.includeTrailers,
releases: options.includeActorScenes,
scenes: options.includeActorScenes,
};
}