DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Fixed qu issues. Fixed media issues. Simplified and expanded date component in search query.

This commit is contained in:
ThePendulum 2020-03-10 00:17:57 +01:00
parent 61a795d634
commit 5c55750c0c
9 changed files with 113 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
config/default.js
View File

@ -66,6 +66,8 @@ module.exports = {
'21sextury', '21sextury',
'julesjordan', 'julesjordan',
'naughtyamerica', 'naughtyamerica',
'cherrypimps',
'pimpxxx',
[ [
'hussiepass', 'hussiepass',
'hushpass', 'hushpass',
@ -75,16 +77,6 @@ module.exports = {
'seehimfuck', 'seehimfuck',
'eyeontheguy', 'eyeontheguy',
], ],
[
'cherrypimps',
'drilledxxx',
'wildoncam',
'bcmxxx',
'familyxxx',
'petitexxx',
'confessionsxxx',
'cuckedxxx',
],
[ [
// Full Porn Network // Full Porn Network
'analized', 'analized',

98
src/media.js
View File

@ -85,87 +85,91 @@ async function extractItem(source) {
const res = await get(source.src); const res = await get(source.src);
if (res.statusCode === 200) { if (res.statusCode === 200) {
const { q } = ex(res.body.toString()); const { qu } = ex(res.body.toString());
return source.extract(q); return source.extract(qu);
} }
return null; return null;
} }
async function fetchSource(source, domain, role, originalSource) {
logger.verbose(`Fetching ${domain} ${role} from ${source.src || source}`);
// const res = await bhttp.get(source.src || source);
const res = await get(source.src || source, {
headers: {
...(source.referer && { referer: source.referer }),
...(source.host && { host: source.host }),
},
});
if (res.statusCode === 200) {
const { pathname } = new URL(source.src || source);
const mimetype = mime.getType(pathname);
const extension = mime.getExtension(mimetype);
const hash = getHash(res.body);
const { entropy, size, width, height } = /image/.test(mimetype) ? await getMeta(res.body) : {};
logger.verbose(`Fetched media item from ${source.src || source}`);
return {
file: res.body,
mimetype,
extension,
hash,
entropy: entropy || null,
size: size || null,
width: width || null,
height: height || null,
quality: source.quality || null,
source: originalSource?.src || originalSource || source.src || source,
scraper: source.scraper,
copyright: source.copyright,
};
}
throw new Error(`Response ${res.statusCode} not OK`);
}
async function fetchItem(source, index, existingItemsBySource, domain, role, attempt = 1, originalSource = null, sourceIndex = 0) { async function fetchItem(source, index, existingItemsBySource, domain, role, attempt = 1, originalSource = null, sourceIndex = 0) {
if (!source) return null; if (!source) return null;
try { try {
if (Array.isArray(source)) { if (Array.isArray(source)) {
if (source.every(sourceX => !!sourceX.quality)) { if (source.every(sourceX => sourceX.quality)) {
// various video qualities provided // various video qualities provided
const selectedSource = pickQuality(source); const selectedSource = pickQuality(source);
return fetchItem(selectedSource, index, existingItemsBySource, domain, role, attempt, originalSource); return fetchItem(selectedSource, index, existingItemsBySource, domain, role, attempt, originalSource);
} }
// fallbacks provided // fallbacks provided
return source.reduce( return source.reduce((outcome, sourceX, sourceIndexX) => outcome.catch(
(outcome, sourceX, sourceIndexX) => outcome.catch(async () => fetchItem(sourceX, index, existingItemsBySource, domain, role, attempt, originalSource, sourceIndexX)), async () => fetchItem(sourceX, index, existingItemsBySource, domain, role, attempt, source, sourceIndexX),
Promise.reject(new Error()), ), Promise.reject(new Error()));
);
} }
if (source.src && source.extract) { if (source.src && source.extract) {
// source links to page containing a (presumably) tokenized photo // source links to page containing a (presumably) tokenized photo
const itemSource = await extractItem(source); const itemSource = await extractItem(source);
return fetchItem(itemSource, index, existingItemsBySource, domain, role, attempt, source); return fetchItem(itemSource, index, existingItemsBySource, domain, role, attempt, source, sourceIndex);
} }
if (existingItemsBySource[source]) { if (existingItemsBySource[source]) {
return null; return null;
} }
logger.verbose(`Fetching ${domain} ${role} from ${source.src || source}`); return fetchSource(source, domain, role, originalSource);
// const res = await bhttp.get(source.src || source);
const res = await get(source.src || source, {
headers: {
...(source.referer && { referer: source.referer }),
...(source.host && { host: source.host }),
},
});
if (res.statusCode === 200) {
const { pathname } = new URL(source.src || source);
const mimetype = mime.getType(pathname);
const extension = mime.getExtension(mimetype);
const hash = getHash(res.body);
const { entropy, size, width, height } = /image/.test(mimetype) ? await getMeta(res.body) : {};
logger.verbose(`Fetched media item from ${source.src || source}`);
return {
file: res.body,
mimetype,
extension,
hash,
entropy: entropy || null,
size: size || null,
width: width || null,
height: height || null,
quality: source.quality || null,
source: originalSource?.src || originalSource || source.src || source,
scraper: source.scraper,
copyright: source.copyright,
};
}
throw new Error(`Response ${res.statusCode} not OK`);
} catch (error) { } catch (error) {
logger.warn(`Failed attempt ${attempt}/3 to fetch ${domain} ${role} ${index + 1} (${source.src || source}): ${error}`); logger.warn(`Failed attempt ${attempt}/3 to fetch ${domain} ${role} ${index + 1} (${source.src || source}): ${error}`);
/*
if (attempt < 3) { if (attempt < 3) {
await Promise.delay(5000); await Promise.delay(5000);
return fetchItem(source, index, existingItemsBySource, domain, role, attempt + 1, originalSource); return fetchItem(source, index, existingItemsBySource, domain, role, attempt + 1, originalSource, sourceIndex);
} }
*/
if (originalSource && sourceIndex < originalSource.length) { if (originalSource && sourceIndex < originalSource.length) {
throw error; throw error;
@ -351,7 +355,7 @@ function associateTargetMedia(targetId, sources, mediaBySource, domain, role, pr
if (!source) return null; if (!source) return null;
const mediaItem = Array.isArray(source) const mediaItem = Array.isArray(source)
? source.reduce((acc, sourceX) => acc || mediaBySource[sourceX.src || sourceX], null) ? mediaBySource[source.map(sourceX => sourceX.src || sourceX).toString()]
: mediaBySource[source.src || source]; : mediaBySource[source.src || source];
// return mediaItem && { [`${domain}_id`]: targetId, media_id: mediaItem.id }; // return mediaItem && { [`${domain}_id`]: targetId, media_id: mediaItem.id };

19
src/releases.js
View File

@ -369,24 +369,19 @@ async function storeReleaseAssets(releases) {
async function updateReleasesSearch(releaseIds) { async function updateReleasesSearch(releaseIds) {
const documents = await knex.raw(` const documents = await knex.raw(`
SELECT SELECT
releases.id as release_id, releases.id AS release_id,
to_tsvector( TO_TSVECTOR(
'traxxx', 'traxxx',
releases.title || ' ' || releases.title || ' ' ||
sites.name || ' ' || sites.name || ' ' ||
sites.slug || ' ' || sites.slug || ' ' ||
networks.name || ' ' || networks.name || ' ' ||
networks.slug || ' ' || networks.slug || ' ' ||
coalesce(releases.shoot_id, '') || ' ' || COALESCE(releases.shoot_id, '') || ' ' ||
EXTRACT(YEAR FROM releases.date) || ' ' || COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMD'), '') || ' ' ||
CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR) || ' ' || STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR) || ' ' || STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
SUBSTRING(CAST(EXTRACT(YEAR FROM releases.date) AS VARCHAR) FROM 3 for 2) || ' ' || STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
LPAD(CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR), 2, '0') || ' ' ||
LPAD(CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR), 2, '0') || ' ' ||
string_agg(coalesce(actors.name, ''), ' ') || ' ' ||
string_agg(coalesce(tags.name, ''), ' ') || ' ' ||
string_agg(coalesce(tags_aliases.name, ''), ' ')
) as document ) as document
FROM releases FROM releases
LEFT JOIN sites ON releases.site_id = sites.id LEFT JOIN sites ON releases.site_id = sites.id

12
src/scrapers/bangbros.js
View File

@ -5,6 +5,7 @@ const bhttp = require('bhttp');
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const moment = require('moment'); const moment = require('moment');
const logger = require('../logger')(__filename);
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
const { ex } = require('../utils/q'); const { ex } = require('../utils/q');
@ -105,7 +106,10 @@ function scrapeScene(html, url, _site) {
release.photos = Array.from({ length: 12 }, (val, index) => firstPhotoUrl.replace(/big\d+/, `big${index + 1}`)); release.photos = Array.from({ length: 12 }, (val, index) => firstPhotoUrl.replace(/big\d+/, `big${index + 1}`));
const [channel] = qu.url('a[href*="/websites"]').match(/\w+$/); const [channel] = qu.url('a[href*="/websites"]').match(/\w+$/);
release.channel = channel === 'bangcasting' ? 'bangbroscasting' : channel;
if (channel === 'bangcasting') release.channel = 'bangbroscasting';
if (channel === 'remaster') release.channel = 'bangbrosremastered';
else release.channel = channel;
return release; return release;
} }
@ -123,8 +127,8 @@ function scrapeProfile(html) {
} }
function scrapeProfileSearch(html, actorName) { function scrapeProfileSearch(html, actorName) {
const { q } = ex(html); const { qu } = ex(html);
const actorLink = q(`a[title="${actorName}"]`, 'href'); const actorLink = qu.url(`a[title="${actorName}" i][href*="model"]`);
return actorLink ? `https://bangbros.com${actorLink}` : null; return actorLink ? `https://bangbros.com${actorLink}` : null;
} }
@ -145,7 +149,7 @@ async function fetchUpcoming(site) {
async function fetchScene(url, site, release) { async function fetchScene(url, site, release) {
if (!release?.date) { if (!release?.date) {
throw new Error(`Cannot fetch Bang Bros scenes from argument URL, as scene pages do not have release dates: ${url}`); logger.warn(`Scraping Bang Bros scene from URL without release date: ${url}`);
} }
const { origin } = new URL(url); const { origin } = new URL(url);

14
src/scrapers/boobpedia.js
View File

@ -5,11 +5,11 @@ const bhttp = require('bhttp');
const { ex } = require('../utils/q'); const { ex } = require('../utils/q');
function scrapeProfile(html) { function scrapeProfile(html) {
const { q, qa, qd, qi, qus } = ex(html); /* eslint-disable-line object-curly-newline */ const { qu } = ex(html); /* eslint-disable-line object-curly-newline */
const profile = {}; const profile = {};
const bio = qa('.infobox tr[valign="top"]') const bio = qu.all('.infobox tr[valign="top"]')
.map(detail => qa(detail, 'td', true)) .map(detail => qu.all(detail, 'td', true))
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {}); .reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
@ -19,9 +19,9 @@ function scrapeProfile(html) {
profile.gender = isTrans ? 'transsexual' : 'female'; profile.gender = isTrans ? 'transsexual' : 'female';
*/ */
profile.birthdate = qd('.bday', 'YYYY-MM-DD'); profile.birthdate = qu.date('.bday', 'YYYY-MM-DD');
profile.description = q('#mw-content-text > p', true); profile.description = qu.q('#mw-content-text > p', true);
if (bio.Born) profile.birthPlace = bio.Born.slice(bio.Born.lastIndexOf(')') + 1); if (bio.Born) profile.birthPlace = bio.Born.slice(bio.Born.lastIndexOf(')') + 1);
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity; if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
@ -62,7 +62,7 @@ function scrapeProfile(html) {
if (bio.Blood_group) profile.blood = bio.Blood_group; if (bio.Blood_group) profile.blood = bio.Blood_group;
if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', '); if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', ');
const avatarThumbPath = qi('.image img'); const avatarThumbPath = qu.img('.image img');
if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) { if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) {
const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', ''); const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', '');
@ -73,7 +73,7 @@ function scrapeProfile(html) {
}; };
} }
profile.social = qus('.infobox a.external'); profile.social = qu.urls('.infobox a.external');
return profile; return profile;
} }

36
src/scrapers/ddfnetwork.js
View File

@ -27,26 +27,26 @@ function scrapeAll(html, site, origin) {
} }
async function scrapeScene(html, url, _site) { async function scrapeScene(html, url, _site) {
const { q, qa, qd, qm, qp, qus } = ex(html); const { qu } = ex(html);
const release = {}; const release = {};
[release.entryId] = url.split('/').slice(-1); [release.entryId] = url.split('/').slice(-1);
release.title = qm('itemprop=name'); release.title = qu.meta('itemprop=name');
release.description = q('.descr-box p', true); release.description = qu.q('.descr-box p', true);
release.date = qd('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content') release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|| qd('.title-border:nth-child(2) p', 'MM.DD.YYYY'); || qu.date('.title-border:nth-child(2) p', 'MM.DD.YYYY');
release.actors = qa('.pornstar-card > a', 'title'); release.actors = qu.all('.pornstar-card > a', 'title');
release.tags = qa('.tags-tab .tags a', true); release.tags = qu.all('.tags-tab .tags a', true);
release.duration = parseInt(q('.icon-video-red + span', true), 10) * 60; release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60;
release.likes = Number(q('.icon-like-red + span', true)); release.likes = Number(qu.q('.icon-like-red + span', true));
release.poster = qp(); release.poster = qu.poster();
release.photos = qus('.photo-slider-guest .card a'); release.photos = qu.urls('.photo-slider-guest .card a');
release.trailer = qa('source[type="video/mp4"]').map(trailer => ({ release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({
src: trailer.src, src: trailer.src,
quality: Number(trailer.attributes.res.value), quality: Number(trailer.attributes.res.value),
})); }));
@ -72,10 +72,10 @@ async function fetchActorReleases(urls) {
} }
async function scrapeProfile(html, _url, actorName) { async function scrapeProfile(html, _url, actorName) {
const { q, qa, qus } = ex(html); const { qu } = ex(html);
const keys = qa('.about-title', true).map(key => slugify(key, { delimiter: '_' })); const keys = qu.all('.about-title', true).map(key => slugify(key, { delimiter: '_' }));
const values = qa('.about-info').map((el) => { const values = qu.all('.about-info').map((el) => {
if (el.children.length > 0) { if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', '); return Array.from(el.children, child => child.textContent.trim()).join(', ');
} }
@ -96,7 +96,7 @@ async function scrapeProfile(html, _url, actorName) {
name: actorName, name: actorName,
}; };
profile.description = q('.description-box', true); profile.description = qu.q('.description-box', true);
profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY'); profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY');
if (bio.nationality) profile.nationality = bio.nationality; if (bio.nationality) profile.nationality = bio.nationality;
@ -118,10 +118,10 @@ async function scrapeProfile(html, _url, actorName) {
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]); if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
const avatarEl = q('.pornstar-details .card-img-top'); const avatarEl = qu.q('.pornstar-details .card-img-top');
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`; if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
profile.releases = await fetchActorReleases(qus('.find-me-tab li a')); profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a'));
return profile; return profile;
} }

16
src/scrapers/naughtyamerica.js
View File

@ -101,22 +101,24 @@ function scrapeScene(html, url, site) {
} }
async function fetchActorReleases(url) { async function fetchActorReleases(url) {
const { qus } = await get(url); const res = await get(url);
return qus('.contain-block:not(.live-scenes) .scene-item > a:first-child'); // live scenes repeat on all pages return res.ok
? res.item.qu.urls('.contain-block:not(.live-scenes) .scene-item > a:first-child') // live scenes repeat on all pages
: [];
} }
async function scrapeProfile(html) { async function scrapeProfile(html) {
const { q, qus } = ex(html); const { qu } = ex(html);
const profile = {}; const profile = {};
profile.description = q('.bio_about_text', true); profile.description = qu.q('.bio_about_text', true);
const avatar = q('img.performer-pic', 'src'); const avatar = qu.q('img.performer-pic', 'src');
if (avatar) profile.avatar = `https:${avatar}`; if (avatar) profile.avatar = `https:${avatar}`;
const releases = qus('.scene-item > a:first-child'); const releases = qu.urls('.scene-item > a:first-child');
const otherPages = qus('.pagination a:not([rel=next]):not([rel=prev])'); const otherPages = qu.urls('.pagination a:not([rel=next]):not([rel=prev])');
const olderReleases = await Promise.all(otherPages.map(async page => fetchActorReleases(page))); const olderReleases = await Promise.all(otherPages.map(async page => fetchActorReleases(page)));
profile.releases = releases.concat(olderReleases.flat()); profile.releases = releases.concat(olderReleases.flat());

2
src/scrapers/nubiles.js
View File

@ -58,7 +58,7 @@ function scrapeAll(scenes, site, origin) {
}); });
} }
async function scrapeScene(qu, url, site) { async function scrapeScene({ qu }, url, site) {
const release = {}; const release = {};
const { origin, pathname } = new URL(url); const { origin, pathname } = new URL(url);

20
src/scrapers/vogov.js
View File

@ -116,23 +116,23 @@ function scrapeLatest(html) {
} }
function scrapeScene(html, url) { function scrapeScene(html, url) {
const { q, qa, qd, qus, ql, qm } = ex(html); const { qu } = ex(html);
const release = { url }; const release = { url };
// release.entryId = slugify(release.title); // release.entryId = slugify(release.title);
[release.entryId] = q('link[rel="canonical"]').href.match(/\d+/); [release.entryId] = qu.q('link[rel="canonical"]').href.match(/\d+/);
release.title = qm('meta[property="og:title"]') || q('.video-page-header h1', true); release.title = qu.meta('meta[property="og:title"]') || qu.q('.video-page-header h1', true);
release.description = qu.meta('meta[property="og:description"]') || qu.q('.info-video-description', true);
release.description = qm('meta[property="og:description"]') || q('.info-video-description', true); release.date = qu.date('.info-video-details li:first-child span', 'MMM DD, YYYY');
release.date = qd('.info-video-details li:first-child span', 'MMM DD, YYYY'); release.duration = qu.dur('.info-video-details li:nth-child(2) span');
release.duration = ql('.info-video-details li:nth-child(2) span');
release.actors = qa('.info-video-models a', true); release.actors = qu.all('.info-video-models a', true);
release.tags = qa('.info-video-category a', true); release.tags = qu.all('.info-video-category a', true);
release.photos = qus('.swiper-wrapper .swiper-slide a').map(source => source.replace('.jpg/', '.jpg')); release.photos = qu.urls('.swiper-wrapper .swiper-slide a').map(source => source.replace('.jpg/', '.jpg'));
release.poster = qm('meta[property="og:image"'); release.poster = qu.meta('meta[property="og:image"');
if (!release.poster) { if (!release.poster) {
const previewStart = html.indexOf('preview_url'); const previewStart = html.indexOf('preview_url');