Added Insex. Renamed q's stand-alone date function. Separated q's trim function. Release tile uses cover if available, and poster is not available.

This commit is contained in:
ThePendulum 2020-02-12 04:39:57 +01:00
parent 2f894edda5
commit b8074205ef
21 changed files with 212 additions and 23 deletions

View File

@ -352,6 +352,7 @@ export default {
.avatar-link {
font-size: 0;
padding: 0 0 1rem 1rem;
flex-shrink: 0;
}
.avatar {

View File

@ -27,6 +27,7 @@ function curateRelease(release) {
};
if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media);
if (release.trailer) curatedRelease.trailer = release.trailer.media;
if (release.teaser) curatedRelease.teaser = release.teaser.media;
if (release.actors) curatedRelease.actors = release.actors.map(({ actor }) => curateActor(actor, curatedRelease));

View File

@ -75,6 +75,17 @@ const releasePosterFragment = `
}
`;
const releaseCoversFragment = `
covers: releasesCovers {
media {
index
path
thumbnail
comment
}
}
`;
const releasePhotosFragment = `
photos: releasesPhotos {
media {
@ -119,6 +130,7 @@ const releaseFields = `
${releaseActorsFragment}
${releaseTagsFragment}
${releasePosterFragment}
${releaseCoversFragment}
${siteFragment}
studio {
id

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 137 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

View File

@ -121,6 +121,11 @@ function getTags(groupsMap) {
description: 'Stuffing a toy, such as a dildo or buttplug, into the ass',
alias_for: null,
},
{
name: 'animated',
slug: 'animated',
alias_for: null,
},
{
name: 'asian',
slug: 'asian',

View File

@ -114,6 +114,12 @@ const networks = [
url: 'https://www.girlsway.com',
description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!',
},
{
slug: 'insex',
name: 'Insex',
description: 'The original bondage and BDSM transgression.',
url: 'http://www.insex.com',
},
{
slug: 'jayrock',
name: 'JayRock Productions',

View File

@ -1775,6 +1775,63 @@ const sites = [
scene: 'https://www.girlsway.com/en/video',
},
},
// INSEX
{
slug: 'sexuallybroken',
name: 'Sexually Broken',
url: 'http://www.sexuallybroken.com',
tags: ['bdsm'],
network: 'insex',
},
{
slug: 'infernalrestraints',
name: 'Infernal Restraints',
url: 'http://www.infernalrestraints.com',
tags: ['bdsm'],
network: 'insex',
},
{
slug: 'hardtied',
name: 'Hardtied',
url: 'http://www.hardtied.com',
tags: ['bdsm'],
network: 'insex',
},
{
slug: 'realtimebondage',
name: 'Real Time Bondage',
url: 'http://www.realtimebondage.com',
tags: ['bdsm'],
network: 'insex',
},
{
slug: 'topgrl',
name: 'TopGrl',
url: 'http://www.topgrl.com',
tags: ['bdsm', 'femdom'],
network: 'insex',
},
{
slug: 'paintoy',
name: 'Paintoy',
url: 'http://www.paintoy.com',
tags: ['bdsm'],
network: 'insex',
},
{
slug: 'aganmedon',
name: 'Agan Medon',
url: 'http://www.aganmedon.com',
tags: ['bdsm', 'animated'],
network: 'insex',
},
{
slug: 'sensualpain',
name: 'Sensual Pain',
url: 'http://www.sensualpain.com',
tags: ['bdsm'],
network: 'insex',
},
// JAYS POV
{
slug: 'jayspov',

View File

@ -2,7 +2,7 @@
const bhttp = require('bhttp');
const { d, ex, exa, get } = require('../utils/q');
const { fd, ex, exa, get } = require('../utils/q');
const slugify = require('../utils/slugify');
/* eslint-disable newline-per-chained-call */
@ -97,7 +97,7 @@ async function scrapeProfile(html, _url, actorName) {
};
profile.description = q('.description-box', true);
profile.birthdate = d(bio.birthday, 'MMMM DD, YYYY');
profile.birthdate = fd(bio.birthday, 'MMMM DD, YYYY');
if (bio.nationality) profile.nationality = bio.nationality;

100
src/scrapers/insex.js Normal file
View File

@ -0,0 +1,100 @@
'use strict';
const bhttp = require('bhttp');
const { get, exa, fd } = require('../utils/q');
function scrapeLatest(html, site) {
const scenes = exa(html, 'body > table');
return scenes.map(({ q, qd, qi, qu, ql }) => {
// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
const release = {};
const titleEl = q('.galleryTitleText, .articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const date = qd('.articlePostDateText', 'MMM D, YYYY');
const url = qu(titleEl, 'a');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
if (date) {
release.title = title.trim();
release.date = date;
} else {
// title should contain date instead
release.title = title.slice(title.indexOf(':') + 1).trim();
release.date = fd(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
}
release.actors = actors.map(actor => actor.trim());
const description = q('.articleCopyText .articleCopyText', true);
if (description) release.description = description;
const duration = ql('.articleCopyText a:nth-child(2)');
if (duration) release.duration = duration;
const poster = qi('a img');
release.poster = [
poster.replace('_thumbnail', ''),
poster,
];
return release;
});
}
function scrapeScene({ q, qd, ql, qu, qis, qp, qt }, site) {
const release = {};
const titleEl = q('.articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const url = qu(titleEl, 'a');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
release.title = title.trim();
release.description = q('.articleCopyText', true);
release.actors = actors.map(actor => actor.trim());
release.date = qd('.articlePostDateText', 'MMMM D, YYYY');
release.duration = ql('.articlePostDateText a:nth-child(2)');
const [cover, ...photos] = qis('img[src*="images"]');
release.covers = [cover];
release.photos = photos;
release.poster = qp();
const trailer = qt();
release.trailer = { src: trailer };
return release;
}
async function fetchLatest(site, page = 1) {
const url = `${site.url}/scripts/switch_tour.php?page=${page}`;
const res = await bhttp.get(url, {
type: 'gallery',
page,
});
if (res.statusCode === 200) {
return scrapeLatest(res.body.html, site);
}
return null;
}
async function fetchScene(url, site) {
const qScene = await get(url);
return qScene && scrapeScene(qScene, site);
}
module.exports = {
fetchLatest,
fetchScene,
};

View File

@ -19,6 +19,7 @@ const freeones = require('./freeones');
const freeonesLegacy = require('./freeones_legacy');
const girlsway = require('./girlsway');
const iconmale = require('./iconmale');
const insex = require('./insex');
const jayrock = require('./jayrock');
const julesjordan = require('./julesjordan');
const kellymadison = require('./kellymadison');
@ -73,6 +74,7 @@ module.exports = {
fakehub,
fantasymassage,
girlsway,
insex,
jayrock,
julesjordan,
kellymadison,

View File

@ -3,7 +3,7 @@
/* eslint-disable no-unused-vars */
const bhttp = require('bhttp');
const { get, date } = require('../utils/q');
const { get, fd } = require('../utils/q');
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
const slugify = require('../utils/slugify');
@ -15,7 +15,7 @@ function scrapeLatestNative(scenes, site) {
release.url = `${site.url}${scene.url}`;
release.title = scene.name;
release.date = date(scene.release_date, 'YYYY-MM-DD');
release.date = fd(scene.release_date, 'YYYY-MM-DD');
release.duration = parseInt(scene.runtime, 10) * 60;
release.actors = scene.cast?.map(actor => ({
@ -40,7 +40,7 @@ function scrapeSceneNative({ html, q, qa }, url, _site) {
release.description = q('.indie-model-p', true);
const dateString = qa('h5').find(el => /Released/.test(el.textContent)).textContent;
release.date = date(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.date = fd(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
const duration = qa('h5').find(el => /Runtime/.test(el.textContent)).textContent;
const [hours, minutes] = duration.match(/\d+/g);
@ -118,7 +118,7 @@ async function fetchSceneWrapper(url, site, release) {
return {
...scene,
url: `${site.url}${sceneMatch.url}`,
date: date(sceneMatch.release_date, 'YYYY-MM-DD'),
date: fd(sceneMatch.release_date, 'YYYY-MM-DD'),
};
}
}

View File

@ -4,6 +4,10 @@ const { JSDOM } = require('jsdom');
const moment = require('moment');
const bhttp = require('bhttp');
function trim(str) {
return str.trim().replace(/\s+/g, ' ');
}
function prefixProtocol(url, protocol = 'https') {
if (protocol && /^\/\//.test(url)) {
return `${protocol}:${url}`;
@ -12,7 +16,7 @@ function prefixProtocol(url, protocol = 'https') {
return url;
}
function q(context, selector, attrArg, trim = true) {
function q(context, selector, attrArg, applyTrim = true) {
const attr = attrArg === true ? 'textContent' : attrArg;
if (attr) {
@ -20,52 +24,52 @@ function q(context, selector, attrArg, trim = true) {
? context.querySelector(selector)?.[attr] || context.querySelector(selector)?.attributes[attr]?.value
: context[attr] || context[attr]?.attributes[attr]?.value;
return trim ? value?.trim() : value;
return applyTrim && value ? trim(value) : value;
}
return selector ? context.querySelector(selector) : context;
}
function qall(context, selector, attrArg, trim = true) {
function qall(context, selector, attrArg, applyTrim = true) {
const attr = attrArg === true ? 'textContent' : attrArg;
if (attr) {
return Array.from(context.querySelectorAll(selector), el => (trim ? el[attr]?.trim() : el[attr]));
return Array.from(context.querySelectorAll(selector), el => (applyTrim && el[attr] ? trim(el[attr]) : el[attr]));
}
return Array.from(context.querySelectorAll(selector));
}
function qtext(context, selector, trim = true) {
const el = q(context, selector, null, trim);
function qtext(context, selector, applyTrim = true) {
const el = q(context, selector, null, applyTrim);
if (!el) return null;
const text = Array.from(el.childNodes)
.filter(node => node.nodeName === '#text')
.map(node => (trim ? node.textContent : node.textContent.trim()))
.map(node => (applyTrim ? node.textContent : trim(node.textContent)))
.join(' ');
if (trim) return text.trim();
if (applyTrim) return trim(text);
return text;
}
function qmeta(context, selector, attrArg = 'content', trim = true) {
function qmeta(context, selector, attrArg = 'content', applyTrim = true) {
if (/meta\[.*\]/.test(selector)) {
return q(context, selector, attrArg, trim);
return q(context, selector, attrArg, applyTrim);
}
return q(context, `meta[${selector}]`, attrArg, trim);
return q(context, `meta[${selector}]`, attrArg, applyTrim);
}
function date(dateString, format, match) {
function formatDate(dateString, format, match) {
if (match) {
const dateStamp = dateString.trim().match(match);
const dateStamp = trim(dateString).match(match);
if (dateStamp) return moment.utc(dateStamp[0], format).toDate();
return null;
}
return moment.utc(dateString.trim(), format).toDate();
return moment.utc(trim(dateString), format).toDate();
}
function qdate(context, selector, format, match, attr = 'textContent') {
@ -73,7 +77,7 @@ function qdate(context, selector, format, match, attr = 'textContent') {
if (!dateString) return null;
return date(dateString, format, match);
return formatDate(dateString, format, match);
}
function qimage(context, selector = 'img', attr = 'src', protocol = 'https') {
@ -226,7 +230,7 @@ async function getAll(url, selector, headers) {
}
module.exports = {
date,
formatDate,
extract,
extractAll,
init,
@ -235,11 +239,12 @@ module.exports = {
getAll,
context: init,
contextAll: initAll,
d: date,
fd: formatDate,
ex: extract,
exa: extractAll,
ctx: init,
ctxa: initAll,
geta: getAll,
fdate: formatDate,
...funcs,
};