DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added profile releases for classic Fame Digital sites (Silvia Saint and Silverstone DVD).

This commit is contained in:
ThePendulum 2020-02-07 03:40:11 +01:00
parent 49405b953f
commit 30963b94dd
7 changed files with 75 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
assets/js/networks/actions.js
View File

@ -21,7 +21,7 @@ function initNetworksActions(store, _router) {
sites(
orderBy: [PRIORITY_DESC, NAME_ASC],
filter: {
enabled: {
show: {
equalTo: true,
},
},

4
migrations/20190325001339_releases.js
View File

@ -144,7 +144,9 @@ exports.up = knex => Promise.resolve()
table.integer('priority', 3)
.defaultTo(0);
table.boolean('enabled')
table.boolean('show')
.defaultTo(true);
table.boolean('scrape')
.defaultTo(true);
table.string('slug', 32)

BIN
public/img/logos/blowpass/sunlustxxx.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 38 KiB

30
seeds/01_sites.js
View File

@ -732,6 +732,15 @@ const sites = [
upcoming: '/en/videos/upcoming',
},
},
{
slug: 'sunlustxxx',
name: 'Sun Lust XXX',
url: 'https://www.sunlustxxx.com',
description: '',
network: 'blowpass',
scrape: false,
show: true, // site offline, use only for indexing old scenes
},
// BRAZZERS
{
slug: 'momsincontrol',
@ -1035,7 +1044,8 @@ const sites = [
description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.',
network: 'ddfnetwork',
parameters: { native: true },
enabled: false, // appears to be re-releases only
scrape: false,
show: true, // appears to be re-releases only
},
{
slug: 'bustylover',
@ -1043,7 +1053,8 @@ const sites = [
url: 'https://bustylover.com',
network: 'ddfnetwork',
parameters: { native: true },
enabled: false, // appears to be re-releases only
scrape: false,
show: true, // appears to be re-releases only
},
// DIGITAL PLAYGROUND
{
@ -1372,7 +1383,8 @@ const sites = [
description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.',
network: 'famedigital',
parameters: { api: true },
enabled: false,
scrape: false,
show: false, // no data sources
},
{
slug: 'peternorth',
@ -3235,7 +3247,8 @@ const sites = [
slug: 'bigboobbundle',
url: 'https://www.bigboobbundle.com',
network: 'score',
enabled: false, // all content appears to be on subsites
scrape: false,
show: false, // all content appears to be on subsites
},
{
name: 'Big Boobs POV',
@ -3630,7 +3643,8 @@ const sites = [
url: 'https://www.scorepass.com/scorelandtv',
network: 'score',
priority: 1,
enabled: false, // appears to be streaming service for other sites
scrape: false,
show: false, // appears to be streaming service for other sites
},
{
name: 'ScoreTV',
@ -3638,7 +3652,8 @@ const sites = [
url: 'https://www.scoretv.tv',
network: 'score',
priority: 1,
enabled: false, // similar to or same as Scoreland TV
scrape: false,
show: false, // similar to or same as Scoreland TV
},
{
name: 'Score Videos',
@ -4079,7 +4094,8 @@ exports.seed = knex => Promise.resolve()
parameters: JSON.stringify(site.parameters),
network_id: networksMap[site.network],
priority: site.priority,
enabled: site.enabled,
scrape: site.scrape,
show: site.show,
}));
return upsert('sites', sitesWithNetworks, 'slug', knex);

41
src/scrapers/famedigital.js
View File

@ -1,7 +1,18 @@
'use strict';
const argv = require('../argv');
const { fetchLatest, fetchApiLatest, fetchUpcoming, fetchApiUpcoming, fetchScene, fetchProfile, fetchApiProfile } = require('./gamma');
const {
fetchLatest,
fetchApiLatest,
fetchUpcoming,
fetchApiUpcoming,
fetchScene,
fetchProfile,
fetchApiProfile,
scrapeAll,
} = require('./gamma');
const { get } = require('../utils/q');
const slugify = require('../utils/slugify');
function extractLowArtActors(release) {
const actors = release.title
@ -47,16 +58,40 @@ function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
}
async function fetchClassicProfile(actorName, siteSlug) {
const actorSlug = slugify(actorName);
const url = `https://${siteSlug}.com/en/pornstars`;
const { qa } = await get(url);
const actorPath = qa('option[value*="/pornstar"]')
.find(el => slugify(el.textContent) === actorSlug)
?.value;
if (actorPath) {
const actorUrl = `https://${siteSlug}.com${actorPath}`;
const { html } = await get(actorUrl);
const releases = scrapeAll(html, null, `https://www.${siteSlug}.com`, false);
return { releases };
}
return null;
}
async function networkFetchProfile(actorName) {
// not all Fame Digital sites offer Gamma actors
const [devils, rocco, peter] = await Promise.all([
const [devils, rocco, peter, silvia] = await Promise.all([
fetchApiProfile(actorName, 'devilsfilm', true),
fetchApiProfile(actorName, 'roccosiffredi'),
argv.withReleases ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl) : [],
argv.withReleases ? fetchClassicProfile(actorName, 'silviasaint') : [],
argv.withReleases ? fetchClassicProfile(actorName, 'silverstonedvd') : [],
]);
if (devils || rocco || peter) {
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || []);
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []);
return {
...peter,

14
src/scrapers/gamma.js
View File

@ -110,7 +110,7 @@ async function scrapeApiReleases(json, site) {
});
}
function scrapeAll(html, site, networkUrl) {
function scrapeAll(html, site, networkUrl, hasTeaser = true) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const scenesElements = $('li[data-itemtype=scene]').toArray();
@ -146,10 +146,12 @@ function scrapeAll(html, site, networkUrl) {
const posterEl = $(element).find('.imgLink img');
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
release.teaser = {
src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4`,
quality: 224,
};
if (hasTeaser) {
release.teaser = {
src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4`,
quality: 224,
};
}
return release;
});
@ -166,7 +168,7 @@ async function scrapeScene(html, url, site) {
const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
[release.entryId] = new URL(url).pathname.split('/').slice(-1);
release.title = data?.name || videoData.playerOptions.sceneInfos.sceneTitle;
release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name;
// date in data object is not the release date of the scene, but the date the entry was added; only use as fallback
const dateString = $('.updatedDate').first().text().trim();

3
src/sites.js
View File

@ -113,6 +113,7 @@ async function fetchSitesFromArgv() {
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.where('sites.scrape', true)
.whereIn('sites.slug', argv.sites || [])
.orWhereIn('networks.slug', argv.networks || [])
.leftJoin('networks', 'sites.network_id', 'networks.id');
@ -133,7 +134,7 @@ async function fetchSitesFromConfig() {
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.leftJoin('networks', 'sites.network_id', 'networks.id')
.where('sites.enabled')
.where('sites.scrape', true)
.where((builder) => {
if (config.include) {
builder