DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added Black for Wife to JayRock. Switched parameters field to JSON type.

Browse Source
This commit is contained in:
ThePendulum
2020-02-09 19:41:39 +01:00
parent f7f4da24ec
commit 0f513266a0
18 changed files with 173 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/actors.js
View File

@@ -15,10 +15,15 @@ const slugify = require('./utils/slugify');
const { createMediaDirectory, storePhotos } = require('./media');
async function curateActor(actor) {
const [aliases, photos, social] = await Promise.all([
const [aliases, avatar, photos, social] = await Promise.all([
knex('actors').where({ alias_for: actor.id }),
knex('media')
.where({ domain: 'actors', target_id: actor.id })
knex('actors_avatars')
.where('actor_id', actor.id)
.join('media', 'media.id', 'actors_avatars.media_id')
.first(),
knex('actors_photos')
.where('actor_id', actor.id)
.join('media', 'media.id', 'actors_photos.media_id')
.orderBy('index'),
knex('actors_social')
.where('actor_id', actor.id)
@@ -43,8 +48,8 @@ async function curateActor(actor) {
naturalBoobs: actor.natural_boobs,
aliases: aliases.map(({ name }) => name),
slug: actor.slug,
avatar: photos.find(photo => photo.role === 'avatar'),
photos: photos.filter(photo => photo.role === 'photo'),
avatar,
photos,
hasTattoos: actor.has_tattoos,
hasPiercings: actor.has_piercings,
tattoos: actor.tattoos,

2
src/networks.js
View File

@@ -18,7 +18,7 @@ async function curateNetwork(network, includeParameters = false) {
description: network.description,
slug: network.slug,
sites,
parameters: includeParameters ? JSON.parse(network.parameters) : null,
parameters: includeParameters ? network.parameters : null,
studios: studios.map(studio => ({
id: studio.id,
name: studio.name,

4
src/releases.js
View File

@@ -125,9 +125,7 @@ async function curateRelease(release) {
site: {
id: release.site_id,
name: release.site_name,
independent: release.site_parameters
? (JSON.parse(release.site_parameters).independent || false)
: false,
independent: !!release.site_parameters?.independent,
slug: release.site_slug,
url: release.site_url,
},

1
src/scrape-releases.js
View File

@@ -65,7 +65,6 @@ async function scrapeRelease(source, basicRelease = null, type = 'scene') {
: await scraper.fetchMovie(url, site, release);
return {
url,
...release,
...scrapedRelease,
...(scrapedRelease && release?.tags && {

10
src/scrape-sites.js
View File

@@ -68,11 +68,13 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a
return scrapeUniqueReleases(scraper, site, afterDate, accReleases.concat(uniqueReleases), page + 1);
}
const uniqueReleasesWithSite = uniqueReleases.map(release => ({ ...release, site }));
if (oldestReleaseOnPage) {
return accReleases.concat(uniqueReleases);
return accReleases.concat(uniqueReleasesWithSite);
}
return accReleases.concat(uniqueReleases).slice(0, argv.nullDateLimit);
return accReleases.concat(uniqueReleasesWithSite).slice(0, argv.nullDateLimit);
}
async function scrapeUpcomingReleases(scraper, site) {
@@ -87,9 +89,9 @@ async function scrapeUpcomingReleases(scraper, site) {
async function deepFetchReleases(baseReleases) {
return Promise.map(baseReleases, async (release) => {
if (release.url) {
if (release.url || (release.path && release.site)) {
try {
const fullRelease = await scrapeRelease(release.url, release, 'scene');
const fullRelease = await scrapeRelease(release.url || release.path, release, 'scene');
if (fullRelease) {
return {

26
src/scrapers/gamma.js
View File

@@ -100,13 +100,14 @@ async function scrapeApiReleases(json, site) {
dislikes: scene.ratings_down,
};
release.url = site.parameters?.scene
? `${site.parameters.scene}/${scene.url_title}/${release.entryId}`
: `${site.url}/en/video/${scene.url_title}/${release.entryId}`;
release.path = `/${scene.url_title}/${release.entryId}`;
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.actors = scene.actors.map(({ name }) => name);
release.director = scene.directors[0].name;
release.director = scene.directors[0]?.name || null;
release.tags = scene.master_categories
.concat(scene.categories?.map(category => category.name))
@@ -171,9 +172,9 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
});
}
async function scrapeScene(html, url, site) {
async function scrapeScene(html, url, site, scrapedRelease) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const release = { $, url };
const release = { $ };
const json = $('script[type="application/ld+json"]').html();
const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
@@ -181,7 +182,7 @@ async function scrapeScene(html, url, site) {
const [data, data2] = json ? JSON.parse(json) : [];
const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
[release.entryId] = new URL(url).pathname.split('/').slice(-1);
[release.entryId] = (scrapedRelease?.path || new URL(url).pathname).split('/').slice(-1);
release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name;
// date in data object is not the release date of the scene, but the date the entry was added; only use as fallback
@@ -194,7 +195,8 @@ async function scrapeScene(html, url, site) {
if (data) {
release.description = data.description;
release.director = data.director?.[0].name || data2?.director?.[0].name;
if (data.director?.[0]?.name) release.director = data.director[0].name;
else if (data2?.director?.[0]?.name) release.director = data2.director[0].name;
const actors = data?.actor || data2?.actor || [];
release.actors = actors.map(actor => actor.name);
@@ -437,8 +439,8 @@ async function fetchUpcoming(site) {
return scrapeAll(res.body.toString(), site);
}
function getDeepUrl(url, site) {
const { pathname } = new URL(url);
function getDeepUrl(url, site, release) {
const pathname = release.path || new URL(url).pathname;
if (site.parameters?.deep === 'network') {
return `${site.network.url}${pathname}`;
@@ -456,11 +458,11 @@ async function fetchScene(url, site, release) {
return release;
}
const deepUrl = getDeepUrl(url, site);
const deepUrl = getDeepUrl(url, site, release);
const res = await bhttp.get(deepUrl);
if (res.statusCode === 200) {
return scrapeScene(res.body.toString(), url, site, deepUrl);
return scrapeScene(res.body.toString(), url, site, release);
}
return null;

13
src/scrapers/jayrock.js
View File

@@ -6,6 +6,7 @@ const bhttp = require('bhttp');
const logger = require('../logger')(__filename);
const slugify = require('../utils/slugify');
const { fetchApiLatest, fetchScene } = require('./gamma');
async function fetchToken(site) {
const res = await bhttp.get(site.url);
@@ -81,6 +82,10 @@ function scrapeLatest(scenes, site, tokens) {
}
async function fetchLatest(site, page = 1) {
if (site.parameters?.useGamma) {
return fetchApiLatest(site, page);
}
const { time, token } = await fetchToken(site);
// transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory
@@ -94,7 +99,11 @@ async function fetchLatest(site, page = 1) {
return null;
}
async function fetchScene(url, site, release) {
async function fetchNetworkScene(url, site, release) {
if (site.parameters?.useGamma) {
return fetchScene(url, site, release);
}
const { time, token } = release?.meta.tokens || await fetchToken(site); // use attached tokens when deep fetching
const { pathname } = new URL(url);
const entryId = pathname.split('/')[2];
@@ -111,5 +120,5 @@ async function fetchScene(url, site, release) {
module.exports = {
fetchLatest,
fetchScene,
fetchScene: fetchNetworkScene,
};

8
src/sites.js
View File

@@ -13,8 +13,6 @@ async function curateSite(site, includeParameters = false) {
.where('site_id', site.id)
.join('tags', 'tags.id', 'sites_tags.tag_id');
const parameters = JSON.parse(site.parameters);
return {
id: site.id,
name: site.name,
@@ -22,15 +20,15 @@ async function curateSite(site, includeParameters = false) {
description: site.description,
slug: site.slug,
tags,
independent: !!parameters && parameters.independent,
parameters: includeParameters ? parameters : null,
independent: !!site.parameters && site.parameters.independent,
parameters: includeParameters ? site.parameters : null,
network: {
id: site.network_id,
name: site.network_name,
description: site.network_description,
slug: site.network_slug,
url: site.network_url,
parameters: includeParameters ? JSON.parse(site.network_parameters) : null,
parameters: includeParameters ? site.network_parameters : null,
},
};
}

2
src/utils/list.js
View File

@@ -23,7 +23,7 @@ async function listSites() {
sites.forEach((site) => {
const rkSpecial = network.id === 'realitykings'
&& (new URL(site.url).hostname === 'www.realitykings.com'
|| (site.parameters && JSON.parse(site.parameters).altLayout))
|| (site.parameters?.altLayout))
? '\\*' : ''; // Reality Kings alt layout sites do not support scene fetch by URL
console.log(` * ${site.name}${rkSpecial}`);

8
src/web/plugins/sites.js
View File

@@ -11,13 +11,7 @@ const schemaExtender = makeExtendSchemaPlugin(_build => ({
resolvers: {
Site: {
independent(parent, _args, _context, _info) {
if (!parent.parameters) {
return false;
}
const parameters = JSON.parse(parent.parameters);
return !!parameters.independent;
return !!parent.parameters?.independent;
},
},
},