DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Redesigned release page. Added 'single site' marker, linking directly to network page. Fixed Dogfart scraper duration.

Browse Source
This commit is contained in:
ThePendulum
2019-11-14 05:13:38 +01:00
parent 4fa13bb163
commit bf223adc55
38 changed files with 624 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/networks.js
View File

@@ -2,11 +2,11 @@
const knex = require('./knex');
const whereOr = require('./utils/where-or');
const { fetchSites } = require('./sites');
async function curateNetwork(network) {
const [sites, studios] = await Promise.all([
knex('sites')
.where({ network_id: network.id }),
fetchSites({ network_id: network.id }),
knex('studios')
.where({ network_id: network.id }),
]);
@@ -17,20 +17,7 @@ async function curateNetwork(network) {
url: network.url,
description: network.description,
slug: network.slug,
sites: sites.map(site => ({
id: site.id,
name: site.name,
url: site.url,
description: site.description,
slug: site.slug,
network: {
id: network.id,
name: network.name,
url: network.url,
description: network.description,
slug: network.slug,
},
})),
sites,
studios: studios.map(studio => ({
id: studio.id,
name: studio.name,

14
src/releases.js
View File

@@ -42,6 +42,9 @@ async function curateRelease(release) {
site: {
id: release.site_id,
name: release.site_name,
independent: release.site_parameters
? (JSON.parse(release.site_parameters).independent || false)
: false,
slug: release.site_slug,
url: release.site_url,
},
@@ -56,6 +59,7 @@ async function curateRelease(release) {
network: {
id: release.network_id,
name: release.network_name,
description: release.network_description,
slug: release.network_slug,
url: release.network_url,
},
@@ -75,9 +79,9 @@ async function fetchReleases(releaseId, filter = []) {
.leftJoin('networks', 'sites.network_id', 'networks.id')
.select(
'releases.*',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description',
)
.whereNotExists((builder) => {
// apply filters
@@ -101,7 +105,7 @@ async function fetchSiteReleases(siteId, siteSlug) {
.orWhere({ 'sites.slug': siteSlug })
.select(
'releases.*',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url',
)
@@ -120,7 +124,7 @@ async function fetchNetworkReleases(networkId, networkSlug) {
.orWhere({ 'networks.slug': networkSlug })
.select(
'releases.*',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url',
)
@@ -140,7 +144,7 @@ async function fetchActorReleases(actorId, actorSlug) {
.select(
'releases.*',
'actors.name as actor_name',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url',
)

4
src/scrapers/dogfart.js
View File

@@ -104,10 +104,10 @@ async function scrapeScene(html, url, site) {
const siteSlug = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
const duration = moment
.duration(document
.duration(`00:${document
.querySelectorAll('.extra-info p')[1]
.textContent
.match(/\d+:\d+$/)[0])
.match(/\d+:\d+$/)[0]}`)
.asSeconds();
const trailerElement = document.querySelector('.html5-video');

6
src/sites.js
View File

@@ -4,15 +4,19 @@ const knex = require('./knex');
const whereOr = require('./utils/where-or');
async function curateSite(site) {
const parameters = JSON.parse(site.parameters);
return {
id: site.id,
name: site.name,
url: site.url,
description: site.description,
slug: site.slug,
independent: !!parameters && parameters.independent,
network: {
id: site.network_id,
name: site.network_name,
description: site.network_description,
slug: site.network_slug,
url: site.network_url,
},
@@ -28,7 +32,7 @@ async function fetchSites(queryObject) {
.where(builder => whereOr(queryObject, 'sites', builder))
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as networks_description',
)
.leftJoin('networks', 'sites.network_id', 'networks.id')
.limit(100);