DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Improved site and network pages. Fixed various issues.

Browse Source
This commit is contained in:
ThePendulum
2019-11-12 01:22:20 +01:00
parent 3c76d39301
commit 832e96ced1
118 changed files with 327 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
src/fetch-releases.js
View File

@@ -88,55 +88,65 @@ async function findDuplicateReleases(latestReleases, _siteId) {
.orWhereIn('entry_id', latestReleasesEntryIds);
}
async function storeReleases(releases = []) {
return Promise.map(releases, async (release) => {
const curatedRelease = {
site_id: release.site.id,
studio_id: release.studio ? release.studio.id : null,
shoot_id: release.shootId || null,
entry_id: release.entryId || null,
url: release.url,
title: release.title,
date: release.date,
description: release.description,
// director: release.director,
duration: release.duration,
likes: release.rating && release.rating.likes,
dislikes: release.rating && release.rating.dislikes,
rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: argv.deep,
};
async function storeRelease(release) {
const curatedRelease = {
site_id: release.site.id,
studio_id: release.studio ? release.studio.id : null,
shoot_id: release.shootId || null,
entry_id: release.entryId || null,
url: release.url,
title: release.title,
date: release.date,
description: release.description,
// director: release.director,
duration: release.duration,
likes: release.rating && release.rating.likes,
dislikes: release.rating && release.rating.dislikes,
rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: argv.deep,
};
const releaseEntries = await knex('releases')
.insert(curatedRelease)
.returning('*');
const releaseEntries = await knex('releases')
.insert(curatedRelease)
.returning('*');
if (releaseEntries.length) {
const releaseEntry = releaseEntries[0];
if (releaseEntries.length) {
const releaseEntry = releaseEntries[0];
console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
if (release.poster || (release.photos && release.photos.length)) {
await fs.mkdir(path.join(config.media.path, release.site.network.slug, release.site.slug, releaseEntry.id.toString()), { recursive: true });
}
await Promise.all([
release.actors && release.actors.length > 0
? storeActors(release, releaseEntry) : Promise.resolve(),
release.tags && release.tags.length > 0
? storeTags(release, releaseEntry) : Promise.resolve(),
release.photos && release.photos.length > 0
? storePhotos(release, releaseEntry) : Promise.resolve(),
release.poster
? storePoster(release, releaseEntry) : Promise.resolve(),
release.trailer
? storeTrailer(release, releaseEntry) : Promise.resolve(),
]);
return;
if (release.poster || (release.photos && release.photos.length)) {
await fs.mkdir(path.join(config.media.path, release.site.network.slug, release.site.slug, releaseEntry.id.toString()), { recursive: true });
}
console.error(`Unable to save scene to database, possible collision: "${release.title}" (${release.site.name})`);
await Promise.all([
release.actors && release.actors.length > 0
? storeActors(release, releaseEntry) : Promise.resolve(),
release.tags && release.tags.length > 0
? storeTags(release, releaseEntry) : Promise.resolve(),
release.photos && release.photos.length > 0
? storePhotos(release, releaseEntry) : Promise.resolve(),
release.poster
? storePoster(release, releaseEntry) : Promise.resolve(),
release.trailer
? storeTrailer(release, releaseEntry) : Promise.resolve(),
]);
return;
}
console.error(`Unable to save scene to database, possible collision: "${release.title}" (${release.site.name})`);
}
async function storeReleases(releases = []) {
return Promise.map(releases, async (release) => {
try {
return storeRelease(release);
} catch (error) {
console.error(error);
return null;
}
}, {
concurrency: 2,
});

61
src/media.js
View File

@@ -19,11 +19,20 @@ function getHash(buffer) {
return hash.digest('hex');
}
async function getThumbnail(buffer) {
return sharp(buffer)
.resize({
height: config.media.thumbnailSize,
withoutEnlargement: true,
})
.toBuffer();
}
async function storePoster(release, releaseEntry) {
console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
const res = await bhttp.get(release.poster);
const thumbnail = await sharp(res.body).resize({ height: config.media.thumbnailSize }).toBuffer();
const thumbnail = await getThumbnail(res.body);
if (res.statusCode === 200) {
const { pathname } = new URL(release.poster);
@@ -44,6 +53,7 @@ async function storePoster(release, releaseEntry) {
thumbnail: thumbpath,
mime: mimetype,
hash,
source: release.poster,
domain: 'releases',
target_id: releaseEntry.id,
role: 'poster',
@@ -62,32 +72,37 @@ async function storePhotos(release, releaseEntry) {
const { pathname } = new URL(photoUrl);
const mimetype = mime.getType(pathname);
const res = await bhttp.get(photoUrl);
const thumbnail = await sharp(res.body).resize({ height: config.media.thumbnailSize }).toBuffer();
try {
const res = await bhttp.get(photoUrl);
const thumbnail = await getThumbnail(res.body);
if (res.statusCode === 200) {
const extension = mime.getExtension(mimetype);
if (res.statusCode === 200) {
const extension = mime.getExtension(mimetype);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}_thumb.${extension}`);
const hash = getHash(res.body);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}_thumb.${extension}`);
const hash = getHash(res.body);
await Promise.all([
fs.writeFile(path.join(config.media.path, filepath), res.body),
fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
]);
await Promise.all([
fs.writeFile(path.join(config.media.path, filepath), res.body),
fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
]);
return {
filepath,
thumbpath,
mimetype,
hash,
};
return {
filepath,
thumbpath,
mimetype,
hash,
source: photoUrl,
};
}
throw new Error(`Response ${res.statusCode} not OK`);
} catch (error) {
console.warn(`Failed to store photo ${index + 1} for "${release.title}" (${photoUrl}, ${release.url}, ${release.site.name}, ${releaseEntry.id}): ${error}`);
return null;
}
console.warn(`Failed to store photo ${index + 1} for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`);
return null;
}, {
concurrency: 2,
});
@@ -99,6 +114,7 @@ async function storePhotos(release, releaseEntry) {
thumbnail: file.thumbpath,
mime: file.mimetype,
hash: file.hash,
source: file.source,
index,
domain: 'releases',
target_id: releaseEntry.id,
@@ -120,6 +136,7 @@ async function storeTrailer(release, releaseEntry) {
knex('media').insert({
path: filepath,
mime: mimetype,
source: release.trailer.src,
domain: 'releases',
target_id: releaseEntry.id,
role: 'trailer',

3
src/releases.js
View File

@@ -11,7 +11,8 @@ async function curateRelease(release) {
knex('tags_associated')
.select('tags.name', 'tags.slug')
.where({ release_id: release.id })
.leftJoin('tags', 'tags.id', 'tags_associated.tag_id'),
.leftJoin('tags', 'tags.id', 'tags_associated.tag_id')
.orderBy('tags.priority', 'desc'),
knex('media')
.where({ target_id: release.id })
.orderBy('role'),

8
src/scrapers/brazzers.js
View File

@@ -5,7 +5,7 @@ const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('../knex');
const fetchSites = require('../sites');
const { matchTags } = require('../tags');
function scrape(html, site, upcoming) {
@@ -75,6 +75,7 @@ async function scrapeScene(html, url, site) {
const siteElement = $('.niche-site-logo');
const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
const siteName = siteElement.attr('title');
const siteSlug = siteName.replace(/\s+/g, '');
const rawTags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
@@ -84,10 +85,7 @@ async function scrapeScene(html, url, site) {
const [tags, channelSite] = await Promise.all([
matchTags(rawTags),
knex('sites')
.where({ url: siteUrl })
.orWhere({ name: siteName })
.first(),
site.isFallback ? fetchSites(null, siteSlug, siteName, siteUrl) : site,
]);
return {

2
src/scrapers/dogfart.js
View File

@@ -54,7 +54,7 @@ function scrapeLatest(html, site) {
return sceneElements.reduce((acc, element) => {
const siteUrl = element.querySelector('.help-block').textContent;
if (siteUrl.toLowerCase() !== new URL(site.url).host) {
if (`www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
// different dogfart site
return acc;
}

4
src/scrapers/xempire.js
View File

@@ -22,8 +22,8 @@ function scrapePhotos(html) {
const lockedThumbnails = $('.preview .imgLink.lockedPicture img')
.map((photoIndex, photoElement) => $(photoElement)
.attr('src')
.replace('_tb.jpg', '.jpg'))
.attr('src'))
// .replace('_tb.jpg', '.jpg')) does not always work
.toArray();
return unlockedPhotos.concat(lockedThumbnails);

6
src/sites.js
View File

@@ -1,6 +1,7 @@
'use strict';
const knex = require('./knex');
const whereOr = require('./utils/where-or');
async function curateSite(site) {
return {
@@ -22,10 +23,9 @@ function curateSites(sites) {
return Promise.all(sites.map(async site => curateSite(site)));
}
async function fetchSites(siteId, siteSlug) {
async function fetchSites(query) {
const sites = await knex('sites')
.where({ 'sites.id': siteId })
.orWhere({ 'sites.slug': siteSlug })
.where(builder => whereOr(query, builder))
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url',

11
src/utils/where-or.js Normal file
View File

@@ -0,0 +1,11 @@
'use strict';
function whereOr(query, builder) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined) {
builder.orWhere(`sites.${key}`, value);
}
});
}
module.exports = whereOr;

9
src/web/sites.js
View File

@@ -3,10 +3,13 @@
const { fetchSites, fetchSitesFromReleases } = require('../sites');
async function fetchSitesApi(req, res) {
const siteId = typeof req.params.siteId === 'number' ? req.params.siteId : null;
const siteSlug = typeof req.params.siteId === 'string' ? req.params.siteId : null;
const siteId = typeof req.params.siteId === 'number' ? req.params.siteId : undefined;
const siteSlug = typeof req.params.siteId === 'string' ? req.params.siteId : undefined;
const sites = await fetchSites(siteId, siteSlug);
const sites = await fetchSites({
id: siteId,
slug: siteSlug,
});
res.send(sites);
}