DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Removed upsert from store releases, observing effects; fixes ? inserted as . Removed query from stored Dogfart URLs.

This commit is contained in:
ThePendulum 2019-11-05 00:54:47 +01:00
parent 877fcb01ad
commit c466d791f5
4 changed files with 49 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
seeds/02_tags.js
View File

@ -88,8 +88,8 @@ exports.seed = knex => Promise.resolve()
alias_for: null, alias_for: null,
}, },
{ {
name: 'anal sex', name: 'anal',
slug: 'anal-sex', slug: 'anal',
alias_for: null, alias_for: null,
}, },
{ {
@ -192,8 +192,8 @@ exports.seed = knex => Promise.resolve()
group_id: groupsMap['hair'], group_id: groupsMap['hair'],
}, },
{ {
name: 'blonde hair', name: 'blonde',
slug: 'blonde-hair', slug: 'blonde',
alias_for: null, alias_for: null,
group_id: groupsMap['hair'], group_id: groupsMap['hair'],
}, },
@ -214,8 +214,8 @@ exports.seed = knex => Promise.resolve()
alias_for: null, alias_for: null,
}, },
{ {
name: 'brown hair', name: 'brunette',
slug: 'brown-hair', slug: 'brunette',
alias_for: null, alias_for: null,
group_id: groupsMap['hair'], group_id: groupsMap['hair'],
}, },
@ -585,8 +585,8 @@ exports.seed = knex => Promise.resolve()
alias_for: null, alias_for: null,
}, },
{ {
name: 'red hair', name: 'redhead',
slug: 'red-hair', slug: 'redhead',
alias_for: null, alias_for: null,
group_id: groupsMap['hair'], group_id: groupsMap['hair'],
}, },
@ -814,8 +814,8 @@ exports.seed = knex => Promise.resolve()
alias_for: tagsMap['gangbang'], alias_for: tagsMap['gangbang'],
}, },
{ {
name: 'anal', name: 'anal sex',
alias_for: tagsMap['anal-sex'], alias_for: tagsMap['anal'],
}, },
{ {
name: 'anal gape', name: 'anal gape',
@ -831,7 +831,7 @@ exports.seed = knex => Promise.resolve()
}, },
{ {
name: 'ass fucking', name: 'ass fucking',
alias_for: tagsMap['anal-sex'], alias_for: tagsMap['anal'],
}, },
{ {
name: 'atm', name: 'atm',
@ -898,12 +898,12 @@ exports.seed = knex => Promise.resolve()
alias_for: tagsMap['ebony'], alias_for: tagsMap['ebony'],
}, },
{ {
name: 'blonde', name: 'blonde hair',
alias_for: tagsMap['blonde-hair'], alias_for: tagsMap['blonde'],
}, },
{ {
name: 'blondes', name: 'blondes',
alias_for: tagsMap['blonde-hair'], alias_for: tagsMap['blonde'],
}, },
{ {
name: 'blow job', name: 'blow job',
@ -934,12 +934,12 @@ exports.seed = knex => Promise.resolve()
alias_for: tagsMap['enhanced-boobs'], alias_for: tagsMap['enhanced-boobs'],
}, },
{ {
name: 'brunette', name: 'brown hair',
alias_for: tagsMap['brown-hair'], alias_for: tagsMap['brunette'],
}, },
{ {
name: 'brunettes', name: 'brunettes',
alias_for: tagsMap['brown-hair'], alias_for: tagsMap['brunette'],
}, },
{ {
name: 'buttplug', name: 'buttplug',
@ -1246,12 +1246,12 @@ exports.seed = knex => Promise.resolve()
alias_for: tagsMap['black-hair'], alias_for: tagsMap['black-hair'],
}, },
{ {
name: 'redhead', name: 'red hair',
alias_for: tagsMap['red-hair'], alias_for: tagsMap['redhead'],
}, },
{ {
name: 'red head', name: 'red head',
alias_for: tagsMap['red-hair'], alias_for: tagsMap['redhead'],
}, },
{ {
name: 'rimming', name: 'rimming',

27
src/fetch-releases.js
View File

@ -206,34 +206,41 @@ async function storeReleases(releases = []) {
description: release.description, description: release.description,
// director: release.director, // director: release.director,
duration: release.duration, duration: release.duration,
// photos: release.photos ? release.photos.length : 0,
likes: release.rating && release.rating.likes, likes: release.rating && release.rating.likes,
dislikes: release.rating && release.rating.dislikes, dislikes: release.rating && release.rating.dislikes,
rating: release.rating && release.rating.stars, rating: release.rating && Math.floor(release.rating.stars),
deep: argv.deep, deep: argv.deep,
}; };
/*
const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`; const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`;
const releaseEntry = await knex.raw(releaseQuery); const releaseEntry = await knex.raw(releaseQuery);
*/
if (releaseEntry.rows.length > 0) { const releaseEntries = await knex('releases')
console.log(`Stored (${release.site.name}, ${releaseEntry.rows[0].id}) "${release.title}"`); .insert(curatedRelease)
.returning('*');
if (releaseEntries.length) {
const releaseEntry = releaseEntries[0];
console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
if (release.poster || (release.photos && release.photos.length)) { if (release.poster || (release.photos && release.photos.length)) {
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.rows[0].id.toString()), { recursive: true }); await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
} }
await Promise.all([ await Promise.all([
release.actors && release.actors.length > 0 release.actors && release.actors.length > 0
? storeActors(release, releaseEntry.rows[0]) : Promise.resolve(), ? storeActors(release, releaseEntry) : Promise.resolve(),
release.tags && release.tags.length > 0 release.tags && release.tags.length > 0
? storeTags(release, releaseEntry.rows[0]) : Promise.resolve(), ? storeTags(release, releaseEntry) : Promise.resolve(),
release.photos && release.photos.length > 0 release.photos && release.photos.length > 0
? storePhotos(release, releaseEntry.rows[0]) : Promise.resolve(), ? storePhotos(release, releaseEntry) : Promise.resolve(),
release.poster release.poster
? storePoster(release, releaseEntry.rows[0]) : Promise.resolve(), ? storePoster(release, releaseEntry) : Promise.resolve(),
release.trailer release.trailer
? storeTrailer(release, releaseEntry.rows[0]) : Promise.resolve(), ? storeTrailer(release, releaseEntry) : Promise.resolve(),
]); ]);
return; return;

7
src/scrapers/dogfart.js
View File

@ -8,6 +8,7 @@ const moment = require('moment');
const knex = require('knex'); const knex = require('knex');
const { matchTags } = require('../tags'); const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');
async function getPhoto(url) { async function getPhoto(url) {
const res = await bhttp.get(url); const res = await bhttp.get(url);
@ -29,7 +30,7 @@ async function getPhotos(albumUrl, site, siteUrl) {
// dogfart has massive albums, pick 25 or specified number of photos: first, last and evenly inbetween // dogfart has massive albums, pick 25 or specified number of photos: first, last and evenly inbetween
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25; const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
const photoIndexes = [1].concat(Array.from({ length: photoLimit - 1 }, (value, index) => Math.floor((index + 1) * (lastPhotoIndex / (photoLimit - 1))))); const photoIndexes = pluckPhotos(lastPhotoIndex, photoLimit);
if (photoLimit > 25) { if (photoLimit > 25) {
console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`); console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`);
@ -117,7 +118,7 @@ async function scrapeScene(html, url, site) {
const { origin, pathname } = new URL(url); const { origin, pathname } = new URL(url);
const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url); const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
const stars = Number(document.querySelector('span[itemprop="average"]').textContent) / 2; const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]').textContent) / 2);
const rawTags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent); const rawTags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
const [channelSite, tags] = await Promise.all([ const [channelSite, tags] = await Promise.all([
@ -131,7 +132,7 @@ async function scrapeScene(html, url, site) {
]); ]);
return { return {
url, url: `${origin}${pathname}`,
title, title,
description, description,
actors, actors,

8
src/utils/pluck-photos.js Normal file
View File

@ -0,0 +1,8 @@
'use strict';
// pick {photoLimit} photos evenly distributed photos from a set with {photoTotal} photos, return array of indexes starting at 1
function pluckPhotos(photoTotal, photoLimit) {
return [1].concat(Array.from({ length: photoLimit - 1 }, (value, index) => Math.floor((index + 1) * (photoTotal / (photoLimit - 1)))));
}
module.exports = pluckPhotos;