Removed upsert from store releases, observing effects; fixes ? inserted as . Removed query from stored Dogfart URLs.
This commit is contained in:
parent
877fcb01ad
commit
c466d791f5
|
|
@ -88,8 +88,8 @@ exports.seed = knex => Promise.resolve()
|
|||
alias_for: null,
|
||||
},
|
||||
{
|
||||
name: 'anal sex',
|
||||
slug: 'anal-sex',
|
||||
name: 'anal',
|
||||
slug: 'anal',
|
||||
alias_for: null,
|
||||
},
|
||||
{
|
||||
|
|
@ -192,8 +192,8 @@ exports.seed = knex => Promise.resolve()
|
|||
group_id: groupsMap['hair'],
|
||||
},
|
||||
{
|
||||
name: 'blonde hair',
|
||||
slug: 'blonde-hair',
|
||||
name: 'blonde',
|
||||
slug: 'blonde',
|
||||
alias_for: null,
|
||||
group_id: groupsMap['hair'],
|
||||
},
|
||||
|
|
@ -214,8 +214,8 @@ exports.seed = knex => Promise.resolve()
|
|||
alias_for: null,
|
||||
},
|
||||
{
|
||||
name: 'brown hair',
|
||||
slug: 'brown-hair',
|
||||
name: 'brunette',
|
||||
slug: 'brunette',
|
||||
alias_for: null,
|
||||
group_id: groupsMap['hair'],
|
||||
},
|
||||
|
|
@ -585,8 +585,8 @@ exports.seed = knex => Promise.resolve()
|
|||
alias_for: null,
|
||||
},
|
||||
{
|
||||
name: 'red hair',
|
||||
slug: 'red-hair',
|
||||
name: 'redhead',
|
||||
slug: 'redhead',
|
||||
alias_for: null,
|
||||
group_id: groupsMap['hair'],
|
||||
},
|
||||
|
|
@ -814,8 +814,8 @@ exports.seed = knex => Promise.resolve()
|
|||
alias_for: tagsMap['gangbang'],
|
||||
},
|
||||
{
|
||||
name: 'anal',
|
||||
alias_for: tagsMap['anal-sex'],
|
||||
name: 'anal sex',
|
||||
alias_for: tagsMap['anal'],
|
||||
},
|
||||
{
|
||||
name: 'anal gape',
|
||||
|
|
@ -831,7 +831,7 @@ exports.seed = knex => Promise.resolve()
|
|||
},
|
||||
{
|
||||
name: 'ass fucking',
|
||||
alias_for: tagsMap['anal-sex'],
|
||||
alias_for: tagsMap['anal'],
|
||||
},
|
||||
{
|
||||
name: 'atm',
|
||||
|
|
@ -898,12 +898,12 @@ exports.seed = knex => Promise.resolve()
|
|||
alias_for: tagsMap['ebony'],
|
||||
},
|
||||
{
|
||||
name: 'blonde',
|
||||
alias_for: tagsMap['blonde-hair'],
|
||||
name: 'blonde hair',
|
||||
alias_for: tagsMap['blonde'],
|
||||
},
|
||||
{
|
||||
name: 'blondes',
|
||||
alias_for: tagsMap['blonde-hair'],
|
||||
alias_for: tagsMap['blonde'],
|
||||
},
|
||||
{
|
||||
name: 'blow job',
|
||||
|
|
@ -934,12 +934,12 @@ exports.seed = knex => Promise.resolve()
|
|||
alias_for: tagsMap['enhanced-boobs'],
|
||||
},
|
||||
{
|
||||
name: 'brunette',
|
||||
alias_for: tagsMap['brown-hair'],
|
||||
name: 'brown hair',
|
||||
alias_for: tagsMap['brunette'],
|
||||
},
|
||||
{
|
||||
name: 'brunettes',
|
||||
alias_for: tagsMap['brown-hair'],
|
||||
alias_for: tagsMap['brunette'],
|
||||
},
|
||||
{
|
||||
name: 'buttplug',
|
||||
|
|
@ -1246,12 +1246,12 @@ exports.seed = knex => Promise.resolve()
|
|||
alias_for: tagsMap['black-hair'],
|
||||
},
|
||||
{
|
||||
name: 'redhead',
|
||||
alias_for: tagsMap['red-hair'],
|
||||
name: 'red hair',
|
||||
alias_for: tagsMap['redhead'],
|
||||
},
|
||||
{
|
||||
name: 'red head',
|
||||
alias_for: tagsMap['red-hair'],
|
||||
alias_for: tagsMap['redhead'],
|
||||
},
|
||||
{
|
||||
name: 'rimming',
|
||||
|
|
|
|||
|
|
@ -206,34 +206,41 @@ async function storeReleases(releases = []) {
|
|||
description: release.description,
|
||||
// director: release.director,
|
||||
duration: release.duration,
|
||||
// photos: release.photos ? release.photos.length : 0,
|
||||
likes: release.rating && release.rating.likes,
|
||||
dislikes: release.rating && release.rating.dislikes,
|
||||
rating: release.rating && release.rating.stars,
|
||||
rating: release.rating && Math.floor(release.rating.stars),
|
||||
deep: argv.deep,
|
||||
};
|
||||
|
||||
/*
|
||||
const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`;
|
||||
const releaseEntry = await knex.raw(releaseQuery);
|
||||
*/
|
||||
|
||||
if (releaseEntry.rows.length > 0) {
|
||||
console.log(`Stored (${release.site.name}, ${releaseEntry.rows[0].id}) "${release.title}"`);
|
||||
const releaseEntries = await knex('releases')
|
||||
.insert(curatedRelease)
|
||||
.returning('*');
|
||||
|
||||
if (releaseEntries.length) {
|
||||
const releaseEntry = releaseEntries[0];
|
||||
|
||||
console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
|
||||
|
||||
if (release.poster || (release.photos && release.photos.length)) {
|
||||
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.rows[0].id.toString()), { recursive: true });
|
||||
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
release.actors && release.actors.length > 0
|
||||
? storeActors(release, releaseEntry.rows[0]) : Promise.resolve(),
|
||||
? storeActors(release, releaseEntry) : Promise.resolve(),
|
||||
release.tags && release.tags.length > 0
|
||||
? storeTags(release, releaseEntry.rows[0]) : Promise.resolve(),
|
||||
? storeTags(release, releaseEntry) : Promise.resolve(),
|
||||
release.photos && release.photos.length > 0
|
||||
? storePhotos(release, releaseEntry.rows[0]) : Promise.resolve(),
|
||||
? storePhotos(release, releaseEntry) : Promise.resolve(),
|
||||
release.poster
|
||||
? storePoster(release, releaseEntry.rows[0]) : Promise.resolve(),
|
||||
? storePoster(release, releaseEntry) : Promise.resolve(),
|
||||
release.trailer
|
||||
? storeTrailer(release, releaseEntry.rows[0]) : Promise.resolve(),
|
||||
? storeTrailer(release, releaseEntry) : Promise.resolve(),
|
||||
]);
|
||||
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ const moment = require('moment');
|
|||
const knex = require('knex');
|
||||
|
||||
const { matchTags } = require('../tags');
|
||||
const pluckPhotos = require('../utils/pluck-photos');
|
||||
|
||||
async function getPhoto(url) {
|
||||
const res = await bhttp.get(url);
|
||||
|
|
@ -29,7 +30,7 @@ async function getPhotos(albumUrl, site, siteUrl) {
|
|||
|
||||
// dogfart has massive albums, pick 25 or specified number of photos: first, last and evenly inbetween
|
||||
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
|
||||
const photoIndexes = [1].concat(Array.from({ length: photoLimit - 1 }, (value, index) => Math.floor((index + 1) * (lastPhotoIndex / (photoLimit - 1)))));
|
||||
const photoIndexes = pluckPhotos(lastPhotoIndex, photoLimit);
|
||||
|
||||
if (photoLimit > 25) {
|
||||
console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`);
|
||||
|
|
@ -117,7 +118,7 @@ async function scrapeScene(html, url, site) {
|
|||
const { origin, pathname } = new URL(url);
|
||||
const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
|
||||
|
||||
const stars = Number(document.querySelector('span[itemprop="average"]').textContent) / 2;
|
||||
const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]').textContent) / 2);
|
||||
const rawTags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
|
||||
|
||||
const [channelSite, tags] = await Promise.all([
|
||||
|
|
@ -131,7 +132,7 @@ async function scrapeScene(html, url, site) {
|
|||
]);
|
||||
|
||||
return {
|
||||
url,
|
||||
url: `${origin}${pathname}`,
|
||||
title,
|
||||
description,
|
||||
actors,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
'use strict';
|
||||
|
||||
// pick {photoLimit} photos evenly distributed photos from a set with {photoTotal} photos, return array of indexes starting at 1
|
||||
function pluckPhotos(photoTotal, photoLimit) {
|
||||
return [1].concat(Array.from({ length: photoLimit - 1 }, (value, index) => Math.floor((index + 1) * (photoTotal / (photoLimit - 1)))));
|
||||
}
|
||||
|
||||
module.exports = pluckPhotos;
|
||||
Loading…
Reference in New Issue