Removed upsert from store releases, observing effects; fixes ? inserted as . Removed query from stored Dogfart URLs.

seeds/02_tags.js

@@ -88,8 +88,8 @@ exports.seed = knex => Promise.resolve()
                 alias_for: null,
             },
             {
-                name: 'anal sex',
-                slug: 'anal-sex',
+                name: 'anal',
+                slug: 'anal',
                 alias_for: null,
             },
             {
@@ -192,8 +192,8 @@ exports.seed = knex => Promise.resolve()
                 group_id: groupsMap['hair'],
             },
             {
-                name: 'blonde hair',
-                slug: 'blonde-hair',
+                name: 'blonde',
+                slug: 'blonde',
                 alias_for: null,
                 group_id: groupsMap['hair'],
             },
@@ -214,8 +214,8 @@ exports.seed = knex => Promise.resolve()
                 alias_for: null,
             },
             {
-                name: 'brown hair',
-                slug: 'brown-hair',
+                name: 'brunette',
+                slug: 'brunette',
                 alias_for: null,
                 group_id: groupsMap['hair'],
             },
@@ -585,8 +585,8 @@ exports.seed = knex => Promise.resolve()
                 alias_for: null,
             },
             {
-                name: 'red hair',
-                slug: 'red-hair',
+                name: 'redhead',
+                slug: 'redhead',
                 alias_for: null,
                 group_id: groupsMap['hair'],
             },
@@ -814,8 +814,8 @@ exports.seed = knex => Promise.resolve()
                 alias_for: tagsMap['gangbang'],
             },
             {
-                name: 'anal',
-                alias_for: tagsMap['anal-sex'],
+                name: 'anal sex',
+                alias_for: tagsMap['anal'],
             },
             {
                 name: 'anal gape',
@@ -831,7 +831,7 @@ exports.seed = knex => Promise.resolve()
             },
             {
                 name: 'ass fucking',
-                alias_for: tagsMap['anal-sex'],
+                alias_for: tagsMap['anal'],
             },
             {
                 name: 'atm',
@@ -898,12 +898,12 @@ exports.seed = knex => Promise.resolve()
                 alias_for: tagsMap['ebony'],
             },
             {
-                name: 'blonde',
-                alias_for: tagsMap['blonde-hair'],
+                name: 'blonde hair',
+                alias_for: tagsMap['blonde'],
             },
             {
                 name: 'blondes',
-                alias_for: tagsMap['blonde-hair'],
+                alias_for: tagsMap['blonde'],
             },
             {
                 name: 'blow job',
@@ -934,12 +934,12 @@ exports.seed = knex => Promise.resolve()
                 alias_for: tagsMap['enhanced-boobs'],
             },
             {
-                name: 'brunette',
-                alias_for: tagsMap['brown-hair'],
+                name: 'brown hair',
+                alias_for: tagsMap['brunette'],
             },
             {
                 name: 'brunettes',
-                alias_for: tagsMap['brown-hair'],
+                alias_for: tagsMap['brunette'],
             },
             {
                 name: 'buttplug',
@@ -1246,12 +1246,12 @@ exports.seed = knex => Promise.resolve()
                 alias_for: tagsMap['black-hair'],
             },
             {
-                name: 'redhead',
-                alias_for: tagsMap['red-hair'],
+                name: 'red hair',
+                alias_for: tagsMap['redhead'],
             },
             {
                 name: 'red head',
-                alias_for: tagsMap['red-hair'],
+                alias_for: tagsMap['redhead'],
             },
             {
                 name: 'rimming',

src/fetch-releases.js

@@ -206,34 +206,41 @@ async function storeReleases(releases = []) {
             description: release.description,
             // director: release.director,
             duration: release.duration,
-            // photos: release.photos ? release.photos.length : 0,
             likes: release.rating && release.rating.likes,
             dislikes: release.rating && release.rating.dislikes,
-            rating: release.rating && release.rating.stars,
+            rating: release.rating && Math.floor(release.rating.stars),
             deep: argv.deep,
         };
 
+        /*
         const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`;
         const releaseEntry = await knex.raw(releaseQuery);
+        */
 
-        if (releaseEntry.rows.length > 0) {
-            console.log(`Stored (${release.site.name}, ${releaseEntry.rows[0].id}) "${release.title}"`);
+        const releaseEntries = await knex('releases')
+            .insert(curatedRelease)
+            .returning('*');
+
+        if (releaseEntries.length) {
+            const releaseEntry = releaseEntries[0];
+
+            console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
 
             if (release.poster || (release.photos && release.photos.length)) {
-                await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.rows[0].id.toString()), { recursive: true });
+                await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
             }
 
             await Promise.all([
                 release.actors && release.actors.length > 0
-                    ? storeActors(release, releaseEntry.rows[0]) : Promise.resolve(),
+                    ? storeActors(release, releaseEntry) : Promise.resolve(),
                 release.tags && release.tags.length > 0
-                    ? storeTags(release, releaseEntry.rows[0]) : Promise.resolve(),
+                    ? storeTags(release, releaseEntry) : Promise.resolve(),
                 release.photos && release.photos.length > 0
-                    ? storePhotos(release, releaseEntry.rows[0]) : Promise.resolve(),
+                    ? storePhotos(release, releaseEntry) : Promise.resolve(),
                 release.poster
-                    ? storePoster(release, releaseEntry.rows[0]) : Promise.resolve(),
+                    ? storePoster(release, releaseEntry) : Promise.resolve(),
                 release.trailer
-                    ? storeTrailer(release, releaseEntry.rows[0]) : Promise.resolve(),
+                    ? storeTrailer(release, releaseEntry) : Promise.resolve(),
             ]);
 
             return;

src/scrapers/dogfart.js

@@ -8,6 +8,7 @@ const moment = require('moment');
 const knex = require('knex');
 
 const { matchTags } = require('../tags');
+const pluckPhotos = require('../utils/pluck-photos');
 
 async function getPhoto(url) {
     const res = await bhttp.get(url);
@@ -29,7 +30,7 @@ async function getPhotos(albumUrl, site, siteUrl) {
 
     // dogfart has massive albums, pick 25 or specified number of photos: first, last and evenly inbetween
     const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
-    const photoIndexes = [1].concat(Array.from({ length: photoLimit - 1 }, (value, index) => Math.floor((index + 1) * (lastPhotoIndex / (photoLimit - 1)))));
+    const photoIndexes = pluckPhotos(lastPhotoIndex, photoLimit);
 
     if (photoLimit > 25) {
         console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`);
@@ -117,7 +118,7 @@ async function scrapeScene(html, url, site) {
     const { origin, pathname } = new URL(url);
     const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
 
-    const stars = Number(document.querySelector('span[itemprop="average"]').textContent) / 2;
+    const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]').textContent) / 2);
     const rawTags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
 
     const [channelSite, tags] = await Promise.all([
@@ -131,7 +132,7 @@ async function scrapeScene(html, url, site) {
     ]);
 
     return {
-        url,
+        url: `${origin}${pathname}`,
         title,
         description,
         actors,

src/utils/pluck-photos.js

@@ -0,0 +1,8 @@
+'use strict';
+
+// pick {photoLimit} photos evenly distributed photos from a set with {photoTotal} photos, return array of indexes starting at 1
+function pluckPhotos(photoTotal, photoLimit) {
+    return [1].concat(Array.from({ length: photoLimit - 1 }, (value, index) => Math.floor((index + 1) * (photoTotal / (photoLimit - 1)))));
+}
+
+module.exports = pluckPhotos;