Generating thumbnails. Added site overview page.

src/.eslintrc

@@ -8,6 +8,6 @@
         "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
         "no-console": 0,
         "indent": ["error", 4],
-        "max-len": [2, {"code": 200, "tabWidth": 4, "ignoreUrls": true}]
+        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}]
     }
 }

src/actors.js

@@ -35,6 +35,24 @@ async function fetchActors(actorId, actorSlug) {
     return curateActors(releases);
 }
 
+async function storeActors(release, releaseEntry) {
+    const actors = await knex('actors').whereIn('name', release.actors);
+    const newActors = release.actors.filter(actorName => !actors.some(actor => actor.name === actorName));
+
+    const { rows: insertedActors } = newActors.length
+        ? await knex.raw(`${knex('actors').insert(newActors.map(actorName => ({
+            name: actorName,
+            slug: actorName.toLowerCase().replace(/\s+/g, '-'),
+        })))} ON CONFLICT DO NOTHING RETURNING *`)
+        : { rows: [] };
+
+    return knex('actors_associated').insert(actors.concat(insertedActors).map(actor => ({
+        release_id: releaseEntry.id,
+        actor_id: actor.id,
+    })), '*');
+}
+
 module.exports = {
     fetchActors,
+    storeActors,
 };

src/fetch-releases.js

@@ -5,13 +5,14 @@ const fs = require('fs-extra');
 const path = require('path');
 const Promise = require('bluebird');
 const moment = require('moment');
-const mime = require('mime');
-const bhttp = require('bhttp');
 
 const argv = require('./argv');
 const knex = require('./knex');
 const scrapers = require('./scrapers');
 const fetchScene = require('./fetch-scene');
+const { storeTags } = require('./tags');
+const { storeActors } = require('./actors');
+const { storePoster, storePhotos, storeTrailer } = require('./media');
 
 function destructConfigNetworks(networks) {
     return networks.reduce((acc, network) => {
@@ -87,112 +88,6 @@ async function findDuplicateReleases(latestReleases, _siteId) {
         .orWhereIn('entry_id', latestReleasesEntryIds);
 }
 
-async function storeActors(release, releaseEntry) {
-    const actors = await knex('actors').whereIn('name', release.actors);
-    const newActors = release.actors.filter(actorName => !actors.some(actor => actor.name === actorName));
-
-    const { rows: insertedActors } = newActors.length
-        ? await knex.raw(`${knex('actors').insert(newActors.map(actorName => ({
-            name: actorName,
-            slug: actorName.toLowerCase().replace(/\s+/g, '-'),
-        })))} ON CONFLICT DO NOTHING RETURNING *`)
-        : { rows: [] };
-
-    return knex('actors_associated').insert(actors.concat(insertedActors).map(actor => ({
-        release_id: releaseEntry.id,
-        actor_id: actor.id,
-    })), '*');
-}
-
-async function storeTags(release, releaseEntry) {
-    return knex('tags_associated').insert(release.tags.map(tagId => ({
-        tag_id: tagId,
-        release_id: releaseEntry.id,
-    })));
-}
-
-async function storePhotos(release, releaseEntry) {
-    console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
-
-    const files = await Promise.map(release.photos, async (photoUrl, index) => {
-        const { pathname } = new URL(photoUrl);
-        const mimetype = mime.getType(pathname);
-
-        const res = await bhttp.get(photoUrl);
-
-        if (res.statusCode === 200) {
-            const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.${mime.getExtension(mimetype)}`);
-            await fs.writeFile(path.join(config.photoPath, filepath), res.body);
-
-            return {
-                filepath,
-                mimetype,
-            };
-        }
-
-        console.warn(`Failed to store photo ${index + 1} for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`);
-
-        return null;
-    }, {
-        concurrency: 2,
-    });
-
-    await knex('media').insert(files.filter(file => file).map(({ filepath, mimetype }, index) => ({
-        path: filepath,
-        mime: mimetype,
-        index,
-        domain: 'releases',
-        target_id: releaseEntry.id,
-        role: 'photo',
-    })));
-}
-
-async function storePoster(release, releaseEntry) {
-    console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
-
-    const res = await bhttp.get(release.poster);
-
-    if (res.statusCode === 200) {
-        const { pathname } = new URL(release.poster);
-        const mimetype = res.headers['content-type'] || mime.getType(pathname) || 'image/jpeg';
-
-        const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `poster.${mime.getExtension(mimetype)}`);
-        await fs.writeFile(path.join(config.photoPath, filepath), res.body);
-
-        await knex('media').insert({
-            path: filepath,
-            mime: mimetype,
-            domain: 'releases',
-            target_id: releaseEntry.id,
-            role: 'poster',
-        });
-
-        return;
-    }
-
-    console.warn(`Failed to store poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`);
-}
-
-async function storeTrailer(release, releaseEntry) {
-    console.log(`Storing trailer for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
-
-    const { pathname } = new URL(release.trailer.src);
-    const mimetype = release.trailer.type || mime.getType(pathname);
-
-    const res = await bhttp.get(release.trailer.src);
-    const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
-    await fs.writeFile(path.join(config.photoPath, filepath), res.body);
-
-    await knex('media').insert({
-        path: filepath,
-        mime: mimetype,
-        domain: 'releases',
-        target_id: releaseEntry.id,
-        role: 'trailer',
-        quality: release.trailer.quality || null,
-    });
-}
-
 async function storeReleases(releases = []) {
     return Promise.map(releases, async (release) => {
         const curatedRelease = {
@@ -212,11 +107,6 @@ async function storeReleases(releases = []) {
             deep: argv.deep,
         };
 
-        /*
-        const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`;
-        const releaseEntry = await knex.raw(releaseQuery);
-        */
-
         const releaseEntries = await knex('releases')
             .insert(curatedRelease)
             .returning('*');
@@ -227,7 +117,7 @@ async function storeReleases(releases = []) {
             console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
 
             if (release.poster || (release.photos && release.photos.length)) {
-                await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
+                await fs.mkdir(path.join(config.media.path, release.site.network.slug, release.site.slug, releaseEntry.id.toString()), { recursive: true });
             }
 
             await Promise.all([

src/media.js

@@ -0,0 +1,137 @@
+'use strict';
+
+const config = require('config');
+const Promise = require('bluebird');
+const path = require('path');
+const fs = require('fs-extra');
+const bhttp = require('bhttp');
+const mime = require('mime');
+const sharp = require('sharp');
+const blake2 = require('blake2');
+
+const knex = require('./knex');
+
+function getHash(buffer) {
+    const hash = blake2.createHash('blake2b', { digestLength: 24 });
+
+    hash.update(buffer);
+
+    return hash.digest('hex');
+}
+
+async function storePoster(release, releaseEntry) {
+    console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
+
+    const res = await bhttp.get(release.poster);
+    const thumbnail = await sharp(res.body)
+        .resize({ width: Math.floor((config.media.thumbnailSize / 9) * 16), height: config.media.thumbnailSize }) // ensure thumbnail is 16:9
+        .toBuffer();
+
+    if (res.statusCode === 200) {
+        const { pathname } = new URL(release.poster);
+        const mimetype = res.headers['content-type'] || mime.getType(pathname) || 'image/jpeg';
+        const extension = mime.getExtension(mimetype);
+
+        const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `poster.${extension}`);
+        const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `poster_thumb.${extension}`);
+        const hash = getHash(res.body);
+
+        await Promise.all([
+            fs.writeFile(path.join(config.media.path, filepath), res.body),
+            fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
+        ]);
+
+        await knex('media').insert({
+            path: filepath,
+            thumbnail: thumbpath,
+            mime: mimetype,
+            hash,
+            domain: 'releases',
+            target_id: releaseEntry.id,
+            role: 'poster',
+        });
+
+        return;
+    }
+
+    console.warn(`Failed to store poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`);
+}
+
+async function storePhotos(release, releaseEntry) {
+    console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
+
+    const files = await Promise.map(release.photos, async (photoUrl, index) => {
+        const { pathname } = new URL(photoUrl);
+        const mimetype = mime.getType(pathname);
+
+        const res = await bhttp.get(photoUrl);
+        const thumbnail = await sharp(res.body).resize({ height: config.media.thumbnailSize }).toBuffer();
+
+        if (res.statusCode === 200) {
+            const extension = mime.getExtension(mimetype);
+
+            const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}.${extension}`);
+            const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}_thumb.${extension}`);
+            const hash = getHash(res.body);
+
+            await Promise.all([
+                fs.writeFile(path.join(config.media.path, filepath), res.body),
+                fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
+            ]);
+
+            return {
+                filepath,
+                thumbpath,
+                mimetype,
+                hash,
+            };
+        }
+
+        console.warn(`Failed to store photo ${index + 1} for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`);
+
+        return null;
+    }, {
+        concurrency: 2,
+    });
+
+    await knex('media')
+        .insert(files.filter(file => file)
+            .map((file, index) => ({
+                path: file.filepath,
+                thumbnail: file.thumbpath,
+                mime: file.mimetype,
+                hash: file.hash,
+                index,
+                domain: 'releases',
+                target_id: releaseEntry.id,
+                role: 'photo',
+            })));
+}
+
+async function storeTrailer(release, releaseEntry) {
+    console.log(`Storing trailer for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
+
+    const { pathname } = new URL(release.trailer.src);
+    const mimetype = release.trailer.type || mime.getType(pathname);
+
+    const res = await bhttp.get(release.trailer.src);
+    const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
+
+    await Promise.all([
+        fs.writeFile(path.join(config.media.path, filepath), res.body),
+        knex('media').insert({
+            path: filepath,
+            mime: mimetype,
+            domain: 'releases',
+            target_id: releaseEntry.id,
+            role: 'trailer',
+            quality: release.trailer.quality || null,
+        }),
+    ]);
+}
+
+module.exports = {
+    storePoster,
+    storePhotos,
+    storeTrailer,
+};

src/sites.js

@@ -3,8 +3,6 @@
 const knex = require('./knex');
 
 async function curateSite(site) {
-    const network = await knex('network').where({ id: site.network_id });
-
     return {
         id: site.id,
         name: site.name,
@@ -12,36 +10,40 @@ async function curateSite(site) {
         description: site.description,
         slug: site.slug,
         network: {
-            id: network.id,
-            name: network.name,
-            url: network.url,
-            description: network.description,
-            slug: network.slug,
+            id: site.network_id,
+            name: site.network_name,
+            slug: site.network_slug,
+            url: site.network_url,
         },
     };
 }
 
-function curateSites(releases) {
-    return Promise.all(releases.map(async site => curateSite(site)));
+function curateSites(sites) {
+    return Promise.all(sites.map(async site => curateSite(site)));
 }
 
 async function fetchSites(siteId, siteSlug) {
-    const releases = await knex('sites')
-        .where({ id: siteId })
-        .orWhere({ slug: siteSlug })
+    const sites = await knex('sites')
+        .where({ 'sites.id': siteId })
+        .orWhere({ 'sites.slug': siteSlug })
+        .select(
+            'sites.*',
+            'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url',
+        )
+        .leftJoin('networks', 'sites.network_id', 'networks.id')
         .limit(100);
 
-    return curateSites(releases);
+    return curateSites(sites);
 }
 
 async function fetchSitesFromReleases() {
-    const releases = await knex('releases')
+    const sites = await knex('releases')
         .select('site_id', '')
         .leftJoin('sites', 'sites.id', 'releases.site_id')
         .groupBy('sites.id')
         .limit(100);
 
-    return curateSites(releases);
+    return curateSites(sites);
 }
 
 module.exports = {

src/tags.js

@@ -25,4 +25,14 @@ async function matchTags(rawTags) {
     return tagEntries;
 }
 
-module.exports = { matchTags };
+async function storeTags(release, releaseEntry) {
+    return knex('tags_associated').insert(release.tags.map(tagId => ({
+        tag_id: tagId,
+        release_id: releaseEntry.id,
+    })));
+}
+
+module.exports = {
+    matchTags,
+    storeTags,
+};

src/web/server.js

@@ -25,7 +25,7 @@ function initServer() {
     const app = express();
     const router = Router();
 
-    router.use('/media', express.static(config.photoPath));
+    router.use('/media', express.static(config.media.path));
     router.use(express.static('public'));
 
     router.use('/img', (req, res) => {

src/web/sites.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { fetchSites, fetchSitesFromReleases } = require('../networks');
+const { fetchSites, fetchSitesFromReleases } = require('../sites');
 
 async function fetchSitesApi(req, res) {
     const siteId = typeof req.params.siteId === 'number' ? req.params.siteId : null;