From 1f444e58cea034e30d975c2d8708c659926c788d Mon Sep 17 00:00:00 2001
From: Niels Simenon <niels.simenon@gmail.com>
Date: Wed, 1 Jul 2020 04:47:05 +0200
Subject: [PATCH] Allowing image sources to specify queue method. Using 5s
 queue for Whale Member to avoid CDN time-outs.

---
 assets/components/expand/expand.vue    |   6 +-
 assets/components/header/filters.vue   |  12 +++
 assets/components/releases/media.vue   |  17 ++--
 assets/components/releases/release.vue | 127 ++++++++++++++-----------
 assets/components/scroll/scroll.vue    |  20 ++--
 assets/js/config/default.js            |   3 +
 src/argv.js                            |   3 +-
 src/media.js                           |   9 +-
 src/scrapers/whalemember.js            |  65 +++++++++++--
 src/store-releases.js                  |   2 +-
 src/utils/http.js                      |  17 ++--
 src/utils/resolve-place.js             |   2 +-
 12 files changed, 183 insertions(+), 100 deletions(-)

diff --git a/assets/components/expand/expand.vue b/assets/components/expand/expand.vue
index c721561d..a7cb606a 100644
--- a/assets/components/expand/expand.vue
+++ b/assets/components/expand/expand.vue
@@ -2,13 +2,13 @@
 	<div>
 		<div
 			v-show="expanded"
-			class="expand expanded"
+			class="expand-button expanded noselect"
 			@click="$emit('expand', false)"
 		><Icon icon="arrow-up3" /></div>
 
 		<div
 			v-show="!expanded"
-			class="expand"
+			class="expand-button noselect"
 			@click="$emit('expand', true)"
 		><Icon icon="arrow-down3" /></div>
 	</div>
@@ -26,7 +26,7 @@ export default {
 </script>
 
 <style lang="scss" scoped>
-.expand {
+.expand-button {
 	width: 100%;
 	display: flex;
 	align-items: center;
diff --git a/assets/components/header/filters.vue b/assets/components/header/filters.vue
index 0801c441..aa5c3281 100644
--- a/assets/components/header/filters.vue
+++ b/assets/components/header/filters.vue
@@ -91,6 +91,18 @@ export default {
 		fill: var(--shadow);
 		margin: 0 .5rem 0 0;
 	}
+
+	&:hover {
+		cursor: pointer;
+
+		.applied {
+			color: var(--shadow-strong);
+		}
+
+		.icon {
+			fill: var(--shadow-strong);
+		}
+	}
 }
 
 .applied {
diff --git a/assets/components/releases/media.vue b/assets/components/releases/media.vue
index 0a7a40f5..4f3d9615 100644
--- a/assets/components/releases/media.vue
+++ b/assets/components/releases/media.vue
@@ -1,13 +1,16 @@
 <template>
 	<div class="media">
-		<div class="trailer-container">
+		<div
+			v-if="release.trailer || release.teaser"
+			class="trailer-container"
+		>
 			<video
 				v-if="release.trailer"
 				:src="`/media/${release.trailer.path}`"
 				:poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)"
 				:alt="release.title"
 				:class="{ sfw: sfw && paused }"
-				class="item trailer-video"
+				class="item trailer"
 				controls
 				@playing="playing = true; paused = false;"
 				@pause="playing = false; paused = true;"
@@ -19,7 +22,7 @@
 				:poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)"
 				:alt="release.title"
 				:class="{ sfw: sfw && paused }"
-				class="item trailer-video"
+				class="item trailer"
 				controls
 				@playing="playing = true; paused = false;"
 				@pause="playing = false; paused = true;"
@@ -29,7 +32,7 @@
 				v-else-if="release.teaser && /^image\//.test(release.teaser.mime)"
 				:src="sfw ? `/img/${release.teaser.sfw.thumbnail}` : `/media/${release.teaser.path}`"
 				:alt="release.title"
-				class="item trailer-video"
+				class="item trailer"
 			>
 
 			<a
@@ -242,9 +245,9 @@ export default {
     max-width: 100%;
 }
 
-.trailer-video {
-    width: 100%;
-	height: 100%;
+.trailer {
+	max-width: 100%;
+	max-height: 100%;
 	object-fit: cover;
 
     &.sfw {
diff --git a/assets/components/releases/release.vue b/assets/components/releases/release.vue
index 8f13d23f..a752ebad 100644
--- a/assets/components/releases/release.vue
+++ b/assets/components/releases/release.vue
@@ -3,18 +3,21 @@
 		v-if="release"
 		class="content"
 	>
+		<Expand
+			v-if="expanded"
+			class="expand"
+			:expanded="expanded"
+			@expand="(state) => expanded = state"
+		/>
+
 		<Scroll
 			class="scroll-light"
 			:expandable="false"
 		>
-			<Media :release="release" />
-
-			<template v-slot:expanded>
-				<Media
-					:release="release"
-					class="expanded"
-				/>
-			</template>
+			<Media
+				:release="release"
+				:class="{ expanded }"
+			/>
 		</Scroll>
 
 		<div class="details">
@@ -82,6 +85,13 @@
 			</div>
 		</div>
 
+		<Expand
+			v-if="release.photos.length > 0 || release.trailer || release.teaser"
+			class="expand-bottom"
+			:expanded="expanded"
+			@expand="(state) => expanded = state"
+		/>
+
 		<div class="info column">
 			<div class="row title-container">
 				<h2 class="title">{{ release.title }}</h2>
@@ -92,6 +102,24 @@
 				>{{ release.shootId }}</span>
 			</div>
 
+			<div
+				v-if="release.tags.length > 0"
+				class="row"
+			>
+				<ul class="tags nolist">
+					<li
+						v-for="tag in release.tags"
+						:key="`tag-${tag.slug}`"
+						class="tag"
+					>
+						<a
+							:href="`/tag/${tag.slug}`"
+							class="link"
+						>{{ tag.name }}</a>
+					</li>
+				</ul>
+			</div>
+
 			<div class="row associations">
 				<ul
 					v-lazy-container="{ selector: '.lazy' }"
@@ -124,66 +152,48 @@
 				/>
 			</div>
 
-			<span class="row-label">Tags</span>
-
 			<div
-				v-if="release.tags.length > 0"
+				v-if="release.description"
 				class="row"
 			>
-				<ul class="tags nolist">
-					<li
-						v-for="tag in release.tags"
-						:key="`tag-${tag.slug}`"
-						class="tag"
-					>
-						<a
-							:href="`/tag/${tag.slug}`"
-							class="link"
-						>{{ tag.name }}</a>
-					</li>
-				</ul>
-			</div>
+				<span class="row-label">Description</span>
 
-			<span class="row-label">Duration</span>
+				<p class="description">{{ release.description }}</p>
+			</div>
 
 			<div
 				v-if="release.duration"
-				class="row duration"
+				class="row"
 			>
-				<span
-					v-if="release.duration >= 3600"
-					class="duration-segment"
-				>{{ Math.floor(release.duration / 3600).toString().padStart(2, '0') }}:</span>
-				<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
-				<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
+				<span class="row-label">Duration</span>
+
+				<div class="duration">
+					<span
+						v-if="release.duration >= 3600"
+						class="duration-segment"
+					>{{ Math.floor(release.duration / 3600).toString().padStart(2, '0') }}:</span>
+					<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
+					<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
+				</div>
 			</div>
 
-			<span class="row-label">Description</span>
-
-			<p
-				v-if="release.description"
-				class="row description"
-			>{{ release.description }}</p>
-
-			<span class="row-label">Studio</span>
-
 			<div
 				v-if="release.studio"
 				class="row"
 			>
+				<span class="row-label">Studio</span>
+
 				<router-link
 					:to="`/studio/${release.studio.slug}`"
-					class="link"
+					class="link studio"
 				>{{ release.studio.name }}</router-link>
 			</div>
 
-			<span class="row-label">Shoot #</span>
-
 			<div
 				v-if="release.shootId"
 				class="row"
 			>
-				<Icon icon="clapboard-play" />
+				<span class="row-label">Shoot #</span>
 
 				<a
 					:href="release.url"
@@ -194,8 +204,9 @@
 				>{{ release.shootId }}</a>
 			</div>
 
-			<span class="row-label">Added</span>
 			<span class="row">
+				<span class="row-label">Added</span>
+
 				<a
 					:href="`/added/${formatDate(release.dateAdded, 'YYYY-MM-DD')}`"
 					:title="`Added on ${formatDate(release.dateAdded, 'MMMM D, YYYY')}`"
@@ -217,6 +228,7 @@ import Actor from '../actors/tile.vue';
 import Release from './tile.vue';
 import Releases from './releases.vue';
 import Scroll from '../scroll/scroll.vue';
+import Expand from '../expand/expand.vue';
 
 function pageTitle() {
 	return this.release && this.release.title;
@@ -233,11 +245,12 @@ export default {
 		Release,
 		Releases,
 		Scroll,
+		Expand,
 	},
 	data() {
 		return {
 			release: null,
-			filename: null,
+			expanded: false,
 		};
 	},
 	computed: {
@@ -315,7 +328,6 @@ export default {
 
 .logo-site {
     height: 2.5rem;
-	width: 100%;
     max-width: 15rem;
 	margin: .25rem 0;
     object-fit: contain;
@@ -336,6 +348,10 @@ export default {
     font-size: .8rem;
 }
 
+.expand-bottom {
+	border-bottom: solid 1px var(--shadow-hint);
+}
+
 .info {
     padding: 1rem;
     border-left: solid 1px var(--shadow-hint);
@@ -344,8 +360,6 @@ export default {
 }
 
 .row {
-    display: flex;
-    align-items: center;
     margin: 0 0 1rem 0;
 
     &.associations {
@@ -390,6 +404,7 @@ export default {
 
 .description {
     line-height: 1.5;
+	margin: -.25rem 0 0 0;
 }
 
 .duration {
@@ -408,14 +423,6 @@ export default {
     flex-wrap: wrap;
 }
 
-.filename {
-    width: 100%;
-    padding: .5rem;
-    color: var(--text);
-    border: solid 1px var(--shadow-weak);
-    background: var(--background);
-}
-
 .link {
     display: inline-flex;
     color: var(--link);
@@ -453,6 +460,10 @@ export default {
     .chain {
         display: none;
     }
+
+	.logo-site {
+		width: 100%;
+	}
 }
 
 @media(max-width: $breakpoint) {
diff --git a/assets/components/scroll/scroll.vue b/assets/components/scroll/scroll.vue
index fbb3e42d..dc083c48 100644
--- a/assets/components/scroll/scroll.vue
+++ b/assets/components/scroll/scroll.vue
@@ -4,7 +4,7 @@
 			v-if="expanded"
 			:expanded="expanded"
 			class="expand-dark"
-			@expand="(state) => expanded = state"
+			@expand="(state) => $emit('expand', state)"
 		/>
 
 		<div class="scrollable">
@@ -12,7 +12,7 @@
 				v-if="expanded"
 				:expanded="expanded"
 				class="expand-light"
-				@expand="(state) => expanded = state"
+				@expand="(state) => $emit('expand', state)"
 			/>
 
 			<div
@@ -22,12 +22,7 @@
 				@click="scroll('left')"
 			><Icon icon="arrow-left3" /></div>
 
-			<slot v-if="!expanded" />
-
-			<slot
-				v-if="expanded"
-				name="expanded"
-			/>
+			<slot />
 
 			<div
 				v-show="enabled && !expanded"
@@ -41,14 +36,14 @@
 			v-if="expanded || (expandable && scrollable)"
 			:expanded="expanded"
 			class="expand-dark"
-			@expand="(state) => expanded = state"
+			@expand="(state) => $emit('expand', state)"
 		/>
 
 		<Expand
 			v-if="expanded || (expandable && scrollable)"
 			:expanded="expanded"
 			class="expand-light"
-			@expand="(state) => expanded = state"
+			@expand="(state) => $emit('expand', state)"
 		/>
 	</div>
 </template>
@@ -107,6 +102,10 @@ export default {
 			type: Boolean,
 			default: true,
 		},
+		expanded: {
+			type: Boolean,
+			default: false,
+		},
 	},
 	data() {
 		return {
@@ -114,7 +113,6 @@ export default {
 			scrollable: true,
 			scrollAtStart: true,
 			scrollAtEnd: false,
-			expanded: false,
 		};
 	},
 	mounted,
diff --git a/assets/js/config/default.js b/assets/js/config/default.js
index 8fb00fd6..8aa0be2c 100644
--- a/assets/js/config/default.js
+++ b/assets/js/config/default.js
@@ -16,11 +16,14 @@ export default {
 		'fisting',
 		'gaping',
 		'gangbang',
+		'interracial',
 		'lesbian',
 		'mff',
 		'mfm',
 		'orgy',
 		'pov',
+		'solo',
 		'squirting',
+		'swallowing',
 	],
 };
diff --git a/src/argv.js b/src/argv.js
index 906d3119..6ebddfa3 100644
--- a/src/argv.js
+++ b/src/argv.js
@@ -10,9 +10,10 @@ const { argv } = yargs
 		type: 'boolean',
 		alias: 'web',
 	})
-	.option('scrape', {
+	.option('all', {
 		describe: 'Scrape channels and networks defined in configuration',
 		type: 'boolean',
+		alias: 'scrape',
 	})
 	.option('networks', {
 		describe: 'Network to scrape all channels from (overrides configuration)',
diff --git a/src/media.js b/src/media.js
index 3fae617b..f4fc1afe 100644
--- a/src/media.js
+++ b/src/media.js
@@ -89,6 +89,8 @@ function toBaseSource(rawSource) {
 
 		if (rawSource.referer) baseSource.referer = rawSource.referer;
 		if (rawSource.host) baseSource.host = rawSource.host;
+		if (rawSource.attempts) baseSource.attempts = rawSource.attempts;
+		if (rawSource.queueMethod) baseSource.queueMethod = rawSource.queueMethod;
 
 		if (rawSource.copyright) baseSource.copyright = rawSource.copyright;
 		if (rawSource.comment) baseSource.comment = rawSource.comment;
@@ -393,6 +395,7 @@ async function fetchSource(source, baseMedia) {
 				stream: true, // sources are fetched in parallel, don't gobble up memory
 				transforms: [hashStream],
 				destination: tempFileTarget,
+				queueMethod: source.queueMethod || null, // use http module's default
 			});
 
 			hasher.end();
@@ -422,9 +425,11 @@ async function fetchSource(source, baseMedia) {
 				},
 			};
 		} catch (error) {
-			logger.warn(`Failed attempt ${attempts}/3 to fetch ${source.src}: ${error.message}`);
+			const maxAttempts = source.attempts || 3;
 
-			if (attempts < 3) {
+			logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);
+
+			if (attempts < maxAttempts) {
 				await Promise.delay(1000);
 
 				return attempt(attempts + 1);
diff --git a/src/scrapers/whalemember.js b/src/scrapers/whalemember.js
index 14f40d37..9d7b8c6c 100644
--- a/src/scrapers/whalemember.js
+++ b/src/scrapers/whalemember.js
@@ -19,13 +19,32 @@ function scrapeLatest(html, site) {
 		release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
 		release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent);
 
+		// slow CDN?
 		const poster = scene.querySelector('.single-image').dataset.src;
-		release.poster = /^http/.test(poster) ? poster : `https:${poster}`;
+		const teaserEl = scene.querySelector('source');
 
-		release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`));
+		release.poster = {
+			src: /^http/.test(poster) ? poster : `https:${poster}`,
+			referer: site.url,
+			attempts: 5,
+			queueMethod: '5s',
+		};
 
-		const trailerEl = scene.querySelector('source');
-		if (trailerEl) release.trailer = { src: trailerEl.dataset.src };
+		release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({
+			src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
+			referer: site.url,
+			attempts: 5,
+			queueMethod: '5s',
+		}));
+
+		if (teaserEl) {
+			release.teaser = {
+				src: teaserEl.dataset.src,
+				referer: site.url,
+				attempts: 5,
+				queueMethod: '5s',
+			};
+		}
 
 		return release;
 	});
@@ -51,16 +70,42 @@ function scrapeScene(html, site, url) {
 		release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
 	}
 
-	release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => (/^http/.test(el.src) ? el.src : `https:${el.src}`));
+	// unreliable CDN
+	release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => ({
+		src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
+		referer: site.url,
+		attempts: 5,
+		queueMethod: '5s',
+	}));
 
 	const posterEl = scene.querySelector('#no-player-image');
 	const videoEl = scene.querySelector('video');
-
-	if (posterEl) release.poster = /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`;
-	else if (videoEl) release.poster = /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`;
-
 	const trailerEl = scene.querySelector('#t2019-video source');
-	if (trailerEl) release.trailer = { src: trailerEl.src };
+
+	if (posterEl) {
+		release.poster = {
+			src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`,
+			referer: site.url,
+			attempts: 5,
+			queueMethod: '5s',
+		};
+	} else if (videoEl) {
+		release.poster = {
+			src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`,
+			referer: site.url,
+			attempts: 5,
+			queueMethod: '5s',
+		};
+	}
+
+	if (trailerEl) {
+		release.trailer = {
+			src: trailerEl.src,
+			referer: site.url,
+			attempts: 5,
+			queueMethod: '5s',
+		};
+	}
 
 	return release;
 }
diff --git a/src/store-releases.js b/src/store-releases.js
index 288294d0..0ee671f9 100644
--- a/src/store-releases.js
+++ b/src/store-releases.js
@@ -46,7 +46,7 @@ function curateReleaseEntry(release, batchId, existingRelease) {
 }
 
 async function attachChannelEntities(releases) {
-	const releasesWithoutEntity = releases.filter(release => release.channel && !release.entity && release.entity.type !== 1);
+	const releasesWithoutEntity = releases.filter(release => release.channel && (!release.entity || release.entity.type === 'network'));
 
 	const channelEntities = await knex('entities')
 		.select(knex.raw('entities.*, row_to_json(parents) as parent'))
diff --git a/src/utils/http.js b/src/utils/http.js
index 960b127a..052983d1 100644
--- a/src/utils/http.js
+++ b/src/utils/http.js
@@ -35,6 +35,7 @@ function useProxy(url) {
 }
 
 const queue = taskQueue();
+const defaultQueueMethod = '20p';
 
 async function handler({
 	url,
@@ -44,9 +45,9 @@ async function handler({
 	options = {},
 }) {
 	if (body) {
-		logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
+		logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)} ${options.queueMethod || defaultQueueMethod}`);
 	} else {
-		logger.silly(`${method.toUpperCase()} ${url}`);
+		logger.silly(`${method.toUpperCase()} ${url} ${options.queueMethod || defaultQueueMethod}`);
 	}
 
 	const reqOptions = {
@@ -98,8 +99,12 @@ queue.define('1s', handler, {
 	interval: 1,
 });
 
-async function get(url, headers, options, queueMethod = '20p') {
-	return queue.push(queueMethod, {
+queue.define('5s', handler, {
+	interval: 5,
+});
+
+async function get(url, headers, options) {
+	return queue.push(options.queueMethod || defaultQueueMethod, {
 		method: 'GET',
 		url,
 		headers,
@@ -107,8 +112,8 @@ async function get(url, headers, options, queueMethod = '20p') {
 	});
 }
 
-async function post(url, body, headers, options, queueMethod = '20p') {
-	return queue.push(queueMethod, {
+async function post(url, body, headers, options) {
+	return queue.push(options.queueMethod || defaultQueueMethod, {
 		method: 'POST',
 		url,
 		body,
diff --git a/src/utils/resolve-place.js b/src/utils/resolve-place.js
index 9a29e224..4c455995 100644
--- a/src/utils/resolve-place.js
+++ b/src/utils/resolve-place.js
@@ -12,7 +12,7 @@ async function resolvePlace(query) {
 		// https://operations.osmfoundation.org/policies/nominatim/
 		const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, {
 			'User-Agent': 'contact at moonloop.adult@protonmail.com',
-		}, null, '1s');
+		}, { queueMethod: '1s' });
 
 		const [item] = res.body;