DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Allowing image sources to specify queue method. Using 5s queue for Whale Member to avoid CDN time-outs.

This commit is contained in:
ThePendulum 2020-07-01 04:47:05 +02:00
parent 53870fda89
commit 1f444e58ce
12 changed files with 183 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
assets/components/expand/expand.vue
View File

@ -2,13 +2,13 @@
<div>
<div
v-show="expanded"
class="expand expanded"
class="expand-button expanded noselect"
@click="$emit('expand', false)"
><Icon icon="arrow-up3" /></div>
<div
v-show="!expanded"
class="expand"
class="expand-button noselect"
@click="$emit('expand', true)"
><Icon icon="arrow-down3" /></div>
</div>
@ -26,7 +26,7 @@ export default {
</script>
<style lang="scss" scoped>
.expand {
.expand-button {
width: 100%;
display: flex;
align-items: center;

12
assets/components/header/filters.vue
View File

@ -91,6 +91,18 @@ export default {
fill: var(--shadow);
margin: 0 .5rem 0 0;
}
&:hover {
cursor: pointer;
.applied {
color: var(--shadow-strong);
}
.icon {
fill: var(--shadow-strong);
}
}
}
.applied {

17
assets/components/releases/media.vue
View File

@ -1,13 +1,16 @@
<template>
<div class="media">
<div class="trailer-container">
<div
v-if="release.trailer || release.teaser"
class="trailer-container"
>
<video
v-if="release.trailer"
:src="`/media/${release.trailer.path}`"
:poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)"
:alt="release.title"
:class="{ sfw: sfw && paused }"
class="item trailer-video"
class="item trailer"
controls
@playing="playing = true; paused = false;"
@pause="playing = false; paused = true;"
@ -19,7 +22,7 @@
:poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)"
:alt="release.title"
:class="{ sfw: sfw && paused }"
class="item trailer-video"
class="item trailer"
controls
@playing="playing = true; paused = false;"
@pause="playing = false; paused = true;"
@ -29,7 +32,7 @@
v-else-if="release.teaser && /^image\//.test(release.teaser.mime)"
:src="sfw ? `/img/${release.teaser.sfw.thumbnail}` : `/media/${release.teaser.path}`"
:alt="release.title"
class="item trailer-video"
class="item trailer"
>
<a
@ -242,9 +245,9 @@ export default {
max-width: 100%;
}
.trailer-video {
width: 100%;
height: 100%;
.trailer {
max-width: 100%;
max-height: 100%;
object-fit: cover;
&.sfw {

127
assets/components/releases/release.vue
View File

@ -3,18 +3,21 @@
v-if="release"
class="content"
>
<Expand
v-if="expanded"
class="expand"
:expanded="expanded"
@expand="(state) => expanded = state"
/>
<Scroll
class="scroll-light"
:expandable="false"
>
<Media :release="release" />
<template v-slot:expanded>
<Media
:release="release"
class="expanded"
/>
</template>
<Media
:release="release"
:class="{ expanded }"
/>
</Scroll>
<div class="details">
@ -82,6 +85,13 @@
</div>
</div>
<Expand
v-if="release.photos.length > 0 || release.trailer || release.teaser"
class="expand-bottom"
:expanded="expanded"
@expand="(state) => expanded = state"
/>
<div class="info column">
<div class="row title-container">
<h2 class="title">{{ release.title }}</h2>
@ -92,6 +102,24 @@
>{{ release.shootId }}</span>
</div>
<div
v-if="release.tags.length > 0"
class="row"
>
<ul class="tags nolist">
<li
v-for="tag in release.tags"
:key="`tag-${tag.slug}`"
class="tag"
>
<a
:href="`/tag/${tag.slug}`"
class="link"
>{{ tag.name }}</a>
</li>
</ul>
</div>
<div class="row associations">
<ul
v-lazy-container="{ selector: '.lazy' }"
@ -124,66 +152,48 @@
/>
</div>
<span class="row-label">Tags</span>
<div
v-if="release.tags.length > 0"
v-if="release.description"
class="row"
>
<ul class="tags nolist">
<li
v-for="tag in release.tags"
:key="`tag-${tag.slug}`"
class="tag"
>
<a
:href="`/tag/${tag.slug}`"
class="link"
>{{ tag.name }}</a>
</li>
</ul>
</div>
<span class="row-label">Description</span>
<span class="row-label">Duration</span>
<p class="description">{{ release.description }}</p>
</div>
<div
v-if="release.duration"
class="row duration"
class="row"
>
<span
v-if="release.duration >= 3600"
class="duration-segment"
>{{ Math.floor(release.duration / 3600).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
<span class="row-label">Duration</span>
<div class="duration">
<span
v-if="release.duration >= 3600"
class="duration-segment"
>{{ Math.floor(release.duration / 3600).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
</div>
</div>
<span class="row-label">Description</span>
<p
v-if="release.description"
class="row description"
>{{ release.description }}</p>
<span class="row-label">Studio</span>
<div
v-if="release.studio"
class="row"
>
<span class="row-label">Studio</span>
<router-link
:to="`/studio/${release.studio.slug}`"
class="link"
class="link studio"
>{{ release.studio.name }}</router-link>
</div>
<span class="row-label">Shoot #</span>
<div
v-if="release.shootId"
class="row"
>
<Icon icon="clapboard-play" />
<span class="row-label">Shoot #</span>
<a
:href="release.url"
@ -194,8 +204,9 @@
>{{ release.shootId }}</a>
</div>
<span class="row-label">Added</span>
<span class="row">
<span class="row-label">Added</span>
<a
:href="`/added/${formatDate(release.dateAdded, 'YYYY-MM-DD')}`"
:title="`Added on ${formatDate(release.dateAdded, 'MMMM D, YYYY')}`"
@ -217,6 +228,7 @@ import Actor from '../actors/tile.vue';
import Release from './tile.vue';
import Releases from './releases.vue';
import Scroll from '../scroll/scroll.vue';
import Expand from '../expand/expand.vue';
function pageTitle() {
return this.release && this.release.title;
@ -233,11 +245,12 @@ export default {
Release,
Releases,
Scroll,
Expand,
},
data() {
return {
release: null,
filename: null,
expanded: false,
};
},
computed: {
@ -315,7 +328,6 @@ export default {
.logo-site {
height: 2.5rem;
width: 100%;
max-width: 15rem;
margin: .25rem 0;
object-fit: contain;
@ -336,6 +348,10 @@ export default {
font-size: .8rem;
}
.expand-bottom {
border-bottom: solid 1px var(--shadow-hint);
}
.info {
padding: 1rem;
border-left: solid 1px var(--shadow-hint);
@ -344,8 +360,6 @@ export default {
}
.row {
display: flex;
align-items: center;
margin: 0 0 1rem 0;
&.associations {
@ -390,6 +404,7 @@ export default {
.description {
line-height: 1.5;
margin: -.25rem 0 0 0;
}
.duration {
@ -408,14 +423,6 @@ export default {
flex-wrap: wrap;
}
.filename {
width: 100%;
padding: .5rem;
color: var(--text);
border: solid 1px var(--shadow-weak);
background: var(--background);
}
.link {
display: inline-flex;
color: var(--link);
@ -453,6 +460,10 @@ export default {
.chain {
display: none;
}
.logo-site {
width: 100%;
}
}
@media(max-width: $breakpoint) {

20
assets/components/scroll/scroll.vue
View File

@ -4,7 +4,7 @@
v-if="expanded"
:expanded="expanded"
class="expand-dark"
@expand="(state) => expanded = state"
@expand="(state) => $emit('expand', state)"
/>
<div class="scrollable">
@ -12,7 +12,7 @@
v-if="expanded"
:expanded="expanded"
class="expand-light"
@expand="(state) => expanded = state"
@expand="(state) => $emit('expand', state)"
/>
<div
@ -22,12 +22,7 @@
@click="scroll('left')"
><Icon icon="arrow-left3" /></div>
<slot v-if="!expanded" />
<slot
v-if="expanded"
name="expanded"
/>
<slot />
<div
v-show="enabled && !expanded"
@ -41,14 +36,14 @@
v-if="expanded || (expandable && scrollable)"
:expanded="expanded"
class="expand-dark"
@expand="(state) => expanded = state"
@expand="(state) => $emit('expand', state)"
/>
<Expand
v-if="expanded || (expandable && scrollable)"
:expanded="expanded"
class="expand-light"
@expand="(state) => expanded = state"
@expand="(state) => $emit('expand', state)"
/>
</div>
</template>
@ -107,6 +102,10 @@ export default {
type: Boolean,
default: true,
},
expanded: {
type: Boolean,
default: false,
},
},
data() {
return {
@ -114,7 +113,6 @@ export default {
scrollable: true,
scrollAtStart: true,
scrollAtEnd: false,
expanded: false,
};
},
mounted,

3
assets/js/config/default.js
View File

@ -16,11 +16,14 @@ export default {
'fisting',
'gaping',
'gangbang',
'interracial',
'lesbian',
'mff',
'mfm',
'orgy',
'pov',
'solo',
'squirting',
'swallowing',
],
};

3
src/argv.js
View File

@ -10,9 +10,10 @@ const { argv } = yargs
type: 'boolean',
alias: 'web',
})
.option('scrape', {
.option('all', {
describe: 'Scrape channels and networks defined in configuration',
type: 'boolean',
alias: 'scrape',
})
.option('networks', {
describe: 'Network to scrape all channels from (overrides configuration)',

9
src/media.js
View File

@ -89,6 +89,8 @@ function toBaseSource(rawSource) {
if (rawSource.referer) baseSource.referer = rawSource.referer;
if (rawSource.host) baseSource.host = rawSource.host;
if (rawSource.attempts) baseSource.attempts = rawSource.attempts;
if (rawSource.queueMethod) baseSource.queueMethod = rawSource.queueMethod;
if (rawSource.copyright) baseSource.copyright = rawSource.copyright;
if (rawSource.comment) baseSource.comment = rawSource.comment;
@ -393,6 +395,7 @@ async function fetchSource(source, baseMedia) {
stream: true, // sources are fetched in parallel, don't gobble up memory
transforms: [hashStream],
destination: tempFileTarget,
queueMethod: source.queueMethod || null, // use http module's default
});
hasher.end();
@ -422,9 +425,11 @@ async function fetchSource(source, baseMedia) {
},
};
} catch (error) {
logger.warn(`Failed attempt ${attempts}/3 to fetch ${source.src}: ${error.message}`);
const maxAttempts = source.attempts || 3;
if (attempts < 3) {
logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);
if (attempts < maxAttempts) {
await Promise.delay(1000);
return attempt(attempts + 1);

65
src/scrapers/whalemember.js
View File

@ -19,13 +19,32 @@ function scrapeLatest(html, site) {
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent);
// slow CDN?
const poster = scene.querySelector('.single-image').dataset.src;
release.poster = /^http/.test(poster) ? poster : `https:${poster}`;
const teaserEl = scene.querySelector('source');
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`));
release.poster = {
src: /^http/.test(poster) ? poster : `https:${poster}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
const trailerEl = scene.querySelector('source');
if (trailerEl) release.trailer = { src: trailerEl.dataset.src };
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
referer: site.url,
attempts: 5,
queueMethod: '5s',
}));
if (teaserEl) {
release.teaser = {
src: teaserEl.dataset.src,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
return release;
});
@ -51,16 +70,42 @@ function scrapeScene(html, site, url) {
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
}
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => (/^http/.test(el.src) ? el.src : `https:${el.src}`));
// unreliable CDN
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => ({
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
referer: site.url,
attempts: 5,
queueMethod: '5s',
}));
const posterEl = scene.querySelector('#no-player-image');
const videoEl = scene.querySelector('video');
if (posterEl) release.poster = /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`;
else if (videoEl) release.poster = /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`;
const trailerEl = scene.querySelector('#t2019-video source');
if (trailerEl) release.trailer = { src: trailerEl.src };
if (posterEl) {
release.poster = {
src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
} else if (videoEl) {
release.poster = {
src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
if (trailerEl) {
release.trailer = {
src: trailerEl.src,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
return release;
}

2
src/store-releases.js
View File

@ -46,7 +46,7 @@ function curateReleaseEntry(release, batchId, existingRelease) {
}
async function attachChannelEntities(releases) {
const releasesWithoutEntity = releases.filter(release => release.channel && !release.entity && release.entity.type !== 1);
const releasesWithoutEntity = releases.filter(release => release.channel && (!release.entity || release.entity.type === 'network'));
const channelEntities = await knex('entities')
.select(knex.raw('entities.*, row_to_json(parents) as parent'))

17
src/utils/http.js
View File

@ -35,6 +35,7 @@ function useProxy(url) {
}
const queue = taskQueue();
const defaultQueueMethod = '20p';
async function handler({
url,
@ -44,9 +45,9 @@ async function handler({
options = {},
}) {
if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)} ${options.queueMethod || defaultQueueMethod}`);
} else {
logger.silly(`${method.toUpperCase()} ${url}`);
logger.silly(`${method.toUpperCase()} ${url} ${options.queueMethod || defaultQueueMethod}`);
}
const reqOptions = {
@ -98,8 +99,12 @@ queue.define('1s', handler, {
interval: 1,
});
async function get(url, headers, options, queueMethod = '20p') {
return queue.push(queueMethod, {
queue.define('5s', handler, {
interval: 5,
});
async function get(url, headers, options) {
return queue.push(options.queueMethod || defaultQueueMethod, {
method: 'GET',
url,
headers,
@ -107,8 +112,8 @@ async function get(url, headers, options, queueMethod = '20p') {
});
}
async function post(url, body, headers, options, queueMethod = '20p') {
return queue.push(queueMethod, {
async function post(url, body, headers, options) {
return queue.push(options.queueMethod || defaultQueueMethod, {
method: 'POST',
url,
body,

2
src/utils/resolve-place.js
View File

@ -12,7 +12,7 @@ async function resolvePlace(query) {
// https://operations.osmfoundation.org/policies/nominatim/
const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, {
'User-Agent': 'contact at moonloop.adult@protonmail.com',
}, null, '1s');
}, { queueMethod: '1s' });
const [item] = res.body;