Allowing image sources to specify queue method. Using 5s queue for Whale Member to avoid CDN time-outs.
This commit is contained in:
parent
53870fda89
commit
1f444e58ce
|
|
@ -2,13 +2,13 @@
|
|||
<div>
|
||||
<div
|
||||
v-show="expanded"
|
||||
class="expand expanded"
|
||||
class="expand-button expanded noselect"
|
||||
@click="$emit('expand', false)"
|
||||
><Icon icon="arrow-up3" /></div>
|
||||
|
||||
<div
|
||||
v-show="!expanded"
|
||||
class="expand"
|
||||
class="expand-button noselect"
|
||||
@click="$emit('expand', true)"
|
||||
><Icon icon="arrow-down3" /></div>
|
||||
</div>
|
||||
|
|
@ -26,7 +26,7 @@ export default {
|
|||
</script>
|
||||
|
||||
<style lang="scss" scoped>
|
||||
.expand {
|
||||
.expand-button {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
|
|
|||
|
|
@ -91,6 +91,18 @@ export default {
|
|||
fill: var(--shadow);
|
||||
margin: 0 .5rem 0 0;
|
||||
}
|
||||
|
||||
&:hover {
|
||||
cursor: pointer;
|
||||
|
||||
.applied {
|
||||
color: var(--shadow-strong);
|
||||
}
|
||||
|
||||
.icon {
|
||||
fill: var(--shadow-strong);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.applied {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
<template>
|
||||
<div class="media">
|
||||
<div class="trailer-container">
|
||||
<div
|
||||
v-if="release.trailer || release.teaser"
|
||||
class="trailer-container"
|
||||
>
|
||||
<video
|
||||
v-if="release.trailer"
|
||||
:src="`/media/${release.trailer.path}`"
|
||||
:poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)"
|
||||
:alt="release.title"
|
||||
:class="{ sfw: sfw && paused }"
|
||||
class="item trailer-video"
|
||||
class="item trailer"
|
||||
controls
|
||||
@playing="playing = true; paused = false;"
|
||||
@pause="playing = false; paused = true;"
|
||||
|
|
@ -19,7 +22,7 @@
|
|||
:poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)"
|
||||
:alt="release.title"
|
||||
:class="{ sfw: sfw && paused }"
|
||||
class="item trailer-video"
|
||||
class="item trailer"
|
||||
controls
|
||||
@playing="playing = true; paused = false;"
|
||||
@pause="playing = false; paused = true;"
|
||||
|
|
@ -29,7 +32,7 @@
|
|||
v-else-if="release.teaser && /^image\//.test(release.teaser.mime)"
|
||||
:src="sfw ? `/img/${release.teaser.sfw.thumbnail}` : `/media/${release.teaser.path}`"
|
||||
:alt="release.title"
|
||||
class="item trailer-video"
|
||||
class="item trailer"
|
||||
>
|
||||
|
||||
<a
|
||||
|
|
@ -242,9 +245,9 @@ export default {
|
|||
max-width: 100%;
|
||||
}
|
||||
|
||||
.trailer-video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
.trailer {
|
||||
max-width: 100%;
|
||||
max-height: 100%;
|
||||
object-fit: cover;
|
||||
|
||||
&.sfw {
|
||||
|
|
|
|||
|
|
@ -3,18 +3,21 @@
|
|||
v-if="release"
|
||||
class="content"
|
||||
>
|
||||
<Expand
|
||||
v-if="expanded"
|
||||
class="expand"
|
||||
:expanded="expanded"
|
||||
@expand="(state) => expanded = state"
|
||||
/>
|
||||
|
||||
<Scroll
|
||||
class="scroll-light"
|
||||
:expandable="false"
|
||||
>
|
||||
<Media :release="release" />
|
||||
|
||||
<template v-slot:expanded>
|
||||
<Media
|
||||
:release="release"
|
||||
class="expanded"
|
||||
:class="{ expanded }"
|
||||
/>
|
||||
</template>
|
||||
</Scroll>
|
||||
|
||||
<div class="details">
|
||||
|
|
@ -82,6 +85,13 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<Expand
|
||||
v-if="release.photos.length > 0 || release.trailer || release.teaser"
|
||||
class="expand-bottom"
|
||||
:expanded="expanded"
|
||||
@expand="(state) => expanded = state"
|
||||
/>
|
||||
|
||||
<div class="info column">
|
||||
<div class="row title-container">
|
||||
<h2 class="title">{{ release.title }}</h2>
|
||||
|
|
@ -92,6 +102,24 @@
|
|||
>{{ release.shootId }}</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-if="release.tags.length > 0"
|
||||
class="row"
|
||||
>
|
||||
<ul class="tags nolist">
|
||||
<li
|
||||
v-for="tag in release.tags"
|
||||
:key="`tag-${tag.slug}`"
|
||||
class="tag"
|
||||
>
|
||||
<a
|
||||
:href="`/tag/${tag.slug}`"
|
||||
class="link"
|
||||
>{{ tag.name }}</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="row associations">
|
||||
<ul
|
||||
v-lazy-container="{ selector: '.lazy' }"
|
||||
|
|
@ -124,32 +152,22 @@
|
|||
/>
|
||||
</div>
|
||||
|
||||
<span class="row-label">Tags</span>
|
||||
|
||||
<div
|
||||
v-if="release.tags.length > 0"
|
||||
v-if="release.description"
|
||||
class="row"
|
||||
>
|
||||
<ul class="tags nolist">
|
||||
<li
|
||||
v-for="tag in release.tags"
|
||||
:key="`tag-${tag.slug}`"
|
||||
class="tag"
|
||||
>
|
||||
<a
|
||||
:href="`/tag/${tag.slug}`"
|
||||
class="link"
|
||||
>{{ tag.name }}</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<span class="row-label">Description</span>
|
||||
|
||||
<span class="row-label">Duration</span>
|
||||
<p class="description">{{ release.description }}</p>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-if="release.duration"
|
||||
class="row duration"
|
||||
class="row"
|
||||
>
|
||||
<span class="row-label">Duration</span>
|
||||
|
||||
<div class="duration">
|
||||
<span
|
||||
v-if="release.duration >= 3600"
|
||||
class="duration-segment"
|
||||
|
|
@ -157,33 +175,25 @@
|
|||
<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
|
||||
<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
|
||||
</div>
|
||||
|
||||
<span class="row-label">Description</span>
|
||||
|
||||
<p
|
||||
v-if="release.description"
|
||||
class="row description"
|
||||
>{{ release.description }}</p>
|
||||
|
||||
<span class="row-label">Studio</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-if="release.studio"
|
||||
class="row"
|
||||
>
|
||||
<span class="row-label">Studio</span>
|
||||
|
||||
<router-link
|
||||
:to="`/studio/${release.studio.slug}`"
|
||||
class="link"
|
||||
class="link studio"
|
||||
>{{ release.studio.name }}</router-link>
|
||||
</div>
|
||||
|
||||
<span class="row-label">Shoot #</span>
|
||||
|
||||
<div
|
||||
v-if="release.shootId"
|
||||
class="row"
|
||||
>
|
||||
<Icon icon="clapboard-play" />
|
||||
<span class="row-label">Shoot #</span>
|
||||
|
||||
<a
|
||||
:href="release.url"
|
||||
|
|
@ -194,8 +204,9 @@
|
|||
>{{ release.shootId }}</a>
|
||||
</div>
|
||||
|
||||
<span class="row-label">Added</span>
|
||||
<span class="row">
|
||||
<span class="row-label">Added</span>
|
||||
|
||||
<a
|
||||
:href="`/added/${formatDate(release.dateAdded, 'YYYY-MM-DD')}`"
|
||||
:title="`Added on ${formatDate(release.dateAdded, 'MMMM D, YYYY')}`"
|
||||
|
|
@ -217,6 +228,7 @@ import Actor from '../actors/tile.vue';
|
|||
import Release from './tile.vue';
|
||||
import Releases from './releases.vue';
|
||||
import Scroll from '../scroll/scroll.vue';
|
||||
import Expand from '../expand/expand.vue';
|
||||
|
||||
function pageTitle() {
|
||||
return this.release && this.release.title;
|
||||
|
|
@ -233,11 +245,12 @@ export default {
|
|||
Release,
|
||||
Releases,
|
||||
Scroll,
|
||||
Expand,
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
release: null,
|
||||
filename: null,
|
||||
expanded: false,
|
||||
};
|
||||
},
|
||||
computed: {
|
||||
|
|
@ -315,7 +328,6 @@ export default {
|
|||
|
||||
.logo-site {
|
||||
height: 2.5rem;
|
||||
width: 100%;
|
||||
max-width: 15rem;
|
||||
margin: .25rem 0;
|
||||
object-fit: contain;
|
||||
|
|
@ -336,6 +348,10 @@ export default {
|
|||
font-size: .8rem;
|
||||
}
|
||||
|
||||
.expand-bottom {
|
||||
border-bottom: solid 1px var(--shadow-hint);
|
||||
}
|
||||
|
||||
.info {
|
||||
padding: 1rem;
|
||||
border-left: solid 1px var(--shadow-hint);
|
||||
|
|
@ -344,8 +360,6 @@ export default {
|
|||
}
|
||||
|
||||
.row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
margin: 0 0 1rem 0;
|
||||
|
||||
&.associations {
|
||||
|
|
@ -390,6 +404,7 @@ export default {
|
|||
|
||||
.description {
|
||||
line-height: 1.5;
|
||||
margin: -.25rem 0 0 0;
|
||||
}
|
||||
|
||||
.duration {
|
||||
|
|
@ -408,14 +423,6 @@ export default {
|
|||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.filename {
|
||||
width: 100%;
|
||||
padding: .5rem;
|
||||
color: var(--text);
|
||||
border: solid 1px var(--shadow-weak);
|
||||
background: var(--background);
|
||||
}
|
||||
|
||||
.link {
|
||||
display: inline-flex;
|
||||
color: var(--link);
|
||||
|
|
@ -453,6 +460,10 @@ export default {
|
|||
.chain {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.logo-site {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
@media(max-width: $breakpoint) {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
v-if="expanded"
|
||||
:expanded="expanded"
|
||||
class="expand-dark"
|
||||
@expand="(state) => expanded = state"
|
||||
@expand="(state) => $emit('expand', state)"
|
||||
/>
|
||||
|
||||
<div class="scrollable">
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
v-if="expanded"
|
||||
:expanded="expanded"
|
||||
class="expand-light"
|
||||
@expand="(state) => expanded = state"
|
||||
@expand="(state) => $emit('expand', state)"
|
||||
/>
|
||||
|
||||
<div
|
||||
|
|
@ -22,12 +22,7 @@
|
|||
@click="scroll('left')"
|
||||
><Icon icon="arrow-left3" /></div>
|
||||
|
||||
<slot v-if="!expanded" />
|
||||
|
||||
<slot
|
||||
v-if="expanded"
|
||||
name="expanded"
|
||||
/>
|
||||
<slot />
|
||||
|
||||
<div
|
||||
v-show="enabled && !expanded"
|
||||
|
|
@ -41,14 +36,14 @@
|
|||
v-if="expanded || (expandable && scrollable)"
|
||||
:expanded="expanded"
|
||||
class="expand-dark"
|
||||
@expand="(state) => expanded = state"
|
||||
@expand="(state) => $emit('expand', state)"
|
||||
/>
|
||||
|
||||
<Expand
|
||||
v-if="expanded || (expandable && scrollable)"
|
||||
:expanded="expanded"
|
||||
class="expand-light"
|
||||
@expand="(state) => expanded = state"
|
||||
@expand="(state) => $emit('expand', state)"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
|
@ -107,6 +102,10 @@ export default {
|
|||
type: Boolean,
|
||||
default: true,
|
||||
},
|
||||
expanded: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
|
|
@ -114,7 +113,6 @@ export default {
|
|||
scrollable: true,
|
||||
scrollAtStart: true,
|
||||
scrollAtEnd: false,
|
||||
expanded: false,
|
||||
};
|
||||
},
|
||||
mounted,
|
||||
|
|
|
|||
|
|
@ -16,11 +16,14 @@ export default {
|
|||
'fisting',
|
||||
'gaping',
|
||||
'gangbang',
|
||||
'interracial',
|
||||
'lesbian',
|
||||
'mff',
|
||||
'mfm',
|
||||
'orgy',
|
||||
'pov',
|
||||
'solo',
|
||||
'squirting',
|
||||
'swallowing',
|
||||
],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,9 +10,10 @@ const { argv } = yargs
|
|||
type: 'boolean',
|
||||
alias: 'web',
|
||||
})
|
||||
.option('scrape', {
|
||||
.option('all', {
|
||||
describe: 'Scrape channels and networks defined in configuration',
|
||||
type: 'boolean',
|
||||
alias: 'scrape',
|
||||
})
|
||||
.option('networks', {
|
||||
describe: 'Network to scrape all channels from (overrides configuration)',
|
||||
|
|
|
|||
|
|
@ -89,6 +89,8 @@ function toBaseSource(rawSource) {
|
|||
|
||||
if (rawSource.referer) baseSource.referer = rawSource.referer;
|
||||
if (rawSource.host) baseSource.host = rawSource.host;
|
||||
if (rawSource.attempts) baseSource.attempts = rawSource.attempts;
|
||||
if (rawSource.queueMethod) baseSource.queueMethod = rawSource.queueMethod;
|
||||
|
||||
if (rawSource.copyright) baseSource.copyright = rawSource.copyright;
|
||||
if (rawSource.comment) baseSource.comment = rawSource.comment;
|
||||
|
|
@ -393,6 +395,7 @@ async function fetchSource(source, baseMedia) {
|
|||
stream: true, // sources are fetched in parallel, don't gobble up memory
|
||||
transforms: [hashStream],
|
||||
destination: tempFileTarget,
|
||||
queueMethod: source.queueMethod || null, // use http module's default
|
||||
});
|
||||
|
||||
hasher.end();
|
||||
|
|
@ -422,9 +425,11 @@ async function fetchSource(source, baseMedia) {
|
|||
},
|
||||
};
|
||||
} catch (error) {
|
||||
logger.warn(`Failed attempt ${attempts}/3 to fetch ${source.src}: ${error.message}`);
|
||||
const maxAttempts = source.attempts || 3;
|
||||
|
||||
if (attempts < 3) {
|
||||
logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);
|
||||
|
||||
if (attempts < maxAttempts) {
|
||||
await Promise.delay(1000);
|
||||
|
||||
return attempt(attempts + 1);
|
||||
|
|
|
|||
|
|
@ -19,13 +19,32 @@ function scrapeLatest(html, site) {
|
|||
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
|
||||
release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent);
|
||||
|
||||
// slow CDN?
|
||||
const poster = scene.querySelector('.single-image').dataset.src;
|
||||
release.poster = /^http/.test(poster) ? poster : `https:${poster}`;
|
||||
const teaserEl = scene.querySelector('source');
|
||||
|
||||
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`));
|
||||
release.poster = {
|
||||
src: /^http/.test(poster) ? poster : `https:${poster}`,
|
||||
referer: site.url,
|
||||
attempts: 5,
|
||||
queueMethod: '5s',
|
||||
};
|
||||
|
||||
const trailerEl = scene.querySelector('source');
|
||||
if (trailerEl) release.trailer = { src: trailerEl.dataset.src };
|
||||
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({
|
||||
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
|
||||
referer: site.url,
|
||||
attempts: 5,
|
||||
queueMethod: '5s',
|
||||
}));
|
||||
|
||||
if (teaserEl) {
|
||||
release.teaser = {
|
||||
src: teaserEl.dataset.src,
|
||||
referer: site.url,
|
||||
attempts: 5,
|
||||
queueMethod: '5s',
|
||||
};
|
||||
}
|
||||
|
||||
return release;
|
||||
});
|
||||
|
|
@ -51,16 +70,42 @@ function scrapeScene(html, site, url) {
|
|||
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
|
||||
}
|
||||
|
||||
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => (/^http/.test(el.src) ? el.src : `https:${el.src}`));
|
||||
// unreliable CDN
|
||||
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => ({
|
||||
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
|
||||
referer: site.url,
|
||||
attempts: 5,
|
||||
queueMethod: '5s',
|
||||
}));
|
||||
|
||||
const posterEl = scene.querySelector('#no-player-image');
|
||||
const videoEl = scene.querySelector('video');
|
||||
|
||||
if (posterEl) release.poster = /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`;
|
||||
else if (videoEl) release.poster = /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`;
|
||||
|
||||
const trailerEl = scene.querySelector('#t2019-video source');
|
||||
if (trailerEl) release.trailer = { src: trailerEl.src };
|
||||
|
||||
if (posterEl) {
|
||||
release.poster = {
|
||||
src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`,
|
||||
referer: site.url,
|
||||
attempts: 5,
|
||||
queueMethod: '5s',
|
||||
};
|
||||
} else if (videoEl) {
|
||||
release.poster = {
|
||||
src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`,
|
||||
referer: site.url,
|
||||
attempts: 5,
|
||||
queueMethod: '5s',
|
||||
};
|
||||
}
|
||||
|
||||
if (trailerEl) {
|
||||
release.trailer = {
|
||||
src: trailerEl.src,
|
||||
referer: site.url,
|
||||
attempts: 5,
|
||||
queueMethod: '5s',
|
||||
};
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ function curateReleaseEntry(release, batchId, existingRelease) {
|
|||
}
|
||||
|
||||
async function attachChannelEntities(releases) {
|
||||
const releasesWithoutEntity = releases.filter(release => release.channel && !release.entity && release.entity.type !== 1);
|
||||
const releasesWithoutEntity = releases.filter(release => release.channel && (!release.entity || release.entity.type === 'network'));
|
||||
|
||||
const channelEntities = await knex('entities')
|
||||
.select(knex.raw('entities.*, row_to_json(parents) as parent'))
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ function useProxy(url) {
|
|||
}
|
||||
|
||||
const queue = taskQueue();
|
||||
const defaultQueueMethod = '20p';
|
||||
|
||||
async function handler({
|
||||
url,
|
||||
|
|
@ -44,9 +45,9 @@ async function handler({
|
|||
options = {},
|
||||
}) {
|
||||
if (body) {
|
||||
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
|
||||
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)} ${options.queueMethod || defaultQueueMethod}`);
|
||||
} else {
|
||||
logger.silly(`${method.toUpperCase()} ${url}`);
|
||||
logger.silly(`${method.toUpperCase()} ${url} ${options.queueMethod || defaultQueueMethod}`);
|
||||
}
|
||||
|
||||
const reqOptions = {
|
||||
|
|
@ -98,8 +99,12 @@ queue.define('1s', handler, {
|
|||
interval: 1,
|
||||
});
|
||||
|
||||
async function get(url, headers, options, queueMethod = '20p') {
|
||||
return queue.push(queueMethod, {
|
||||
queue.define('5s', handler, {
|
||||
interval: 5,
|
||||
});
|
||||
|
||||
async function get(url, headers, options) {
|
||||
return queue.push(options.queueMethod || defaultQueueMethod, {
|
||||
method: 'GET',
|
||||
url,
|
||||
headers,
|
||||
|
|
@ -107,8 +112,8 @@ async function get(url, headers, options, queueMethod = '20p') {
|
|||
});
|
||||
}
|
||||
|
||||
async function post(url, body, headers, options, queueMethod = '20p') {
|
||||
return queue.push(queueMethod, {
|
||||
async function post(url, body, headers, options) {
|
||||
return queue.push(options.queueMethod || defaultQueueMethod, {
|
||||
method: 'POST',
|
||||
url,
|
||||
body,
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ async function resolvePlace(query) {
|
|||
// https://operations.osmfoundation.org/policies/nominatim/
|
||||
const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, {
|
||||
'User-Agent': 'contact at moonloop.adult@protonmail.com',
|
||||
}, null, '1s');
|
||||
}, { queueMethod: '1s' });
|
||||
|
||||
const [item] = res.body;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue