DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 36 Pull Requests 1 Releases Wiki Activity

Allowing image sources to specify queue method. Using 5s queue for Whale Member to avoid CDN time-outs.

This commit is contained in:
ThePendulum 2020-07-01 04:47:05 +02:00
parent 53870fda89
commit 1f444e58ce
12 changed files with 183 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
assets/components/expand/expand.vue
View File

@ -2,13 +2,13 @@
<div> <div>
<div <div
v-show="expanded" v-show="expanded"
class="expand expanded" class="expand-button expanded noselect"
@click="$emit('expand', false)" @click="$emit('expand', false)"
><Icon icon="arrow-up3" /></div> ><Icon icon="arrow-up3" /></div>
<div <div
v-show="!expanded" v-show="!expanded"
class="expand" class="expand-button noselect"
@click="$emit('expand', true)" @click="$emit('expand', true)"
><Icon icon="arrow-down3" /></div> ><Icon icon="arrow-down3" /></div>
</div> </div>
@ -26,7 +26,7 @@ export default {
</script> </script>
<style lang="scss" scoped> <style lang="scss" scoped>
.expand { .expand-button {
width: 100%; width: 100%;
display: flex; display: flex;
align-items: center; align-items: center;

12
assets/components/header/filters.vue
View File

@ -91,6 +91,18 @@ export default {
fill: var(--shadow); fill: var(--shadow);
margin: 0 .5rem 0 0; margin: 0 .5rem 0 0;
} }
&:hover {
cursor: pointer;
.applied {
color: var(--shadow-strong);
}
.icon {
fill: var(--shadow-strong);
}
}
} }
.applied { .applied {

17
assets/components/releases/media.vue
View File

@ -1,13 +1,16 @@
<template> <template>
<div class="media"> <div class="media">
<div class="trailer-container"> <div
v-if="release.trailer || release.teaser"
class="trailer-container"
>
<video <video
v-if="release.trailer" v-if="release.trailer"
:src="`/media/${release.trailer.path}`" :src="`/media/${release.trailer.path}`"
:poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)" :poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)"
:alt="release.title" :alt="release.title"
:class="{ sfw: sfw && paused }" :class="{ sfw: sfw && paused }"
class="item trailer-video" class="item trailer"
controls controls
@playing="playing = true; paused = false;" @playing="playing = true; paused = false;"
@pause="playing = false; paused = true;" @pause="playing = false; paused = true;"
@ -19,7 +22,7 @@
:poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)" :poster="release.poster && (sfw ? `/img/${release.poster.sfw.thumbnail}` : `/media/${release.poster.thumbnail}`)"
:alt="release.title" :alt="release.title"
:class="{ sfw: sfw && paused }" :class="{ sfw: sfw && paused }"
class="item trailer-video" class="item trailer"
controls controls
@playing="playing = true; paused = false;" @playing="playing = true; paused = false;"
@pause="playing = false; paused = true;" @pause="playing = false; paused = true;"
@ -29,7 +32,7 @@
v-else-if="release.teaser && /^image\//.test(release.teaser.mime)" v-else-if="release.teaser && /^image\//.test(release.teaser.mime)"
:src="sfw ? `/img/${release.teaser.sfw.thumbnail}` : `/media/${release.teaser.path}`" :src="sfw ? `/img/${release.teaser.sfw.thumbnail}` : `/media/${release.teaser.path}`"
:alt="release.title" :alt="release.title"
class="item trailer-video" class="item trailer"
> >
<a <a
@ -242,9 +245,9 @@ export default {
max-width: 100%; max-width: 100%;
} }
.trailer-video { .trailer {
width: 100%; max-width: 100%;
height: 100%; max-height: 100%;
object-fit: cover; object-fit: cover;
&.sfw { &.sfw {

127
assets/components/releases/release.vue
View File

@ -3,18 +3,21 @@
v-if="release" v-if="release"
class="content" class="content"
> >
<Expand
v-if="expanded"
class="expand"
:expanded="expanded"
@expand="(state) => expanded = state"
/>
<Scroll <Scroll
class="scroll-light" class="scroll-light"
:expandable="false" :expandable="false"
> >
<Media :release="release" /> <Media
:release="release"
<template v-slot:expanded> :class="{ expanded }"
<Media />
:release="release"
class="expanded"
/>
</template>
</Scroll> </Scroll>
<div class="details"> <div class="details">
@ -82,6 +85,13 @@
</div> </div>
</div> </div>
<Expand
v-if="release.photos.length > 0 || release.trailer || release.teaser"
class="expand-bottom"
:expanded="expanded"
@expand="(state) => expanded = state"
/>
<div class="info column"> <div class="info column">
<div class="row title-container"> <div class="row title-container">
<h2 class="title">{{ release.title }}</h2> <h2 class="title">{{ release.title }}</h2>
@ -92,6 +102,24 @@
>{{ release.shootId }}</span> >{{ release.shootId }}</span>
</div> </div>
<div
v-if="release.tags.length > 0"
class="row"
>
<ul class="tags nolist">
<li
v-for="tag in release.tags"
:key="`tag-${tag.slug}`"
class="tag"
>
<a
:href="`/tag/${tag.slug}`"
class="link"
>{{ tag.name }}</a>
</li>
</ul>
</div>
<div class="row associations"> <div class="row associations">
<ul <ul
v-lazy-container="{ selector: '.lazy' }" v-lazy-container="{ selector: '.lazy' }"
@ -124,66 +152,48 @@
/> />
</div> </div>
<span class="row-label">Tags</span>
<div <div
v-if="release.tags.length > 0" v-if="release.description"
class="row" class="row"
> >
<ul class="tags nolist"> <span class="row-label">Description</span>
<li
v-for="tag in release.tags"
:key="`tag-${tag.slug}`"
class="tag"
>
<a
:href="`/tag/${tag.slug}`"
class="link"
>{{ tag.name }}</a>
</li>
</ul>
</div>
<span class="row-label">Duration</span> <p class="description">{{ release.description }}</p>
</div>
<div <div
v-if="release.duration" v-if="release.duration"
class="row duration" class="row"
> >
<span <span class="row-label">Duration</span>
v-if="release.duration >= 3600"
class="duration-segment" <div class="duration">
>{{ Math.floor(release.duration / 3600).toString().padStart(2, '0') }}:</span> <span
<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span> v-if="release.duration >= 3600"
<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span> class="duration-segment"
>{{ Math.floor(release.duration / 3600).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
</div>
</div> </div>
<span class="row-label">Description</span>
<p
v-if="release.description"
class="row description"
>{{ release.description }}</p>
<span class="row-label">Studio</span>
<div <div
v-if="release.studio" v-if="release.studio"
class="row" class="row"
> >
<span class="row-label">Studio</span>
<router-link <router-link
:to="`/studio/${release.studio.slug}`" :to="`/studio/${release.studio.slug}`"
class="link" class="link studio"
>{{ release.studio.name }}</router-link> >{{ release.studio.name }}</router-link>
</div> </div>
<span class="row-label">Shoot #</span>
<div <div
v-if="release.shootId" v-if="release.shootId"
class="row" class="row"
> >
<Icon icon="clapboard-play" /> <span class="row-label">Shoot #</span>
<a <a
:href="release.url" :href="release.url"
@ -194,8 +204,9 @@
>{{ release.shootId }}</a> >{{ release.shootId }}</a>
</div> </div>
<span class="row-label">Added</span>
<span class="row"> <span class="row">
<span class="row-label">Added</span>
<a <a
:href="`/added/${formatDate(release.dateAdded, 'YYYY-MM-DD')}`" :href="`/added/${formatDate(release.dateAdded, 'YYYY-MM-DD')}`"
:title="`Added on ${formatDate(release.dateAdded, 'MMMM D, YYYY')}`" :title="`Added on ${formatDate(release.dateAdded, 'MMMM D, YYYY')}`"
@ -217,6 +228,7 @@ import Actor from '../actors/tile.vue';
import Release from './tile.vue'; import Release from './tile.vue';
import Releases from './releases.vue'; import Releases from './releases.vue';
import Scroll from '../scroll/scroll.vue'; import Scroll from '../scroll/scroll.vue';
import Expand from '../expand/expand.vue';
function pageTitle() { function pageTitle() {
return this.release && this.release.title; return this.release && this.release.title;
@ -233,11 +245,12 @@ export default {
Release, Release,
Releases, Releases,
Scroll, Scroll,
Expand,
}, },
data() { data() {
return { return {
release: null, release: null,
filename: null, expanded: false,
}; };
}, },
computed: { computed: {
@ -315,7 +328,6 @@ export default {
.logo-site { .logo-site {
height: 2.5rem; height: 2.5rem;
width: 100%;
max-width: 15rem; max-width: 15rem;
margin: .25rem 0; margin: .25rem 0;
object-fit: contain; object-fit: contain;
@ -336,6 +348,10 @@ export default {
font-size: .8rem; font-size: .8rem;
} }
.expand-bottom {
border-bottom: solid 1px var(--shadow-hint);
}
.info { .info {
padding: 1rem; padding: 1rem;
border-left: solid 1px var(--shadow-hint); border-left: solid 1px var(--shadow-hint);
@ -344,8 +360,6 @@ export default {
} }
.row { .row {
display: flex;
align-items: center;
margin: 0 0 1rem 0; margin: 0 0 1rem 0;
&.associations { &.associations {
@ -390,6 +404,7 @@ export default {
.description { .description {
line-height: 1.5; line-height: 1.5;
margin: -.25rem 0 0 0;
} }
.duration { .duration {
@ -408,14 +423,6 @@ export default {
flex-wrap: wrap; flex-wrap: wrap;
} }
.filename {
width: 100%;
padding: .5rem;
color: var(--text);
border: solid 1px var(--shadow-weak);
background: var(--background);
}
.link { .link {
display: inline-flex; display: inline-flex;
color: var(--link); color: var(--link);
@ -453,6 +460,10 @@ export default {
.chain { .chain {
display: none; display: none;
} }
.logo-site {
width: 100%;
}
} }
@media(max-width: $breakpoint) { @media(max-width: $breakpoint) {

20
assets/components/scroll/scroll.vue
View File

@ -4,7 +4,7 @@
v-if="expanded" v-if="expanded"
:expanded="expanded" :expanded="expanded"
class="expand-dark" class="expand-dark"
@expand="(state) => expanded = state" @expand="(state) => $emit('expand', state)"
/> />
<div class="scrollable"> <div class="scrollable">
@ -12,7 +12,7 @@
v-if="expanded" v-if="expanded"
:expanded="expanded" :expanded="expanded"
class="expand-light" class="expand-light"
@expand="(state) => expanded = state" @expand="(state) => $emit('expand', state)"
/> />
<div <div
@ -22,12 +22,7 @@
@click="scroll('left')" @click="scroll('left')"
><Icon icon="arrow-left3" /></div> ><Icon icon="arrow-left3" /></div>
<slot v-if="!expanded" /> <slot />
<slot
v-if="expanded"
name="expanded"
/>
<div <div
v-show="enabled && !expanded" v-show="enabled && !expanded"
@ -41,14 +36,14 @@
v-if="expanded || (expandable && scrollable)" v-if="expanded || (expandable && scrollable)"
:expanded="expanded" :expanded="expanded"
class="expand-dark" class="expand-dark"
@expand="(state) => expanded = state" @expand="(state) => $emit('expand', state)"
/> />
<Expand <Expand
v-if="expanded || (expandable && scrollable)" v-if="expanded || (expandable && scrollable)"
:expanded="expanded" :expanded="expanded"
class="expand-light" class="expand-light"
@expand="(state) => expanded = state" @expand="(state) => $emit('expand', state)"
/> />
</div> </div>
</template> </template>
@ -107,6 +102,10 @@ export default {
type: Boolean, type: Boolean,
default: true, default: true,
}, },
expanded: {
type: Boolean,
default: false,
},
}, },
data() { data() {
return { return {
@ -114,7 +113,6 @@ export default {
scrollable: true, scrollable: true,
scrollAtStart: true, scrollAtStart: true,
scrollAtEnd: false, scrollAtEnd: false,
expanded: false,
}; };
}, },
mounted, mounted,

3
assets/js/config/default.js
View File

@ -16,11 +16,14 @@ export default {
'fisting', 'fisting',
'gaping', 'gaping',
'gangbang', 'gangbang',
'interracial',
'lesbian', 'lesbian',
'mff', 'mff',
'mfm', 'mfm',
'orgy', 'orgy',
'pov', 'pov',
'solo',
'squirting', 'squirting',
'swallowing',
], ],
}; };

3
src/argv.js
View File

@ -10,9 +10,10 @@ const { argv } = yargs
type: 'boolean', type: 'boolean',
alias: 'web', alias: 'web',
}) })
.option('scrape', { .option('all', {
describe: 'Scrape channels and networks defined in configuration', describe: 'Scrape channels and networks defined in configuration',
type: 'boolean', type: 'boolean',
alias: 'scrape',
}) })
.option('networks', { .option('networks', {
describe: 'Network to scrape all channels from (overrides configuration)', describe: 'Network to scrape all channels from (overrides configuration)',

9
src/media.js
View File

@ -89,6 +89,8 @@ function toBaseSource(rawSource) {
if (rawSource.referer) baseSource.referer = rawSource.referer; if (rawSource.referer) baseSource.referer = rawSource.referer;
if (rawSource.host) baseSource.host = rawSource.host; if (rawSource.host) baseSource.host = rawSource.host;
if (rawSource.attempts) baseSource.attempts = rawSource.attempts;
if (rawSource.queueMethod) baseSource.queueMethod = rawSource.queueMethod;
if (rawSource.copyright) baseSource.copyright = rawSource.copyright; if (rawSource.copyright) baseSource.copyright = rawSource.copyright;
if (rawSource.comment) baseSource.comment = rawSource.comment; if (rawSource.comment) baseSource.comment = rawSource.comment;
@ -393,6 +395,7 @@ async function fetchSource(source, baseMedia) {
stream: true, // sources are fetched in parallel, don't gobble up memory stream: true, // sources are fetched in parallel, don't gobble up memory
transforms: [hashStream], transforms: [hashStream],
destination: tempFileTarget, destination: tempFileTarget,
queueMethod: source.queueMethod || null, // use http module's default
}); });
hasher.end(); hasher.end();
@ -422,9 +425,11 @@ async function fetchSource(source, baseMedia) {
}, },
}; };
} catch (error) { } catch (error) {
logger.warn(`Failed attempt ${attempts}/3 to fetch ${source.src}: ${error.message}`); const maxAttempts = source.attempts || 3;
if (attempts < 3) { logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);
if (attempts < maxAttempts) {
await Promise.delay(1000); await Promise.delay(1000);
return attempt(attempts + 1); return attempt(attempts + 1);

65
src/scrapers/whalemember.js
View File

@ -19,13 +19,32 @@ function scrapeLatest(html, site) {
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate(); release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent); release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent);
// slow CDN?
const poster = scene.querySelector('.single-image').dataset.src; const poster = scene.querySelector('.single-image').dataset.src;
release.poster = /^http/.test(poster) ? poster : `https:${poster}`; const teaserEl = scene.querySelector('source');
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`)); release.poster = {
src: /^http/.test(poster) ? poster : `https:${poster}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
const trailerEl = scene.querySelector('source'); release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({
if (trailerEl) release.trailer = { src: trailerEl.dataset.src }; src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
referer: site.url,
attempts: 5,
queueMethod: '5s',
}));
if (teaserEl) {
release.teaser = {
src: teaserEl.dataset.src,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
return release; return release;
}); });
@ -51,16 +70,42 @@ function scrapeScene(html, site, url) {
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60; release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
} }
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => (/^http/.test(el.src) ? el.src : `https:${el.src}`)); // unreliable CDN
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => ({
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
referer: site.url,
attempts: 5,
queueMethod: '5s',
}));
const posterEl = scene.querySelector('#no-player-image'); const posterEl = scene.querySelector('#no-player-image');
const videoEl = scene.querySelector('video'); const videoEl = scene.querySelector('video');
if (posterEl) release.poster = /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`;
else if (videoEl) release.poster = /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`;
const trailerEl = scene.querySelector('#t2019-video source'); const trailerEl = scene.querySelector('#t2019-video source');
if (trailerEl) release.trailer = { src: trailerEl.src };
if (posterEl) {
release.poster = {
src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
} else if (videoEl) {
release.poster = {
src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
if (trailerEl) {
release.trailer = {
src: trailerEl.src,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
return release; return release;
} }

2
src/store-releases.js
View File

@ -46,7 +46,7 @@ function curateReleaseEntry(release, batchId, existingRelease) {
} }
async function attachChannelEntities(releases) { async function attachChannelEntities(releases) {
const releasesWithoutEntity = releases.filter(release => release.channel && !release.entity && release.entity.type !== 1); const releasesWithoutEntity = releases.filter(release => release.channel && (!release.entity || release.entity.type === 'network'));
const channelEntities = await knex('entities') const channelEntities = await knex('entities')
.select(knex.raw('entities.*, row_to_json(parents) as parent')) .select(knex.raw('entities.*, row_to_json(parents) as parent'))

17
src/utils/http.js
View File

@ -35,6 +35,7 @@ function useProxy(url) {
} }
const queue = taskQueue(); const queue = taskQueue();
const defaultQueueMethod = '20p';
async function handler({ async function handler({
url, url,
@ -44,9 +45,9 @@ async function handler({
options = {}, options = {},
}) { }) {
if (body) { if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`); logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)} ${options.queueMethod || defaultQueueMethod}`);
} else { } else {
logger.silly(`${method.toUpperCase()} ${url}`); logger.silly(`${method.toUpperCase()} ${url} ${options.queueMethod || defaultQueueMethod}`);
} }
const reqOptions = { const reqOptions = {
@ -98,8 +99,12 @@ queue.define('1s', handler, {
interval: 1, interval: 1,
}); });
async function get(url, headers, options, queueMethod = '20p') { queue.define('5s', handler, {
return queue.push(queueMethod, { interval: 5,
});
async function get(url, headers, options) {
return queue.push(options.queueMethod || defaultQueueMethod, {
method: 'GET', method: 'GET',
url, url,
headers, headers,
@ -107,8 +112,8 @@ async function get(url, headers, options, queueMethod = '20p') {
}); });
} }
async function post(url, body, headers, options, queueMethod = '20p') { async function post(url, body, headers, options) {
return queue.push(queueMethod, { return queue.push(options.queueMethod || defaultQueueMethod, {
method: 'POST', method: 'POST',
url, url,
body, body,

2
src/utils/resolve-place.js
View File

@ -12,7 +12,7 @@ async function resolvePlace(query) {
// https://operations.osmfoundation.org/policies/nominatim/ // https://operations.osmfoundation.org/policies/nominatim/
const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, { const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, {
'User-Agent': 'contact at moonloop.adult@protonmail.com', 'User-Agent': 'contact at moonloop.adult@protonmail.com',
}, null, '1s'); }, { queueMethod: '1s' });
const [item] = res.body; const [item] = res.body;