Added photo album support to Blowpass scraper.

This commit is contained in:
ThePendulum 2019-12-13 05:04:04 +01:00
parent c9e0c29d51
commit c998fcf933
8 changed files with 606 additions and 6 deletions

View File

@ -0,0 +1,117 @@
<template>
<div
class="banner"
@wheel.prevent="scrollBanner"
>
<template v-if="release.covers && release.covers.length > 0">
<a
v-for="cover in release.covers"
:key="`cover-${cover.id}`"
:href="`/media/${cover.path}`"
target="_blank"
rel="noopener noreferrer"
>
<img
:src="`/media/${cover.thumbnail}`"
class="cover"
>
</a>
</template>
<div class="trailer">
<video
v-if="release.trailer"
:src="`/media/${release.trailer.path}`"
:poster="`/media/${(release.poster && release.poster.thumbnail) || (release.photos.length && release.photos[Math.floor(Math.random() * release.photos.length)].path)}`"
:alt="release.title"
class="item trailer-video"
controls
>Sorry, the tailer cannot be played in your browser</video>
</div>
<a
v-for="photo in photos"
:key="`banner-${photo.index}`"
:href="`/media/${photo.path}`"
target="_blank"
rel="noopener noreferrer"
>
<img
:src="`/media/${photo.thumbnail}`"
:alt="`Photo ${photo.index + 1}`"
class="item"
>
</a>
</div>
</template>
<script>
function photos() {
if (this.release.photos.length) {
const set = this.release.photos.sort(({ index: indexA }, { index: indexB }) => indexA - indexB);
if (this.release.trailer) {
return set;
}
return [this.release.poster].concat(set);
}
if (this.release.poster && !this.release.trailer) {
return [this.release.poster];
}
return [];
}
function scrollBanner(event) {
event.currentTarget.scrollLeft += event.deltaY; // eslint-disable-line no-param-reassign
}
export default {
props: {
release: {
type: Object,
default: null,
},
},
computed: {
photos,
},
methods: {
scrollBanner,
},
};
</script>
<style lang="scss" scoped>
@import 'theme';
.banner {
background: $empty;
flex-shrink: 0;
white-space: nowrap;
overflow-x: auto;
scrollbar-width: none;
box-shadow: 0 0 3px $shadow;
font-size: 0;
&::-webkit-scrollbar {
display: none;
}
}
.trailer {
display: inline-block;
max-width: 100vw;
}
.trailer-video {
max-width: 100%;
}
.item {
height: 18rem;
vertical-align: middle;
}
</style>

View File

@ -0,0 +1,398 @@
<template>
<div
v-if="release"
class="content"
>
<Banner :release="release" />
<div class="info">
<div class="column">
<a
v-if="release.date"
v-tooltip.bottom="release.url && `View scene on ${release.site.name}`"
:title="release.url && `View scene on ${release.site.name}`"
:href="release.url"
:class="{ link: release.url }"
target="_blank"
rel="noopener noreferrer"
class="tidbit date"
>
<Icon
v-if="isAfter(new Date(), release.date)"
icon="calendar2"
/>
<Icon
v-else
v-tooltip.bottom="'To be released'"
icon="sun3"
/>
<span class="showable">{{ formatDate(release.date, 'MMM D, YYYY') }}</span>
<span class="hideable">{{ formatDate(release.date, 'MMMM D, YYYY') }}</span>
</a>
<span
v-if="release.shootId"
v-tooltip.bottom="`Shoot #`"
class="tidbit shoot hideable"
>
<Icon icon="clapboard-play" />
{{ release.shootId }}
</span>
<span
v-if="release.duration"
v-tooltip.bottom="`Duration`"
class="tidbit duration hideable"
>
<Icon icon="stopwatch" />
<span
v-if="release.duration >= 3600"
class="duration-segment"
>{{ Math.floor(release.duration / 3600) }}:</span>
<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
</span>
<span class="tidbit site">
<a
v-if="release.site.independent"
:href="`/network/${release.network.slug}`"
>
<img
:src="`/img/logos/${release.network.slug}/network.png`"
:title="release.network.name"
class="logo logo-site"
>
</a>
<template v-else>
<a :href="`/network/${release.network.slug}`">
<img
:src="`/img/logos/${release.network.slug}/network.png`"
:title="release.network.name"
:alt="release.network.name"
class="logo logo-network"
>
</a>
<span class="chain">presents</span>
<a :href="`/site/${release.site.slug}`">
<img
:src="`/img/logos/${release.network.slug}/${release.site.slug}.png`"
:title="release.site.name"
class="logo logo-site"
>
</a>
</template>
</span>
</div>
</div>
<div class="column">
<h2 class="row title">{{ release.title }}</h2>
<div class="row">
<ul class="actors nolist">
<li
v-for="actor in release.actors"
:key="actor.id"
class="actor"
>
<Actor :actor="actor" />
</li>
</ul>
</div>
<div
v-if="release.tags.length > 0"
class="row"
>
<Icon icon="price-tags3" />
<ul class="tags nolist">
<li
v-for="tag in release.tags"
:key="`tag-${tag.slug}`"
class="tag"
>
<a
:href="`/tag/${tag.slug}`"
class="link"
>{{ tag.name }}</a>
</li>
</ul>
</div>
<div
v-if="release.duration"
class="row duration showable"
>
<Icon icon="stopwatch" />
<span
v-if="release.duration >= 3600"
class="duration-segment"
>{{ Math.floor(release.duration / 3600) }}:</span>
<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
</div>
<p
v-if="release.description"
class="row description"
>
<Icon icon="info2" />
{{ release.description }}
</p>
<div
v-if="release.studio"
class="row"
>
<Icon icon="video-camera2" />
<a
v-if="release.studio"
:href="release.studio.url"
target="_blank"
rel="noopener noreferrer"
class="link"
>{{ release.studio.name }}</a>
</div>
<div
v-if="release.shootId"
class="row showable"
>
<Icon icon="clapboard-play" />
<a
:href="release.url"
:title="`release.shootId`"
target="_blank"
rel="noopener noreferrer"
class="link shoot"
>{{ release.shootId }}</a>
</div>
<span class="row">
<Icon icon="drawer-in" />
<a
:href="`/added/${formatDate(release.dateAdded, 'YYYY-MM-DD')}`"
:title="`Added on ${formatDate(release.dateAdded, 'MMMM D, YYYY')}`"
target="_blank"
rel="noopener noreferrer"
class="link added"
>{{ formatDate(release.dateAdded, 'MMMM D, YYYY') }}</a>
</span>
</div>
</div>
</template>
<script>
import Actor from '../tile/actor.vue';
import Banner from './banner.vue';
function pageTitle() {
return this.release && this.release.title;
}
async function mounted() {
this.release = await this.$store.dispatch('fetchReleaseById', this.$route.params.releaseId);
}
export default {
components: {
Actor,
Banner,
},
data() {
return {
release: null,
};
},
computed: {
pageTitle,
},
mounted,
};
</script>
<style lang="scss" scoped>
@import 'theme';
.column {
width: 1200px;
max-width: 100%;
padding: 0 1rem;
margin: 0 auto;
box-sizing: border-box;
}
.row {
display: flex;
align-items: center;
margin: 0 0 1rem 0;
.icon {
display: inline-block;
width: 1rem;
fill: $shadow-strong;
margin: 0 1rem 0 0;
}
}
.info {
background: $background;
margin: 0 0 1.5rem 0;
box-shadow: 0 0 3px $shadow-weak;
cursor: default;
.column {
display: flex;
align-items: center;
padding: 0 1rem;
}
}
.tidbit {
display: inline-block;
height: 100%;
&:not(:last-child) {
border-right: solid 1px $shadow-hint;
}
.icon {
fill: $shadow-strong;
margin: 0 .25rem 0 0;
}
&.date,
&.duration,
&.shoot {
flex-shrink: 0;
padding: 1.25rem 1rem 1.25rem 0;
margin: 0 1rem 0 0;
}
}
.site {
display: inline-flex;
flex-grow: 1;
align-items: center;
justify-content: flex-end;
padding: .25rem 0;
font-size: 0;
}
.logo {
display: inline-block;
filter: $logo-outline;
}
.logo-site {
height: 3rem;
max-width: 15rem;
object-fit: contain;
object-position: 100% 50%;
}
.logo-network {
height: 1.5rem;
max-width: 10rem;
object-fit: contain;
object-position: 100% 50%;
}
.chain {
color: $shadow;
padding: 0 .5rem;
font-weight: bold;
font-size: .8rem;
}
.title {
margin: 0 0 1.5rem 0;
}
.description {
line-height: 1.25;
}
.duration {
font-size: 0;
}
.duration-segment {
font-size: 1rem;
}
.actors {
display: flex;
flex-wrap: wrap;
}
.link {
display: inline-block;
color: $link;
text-decoration: none;
&:hover {
color: $primary;
.icon {
fill: $primary;
}
}
}
.tag .link {
background: $background;
display: inline-block;
padding: .5rem;
margin: 0 .25rem .25rem 0;
box-shadow: 0 0 2px $shadow-weak;
text-decoration: none;
text-transform: capitalize;
&:hover {
color: $primary;
}
}
.showable {
display: none;
}
@media(max-width: $breakpoint3) {
.logo-network,
.chain {
display: none;
}
}
@media(max-width: $breakpoint) {
.hideable {
display: none;
}
.row .showable {
display: block;
}
.tidbit .showable {
display: inline-block;
}
.logo-site {
width: 15rem;
max-width: 100%;
}
}
</style>

View File

@ -27,9 +27,21 @@
v-if="actor.age || actor.origin" v-if="actor.age || actor.origin"
class="details" class="details"
> >
<span class="age">{{ actor.age }}</span> <span v-if="actor.age">
<span
v-tooltip="`Born on ${formatDate(actor.birthdate, 'MMMM D, YYYY')}`"
class="age"
>{{ actor.age }}</span>
<span
v-if="actor.ageThen && actor.ageThen < actor.age"
v-tooltip="'Age at scene date'"
class="age-then"
>@ {{ actor.ageThen }}</span>
</span>
<span <span
v-if="actor.origin" v-if="actor.origin"
v-tooltip="`Born in ${actor.origin.country.alias || actor.origin.country.name}`"
class="country" class="country"
> >
{{ actor.origin.country.alpha2 }} {{ actor.origin.country.alpha2 }}
@ -110,4 +122,8 @@ export default {
font-size: .8rem; font-size: .8rem;
font-weight: bold; font-weight: bold;
} }
.age-then {
color: $highlight;
}
</style> </style>

View File

@ -50,7 +50,7 @@
> >
<img <img
v-else-if="release.covers" v-else-if="release.covers && release.covers.length > 0"
:src="`/media/${release.covers[0].thumbnail}`" :src="`/media/${release.covers[0].thumbnail}`"
:alt="release.title" :alt="release.title"
class="thumbnail" class="thumbnail"

View File

@ -326,6 +326,9 @@
font-size: .8rem; font-size: .8rem;
font-weight: bold; font-weight: bold;
} }
.age-then[data-v-6989dc6f] {
color: rgba(255, 255, 255, 0.5);
}
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.banner[data-v-42bb19c4] { .banner[data-v-42bb19c4] {

View File

@ -65,6 +65,7 @@ async function curateRelease(release) {
gender: actor.gender, gender: actor.gender,
birthdate: actor.birthdate, birthdate: actor.birthdate,
age: moment().diff(actor.birthdate, 'years'), age: moment().diff(actor.birthdate, 'years'),
ageThen: moment(release.date).diff(actor.birthdate, 'years'),
avatar: actor.avatar, avatar: actor.avatar,
origin: actor.birth_country_alpha2 origin: actor.birth_country_alpha2
? { ? {

View File

@ -44,6 +44,8 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a
console.log(`\x1b[90m${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases\x1b[0m`); console.log(`\x1b[90m${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases\x1b[0m`);
console.log(oldestReleaseOnPage, afterDate, moment(oldestReleaseOnPage).isAfter(afterDate));
if ( if (
uniqueReleases.length > 0 uniqueReleases.length > 0
&& (oldestReleaseOnPage || page < argv.pages) && (oldestReleaseOnPage || page < argv.pages)

View File

@ -1,11 +1,71 @@
'use strict'; 'use strict';
/* eslint-disable newline-per-chained-call */ /* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp'); const bhttp = require('bhttp');
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const moment = require('moment'); const moment = require('moment');
const { matchTags } = require('../tags'); async function fetchPhotos(url) {
const res = await bhttp.get(url);
return res.body.toString();
}
function scrapePhotos(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
return $('.preview .imgLink').toArray().map((linkEl) => {
const url = $(linkEl).attr('href');
if (url.match('/join')) {
// URL links to join page instead of full photo, extract thumbnail
const src = $(linkEl).find('img').attr('src');
if (src.match('previews/')) {
// resource often serves full photo at a modifier URL anyway, add as primary source
const highRes = src
.replace('previews/', '')
.replace('_tb.jpg', '.jpg');
// keep original thumbnail as fallback in case full photo is not available
return [highRes, src];
}
return src;
}
// URL links to full photo
return url;
});
}
async function getPhotos(albumPath, siteDomain) {
const albumUrl = `https://www.blowpass.com${albumPath}`;
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html);
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
const otherPhotos = await Promise.map(pages, async (page) => {
const pageUrl = `https://${siteDomain}${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml);
}, {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
} catch (error) {
console.error(`Failed to fetch Blowpass photos from ${albumPath}: ${error.message}`);
return [];
}
}
function scrape(html, site) { function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
@ -68,13 +128,14 @@ async function scrapeScene(html, url, site) {
const likes = Number(sceneElement.find('.rating .state_1 .value').text()); const likes = Number(sceneElement.find('.rating .state_1 .value').text());
const dislikes = Number(sceneElement.find('.rating .state_2 .value').text()); const dislikes = Number(sceneElement.find('.rating .state_2 .value').text());
const channel = $('.siteNameSpan').text().trim().toLowerCase();
const poster = playerData.picPreview; const poster = playerData.picPreview;
const trailer = `${playerData.playerOptions.host}${playerData.url}`; const trailer = `${playerData.playerOptions.host}${playerData.url}`;
const photos = await getPhotos($('.picturesItem a').attr('href'), channel, site);
const duration = moment.duration(data.duration.slice(2)).asSeconds(); const duration = moment.duration(data.duration.slice(2)).asSeconds();
const tags = data.keywords.split(', ');
const rawTags = data.keywords.split(', ');
const tags = await matchTags(rawTags);
return { return {
url, url,
@ -86,6 +147,7 @@ async function scrapeScene(html, url, site) {
date, date,
duration, duration,
poster, poster,
photos,
trailer: { trailer: {
src: trailer, src: trailer,
quality: playerData.sizeOnLoad.slice(0, -1), quality: playerData.sizeOnLoad.slice(0, -1),
@ -96,6 +158,7 @@ async function scrapeScene(html, url, site) {
dislikes, dislikes,
}, },
site, site,
channel,
}; };
} }