Using media database for images.

This commit is contained in:
ThePendulum 2019-09-10 16:48:04 +02:00
parent 02e81a8be9
commit cf4987ba48
8 changed files with 121 additions and 70 deletions

View File

@ -33,8 +33,8 @@
class="scene-link" class="scene-link"
> >
<img <img
v-if="release.photos > 0" v-if="release.photos.length > 0"
:src="`/${release.site.id}/${release.id}/1.jpg`" :src="`/${release.photos[0].file}`"
:alt="release.title" :alt="release.title"
class="scene-thumbnail" class="scene-thumbnail"
> >
@ -65,7 +65,7 @@
class="scene-actor" class="scene-actor"
> >
<a <a
:href="`/actor/${actor.id}`" :href="`/actor/${actor.slug}`"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="actor-link" class="actor-link"
@ -85,7 +85,7 @@
class="scene-tag" class="scene-tag"
> >
<a <a
:href="`/tag/${tag.tag}`" :href="`/tag/${tag.slug}`"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="tag-link" class="tag-link"
@ -100,6 +100,10 @@
</template> </template>
<script> <script>
function pageTitle() {
return '';
}
async function mounted() { async function mounted() {
this.releases = await this.$store.dispatch('fetchReleases'); this.releases = await this.$store.dispatch('fetchReleases');
} }
@ -110,6 +114,9 @@ export default {
releases: [], releases: [],
}; };
}, },
computed: {
pageTitle,
},
mounted, mounted,
}; };
</script> </script>

View File

@ -3,25 +3,34 @@
<div <div
class="banner" class="banner"
@wheel.prevent="scrollBanner" @wheel.prevent="scrollBanner"
>
<a
v-for="photo in photos"
:key="`banner-${photo.index}`"
:href="`/${photo.file}`"
target="_blank"
rel="noopener noreferrer"
> >
<img <img
v-for="i in release.photos" :src="`/${photo.file}`"
:key="`banner-${i}`" :alt="`Photo ${photo.index}`"
:src="`/${release.site.id}/${release.id}/${i}.jpg`"
:alt="`Photo ${i}`"
class="banner-item" class="banner-item"
> >
</a>
</div> </div>
<h2 class="row title">{{ release.title }}</h2> <h2 class="row title">{{ release.title }}</h2>
<span class="row">
<a <a
:href="release.url" :href="release.url"
:title="release.shootId || release.entryId" :title="release.shootId || release.entryId"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="row date" class="date"
>{{ formatDate(release.date, 'MMMM D, YYYY') }}</a> >{{ formatDate(release.date, 'MMMM D, YYYY') }}</a>
</span>
<ul class="row actors"> <ul class="row actors">
<li <li
@ -30,7 +39,7 @@
class="actor" class="actor"
> >
<a <a
:href="`/actor/${actor.id}`" :href="`/actor/${actor.slug}`"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="actor-link" class="actor-link"
@ -61,7 +70,7 @@
class="tag" class="tag"
> >
<a <a
:href="`/tag/${tag.tag}`" :href="`/tag/${tag.slug}`"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="tag-link" class="tag-link"
@ -80,10 +89,20 @@ function scrollBanner(event) {
event.currentTarget.scrollLeft += event.deltaY; // eslint-disable-line no-param-reassign event.currentTarget.scrollLeft += event.deltaY; // eslint-disable-line no-param-reassign
} }
function photos() {
if (this.release) {
if (this.release.photos[0].role === 'poster') {
return this.release.photos.slice(1);
}
return this.release.photos;
}
return [];
}
async function mounted() { async function mounted() {
[this.release] = await this.$store.dispatch('fetchReleases', this.$route.params.releaseId); [this.release] = await this.$store.dispatch('fetchReleases', this.$route.params.releaseId);
console.log(this.release);
} }
export default { export default {
@ -94,6 +113,7 @@ export default {
}, },
computed: { computed: {
pageTitle, pageTitle,
photos,
}, },
mounted, mounted,
methods: { methods: {

View File

@ -949,6 +949,13 @@ exports.seed = knex => Promise.resolve()
description: 'Your favorite girls restrained with rope, punished & trained. Hogtied is the original extreme bondage porn website. Watch top pornstars and pain sluts in brutal bondage, getting tormented, and forced to orgasm!', description: 'Your favorite girls restrained with rope, punished & trained. Hogtied is the original extreme bondage porn website. Watch top pornstars and pain sluts in brutal bondage, getting tormented, and forced to orgasm!',
network_id: networksMap['kink'], network_id: networksMap['kink'],
}, },
{
slug: 'kinkfeatures',
name: 'Kink Features',
url: 'https://www.kink.com/channel/kinkfeatures',
description: 'Curated scenes by Kink\'s very best directors.',
network_id: networksMap['kink'],
},
{ {
slug: 'kinkuniversity', slug: 'kinkuniversity',
name: 'Kink University', name: 'Kink University',

View File

@ -36,11 +36,13 @@ function curateSites(sites) {
return sites.map(site => ({ return sites.map(site => ({
id: site.id, id: site.id,
name: site.name, name: site.name,
slug: site.slug,
description: site.description, description: site.description,
url: site.url, url: site.url,
network: { network: {
id: site.network_id, id: site.network_id,
name: site.network_name, name: site.network_name,
slug: site.network_slug,
}, },
parameters: JSON.parse(site.parameters), parameters: JSON.parse(site.parameters),
})); }));
@ -48,11 +50,11 @@ function curateSites(sites) {
async function accumulateIncludedSites() { async function accumulateIncludedSites() {
if (argv.networks || argv.sites) { if (argv.networks || argv.sites) {
const networks = await knex('networks').select('id').whereIn('slug', argv.networks); const networks = await knex('networks').select('id').whereIn('slug', argv.networks || []);
const networkIds = networks.map(network => network.id); const networkIds = networks.map(network => network.id);
const rawSites = await knex('sites') const rawSites = await knex('sites')
.select('sites.*', 'networks.name as network_name') .select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug')
.whereIn('sites.slug', argv.sites || []) .whereIn('sites.slug', argv.sites || [])
.orWhereIn('network_id', networkIds) .orWhereIn('network_id', networkIds)
.leftJoin('networks', 'sites.network_id', 'networks.id'); .leftJoin('networks', 'sites.network_id', 'networks.id');
@ -62,12 +64,12 @@ async function accumulateIncludedSites() {
const included = destructConfigNetworks(config.include); const included = destructConfigNetworks(config.include);
const networks = await knex('networks').select('id').whereIn('slug', included.networks); const networks = await knex('networks').select('id').whereIn('slug', included.networks || []);
const networkIds = networks.map(network => network.id); const networkIds = networks.map(network => network.id);
const rawSites = await knex('sites') const rawSites = await knex('sites')
.select('sites.*', 'networks.name as network_name') .select('sites.*', 'networks.name as network_name')
.whereIn('sites.slug', included.sites) .whereIn('sites.slug', included.sites || [])
.orWhereIn('network_id', networkIds) .orWhereIn('network_id', networkIds)
.leftJoin('networks', 'sites.network_id', 'networks.id'); .leftJoin('networks', 'sites.network_id', 'networks.id');
@ -84,20 +86,42 @@ async function findDuplicateReleases(latestReleases, _siteId) {
} }
async function storePhotos(release, releaseEntry) { async function storePhotos(release, releaseEntry) {
const photoPath = path.join(config.photoPath, release.site.id, releaseEntry.rows[0].id.toString()); await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
await fs.mkdir(photoPath, { recursive: true }); console.log(`Storing photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
console.log(`Storing photos for (${release.site.name}, ${release.id}) "${release.title}"`); const filepaths = await Promise.map(release.photos, async (photoUrl, index) => {
await Promise.map(release.photos, async (photoUrl, index) => {
const res = await bhttp.get(photoUrl); const res = await bhttp.get(photoUrl);
await fs.writeFile(path.join(photoPath, `${index + 1}.jpg`), res.body); const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.jpg`);
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
return photoUrl; return filepath;
}, { }, {
concurrency: 2, concurrency: 2,
}); });
await knex('media').insert(filepaths.map((filepath, index) => ({
file: filepath,
mime: 'image/jpeg',
index,
domain: 'releases',
target_id: releaseEntry.id,
role: null,
})));
if (release.trailer && release.trailer.poster) {
const res = await bhttp.get(release.trailer.poster);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), 'poster.jpg');
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
await knex('media').insert({
file: filepath,
mime: 'image/jpeg',
domain: 'releases',
target_id: releaseEntry.id,
role: 'poster',
});
}
} }
async function storeReleases(releases = []) { async function storeReleases(releases = []) {
@ -112,22 +136,25 @@ async function storeReleases(releases = []) {
description: release.description, description: release.description,
// director: release.director, // director: release.director,
duration: release.duration, duration: release.duration,
photos: release.photos ? release.photos.length : 0, // photos: release.photos ? release.photos.length : 0,
likes: release.rating && release.rating.likes, likes: release.rating && release.rating.likes,
dislikes: release.rating && release.rating.dislikes, dislikes: release.rating && release.rating.dislikes,
rating: release.rating && release.rating.stars, rating: release.rating && release.rating.stars,
}; };
console.log(`Storing (${release.site.name}, ${release.id}) "${release.title}"`);
const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`; const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`;
const releaseEntry = await knex.raw(releaseQuery); const releaseEntry = await knex.raw(releaseQuery);
console.log(`Stored (${release.site.name}, ${releaseEntry.rows[0].id}) "${release.title}"`);
if (release.actors && release.actors.length > 0) { if (release.actors && release.actors.length > 0) {
const actors = await knex('actors').whereIn('name', release.actors); const actors = await knex('actors').whereIn('name', release.actors);
const newActors = release.actors.filter(actorName => !actors.some(actor => actor.name === actorName)); const newActors = release.actors.filter(actorName => !actors.some(actor => actor.name === actorName));
const { rows: insertedActors } = newActors.length const { rows: insertedActors } = newActors.length
? await knex.raw(`${knex('actors').insert(newActors.map(actorName => ({ name: actorName })))} ON CONFLICT DO NOTHING RETURNING *`) ? await knex.raw(`${knex('actors').insert(newActors.map(actorName => ({
name: actorName,
slug: actorName.toLowerCase().replace(/\s+/g, '-'),
})))} ON CONFLICT DO NOTHING RETURNING *`)
: { rows: [] }; : { rows: [] };
await knex('actors_associated').insert(actors.concat(insertedActors).map(actor => ({ await knex('actors_associated').insert(actors.concat(insertedActors).map(actor => ({
@ -144,7 +171,7 @@ async function storeReleases(releases = []) {
} }
if (release.photos && release.photos.length > 0) { if (release.photos && release.photos.length > 0) {
await storePhotos(release, releaseEntry); await storePhotos(release, releaseEntry.rows[0]);
} }
}, { }, {
concurrency: 2, concurrency: 2,
@ -184,12 +211,8 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page
async function fetchReleases() { async function fetchReleases() {
const sites = await accumulateIncludedSites(); const sites = await accumulateIncludedSites();
console.log(sites);
return;
const scenesPerSite = await Promise.map(sites, async (site) => { const scenesPerSite = await Promise.map(sites, async (site) => {
const scraper = scrapers[site.id] || scrapers[site.network.id]; const scraper = scrapers[site.slug] || scrapers[site.network.slug];
if (scraper) { if (scraper) {
try { try {

View File

@ -11,20 +11,10 @@ async function findSite(url) {
const { hostname } = new URL(url); const { hostname } = new URL(url);
const domain = hostname.replace(/^www./, ''); const domain = hostname.replace(/^www./, '');
/*
const site = await knex('sites') const site = await knex('sites')
.where({ url: `${protocol}//www.${domain}` }) .select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug')
.orWhere({ url: `${protocol}//${domain}` }) .where('sites.url', 'like', `%${domain}`)
.first() .leftJoin('networks', 'sites.network_id', 'networks.id')
// scene might use generic network URL, let network scraper determine channel site
|| await knex('networks')
.where({ url: `${protocol}//www.${hostname}` })
.orWhere({ url: `${protocol}//${hostname}` })
.first();
*/
const site = await knex('sites')
.where('url', 'like', `%${domain}`)
.first() .first()
// scene might use generic network URL, let network scraper determine channel site // scene might use generic network URL, let network scraper determine channel site
|| await knex('networks') || await knex('networks')
@ -34,10 +24,12 @@ async function findSite(url) {
return { return {
id: site.id, id: site.id,
name: site.name, name: site.name,
slug: site.slug,
description: site.description, description: site.description,
url: site.url, url: site.url,
network: { network: {
id: site.network_id || site.id, id: site.network_id || site.id,
slug: site.network_slug || site.slug,
}, },
parameters: site.parameters && JSON.parse(site.parameters), parameters: site.parameters && JSON.parse(site.parameters),
isFallback: site.network_id === undefined, isFallback: site.network_id === undefined,
@ -104,7 +96,7 @@ async function storeRelease(release) {
async function fetchScene(url) { async function fetchScene(url) {
const site = await findSite(url); const site = await findSite(url);
const scraper = scrapers[site.id] || scrapers[site.network.id]; const scraper = scrapers[site.slug] || scrapers[site.network.slug];
if (!scraper) { if (!scraper) {
throw new Error('Could not find scraper for URL'); throw new Error('Could not find scraper for URL');

View File

@ -3,15 +3,19 @@
const knex = require('./knex'); const knex = require('./knex');
async function curateRelease(release) { async function curateRelease(release) {
const actors = await knex('actors_associated') const [actors, tags, media] = await Promise.all([
.select('actors.id', 'actors.name', 'actors.gender') knex('actors_associated')
.select('actors.id', 'actors.name', 'actors.gender', 'actors.slug')
.where({ release_id: release.id }) .where({ release_id: release.id })
.leftJoin('actors', 'actors.id', 'actors_associated.actor_id'); .leftJoin('actors', 'actors.id', 'actors_associated.actor_id'),
knex('tags_associated')
const tags = await knex('tags_associated') .select('tags.tag', 'tags.slug')
.select('tags.tag', 'tags.capitalization')
.where({ release_id: release.id }) .where({ release_id: release.id })
.leftJoin('tags', 'tags.tag', 'tags_associated.tag_id'); .leftJoin('tags', 'tags.id', 'tags_associated.tag_id'),
knex('media')
.where({ target_id: release.id })
.orderBy('role'),
]);
return { return {
id: release.id, id: release.id,
@ -24,7 +28,7 @@ async function curateRelease(release) {
actors, actors,
director: release.director, director: release.director,
tags, tags,
photos: release.photos, photos: media,
rating: { rating: {
likes: release.likes, likes: release.likes,
dislikes: release.dislikes, dislikes: release.dislikes,
@ -58,8 +62,6 @@ async function fetchReleases(releaseId) {
.orderBy('date', 'desc') .orderBy('date', 'desc')
.limit(100); .limit(100);
// console.log(curateReleases(releases));
return curateReleases(releases); return curateReleases(releases);
} }

View File

@ -73,7 +73,7 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [channelSite, tags] = await Promise.all([ const [channelSite, tags] = await Promise.all([
knex('sites').where({ id: sitename }).first(), knex('sites').where({ slug: sitename }).first(),
matchTags(rawTags), matchTags(rawTags),
]); ]);

View File

@ -5,7 +5,7 @@ const knex = require('./knex');
async function matchTags(rawTags) { async function matchTags(rawTags) {
const tagEntries = await knex('tags').whereIn('tags.tag', rawTags.map(tag => tag.toLowerCase())); const tagEntries = await knex('tags').whereIn('tags.tag', rawTags.map(tag => tag.toLowerCase()));
return Array.from(new Set(tagEntries.map((tag => tag.alias_for || tag.tag)).sort())); // reduce to tag name and filter duplicates return Array.from(new Set(tagEntries.map((tag => tag.alias_for || tag.id)).sort())); // reduce to tag name and filter duplicates
} }
module.exports = { matchTags }; module.exports = { matchTags };