Merge branch 'experimental' into master

This commit is contained in:
DebaucheryLibrarian 2020-08-21 03:56:01 +02:00
commit c7dc7de0c3
30 changed files with 486 additions and 268 deletions

View File

@ -82,8 +82,8 @@ export default {
<style lang="scss" scoped>
.home {
flex-grow: 1;
display: flex;
flex-direction: column;
flex-grow: 1;
}
</style>

View File

@ -0,0 +1,126 @@
<template>
<ul class="clips nolist">
<li
v-for="clip in clips"
:key="`clip-${clip.id}`"
class="clip"
>
<div class="clip-poster-container">
<a
v-if="clip.poster"
:href="`/media/${clip.poster.path}`"
target="_blank"
rel="noopener noreferrer"
>
<img
:src="`/media/${clip.poster.thumbnail}`"
class="clip-poster"
>
</a>
<span
v-if="clip.duration"
class="clip-duration"
>{{ formatDuration(clip.duration) }}</span>
</div>
<div class="clip-info">
<h3
v-if="clip.title"
class="clip-row clip-title"
:title="clip.title"
>{{ clip.title }}</h3>
<p class="clip-row clip-description">{{ clip.description }}</p>
<Tags
:tags="clip.tags"
class="clip-row clip-tags"
/>
</div>
</li>
</ul>
</template>
<script>
import Tags from './tags.vue';
export default {
components: {
Tags,
},
props: {
clips: {
type: Array,
default: () => [],
},
},
};
</script>
<style lang="scss" scoped>
@import 'breakpoints';
.clips {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(15rem, 1fr));
grid-gap: 1rem;
}
.clip {
background: var(--background);
box-shadow: 0 0 3px var(--shadow-weak);
margin: 0 0 .5rem 0;
}
.clip-poster-container {
position: relative;
margin: 0 0 1rem 0;
}
.clip-poster {
width: 100%;
height: 10rem;
object-fit: cover;
object-position: center;
}
.clip-duration {
background: var(--darken);
color: var(--text-light);
display: block;
position: absolute;
top: 0;
right: 0;
padding: .25rem .5rem;
font-size: .9rem;
font-weight: bold;
text-shadow: 0 0 2px var(--darken-strong);
}
.clip-info {
padding: 0 1rem;
}
.clip-row {
margin: 0 0 .75rem 0;
}
.clip-title {
padding: 0;
font-size: 1rem;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.clip-description {
line-height: 1.5;
}
@media(max-width: $breakpoint-micro) {
.clips {
grid-template-columns: repeat(auto-fill, minmax(12rem, 1fr));
}
}
</style>

View File

@ -3,7 +3,6 @@
<div class="column">
<div class="tidbits">
<a
v-if="release.date"
:title="release.url && `View scene on ${release.entity.name}`"
:href="release.url"
:class="{ link: release.url }"
@ -11,8 +10,8 @@
rel="noopener noreferrer"
class="tidbit date nolink"
>
<span class="date-compact">{{ formatDate(release.date, 'MMM D, YYYY', release.datePrecision) }}</span>
<span class="date-full">{{ formatDate(release.date, 'MMMM D, YYYY', release.datePrecision) }}</span>
<span class="date-compact">{{ release.date ? formatDate(release.date, 'MMM D, YYYY', release.datePrecision) : 'Date N/A' }}</span>
<span class="date-full">{{ release.date ? formatDate(release.date, 'MMMM D, YYYY', release.datePrecision) : 'Date unknown' }}</span>
<Icon
v-if="release.url"

View File

@ -105,20 +105,22 @@ function sfw() {
}
function photos() {
const photosWithChapterPosters = (this.release.photos || []).concat(this.release.chapters ? this.release.chapters.map(chapter => chapter.poster) : []);
const clipPostersById = (this.release.clips || []).reduce((acc, clip) => ({ ...acc, [clip.poster.id]: clip.poster }), {});
const uniqueClipPosters = Array.from(new Set(this.release.clips.map(clip => clip.poster.id) || [])).map(posterId => clipPostersById[posterId]);
const photosWithClipPosters = (this.release.photos || []).concat(uniqueClipPosters);
if (this.release.trailer || this.release.teaser) {
// poster will be on trailer video
return photosWithChapterPosters;
return photosWithClipPosters;
}
if (this.release.poster) {
// no trailer, add poster to photos
return [this.release.poster].concat(this.release.photos).concat(photosWithChapterPosters);
return [this.release.poster].concat(this.release.photos).concat(photosWithClipPosters);
}
// no poster available
return photosWithChapterPosters;
return photosWithClipPosters;
}
export default {

View File

@ -87,10 +87,17 @@
</span>
<div class="labels">
<span
v-if="release.shootId"
<router-link
v-if="release.shootId && release.studio"
:to="{ name: 'studio', params: { entitySlug: release.studio.slug } }"
:title="release.studio && release.studio.name"
class="shoot"
class="shoot nolink"
>{{ release.shootId }}</router-link>
<span
v-else-if="release.shootId"
:title="release.studio && release.studio.name"
class="shoot nolink"
>{{ release.shootId }}</span>
<ul

View File

@ -99,51 +99,6 @@
<p class="description">{{ release.description }}</p>
</div>
<ul
v-if="release.chapters && release.chapters.length > 0"
class="chapters row nolist"
>
<span class="row-label">Chapters</span>
<li
v-for="chapter in release.chapters"
:key="`chapter-${chapter.id}`"
class="chapter"
>
<a
v-if="chapter.poster"
:href="`/media/${chapter.poster.path}`"
target="_blank"
rel="noopener noreferrer"
class="chapter-poster-link"
>
<img
:src="`/media/${chapter.poster.thumbnail}`"
class="chapter-poster"
>
</a>
<div class="chapter-info">
<div class="chapter-header">
<h3 class="chapter-title">{{ chapter.title }}</h3>
<span
v-if="chapter.duration"
class="chapter-duration"
>{{ chapter.duration }}</span>
</div>
<p
v-if="chapter.description"
class="chapter-description"
>{{ chapter.description }}</p>
<ul class="nolist"><li
v-for="tag in chapter.tags"
:key="`chapter-tag-${tag.id}`"
>{{ tag.name }}</li></ul>
</div>
</li>
</ul>
<div class="row row-tidbits">
<div
v-if="release.duration"
@ -151,14 +106,7 @@
>
<span class="row-label">Duration</span>
<div class="duration">
<span
v-if="release.duration >= 3600"
class="duration-segment"
>{{ Math.floor(release.duration / 3600).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ Math.floor((release.duration % 3600) / 60).toString().padStart(2, '0') }}:</span>
<span class="duration-segment">{{ (release.duration % 60).toString().padStart(2, '0') }}</span>
</div>
<div class="duration">{{ formatDuration(release.duration) }}</div>
</div>
<div
@ -216,6 +164,15 @@
</div>
</div>
<div
v-if="release.clips && release.clips.length > 0"
class="row nolist"
>
<span class="row-label">Clips</span>
<Clips :clips="release.clips" />
</div>
<div
v-if="release.comment"
class="row"
@ -241,6 +198,7 @@
import Media from './media.vue';
import Details from './details.vue';
import Tags from './tags.vue';
import Clips from './clips.vue';
import Actor from '../actors/tile.vue';
import Scroll from '../scroll/scroll.vue';
import Expand from '../expand/expand.vue';
@ -262,6 +220,7 @@ export default {
Media,
Scroll,
Expand,
Clips,
Tags,
},
data() {
@ -346,14 +305,6 @@ export default {
margin: -.25rem 0 0 0;
}
.duration {
font-size: 0;
}
.duration-segment {
font-size: 1rem;
}
.actors {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(10rem, 1fr));
@ -395,36 +346,6 @@ export default {
text-overflow: ellipsis;
}
.chapter {
display: flex;
background: var(--background);
box-shadow: 0 0 3px var(--shadow-weak);
margin: 0 0 .5rem 0;
}
.chapter-poster {
width: 12rem;
height: 100%;
object-fit: cover;
object-position: center;
}
.chapter-info {
flex-grow: 1;
padding: 1rem 1rem .5rem 1rem;
}
.chapter-header {
display: flex;
justify-content: space-between;
}
.chapter-title {
display: inline-block;
padding: 0;
margin: 0;
}
.flag {
height: 1rem;
margin: 0 0 -.15rem .1rem;

View File

@ -103,6 +103,7 @@ async function mounted() {
'femdom',
],
toys: [
'anal-toys',
'double-dildo',
'double-dildo-blowjob',
],

View File

@ -69,7 +69,7 @@ function curateRelease(release) {
if (release.scenes) curatedRelease.scenes = release.scenes.map(({ scene }) => curateRelease(scene));
if (release.movies) curatedRelease.movies = release.movies.map(({ movie }) => curateRelease(movie));
if (release.chapters) curatedRelease.chapters = release.chapters.map(chapter => curateRelease(chapter));
if (release.clips) curatedRelease.clips = release.clips.map(clip => curateRelease(clip));
if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media);
if (release.trailer) curatedRelease.trailer = release.trailer.media;

29
assets/js/format.js Normal file
View File

@ -0,0 +1,29 @@
import dayjs from 'dayjs';
export function formatDuration(duration, forceHours) {
const hours = Math.floor(duration / 3600);
const minutes = Math.floor((duration % 3600) / 60);
const seconds = Math.floor(duration % 60);
const [formattedHours, formattedMinutes, formattedSeconds] = [hours, minutes, seconds].map(segment => segment.toString().padStart(2, '0'));
if (duration >= 3600 || forceHours) {
return `${formattedHours}:${formattedMinutes}:${formattedSeconds}`;
}
return `${formattedMinutes}:${formattedSeconds}`;
}
export function formatDate(date, format = 'MMMM D, YYYY', precision = 'day') {
if (precision === 'year') {
const newFormat = format.match(/Y+/);
return dayjs(date).format(newFormat ? newFormat[0] : 'YYYY');
}
if (precision === 'month') {
const newFormat = format.match(/(M{1,4})|(Y{2,4})/g);
return dayjs(date).format(newFormat ? newFormat.join(' ') : 'MMMM YYYY');
}
return dayjs(date).format(format);
}

View File

@ -165,6 +165,7 @@ const releaseTrailerFragment = `
const releaseTeaserFragment = `
teaser: releasesTeaserByReleaseId {
media {
id
index
path
thumbnail
@ -255,20 +256,21 @@ const releaseFragment = `
${releaseTrailerFragment}
${releaseTeaserFragment}
${siteFragment}
chapters {
clips {
id
title
description
duration
tags: chaptersTags {
tags: clipsTags {
tag {
id
name
slug
}
}
poster: chaptersPosterByChapterId {
poster: clipsPosterByClipId {
media {
id
index
path
thumbnail
@ -297,6 +299,7 @@ const releaseFragment = `
slug
covers: moviesCovers {
media {
id
index
path
thumbnail

View File

@ -5,29 +5,16 @@ import dayjs from 'dayjs';
import router from './router';
import initStore from './store';
import initUiObservers from './ui/observers';
import { formatDate, formatDuration } from './format';
import '../css/style.scss';
import Container from '../components/container/container.vue';
import Icon from '../components/icon/icon.vue';
import Footer from '../components/footer/footer.vue';
function formatDate(date, format = 'MMMM D, YYYY', precision = 'day') {
if (precision === 'year') {
const newFormat = format.match(/Y+/);
return dayjs(date).format(newFormat ? newFormat[0] : 'YYYY');
}
if (precision === 'month') {
const newFormat = format.match(/(M{1,4})|(Y{2,4})/g);
return dayjs(date).format(newFormat ? newFormat.join(' ') : 'MMMM YYYY');
}
return dayjs(date).format(format);
}
function init() {
const store = initStore(router);
@ -54,6 +41,7 @@ function init() {
},
methods: {
formatDate,
formatDuration,
isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB),
isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB),
},

View File

@ -853,7 +853,7 @@ exports.up = knex => Promise.resolve()
.references('id')
.inTable('media');
}))
.then(() => knex.schema.createTable('chapters', (table) => {
.then(() => knex.schema.createTable('clips', (table) => {
table.increments('id', 16);
table.integer('release_id', 12)
@ -861,9 +861,9 @@ exports.up = knex => Promise.resolve()
.inTable('releases')
.notNullable();
table.integer('chapter', 6);
table.integer('clip', 6);
table.unique(['release_id', 'chapter']);
table.unique(['release_id', 'clip']);
table.text('title');
table.text('description');
@ -882,44 +882,44 @@ exports.up = knex => Promise.resolve()
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('chapters_posters', (table) => {
table.integer('chapter_id', 16)
.then(() => knex.schema.createTable('clips_posters', (table) => {
table.integer('clip_id', 16)
.notNullable()
.references('id')
.inTable('chapters');
.inTable('clips');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media');
table.unique('chapter_id');
table.unique('clip_id');
}))
.then(() => knex.schema.createTable('chapters_photos', (table) => {
table.integer('chapter_id', 16)
.then(() => knex.schema.createTable('clips_photos', (table) => {
table.integer('clip_id', 16)
.notNullable()
.references('id')
.inTable('chapters');
.inTable('clips');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media');
table.unique(['chapter_id', 'media_id']);
table.unique(['clip_id', 'media_id']);
}))
.then(() => knex.schema.createTable('chapters_tags', (table) => {
.then(() => knex.schema.createTable('clips_tags', (table) => {
table.integer('tag_id', 12)
.notNullable()
.references('id')
.inTable('tags');
table.integer('chapter_id', 16)
table.integer('clip_id', 16)
.notNullable()
.references('id')
.inTable('chapters');
.inTable('clips');
table.unique(['tag_id', 'chapter_id']);
table.unique(['tag_id', 'clip_id']);
}))
// SEARCH
.then(() => { // eslint-disable-line arrow-body-style
@ -1100,9 +1100,9 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS movies_scenes CASCADE;
DROP TABLE IF EXISTS movies_trailers CASCADE;
DROP TABLE IF EXISTS chapters_tags CASCADE;
DROP TABLE IF EXISTS chapters_posters CASCADE;
DROP TABLE IF EXISTS chapters_photos CASCADE;
DROP TABLE IF EXISTS clips_tags CASCADE;
DROP TABLE IF EXISTS clips_posters CASCADE;
DROP TABLE IF EXISTS clips_photos CASCADE;
DROP TABLE IF EXISTS batches CASCADE;
@ -1122,7 +1122,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS tags_posters CASCADE;
DROP TABLE IF EXISTS tags_photos CASCADE;
DROP TABLE IF EXISTS movies CASCADE;
DROP TABLE IF EXISTS chapters CASCADE;
DROP TABLE IF EXISTS clips CASCADE;
DROP TABLE IF EXISTS releases CASCADE;
DROP TABLE IF EXISTS actors CASCADE;
DROP TABLE IF EXISTS directors CASCADE;

Binary file not shown.

After

Width:  |  Height:  |  Size: 372 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 932 KiB

After

Width:  |  Height:  |  Size: 1019 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.1 KiB

After

Width:  |  Height:  |  Size: 6.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 34 KiB

After

Width:  |  Height:  |  Size: 26 KiB

View File

@ -58,7 +58,7 @@ const groups = [
const tags = [
{
name: '3d',
name: '3D',
slug: '3d',
description: 'Available in 3D.',
},

View File

@ -2649,7 +2649,7 @@ const sites = [
{
slug: 'inthecrack',
name: 'InTheCrack',
url: 'https://inthecrack.com/',
url: 'https://inthecrack.com',
},
// INTERRACIAL PASS
{

View File

@ -146,6 +146,12 @@ const studios = [
url: 'https://www.legalporno.com/studios/kinky-sex',
parent: 'legalporno',
},
{
slug: 'sexyangelproductions',
name: 'Sexy Angel Productions',
url: 'https://www.legalporno.com/studios/sexy-angel-productions',
parent: 'legalporno',
},
{
slug: 'nfstudio',
name: 'N&F Studio',

View File

@ -589,6 +589,7 @@ const tagPosters = [
['airtight', 6, 'Remy Lacroix in "Ass Worship 14" for Jules Jordan'],
['anal', 0, 'Adriana Chechik in "Manuel Creampies Their Asses 3" for Jules Jordan'],
['anal-creampie', 1, 'Aleska Diamond in "Aleska Wants More" for Asshole Fever'],
['anal-toys', 0, 'Kira Noir in 1225 for InTheCrack'],
['ass-eating', 0, 'Angelica Heart and Leanna Sweet in "ATM Bitches" for Asshole Fever'],
['asian', 0, 'Jade Kush for Erotica X'],
['atm', 2, 'Jureka Del Mar in "Stretched Out" for Her Limit'],

View File

@ -38,10 +38,10 @@ async function init() {
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const sceneMovies = deepScenes && deepScenes.map(scene => scene.movie).filter(Boolean);
const deepMovies = await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]);
const deepMovies = argv.sceneMovies && await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]);
const movieScenes = deepMovies.map(movie => movie.scenes).flat().filter(Boolean);
const deepMovieScenes = await fetchScenes(movieScenes);
const movieScenes = deepMovies.map(movie => movie.scenes.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean);
const deepMovieScenes = argv.movieScenes ? await fetchScenes(movieScenes) : movieScenes;
if (argv.inspect) {
console.log(util.inspect(deepScenes));
@ -52,7 +52,7 @@ async function init() {
if (deepMovies.length > 0) {
const storedMovieScenes = await storeScenes(deepMovieScenes);
await storeMovies(deepMovies, storedMovieScenes);
await storeMovies(deepMovies, storedMovieScenes || []);
}
if (deepScenes.length > 0 || deepMovieScenes.length > 0) {

View File

@ -67,6 +67,7 @@ const { argv } = yargs
describe: 'Fetch all scenes for an actor',
type: 'boolean',
default: false,
alias: 'actor-scenes',
})
.option('actors-sources', {
describe: 'Use these scrapers for actor data',

View File

@ -135,6 +135,10 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
} catch (error) {
logger.error(`Deep scrape failed for ${baseRelease.url}: ${error.message}`);
if (argv.debug) {
console.error(error);
}
if (error.code === 'NO_ENTRY_ID') {
return null;
}

View File

@ -4,17 +4,19 @@ const moment = require('moment');
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
const { feetInchesToCm, lbsToKg } = require('../utils/convert');
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/Collection\/(\d+)/)[1];
// release.entryId = new URL(release.url).pathname.match(/\/Collection\/(\d+)/)[1]; can't be matched with upcoming scenes
release.shootId = query.cnt('a span:nth-of-type(1)').match(/^\d+/)?.[0];
release.date = query.date('a span:nth-of-type(2)', 'YYYY-MM-DD');
release.entryId = release.shootId;
release.date = query.date('a span:nth-of-type(2)', 'YYYY-MM-DD');
release.actors = (query.q('a img', 'alt') || query.cnt('a span:nth-of-type(1)'))?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g);
release.poster = release.shootId
@ -25,20 +27,152 @@ function scrapeAll(scenes, channel) {
});
}
function scrapeScene({ query, html }, url, channel) {
function scrapeUpcoming(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.entryId = new URL(url).pathname.match(/\/Collection\/(\d+)/)[1];
release.shootId = query.cnt('h2 span').match(/^\d+/)?.[0];
const title = query.cnt('span');
release.actors = query.cnt('h2 span')?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g);
release.entryId = title.match(/^\d+/)[0];
release.actors = title.slice(0, title.indexOf('-')).match(/[a-zA-Z]+(\s[a-zA-Z]+)*/g);
const date = moment.utc(title.match(/\w+ \d+\w+$/)[0], 'MMM Do');
if (date.isBefore()) {
// date is next year
release.date = date.add(1, 'year').toDate();
} else {
release.date = date.toDate();
}
release.poster = [
`https://inthecrack.com/assets/images/posters/collections/${release.entryId}.jpg`,
query.img('img', 'src', { origin: channel.url }),
];
return release;
});
}
function scrapeProfileScenes(items, actorName, channel) {
return items.map(({ query }) => {
const release = {};
if (slugify(query.cnt()) === 'no-other-collections') {
return null;
}
const details = query.cnts('figure p').reduce((acc, info) => {
const [key, value] = info.split(':');
return {
...acc,
[slugify(key, '_')]: value?.trim(),
};
}, {});
release.url = query.url('a', 'href', { origin: channel.url });
release.shootId = details.collection.match(/\d+/)[0];
release.entryId = release.shootId;
release.date = qu.parseDate(details.release_date, 'YYYY-MM-DD');
release.actors = [actorName];
/* rely on clip length
const durationString = Object.keys(details).find(info => /\d+_min_video/.test(info));
release.duration = durationString && Number(durationString.match(/^\d+/)?.[0]) * 60;
*/
release.productionLocation = details.shoot_location;
release.poster = [
`https://inthecrack.com/assets/images/posters/collections/${release.entryId}.jpg`,
query.img('img', 'src', { origin: channel.url }),
];
return release;
}).filter(Boolean);
}
function scrapeProfile({ query }, actorName, actorAvatar, channel, releasesFromScene) {
const profile = {};
const bio = query.cnts(releasesFromScene ? 'ul li' : 'div.modelInfo li').reduce((acc, info) => {
const [key, value] = info.split(':');
return {
...acc,
[slugify(key, '_')]: value.trim(),
};
}, {});
profile.name = actorName || bio.name;
profile.gender = 'female';
profile.birthPlace = bio.nationality;
if (bio.height) profile.height = feetInchesToCm(bio.height);
if (bio.weight) profile.weight = lbsToKg(bio.weight);
profile.releases = releasesFromScene?.[profile.name] || scrapeProfileScenes(qu.initAll(query.all('.Models li')), actorName, channel);
// avatar is the poster of a scene, find scene and use its high quality poster instead
const avatarRelease = profile.releases.find(release => new URL(release.poster[1]).pathname === new URL(actorAvatar).pathname);
profile.avatar = avatarRelease?.poster[0];
return profile;
}
async function fetchSceneActors(entryId, _release, channel) {
const url = `https://inthecrack.com/Collection/Biography/${entryId}`;
const res = await qu.get(url);
if (res.ok) {
const actorTabs = qu.initAll(res.item.query.all('#ModelTabs li')).map(({ query }) => ({
name: query.cnt('a'),
id: query.q('a', 'data-model'),
}));
const actorReleasesByActorName = actorTabs.reduce((acc, { name, id }) => {
const releaseEls = qu.initAll(res.item.query.all(`#Model-${id} li`));
const releases = scrapeProfileScenes(releaseEls, name, channel);
return {
...acc,
[name]: releases,
};
}, {});
const actors = qu.initAll(res.item.query.all('.modelInfo > li')).map((item) => {
const avatar = item.query.img('img', 'src', { origin: channel.url });
const profile = scrapeProfile(item, null, avatar, channel, actorReleasesByActorName);
return profile;
});
return actors;
}
return null;
}
async function scrapeScene({ query, html }, url, channel) {
const release = {};
const entryId = new URL(url).pathname.match(/\/Collection\/(\d+)/)[1];
release.shootId = query.cnt('h2 span').match(/^\d+/)?.[0];
release.entryId = release.shootId; // site entry ID can't be matched with upcoming scenes
const actors = await fetchSceneActors(entryId, release, channel);
release.actors = actors || query.cnt('h2 span')?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g);
release.description = query.cnt('p#CollectionDescription');
release.productionLocation = query.cnt('.modelCollectionHeader p')?.match(/Shoot Location: (.*)/)?.[1];
release.poster = qu.prefixUrl(html.match(/background-image: url\('(.*)'\)/)?.[1], channel.url);
release.chapters = query.all('.ClipOuter').map((el) => {
release.clips = query.all('.ClipOuter').map((el) => {
const chapter = {};
chapter.title = query.text(el, 'h4');
@ -67,22 +201,6 @@ function scrapeScene({ query, html }, url, channel) {
return release;
}
function scrapeProfile({ query, el }, actorName, entity, include) {
const profile = {};
profile.description = query.cnt('.bio-text');
profile.birthPlace = query.cnt('.birth-place span');
profile.avatar = query.img('.actor-photo img');
if (include.releases) {
return scrapeAll(qu.initAll(el, '.scene'));
}
console.log(profile);
return profile;
}
async function fetchLatest(channel, page = 1) {
const year = moment().subtract(page - 1, ' year').year();
@ -96,6 +214,16 @@ async function fetchLatest(channel, page = 1) {
return res.status;
}
async function fetchUpcoming(channel) {
const res = await qu.getAll(channel.url, '#ComingSoon li');
if (res.ok) {
return scrapeUpcoming(res.items, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
@ -106,12 +234,27 @@ async function fetchScene(url, channel) {
return res.status;
}
async function fetchProfile({ name: actorName }, entity, include) {
const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
const res = await qu.get(url);
async function fetchProfile({ name: actorName }, channel, _include) {
const firstLetter = actorName.charAt(0).toUpperCase();
const url = `${channel.url}/Collections/Name/${firstLetter}`;
const res = await qu.getAll(url, '.collectionGridLayout li');
if (res.ok) {
return scrapeProfile(res.item, actorName, entity, include);
const actorItem = res.items.find(({ query }) => slugify(query.cnt('span')) === slugify(actorName));
if (actorItem) {
const actorUrl = actorItem.query.url('a', 'href', { origin: channel.url });
const actorAvatar = actorItem.query.img('img', 'src', { origin: channel.url });
const actorRes = await qu.get(actorUrl);
if (actorRes.ok) {
return scrapeProfile(actorRes.item, actorName, actorAvatar, channel);
}
return actorRes.status;
}
return null;
}
return res.status;
@ -119,6 +262,7 @@ async function fetchProfile({ name: actorName }, entity, include) {
module.exports = {
fetchLatest,
fetchUpcoming,
fetchScene,
// fetchProfile,
fetchProfile,
};

View File

@ -197,6 +197,7 @@ module.exports = {
iconmale,
interracialpass: hush,
interracialpovs: hush,
inthecrack,
jamesdeen: fullpornnetwork,
julesjordan,
kellymadison,

View File

@ -49,7 +49,10 @@ async function getTrailer(scene, site, url) {
file: scene.previewVideoUrl1080P,
sizes: qualities.join('+'),
type: 'trailer',
}, { referer: url });
}, {
referer: url,
origin: site.url,
});
if (!tokenRes.ok) {
return null;

View File

@ -243,44 +243,44 @@ async function updateReleasesSearch(releaseIds) {
}
}
async function storeChapters(releases) {
const chapters = releases.map(release => release.chapters?.map((chapter, index) => ({
title: chapter.title,
description: chapter.description,
async function storeClips(releases) {
const clips = releases.map(release => release.clips?.map((clip, index) => ({
title: clip.title,
description: clip.description,
releaseId: release.id,
chapter: index + 1,
duration: chapter.duration,
poster: chapter.poster,
photos: chapter.photos,
tags: chapter.tags,
clip: index + 1,
duration: clip.duration,
poster: clip.poster,
photos: clip.photos,
tags: clip.tags,
}))).flat().filter(Boolean);
const curatedChapterEntries = chapters.map(chapter => ({
title: chapter.title,
description: chapter.description,
duration: chapter.duration,
release_id: chapter.releaseId,
chapter: chapter.chapter,
const curatedClipEntries = clips.map(clip => ({
title: clip.title,
description: clip.description,
duration: clip.duration,
release_id: clip.releaseId,
clip: clip.clip,
}));
const storedChapters = await bulkInsert('chapters', curatedChapterEntries);
const chapterIdsByReleaseIdAndChapter = storedChapters.reduce((acc, chapter) => ({
const storedClips = await bulkInsert('clips', curatedClipEntries, ['release_id', 'clip']);
const clipIdsByReleaseIdAndClip = storedClips.reduce((acc, clip) => ({
...acc,
[chapter.release_id]: {
...acc[chapter.release_id],
[chapter.chapter]: chapter.id,
[clip.release_id]: {
...acc[clip.release_id],
[clip.clip]: clip.id,
},
}), {});
const chaptersWithId = chapters.map(chapter => ({
...chapter,
id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.chapter],
const clipsWithId = clips.map(clip => ({
...clip,
id: clipIdsByReleaseIdAndClip[clip.releaseId][clip.clip],
}));
await associateReleaseTags(chaptersWithId, 'chapter');
await associateReleaseTags(clipsWithId, 'clip');
// media is more error-prone, associate separately
await associateReleaseMedia(chaptersWithId, 'chapter');
await associateReleaseMedia(clipsWithId, 'clip');
}
async function storeScenes(releases) {
@ -318,7 +318,7 @@ async function storeScenes(releases) {
await scrapeActors(actors.map(actor => actor.name));
}
await storeChapters(releasesWithId);
await storeClips(releasesWithId);
logger.info(`Stored ${storedReleaseEntries.length} releases`);

View File

@ -37,18 +37,16 @@ async function filterUniqueReleases(latestReleases, accReleases) {
return uniqueReleases;
}
function needNextPage(uniqueReleases, pageAccReleases, hasDates) {
if (uniqueReleases.length === 0) {
return false;
function needNextPage(releases, uniqueReleases, totalReleases, hasDates) {
if (argv.last) {
return totalReleases + releases.length < argv.last;
}
if (argv.last && pageAccReleases.length < argv.last) {
// TODO: find a way to paginate if scraper filters page with multiple channels, see Kelly Madison
return true;
if (!hasDates) {
return totalReleases + releases.length < argv.nullDateLimit;
}
if (hasDates) {
const oldestReleaseOnPage = uniqueReleases
const oldestReleaseOnPage = releases
.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0];
@ -56,62 +54,41 @@ function needNextPage(uniqueReleases, pageAccReleases, hasDates) {
// oldest release on page is newer than the specified date cut-off
return true;
}
}
// dates missing, and limit for scenes without dates not yet reached
return pageAccReleases.length <= argv.nullDateLimit;
return false;
}
async function scrapeReleases(scraper, entity, preData, upcoming = false) {
const scrapePage = async (page = 1, accReleases = []) => {
const latestReleases = upcoming
async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, accReleases = [], totalReleases = 0) {
const releases = upcoming
? await scraper.fetchUpcoming(entity, page, include, preData)
: await scraper.fetchLatest(entity, page, include, preData);
if (!Array.isArray(latestReleases)) {
if (!Array.isArray(releases)) {
// scraper is unable to fetch the releases and returned a HTTP code or null
logger.warn(`Scraper returned ${latestReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
logger.warn(`Scraper returned ${releases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
return accReleases;
}
const latestReleasesWithEntity = latestReleases.map(release => ({
const releasesWithEntity = releases.map(release => ({
...release,
entity: release.entity || entity, // allow override
})); // attach entity the release is assigned to when stored
const hasDates = latestReleasesWithEntity.every(release => !!release.date);
const hasDates = releasesWithEntity.every(release => !!release.date);
console.log(hasDates);
const limitedReleases = (argv.last && releasesWithEntity.slice(0, Math.max(argv.last - totalReleases, 0)))
|| (hasDates && releasesWithEntity.filter(release => moment(release.date).isAfter(argv.after)))
|| releasesWithEntity.slice(0, Math.max(argv.nullDateLimit - totalReleases, 0));
const uniqueReleases = argv.redownload
? latestReleasesWithEntity
: await filterUniqueReleases(latestReleasesWithEntity, accReleases);
const uniqueReleases = argv.force
? limitedReleases
: await filterUniqueReleases(limitedReleases, accReleases);
const pageAccReleases = accReleases.concat(uniqueReleases);
logger.verbose(`Scraped '${entity.name}' (${entity.parent?.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`);
if (needNextPage(uniqueReleases, pageAccReleases, hasDates)) {
return scrapePage(page + 1, pageAccReleases);
if (needNextPage(releases, uniqueReleases, totalReleases, hasDates)) {
return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length);
}
return pageAccReleases;
};
const rawReleases = await scrapePage(argv.page || 1, []);
const releases = upcoming
? rawReleases.map(rawRelease => ({ ...rawRelease, upcoming: true }))
: rawReleases;
if (argv.last) {
return releases.slice(0, argv.last);
}
if (releases.every(release => release.date)) {
return releases.filter(release => moment(release.date).isAfter(argv.after));
}
return releases.slice(0, argv.nullDateLimit);
return accReleases.concat(uniqueReleases);
}
async function scrapeLatestReleases(scraper, entity, preData) {
@ -120,9 +97,12 @@ async function scrapeLatestReleases(scraper, entity, preData) {
}
try {
return await scrapeReleases(scraper, entity, preData, false);
return await scrapeReleases(scraper, entity, preData, false, argv.page || 1);
} catch (error) {
if (argv.debug) {
console.trace(error);
}
logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
}
@ -137,6 +117,10 @@ async function scrapeUpcomingReleases(scraper, entity, preData) {
try {
return await scrapeReleases(scraper, entity, preData, true);
} catch (error) {
if (argv.debug) {
console.trace(error);
}
logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
}
@ -159,7 +143,7 @@ async function scrapeMovies(scraper, entity) {
}
async function scrapeChannelReleases(scraper, channelEntity, preData) {
const [latestReleases, upcomingReleases, movies] = await Promise.all([
const [latestReleases, upcomingReleases] = await Promise.all([
argv.latest
? scrapeLatestReleases(scraper, channelEntity, preData)
: [],
@ -171,8 +155,6 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
: [],
]);
console.log(movies);
logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
return [...latestReleases, ...upcomingReleases];