DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Switched to tabs. Adding missing actor entries when scraping actors, with batch ID.

This commit is contained in:
ThePendulum 2020-05-14 04:26:05 +02:00
parent f1eb29c713
commit 11eb66f834
178 changed files with 16594 additions and 16929 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.editorconfig
View File

@ -5,7 +5,7 @@ root = true
[*] [*]
end_of_line = lf end_of_line = lf
insert_final_newline = true insert_final_newline = true
indent_style = space indent_style = tab
indent_size = 4 indent_size = 4
# Matches multiple files with brace expansion notation # Matches multiple files with brace expansion notation

5
.eslintrc
View File

@ -7,13 +7,14 @@
"sourceType": "module" "sourceType": "module"
}, },
"rules": { "rules": {
"indent": ["error", "tab"],
"no-tabs": "off",
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}], "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
"no-console": 0, "no-console": 0,
"indent": "off",
"template-curly-spacing": "off", "template-curly-spacing": "off",
"max-len": 0, "max-len": 0,
"vue/no-v-html": 0, "vue/no-v-html": 0,
"vue/html-indent": ["error", 4], "vue/html-indent": ["error", "tab"],
"vue/multiline-html-element-content-newline": 0, "vue/multiline-html-element-content-newline": 0,
"vue/singleline-html-element-content-newline": 0, "vue/singleline-html-element-content-newline": 0,
"no-param-reassign": ["error", { "no-param-reassign": ["error", {

494
assets/components/actors/actor.vue
View File

@ -1,248 +1,248 @@
<template> <template>
<div <div
v-if="actor" v-if="actor"
class="content actor" class="content actor"
> >
<FilterBar :fetch-releases="fetchActor" /> <FilterBar :fetch-releases="fetchActor" />
<div class="actor-header"> <div class="actor-header">
<h2 class="header-name"> <h2 class="header-name">
<span v-if="actor.network">{{ actor.name }} ({{ actor.network.name }})</span> <span v-if="actor.network">{{ actor.name }} ({{ actor.network.name }})</span>
<span v-else="">{{ actor.name }}</span> <span v-else="">{{ actor.name }}</span>
<Gender <Gender
:gender="actor.gender" :gender="actor.gender"
class="header-gender" class="header-gender"
/> />
</h2> </h2>
<li <li
v-if="actor.aliases.length" v-if="actor.aliases.length"
class="bio-item" class="bio-item"
> >
<dfn class="bio-label">Also known as</dfn> <dfn class="bio-label">Also known as</dfn>
<span>{{ actor.aliases.join(', ') }}</span> <span>{{ actor.aliases.join(', ') }}</span>
</li> </li>
<Social <Social
v-if="actor.social && actor.social.length > 0" v-if="actor.social && actor.social.length > 0"
:actor="actor" :actor="actor"
class="header-social" class="header-social"
/> />
</div> </div>
<div class="actor-inner"> <div class="actor-inner">
<div <div
class="profile" class="profile"
:class="{ expanded, 'with-avatar': !!actor.avatar }" :class="{ expanded, 'with-avatar': !!actor.avatar }"
> >
<a <a
v-if="actor.avatar" v-if="actor.avatar"
:href="`/media/${actor.avatar.path}`" :href="`/media/${actor.avatar.path}`"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="avatar-link" class="avatar-link"
> >
<img <img
:src="`/media/${actor.avatar.thumbnail}`" :src="`/media/${actor.avatar.thumbnail}`"
:title="actor.avatar.copyright && `© ${actor.avatar.copyright}`" :title="actor.avatar.copyright && `© ${actor.avatar.copyright}`"
class="avatar" class="avatar"
> >
</a> </a>
<span <span
v-show="expanded" v-show="expanded"
class="expand collapse-header noselect" class="expand collapse-header noselect"
@click="expanded = false" @click="expanded = false"
><Icon icon="arrow-up3" /></span> ><Icon icon="arrow-up3" /></span>
<ul class="bio nolist"> <ul class="bio nolist">
<li <li
v-if="actor.birthdate" v-if="actor.birthdate"
class="bio-item" class="bio-item"
> >
<dfn class="bio-label"><Icon icon="cake" />Birthdate</dfn> <dfn class="bio-label"><Icon icon="cake" />Birthdate</dfn>
<span <span
v-if="actor.birthdate" v-if="actor.birthdate"
class="birthdate" class="birthdate"
>{{ formatDate(actor.birthdate, 'MMMM D, YYYY') }}<span class="age">{{ actor.age }}</span></span> >{{ formatDate(actor.birthdate, 'MMMM D, YYYY') }}<span class="age">{{ actor.age }}</span></span>
</li> </li>
<li <li
v-if="actor.origin" v-if="actor.origin"
class="bio-item birth" class="bio-item birth"
> >
<dfn class="bio-label"><Icon icon="home2" />Born in</dfn> <dfn class="bio-label"><Icon icon="home2" />Born in</dfn>
<span> <span>
<span <span
v-if="actor.origin.city" v-if="actor.origin.city"
class="city hideable" class="city hideable"
>{{ actor.origin.city }}</span><span >{{ actor.origin.city }}</span><span
v-if="actor.origin.state && (!actor.origin.city || (actor.origin.country && actor.origin.country.alpha2 === 'US'))" v-if="actor.origin.state && (!actor.origin.city || (actor.origin.country && actor.origin.country.alpha2 === 'US'))"
class="state hideable" class="state hideable"
>{{ actor.origin.city ? `, ${actor.origin.state}` : actor.origin.state }}</span> >{{ actor.origin.city ? `, ${actor.origin.state}` : actor.origin.state }}</span>
<span <span
v-if="actor.origin.country" v-if="actor.origin.country"
class="country birthcountry" class="country birthcountry"
> >
<img <img
class="flag" class="flag"
:src="`/img/flags/svg-simple/${actor.origin.country.alpha2.toLowerCase()}.svg`" :src="`/img/flags/svg-simple/${actor.origin.country.alpha2.toLowerCase()}.svg`"
>{{ actor.origin.country.alias || actor.origin.country.name }} >{{ actor.origin.country.alias || actor.origin.country.name }}
</span> </span>
</span> </span>
</li> </li>
<li <li
v-if="actor.residence" v-if="actor.residence"
class="bio-item residence" class="bio-item residence"
> >
<dfn class="bio-label"><Icon icon="location" />Lives in</dfn> <dfn class="bio-label"><Icon icon="location" />Lives in</dfn>
<span> <span>
<span <span
v-if="actor.residence.city" v-if="actor.residence.city"
class="city hideable" class="city hideable"
>{{ actor.residence.city }}</span><span >{{ actor.residence.city }}</span><span
v-if="actor.residence.state && actor.residence.country && actor.residence.country.alpha2 === 'US'" v-if="actor.residence.state && actor.residence.country && actor.residence.country.alpha2 === 'US'"
class="state hideable" class="state hideable"
>{{ actor.residence.city ? `, ${actor.residence.state}` : actor.residence.state }}</span> >{{ actor.residence.city ? `, ${actor.residence.state}` : actor.residence.state }}</span>
<span <span
v-if="actor.residence.country" v-if="actor.residence.country"
class="country" class="country"
> >
<img <img
class="flag" class="flag"
:src="`/img/flags/${actor.residence.country.alpha2.toLowerCase()}.png`" :src="`/img/flags/${actor.residence.country.alpha2.toLowerCase()}.png`"
>{{ actor.residence.country.alias || actor.residence.country.name }} >{{ actor.residence.country.alias || actor.residence.country.name }}
</span> </span>
</span> </span>
</li> </li>
<li <li
v-if="actor.ethnicity" v-if="actor.ethnicity"
class="bio-item ethnicity hideable" class="bio-item ethnicity hideable"
> >
<dfn class="bio-label"><Icon icon="earth2" />Ethnicity</dfn> <dfn class="bio-label"><Icon icon="earth2" />Ethnicity</dfn>
<span>{{ actor.ethnicity }}</span> <span>{{ actor.ethnicity }}</span>
</li> </li>
<li <li
v-if="actor.bust || actor.waist || actor.hip" v-if="actor.bust || actor.waist || actor.hip"
title="bust-waist-hip" title="bust-waist-hip"
class="bio-item" class="bio-item"
> >
<dfn class="bio-label"><Icon icon="ruler" />Figure</dfn> <dfn class="bio-label"><Icon icon="ruler" />Figure</dfn>
<span> <span>
<Icon <Icon
v-if="actor.naturalBoobs === false" v-if="actor.naturalBoobs === false"
v-tooltip="'Boobs enhanced'" v-tooltip="'Boobs enhanced'"
icon="magic-wand" icon="magic-wand"
class="enhanced" class="enhanced"
/>{{ actor.bust || '??' }}-{{ actor.waist || '??' }}-{{ actor.hip || '??' }} />{{ actor.bust || '??' }}-{{ actor.waist || '??' }}-{{ actor.hip || '??' }}
</span> </span>
</li> </li>
<li <li
v-if="actor.height" v-if="actor.height"
class="bio-item height" class="bio-item height"
> >
<dfn class="bio-label"><Icon icon="height" />Height</dfn> <dfn class="bio-label"><Icon icon="height" />Height</dfn>
<span> <span>
<span class="height-metric">{{ actor.height.metric }} cm</span> <span class="height-metric">{{ actor.height.metric }} cm</span>
<span class="height-imperial">{{ actor.height.imperial }}</span> <span class="height-imperial">{{ actor.height.imperial }}</span>
</span> </span>
</li> </li>
<li <li
v-if="actor.weight" v-if="actor.weight"
class="bio-item weight hideable" class="bio-item weight hideable"
> >
<dfn class="bio-label"><Icon icon="scale" />Weight</dfn> <dfn class="bio-label"><Icon icon="scale" />Weight</dfn>
<span> <span>
<span class="weight-metric">{{ actor.weight.metric }} kg</span> <span class="weight-metric">{{ actor.weight.metric }} kg</span>
<span class="weight-imperial">{{ actor.weight.imperial }} lbs</span> <span class="weight-imperial">{{ actor.weight.imperial }} lbs</span>
</span> </span>
</li> </li>
<li <li
v-if="actor.hasTattoos" v-if="actor.hasTattoos"
class="bio-item tattoos hideable" class="bio-item tattoos hideable"
> >
<dfn class="bio-label"><Icon icon="flower" />Tattoos</dfn> <dfn class="bio-label"><Icon icon="flower" />Tattoos</dfn>
<span <span
v-if="actor.tattoos" v-if="actor.tattoos"
v-tooltip="actor.tattoos" v-tooltip="actor.tattoos"
class="bio-value" class="bio-value"
>{{ actor.tattoos }}</span> >{{ actor.tattoos }}</span>
<span v-else>Yes</span> <span v-else>Yes</span>
</li> </li>
<li <li
v-if="actor.hasPiercings" v-if="actor.hasPiercings"
class="bio-item piercings hideable" class="bio-item piercings hideable"
> >
<dfn class="bio-label"><Icon icon="trophy4" />Piercings</dfn> <dfn class="bio-label"><Icon icon="trophy4" />Piercings</dfn>
<span <span
v-if="actor.piercings" v-if="actor.piercings"
v-tooltip="actor.piercings" v-tooltip="actor.piercings"
class="bio-value" class="bio-value"
>{{ actor.piercings }}</span> >{{ actor.piercings }}</span>
<span v-else>Yes</span> <span v-else>Yes</span>
</li> </li>
<li class="bio-item scraped hideable">Updated {{ formatDate(actor.scrapedAt, 'YYYY-MM-DD HH:mm') }}, ID: {{ actor.id }}</li> <li class="bio-item scraped hideable">Updated {{ formatDate(actor.updatedAt, 'YYYY-MM-DD HH:mm') }}, ID: {{ actor.id }}</li>
</ul> </ul>
<span <span
v-show="!expanded" v-show="!expanded"
class="expand expand-header collapse-header noselect" class="expand expand-header collapse-header noselect"
@click="expanded = true" @click="expanded = true"
><Icon icon="arrow-down3" /></span> ><Icon icon="arrow-down3" /></span>
<p <p
v-if="actor.description" v-if="actor.description"
class="description" class="description"
>{{ actor.description }}</p> >{{ actor.description }}</p>
<Social <Social
v-if="actor.social && actor.social.length > 0" v-if="actor.social && actor.social.length > 0"
:actor="actor" :actor="actor"
class="profile-social" class="profile-social"
/> />
<span <span
v-show="expanded" v-show="expanded"
class="expand expand-header collapse-header noselect" class="expand expand-header collapse-header noselect"
@click="expanded = false" @click="expanded = false"
><Icon icon="arrow-up3" /></span> ><Icon icon="arrow-up3" /></span>
</div> </div>
<div class="actor-content"> <div class="actor-content">
<div <div
v-if="actor.avatar || (actor.photos && actor.photos.length > 0)" v-if="actor.avatar || (actor.photos && actor.photos.length > 0)"
class="photos-container" class="photos-container"
> >
<Photos :actor="actor" /> <Photos :actor="actor" />
<Photos <Photos
:actor="actor" :actor="actor"
:class="{ expanded }" :class="{ expanded }"
class="compact" class="compact"
/> />
</div> </div>
<Releases :releases="actor.releases" /> <Releases :releases="actor.releases" />
</div> </div>
</div> </div>
</div> </div>
</template> </template>
<script> <script>
@ -253,52 +253,52 @@ import Gender from './gender.vue';
import Social from './social.vue'; import Social from './social.vue';
async function fetchActor() { async function fetchActor() {
this.actor = await this.$store.dispatch('fetchActorBySlug', { this.actor = await this.$store.dispatch('fetchActorBySlug', {
actorSlug: this.$route.params.actorSlug, actorSlug: this.$route.params.actorSlug,
range: this.$route.params.range, range: this.$route.params.range,
}); });
} }
async function route() { async function route() {
await this.fetchActor(); await this.fetchActor();
} }
function scrollPhotos(event) { function scrollPhotos(event) {
event.currentTarget.scrollLeft += event.deltaY; // eslint-disable-line no-param-reassign event.currentTarget.scrollLeft += event.deltaY; // eslint-disable-line no-param-reassign
} }
async function mounted() { async function mounted() {
await this.fetchActor(); await this.fetchActor();
if (this.actor) { if (this.actor) {
this.pageTitle = this.actor.name; this.pageTitle = this.actor.name;
} }
} }
export default { export default {
components: { components: {
FilterBar, FilterBar,
Photos, Photos,
Releases, Releases,
Gender, Gender,
Social, Social,
}, },
data() { data() {
return { return {
actor: null, actor: null,
releases: null, releases: null,
pageTitle: null, pageTitle: null,
expanded: false, expanded: false,
}; };
}, },
watch: { watch: {
$route: route, $route: route,
}, },
mounted, mounted,
methods: { methods: {
fetchActor, fetchActor,
scrollPhotos, scrollPhotos,
}, },
}; };
</script> </script>
@ -426,7 +426,7 @@ export default {
} }
.bio-value { .bio-value {
margin: 0 0 0 2rem; margin: 0 0 0 2rem;
white-space: nowrap; white-space: nowrap;
text-overflow: ellipsis; text-overflow: ellipsis;
overflow: hidden; overflow: hidden;

296
assets/components/networks/network.vue
View File

@ -1,124 +1,124 @@
<template> <template>
<div <div
v-if="network" v-if="network"
class="content" class="content"
> >
<FilterBar :fetch-releases="fetchNetwork" /> <FilterBar :fetch-releases="fetchNetwork" />
<div <div
class="network" class="network"
:class="{ nosites: sites.length === 0 && networks.length === 0 }" :class="{ nosites: sites.length === 0 && networks.length === 0 }"
> >
<div <div
v-show="sites.length > 0 || networks.length > 0" v-show="sites.length > 0 || networks.length > 0"
class="sidebar" class="sidebar"
:class="{ expanded }" :class="{ expanded }"
> >
<a <a
v-tooltip.bottom="`Go to ${network.url}`" v-tooltip.bottom="`Go to ${network.url}`"
:href="network.url" :href="network.url"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="title" class="title"
> >
<img <img
:src="`/img/logos/${network.slug}/network.png`" :src="`/img/logos/${network.slug}/thumbs/network.png`"
class="logo" class="logo"
> >
</a> </a>
<p <p
v-if="network.description" v-if="network.description"
class="description" class="description"
>{{ network.description }}</p> >{{ network.description }}</p>
<Sites <Sites
v-if="sites.length" v-if="sites.length"
:sites="sites" :sites="sites"
:class="{ expanded }" :class="{ expanded }"
/> />
<div <div
v-if="networks.length > 0" v-if="networks.length > 0"
class="networks" class="networks"
> >
<Network <Network
v-for="childNetwork in networks" v-for="childNetwork in networks"
:key="`network-${childNetwork.id}`" :key="`network-${childNetwork.id}`"
:network="childNetwork" :network="childNetwork"
/> />
</div> </div>
<Network <Network
v-if="network.parent" v-if="network.parent"
:network="network.parent" :network="network.parent"
class="parent" class="parent"
/> />
</div> </div>
<template v-if="sites.length > 0 || networks.length > 0"> <template v-if="sites.length > 0 || networks.length > 0">
<span <span
v-show="!expanded" v-show="!expanded"
class="expand expand-sidebar noselect" class="expand expand-sidebar noselect"
@click="expanded = true" @click="expanded = true"
><Icon icon="arrow-right3" /></span> ><Icon icon="arrow-right3" /></span>
<span <span
v-show="expanded" v-show="expanded"
class="expand expand-sidebar noselect" class="expand expand-sidebar noselect"
@click="expanded = false" @click="expanded = false"
><Icon icon="arrow-left3" /></span> ><Icon icon="arrow-left3" /></span>
</template> </template>
<div <div
class="header" class="header"
:class="{ hideable: sites.length > 0 || networks.length > 0 }" :class="{ hideable: sites.length > 0 || networks.length > 0 }"
> >
<a <a
v-tooltip.bottom="`Go to ${network.url}`" v-tooltip.bottom="`Go to ${network.url}`"
:href="network.url" :href="network.url"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="title" class="title"
> >
<img <img
:src="`/img/logos/${network.slug}/network.png`" :src="`/img/logos/${network.slug}/thumbs/network.png`"
class="logo" class="logo"
> >
</a> </a>
</div> </div>
<div class="content-inner"> <div class="content-inner">
<template v-if="sites.length > 0 || networks.length > 0"> <template v-if="sites.length > 0 || networks.length > 0">
<span <span
v-show="expanded" v-show="expanded"
class="expand collapse-header noselect" class="expand collapse-header noselect"
@click="expanded = false" @click="expanded = false"
><Icon icon="arrow-up3" /></span> ><Icon icon="arrow-up3" /></span>
<Sites <Sites
:sites="sites" :sites="sites"
:class="{ expanded }" :class="{ expanded }"
class="compact" class="compact"
/> />
<span <span
v-show="!expanded" v-show="!expanded"
class="expand expand-header noselect" class="expand expand-header noselect"
@click="expanded = true" @click="expanded = true"
><Icon icon="arrow-down3" /></span> ><Icon icon="arrow-down3" /></span>
<span <span
v-show="expanded" v-show="expanded"
class="expand expand-header noselect" class="expand expand-header noselect"
@click="expanded = false" @click="expanded = false"
><Icon icon="arrow-up3" /></span> ><Icon icon="arrow-up3" /></span>
</template> </template>
<Releases :releases="releases" /> <Releases :releases="releases" />
</div> </div>
</div> </div>
</div> </div>
</template> </template>
<script> <script>
@ -128,59 +128,59 @@ import Sites from '../sites/sites.vue';
import Network from '../tile/network.vue'; import Network from '../tile/network.vue';
async function fetchNetwork() { async function fetchNetwork() {
this.network = await this.$store.dispatch('fetchNetworkBySlug', { this.network = await this.$store.dispatch('fetchNetworkBySlug', {
networkSlug: this.$route.params.networkSlug, networkSlug: this.$route.params.networkSlug,
range: this.$route.params.range, range: this.$route.params.range,
}); });
if (this.network.studios) { if (this.network.studios) {
this.studios = this.network.studios.map(studio => ({ this.studios = this.network.studios.map(studio => ({
...studio, ...studio,
network: this.network, network: this.network,
})); }));
} }
this.networks = this.network.networks; this.networks = this.network.networks;
this.sites = this.network.sites this.sites = this.network.sites
.filter(site => !site.independent); .filter(site => !site.independent);
this.releases = this.network.releases; this.releases = this.network.releases;
} }
async function route() { async function route() {
await this.fetchNetwork(); await this.fetchNetwork();
} }
async function mounted() { async function mounted() {
await this.fetchNetwork(); await this.fetchNetwork();
this.pageTitle = this.network.name; this.pageTitle = this.network.name;
} }
export default { export default {
components: { components: {
FilterBar, FilterBar,
Releases, Releases,
Sites, Sites,
Network, Network,
}, },
data() { data() {
return { return {
network: null, network: null,
sites: [], sites: [],
networks: [], networks: [],
studios: [], studios: [],
releases: [], releases: [],
pageTitle: null, pageTitle: null,
expanded: false, expanded: false,
}; };
}, },
watch: { watch: {
$route: route, $route: route,
}, },
mounted, mounted,
methods: { methods: {
fetchNetwork, fetchNetwork,
}, },
}; };
</script> </script>

34
assets/components/tile/site.vue
View File

@ -1,25 +1,25 @@
<template> <template>
<a <a
:href="`/site/${site.slug}`" :href="`/site/${site.slug}`"
:title="site.name" :title="site.name"
class="tile" class="tile"
> >
<img <img
:src="`/img/logos/${site.network.slug}/${site.slug}.png`" :src="`/img/logos/${site.network.slug}/thumbs/${site.slug}.png`"
:alt="site.name" :alt="site.name"
class="logo" class="logo"
> >
</a> </a>
</template> </template>
<script> <script>
export default { export default {
props: { props: {
site: { site: {
type: Object, type: Object,
default: null, default: null,
}, },
}, },
}; };
</script> </script>

156
assets/js/actors/actions.js
View File

@ -1,59 +1,61 @@
import { graphql, get } from '../api'; import { graphql, get } from '../api';
import { import {
releasePosterFragment, releasePosterFragment,
releaseActorsFragment, releaseActorsFragment,
releaseTagsFragment, releaseTagsFragment,
} from '../fragments'; } from '../fragments';
import { curateRelease } from '../curate'; import { curateRelease } from '../curate';
import getDateRange from '../get-date-range'; import getDateRange from '../get-date-range';
function curateActor(actor) { function curateActor(actor) {
if (!actor) { if (!actor) {
return null; return null;
} }
const curatedActor = { const curatedActor = {
...actor, ...actor,
height: actor.heightMetric && { height: actor.heightMetric && {
metric: actor.heightMetric, metric: actor.heightMetric,
imperial: actor.heightImperial, imperial: actor.heightImperial,
}, },
weight: actor.weightMetric && { weight: actor.weightMetric && {
metric: actor.weightMetric, metric: actor.weightMetric,
imperial: actor.weightImperial, imperial: actor.weightImperial,
}, },
origin: actor.birthCountry && { origin: actor.birthCountry && {
city: actor.birthCity, city: actor.birthCity,
state: actor.birthState, state: actor.birthState,
country: actor.birthCountry, country: actor.birthCountry,
}, },
residence: actor.residenceCountry && { residence: actor.residenceCountry && {
city: actor.residenceCity, city: actor.residenceCity,
state: actor.residenceState, state: actor.residenceState,
country: actor.residenceCountry, country: actor.residenceCountry,
}, },
}; scrapedAt: new Date(actor.createdAt),
updatedAt: new Date(actor.updatedAt),
};
if (actor.avatar) { if (actor.avatar) {
curatedActor.avatar = actor.avatar.media; curatedActor.avatar = actor.avatar.media;
} }
if (actor.releases) { if (actor.releases) {
curatedActor.releases = actor.releases.map(release => curateRelease(release.release)); curatedActor.releases = actor.releases.map(release => curateRelease(release.release));
} }
if (actor.photos) { if (actor.photos) {
curatedActor.photos = actor.photos.map(photo => photo.media); curatedActor.photos = actor.photos.map(photo => photo.media);
} }
return curatedActor; return curatedActor;
} }
function initActorActions(store, _router) { function initActorActions(store, _router) {
async function fetchActorBySlug({ _commit }, { actorSlug, limit = 100, range = 'latest' }) { async function fetchActorBySlug({ _commit }, { actorSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range); const { before, after, orderBy } = getDateRange(range);
const { actors: [actor] } = await graphql(` const { actors: [actor] } = await graphql(`
query Actor( query Actor(
$actorSlug: String! $actorSlug: String!
$limit:Int = 1000, $limit:Int = 1000,
@ -90,6 +92,8 @@ function initActorActions(store, _router) {
tattoos tattoos
piercings piercings
description description
createdAt
updatedAt
network { network {
id id
name name
@ -184,27 +188,27 @@ function initActorActions(store, _router) {
} }
} }
`, { `, {
actorSlug, actorSlug,
limit, limit,
after, after,
before, before,
orderBy: orderBy === 'DATE_DESC' ? 'RELEASE_BY_RELEASE_ID__DATE_DESC' : 'RELEASE_BY_RELEASE_ID__DATE_ASC', orderBy: orderBy === 'DATE_DESC' ? 'RELEASE_BY_RELEASE_ID__DATE_DESC' : 'RELEASE_BY_RELEASE_ID__DATE_ASC',
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });
return curateActor(actor); return curateActor(actor);
} }
async function fetchActors({ _commit }, { async function fetchActors({ _commit }, {
limit = 100, limit = 100,
letter, letter,
gender, gender,
}) { }) {
const genderFilter = gender === null const genderFilter = gender === null
? 'isNull: true' ? 'isNull: true'
: `equalTo: "${gender}"`; : `equalTo: "${gender}"`;
const { actors } = await graphql(` const { actors } = await graphql(`
query Actors( query Actors(
$limit: Int, $limit: Int,
$letter: String! = "", $letter: String! = "",
@ -249,28 +253,28 @@ function initActorActions(store, _router) {
} }
} }
`, { `, {
limit, limit,
letter, letter,
}); });
return actors.map(actor => curateActor(actor)); return actors.map(actor => curateActor(actor));
} }
async function fetchActorReleases({ _commit }, actorId) { async function fetchActorReleases({ _commit }, actorId) {
const releases = await get(`/actors/${actorId}/releases`, { const releases = await get(`/actors/${actorId}/releases`, {
filter: store.state.ui.filter, filter: store.state.ui.filter,
after: store.getters.after, after: store.getters.after,
before: store.getters.before, before: store.getters.before,
}); });
return releases; return releases;
} }
return { return {
fetchActorBySlug, fetchActorBySlug,
fetchActors, fetchActors,
fetchActorReleases, fetchActorReleases,
}; };
} }
export default initActorActions; export default initActorActions;

10
assets/js/actors/actors.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions'; import actions from './actions';
function initActorsStore(store, router) { function initActorsStore(store, router) {
return { return {
state, state,
mutations, mutations,
actions: actions(store, router), actions: actions(store, router),
}; };
} }
export default initActorsStore; export default initActorsStore;

94
assets/js/api.js
View File

@ -1,71 +1,71 @@
import config from 'config'; import config from 'config';
async function get(endpoint, query = {}) { async function get(endpoint, query = {}) {
const curatedQuery = Object.entries(query).reduce((acc, [key, value]) => (value ? { ...acc, [key]: value } : acc), {}); // remove empty values const curatedQuery = Object.entries(query).reduce((acc, [key, value]) => (value ? { ...acc, [key]: value } : acc), {}); // remove empty values
const q = new URLSearchParams(curatedQuery).toString(); const q = new URLSearchParams(curatedQuery).toString();
const res = await fetch(`${config.api.url}${endpoint}?${q}`, { const res = await fetch(`${config.api.url}${endpoint}?${q}`, {
method: 'GET', method: 'GET',
mode: 'cors', mode: 'cors',
credentials: 'same-origin', credentials: 'same-origin',
}); });
if (res.ok) { if (res.ok) {
return res.json(); return res.json();
} }
const errorMsg = await res.text(); const errorMsg = await res.text();
throw new Error(errorMsg); throw new Error(errorMsg);
} }
async function post(endpoint, data) { async function post(endpoint, data) {
const res = await fetch(`${config.api.url}${endpoint}`, { const res = await fetch(`${config.api.url}${endpoint}`, {
method: 'POST', method: 'POST',
mode: 'cors', mode: 'cors',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
}, },
credentials: 'same-origin', credentials: 'same-origin',
body: JSON.stringify(data), body: JSON.stringify(data),
}); });
if (res.ok) { if (res.ok) {
return res.json(); return res.json();
} }
const errorMsg = await res.text(); const errorMsg = await res.text();
throw new Error(errorMsg); throw new Error(errorMsg);
} }
async function graphql(query, variables = null) { async function graphql(query, variables = null) {
const res = await fetch('/graphql', { const res = await fetch('/graphql', {
method: 'POST', method: 'POST',
mode: 'cors', mode: 'cors',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
}, },
credentials: 'same-origin', credentials: 'same-origin',
body: JSON.stringify({ body: JSON.stringify({
query, query,
variables, variables,
}), }),
}); });
if (res.ok) { if (res.ok) {
const { data } = await res.json(); const { data } = await res.json();
return data; return data;
} }
const errorMsg = await res.text(); const errorMsg = await res.text();
throw new Error(errorMsg); throw new Error(errorMsg);
} }
export { export {
get, get,
post, post,
graphql, graphql,
}; };

10
assets/js/auth/auth.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions'; import actions from './actions';
function initAuthStore(store, router) { function initAuthStore(store, router) {
return { return {
state, state,
mutations, mutations,
actions: actions(store, router), actions: actions(store, router),
}; };
} }
export default initAuthStore; export default initAuthStore;

4
assets/js/auth/state.js
View File

@ -1,4 +1,4 @@
export default { export default {
authenticated: false, authenticated: false,
user: null, user: null,
}; };

16
assets/js/config/default.js
View File

@ -1,10 +1,10 @@
export default { export default {
api: { api: {
url: `${window.location.origin}/api`, url: `${window.location.origin}/api`,
}, },
filename: { filename: {
pattern: '{site.name} - {title} ({actors.$n.name}, {date} {shootId})', pattern: '{site.name} - {title} ({actors.$n.name}, {date} {shootId})',
separator: ', ', separator: ', ',
date: 'DD-MM-YYYY', date: 'DD-MM-YYYY',
}, },
}; };

128
assets/js/curate.js
View File

@ -1,94 +1,94 @@
import dayjs from 'dayjs'; import dayjs from 'dayjs';
function curateActor(actor, release) { function curateActor(actor, release) {
const curatedActor = { const curatedActor = {
...actor, ...actor,
origin: actor.originCountry && { origin: actor.originCountry && {
country: actor.originCountry, country: actor.originCountry,
}, },
}; };
if (actor.avatar) curatedActor.avatar = actor.avatar.media; if (actor.avatar) curatedActor.avatar = actor.avatar.media;
if (release && release.date && curatedActor.birthdate) { if (release && release.date && curatedActor.birthdate) {
curatedActor.ageThen = dayjs(release.date).diff(actor.birthdate, 'year'); curatedActor.ageThen = dayjs(release.date).diff(actor.birthdate, 'year');
} }
return curatedActor; return curatedActor;
} }
function curateRelease(release) { function curateRelease(release) {
const curatedRelease = { const curatedRelease = {
...release, ...release,
actors: [], actors: [],
poster: release.poster && release.poster.media, poster: release.poster && release.poster.media,
tags: release.tags ? release.tags.map(({ tag }) => tag) : [], tags: release.tags ? release.tags.map(({ tag }) => tag) : [],
}; };
if (release.site) curatedRelease.network = release.site.network; if (release.site) curatedRelease.network = release.site.network;
if (release.scenes) curatedRelease.scenes = release.scenes.map(({ scene }) => curateRelease(scene)); if (release.scenes) curatedRelease.scenes = release.scenes.map(({ scene }) => curateRelease(scene));
if (release.movies) curatedRelease.movies = release.movies.map(({ movie }) => curateRelease(movie)); if (release.movies) curatedRelease.movies = release.movies.map(({ movie }) => curateRelease(movie));
if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media); if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media); if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media);
if (release.trailer) curatedRelease.trailer = release.trailer.media; if (release.trailer) curatedRelease.trailer = release.trailer.media;
if (release.teaser) curatedRelease.teaser = release.teaser.media; if (release.teaser) curatedRelease.teaser = release.teaser.media;
if (release.actors) curatedRelease.actors = release.actors.map(({ actor }) => curateActor(actor, curatedRelease)); if (release.actors) curatedRelease.actors = release.actors.map(({ actor }) => curateActor(actor, curatedRelease));
if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.map(({ tag }) => tag); if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.map(({ tag }) => tag);
if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.map(({ actor }) => curateActor(actor, curatedRelease)); if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.map(({ actor }) => curateActor(actor, curatedRelease));
return curatedRelease; return curatedRelease;
} }
function curateSite(site, network) { function curateSite(site, network) {
const curatedSite = { const curatedSite = {
id: site.id, id: site.id,
name: site.name, name: site.name,
slug: site.slug, slug: site.slug,
url: site.url, url: site.url,
independent: site.independent, independent: site.independent,
}; };
if (site.releases) curatedSite.releases = site.releases.map(release => curateRelease(release)); if (site.releases) curatedSite.releases = site.releases.map(release => curateRelease(release));
if (site.network || network) curatedSite.network = site.network || network; if (site.network || network) curatedSite.network = site.network || network;
if (site.tags) curatedSite.tags = site.tags.map(({ tag }) => tag); if (site.tags) curatedSite.tags = site.tags.map(({ tag }) => tag);
return curatedSite; return curatedSite;
} }
function curateNetwork(network, releases) { function curateNetwork(network, releases) {
const curatedNetwork = { const curatedNetwork = {
id: network.id, id: network.id,
name: network.name, name: network.name,
slug: network.slug, slug: network.slug,
url: network.url, url: network.url,
networks: [], networks: [],
}; };
if (network.parent) curatedNetwork.parent = curateNetwork(network.parent); if (network.parent) curatedNetwork.parent = curateNetwork(network.parent);
if (network.sites) curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork)); if (network.sites) curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork));
if (network.networks) curatedNetwork.networks = network.networks.map(subNetwork => curateNetwork(subNetwork)); if (network.networks) curatedNetwork.networks = network.networks.map(subNetwork => curateNetwork(subNetwork));
if (network.studios) curatedNetwork.studios = network.studios; if (network.studios) curatedNetwork.studios = network.studios;
if (releases) curatedNetwork.releases = releases.map(release => curateRelease(release)); if (releases) curatedNetwork.releases = releases.map(release => curateRelease(release));
return curatedNetwork; return curatedNetwork;
} }
function curateTag(tag) { function curateTag(tag) {
const curatedTag = { const curatedTag = {
...tag, ...tag,
}; };
if (tag.releases) curatedTag.releases = tag.releases.map(({ release }) => curateRelease(release)); if (tag.releases) curatedTag.releases = tag.releases.map(({ release }) => curateRelease(release));
if (tag.photos) curatedTag.photos = tag.photos.map(({ media }) => media); if (tag.photos) curatedTag.photos = tag.photos.map(({ media }) => media);
if (tag.poster) curatedTag.poster = tag.poster.media; if (tag.poster) curatedTag.poster = tag.poster.media;
return curatedTag; return curatedTag;
} }
export { export {
curateActor, curateActor,
curateRelease, curateRelease,
curateSite, curateSite,
curateNetwork, curateNetwork,
curateTag, curateTag,
}; };

20
assets/js/fragments.js
View File

@ -278,14 +278,14 @@ const releaseFragment = `
`; `;
export { export {
releaseActorsFragment, releaseActorsFragment,
releaseFields, releaseFields,
releaseTagsFragment, releaseTagsFragment,
releasePosterFragment, releasePosterFragment,
releasePhotosFragment, releasePhotosFragment,
releaseTrailerFragment, releaseTrailerFragment,
releasesFragment, releasesFragment,
releaseFragment, releaseFragment,
siteFragment, siteFragment,
sitesFragment, sitesFragment,
}; };

42
assets/js/get-date-range.js
View File

@ -1,30 +1,30 @@
import dayjs from 'dayjs'; import dayjs from 'dayjs';
const dateRanges = { const dateRanges = {
latest: () => ({ latest: () => ({
after: '1900-01-01', after: '1900-01-01',
before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'), before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
orderBy: 'DATE_DESC', orderBy: 'DATE_DESC',
}), }),
upcoming: () => ({ upcoming: () => ({
after: dayjs(new Date()).format('YYYY-MM-DD'), after: dayjs(new Date()).format('YYYY-MM-DD'),
before: '2100-01-01', before: '2100-01-01',
orderBy: 'DATE_ASC', orderBy: 'DATE_ASC',
}), }),
new: () => ({ new: () => ({
after: '1900-01-01', after: '1900-01-01',
before: '2100-01-01', before: '2100-01-01',
orderBy: 'CREATED_AT_DESC', orderBy: 'CREATED_AT_DESC',
}), }),
all: () => ({ all: () => ({
after: '1900-01-01', after: '1900-01-01',
before: '2100-01-01', before: '2100-01-01',
orderBy: 'DATE_DESC', orderBy: 'DATE_DESC',
}), }),
}; };
function getDateRange(range) { function getDateRange(range) {
return dateRanges[range](); return dateRanges[range]();
} }
export default getDateRange; export default getDateRange;

72
assets/js/main.js
View File

@ -14,48 +14,48 @@ import Container from '../components/container/container.vue';
import Icon from '../components/icon/icon.vue'; import Icon from '../components/icon/icon.vue';
function init() { function init() {
const store = initStore(router); const store = initStore(router);
initUiObservers(store, router); initUiObservers(store, router);
if (window.env.sfw) { if (window.env.sfw) {
store.dispatch('setSfw', true); store.dispatch('setSfw', true);
} }
Vue.mixin({ Vue.mixin({
components: { components: {
Icon, Icon,
}, },
watch: { watch: {
pageTitle(title) { pageTitle(title) {
if (title) { if (title) {
document.title = `traxxx - ${title}`; document.title = `traxxx - ${title}`;
return; return;
} }
document.title = 'traxxx'; document.title = 'traxxx';
}, },
}, },
methods: { methods: {
formatDate: (date, format) => dayjs(date).format(format), formatDate: (date, format) => dayjs(date).format(format),
isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB), isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB),
isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB), isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB),
}, },
}); });
Vue.use(VTooltip); Vue.use(VTooltip);
Vue.use(VueLazyLoad, { Vue.use(VueLazyLoad, {
throttleWait: 0, throttleWait: 0,
}); });
new Vue({ // eslint-disable-line no-new new Vue({ // eslint-disable-line no-new
el: '#container', el: '#container',
store, store,
router, router,
render(createElement) { render(createElement) {
return createElement(Container); return createElement(Container);
}, },
}); });
} }
init(); init();

44
assets/js/networks/actions.js
View File

@ -4,10 +4,10 @@ import { curateNetwork } from '../curate';
import getDateRange from '../get-date-range'; import getDateRange from '../get-date-range';
function initNetworksActions(store, _router) { function initNetworksActions(store, _router) {
async function fetchNetworkBySlug({ _commit }, { networkSlug, limit = 100, range = 'latest' }) { async function fetchNetworkBySlug({ _commit }, { networkSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range); const { before, after, orderBy } = getDateRange(range);
const { network, releases } = await graphql(` const { network, releases } = await graphql(`
query Network( query Network(
$networkSlug: String! $networkSlug: String!
$limit:Int = 1000, $limit:Int = 1000,
@ -107,21 +107,21 @@ function initNetworksActions(store, _router) {
} }
} }
`, { `, {
networkSlug, networkSlug,
limit, limit,
after, after,
before, before,
orderBy, orderBy,
afterTime: store.getters.after, afterTime: store.getters.after,
beforeTime: store.getters.before, beforeTime: store.getters.before,
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });
return curateNetwork(network, releases); return curateNetwork(network, releases);
} }
async function fetchNetworks({ _commit }) { async function fetchNetworks({ _commit }) {
const { networks } = await graphql(` const { networks } = await graphql(`
query Networks { query Networks {
networks(orderBy: NAME_ASC) { networks(orderBy: NAME_ASC) {
id id
@ -133,13 +133,13 @@ function initNetworksActions(store, _router) {
} }
`); `);
return networks.map(network => curateNetwork(network)); return networks.map(network => curateNetwork(network));
} }
return { return {
fetchNetworkBySlug, fetchNetworkBySlug,
fetchNetworks, fetchNetworks,
}; };
} }
export default initNetworksActions; export default initNetworksActions;

10
assets/js/networks/networks.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions'; import actions from './actions';
function initNetworksStore(store, router) { function initNetworksStore(store, router) {
return { return {
state, state,
mutations, mutations,
actions: actions(store, router), actions: actions(store, router),
}; };
} }
export default initNetworksStore; export default initNetworksStore;

62
assets/js/releases/actions.js
View File

@ -4,10 +4,10 @@ import { curateRelease } from '../curate';
import getDateRange from '../get-date-range'; import getDateRange from '../get-date-range';
function initReleasesActions(store, _router) { function initReleasesActions(store, _router) {
async function fetchReleases({ _commit }, { limit = 100, range = 'latest' }) { async function fetchReleases({ _commit }, { limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range); const { before, after, orderBy } = getDateRange(range);
const { releases } = await graphql(` const { releases } = await graphql(`
query Releases( query Releases(
$limit:Int = 1000, $limit:Int = 1000,
$after:Date = "1900-01-01", $after:Date = "1900-01-01",
@ -18,18 +18,18 @@ function initReleasesActions(store, _router) {
${releasesFragment} ${releasesFragment}
} }
`, { `, {
limit, limit,
after, after,
before, before,
orderBy, orderBy,
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });
return releases.map(release => curateRelease(release)); return releases.map(release => curateRelease(release));
} }
async function searchReleases({ _commit }, { query, limit = 20 }) { async function searchReleases({ _commit }, { query, limit = 20 }) {
const res = await graphql(` const res = await graphql(`
query SearchReleases( query SearchReleases(
$query: String! $query: String!
$limit: Int = 20 $limit: Int = 20
@ -88,34 +88,34 @@ function initReleasesActions(store, _router) {
} }
} }
`, { `, {
query, query,
limit, limit,
}); });
if (!res) return []; if (!res) return [];
return res.releases.map(release => curateRelease(release)); return res.releases.map(release => curateRelease(release));
} }
async function fetchReleaseById({ _commit }, releaseId) { async function fetchReleaseById({ _commit }, releaseId) {
// const release = await get(`/releases/${releaseId}`); // const release = await get(`/releases/${releaseId}`);
const { release } = await graphql(` const { release } = await graphql(`
query Release($releaseId:Int!) { query Release($releaseId:Int!) {
${releaseFragment} ${releaseFragment}
} }
`, { `, {
releaseId: Number(releaseId), releaseId: Number(releaseId),
}); });
return curateRelease(release); return curateRelease(release);
} }
return { return {
fetchReleases, fetchReleases,
fetchReleaseById, fetchReleaseById,
searchReleases, searchReleases,
}; };
} }
export default initReleasesActions; export default initReleasesActions;

8
assets/js/releases/mutations.js
View File

@ -1,14 +1,14 @@
import Vue from 'vue'; import Vue from 'vue';
function setCache(state, { target, releases }) { function setCache(state, { target, releases }) {
Vue.set(state.cache, target, releases); Vue.set(state.cache, target, releases);
} }
function deleteCache(state, target) { function deleteCache(state, target) {
Vue.delete(state.cache, target); Vue.delete(state.cache, target);
} }
export default { export default {
setCache, setCache,
deleteCache, deleteCache,
}; };

10
assets/js/releases/releases.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions'; import actions from './actions';
function initReleasesStore(store, router) { function initReleasesStore(store, router) {
return { return {
state, state,
mutations, mutations,
actions: actions(store, router), actions: actions(store, router),
}; };
} }
export default initReleasesStore; export default initReleasesStore;

2
assets/js/releases/state.js
View File

@ -1,3 +1,3 @@
export default { export default {
cache: {}, cache: {},
}; };

260
assets/js/router.js
View File

@ -16,139 +16,139 @@ import NotFound from '../components/errors/404.vue';
Vue.use(VueRouter); Vue.use(VueRouter);
const routes = [ const routes = [
{ {
path: '/', path: '/',
redirect: { redirect: {
name: 'latest', name: 'latest',
}, },
}, },
{ {
path: '/home', path: '/home',
redirect: { redirect: {
name: 'latest', name: 'latest',
}, },
}, },
{ {
path: '/latest', path: '/latest',
component: Home, component: Home,
name: 'latest', name: 'latest',
}, },
{ {
path: '/upcoming', path: '/upcoming',
component: Home, component: Home,
name: 'upcoming', name: 'upcoming',
}, },
{ {
path: '/new', path: '/new',
component: Home, component: Home,
name: 'new', name: 'new',
}, },
{ {
path: '/scene/:releaseId/:releaseSlug?', path: '/scene/:releaseId/:releaseSlug?',
component: Release, component: Release,
name: 'scene', name: 'scene',
}, },
{ {
path: '/movie/:releaseId/:releaseSlug?', path: '/movie/:releaseId/:releaseSlug?',
component: Release, component: Release,
name: 'movie', name: 'movie',
}, },
{ {
path: '/actor/:actorSlug', path: '/actor/:actorSlug',
name: 'actor', name: 'actor',
redirect: from => ({ redirect: from => ({
name: 'actorRange', name: 'actorRange',
params: { params: {
...from.params, ...from.params,
range: 'latest', range: 'latest',
}, },
}), }),
}, },
{ {
path: '/actor/:actorSlug/:range', path: '/actor/:actorSlug/:range',
component: Actor, component: Actor,
name: 'actorRange', name: 'actorRange',
}, },
{ {
path: '/site/:siteSlug', path: '/site/:siteSlug',
component: Site, component: Site,
name: 'site', name: 'site',
redirect: from => ({ redirect: from => ({
name: 'siteRange', name: 'siteRange',
params: { params: {
...from.params, ...from.params,
range: 'latest', range: 'latest',
}, },
}), }),
}, },
{ {
path: '/site/:siteSlug/:range', path: '/site/:siteSlug/:range',
component: Site, component: Site,
name: 'siteRange', name: 'siteRange',
}, },
{ {
path: '/network/:networkSlug', path: '/network/:networkSlug',
component: Network, component: Network,
name: 'network', name: 'network',
redirect: from => ({ redirect: from => ({
name: 'networkRange', name: 'networkRange',
params: { params: {
...from.params, ...from.params,
range: 'latest', range: 'latest',
}, },
}), }),
}, },
{ {
path: '/network/:networkSlug/:range', path: '/network/:networkSlug/:range',
component: Network, component: Network,
name: 'networkRange', name: 'networkRange',
}, },
{ {
path: '/tag/:tagSlug', path: '/tag/:tagSlug',
component: Tag, component: Tag,
name: 'tag', name: 'tag',
redirect: from => ({ redirect: from => ({
name: 'tagRange', name: 'tagRange',
params: { params: {
...from.params, ...from.params,
range: 'latest', range: 'latest',
}, },
}), }),
}, },
{ {
path: '/tag/:tagSlug/:range', path: '/tag/:tagSlug/:range',
component: Tag, component: Tag,
name: 'tagRange', name: 'tagRange',
}, },
{ {
path: '/actors/:gender?/:letter?', path: '/actors/:gender?/:letter?',
component: Actors, component: Actors,
name: 'actors', name: 'actors',
}, },
{ {
path: '/networks', path: '/networks',
component: Networks, component: Networks,
name: 'networks', name: 'networks',
}, },
{ {
path: '/tags', path: '/tags',
component: Tags, component: Tags,
name: 'tags', name: 'tags',
}, },
{ {
path: '/search', path: '/search',
component: Search, component: Search,
name: 'search', name: 'search',
}, },
{ {
path: '*', path: '*',
component: NotFound, component: NotFound,
}, },
]; ];
const router = new VueRouter({ const router = new VueRouter({
mode: 'history', mode: 'history',
routes, routes,
}); });
export default router; export default router;

66
assets/js/sites/actions.js
View File

@ -4,10 +4,10 @@ import { curateSite } from '../curate';
import getDateRange from '../get-date-range'; import getDateRange from '../get-date-range';
function initSitesActions(store, _router) { function initSitesActions(store, _router) {
async function fetchSiteBySlug({ _commit }, { siteSlug, limit = 100, range = 'latest' }) { async function fetchSiteBySlug({ _commit }, { siteSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range); const { before, after, orderBy } = getDateRange(range);
const { site } = await graphql(` const { site } = await graphql(`
query Site( query Site(
$siteSlug: String!, $siteSlug: String!,
$limit:Int = 100, $limit:Int = 100,
@ -37,20 +37,20 @@ function initSitesActions(store, _router) {
} }
} }
`, { `, {
siteSlug, siteSlug,
limit, limit,
after, after,
before, before,
orderBy, orderBy,
isNew: store.getters.isNew, isNew: store.getters.isNew,
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });
return curateSite(site); return curateSite(site);
} }
async function fetchSites({ _commit }, { limit = 100 }) { async function fetchSites({ _commit }, { limit = 100 }) {
const { sites } = await graphql(` const { sites } = await graphql(`
query Sites( query Sites(
$actorSlug: String! $actorSlug: String!
$limit:Int = 100, $limit:Int = 100,
@ -64,16 +64,16 @@ function initSitesActions(store, _router) {
} }
} }
`, { `, {
limit, limit,
after: store.getters.after, after: store.getters.after,
before: store.getters.before, before: store.getters.before,
}); });
return sites; return sites;
} }
async function searchSites({ _commit }, { query, limit = 20 }) { async function searchSites({ _commit }, { query, limit = 20 }) {
const { sites } = await graphql(` const { sites } = await graphql(`
query SearchSites( query SearchSites(
$query: String! $query: String!
$limit:Int = 20, $limit:Int = 20,
@ -93,18 +93,18 @@ function initSitesActions(store, _router) {
} }
} }
`, { `, {
query, query,
limit, limit,
}); });
return sites; return sites;
} }
return { return {
fetchSiteBySlug, fetchSiteBySlug,
fetchSites, fetchSites,
searchSites, searchSites,
}; };
} }
export default initSitesActions; export default initSitesActions;

10
assets/js/sites/sites.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions'; import actions from './actions';
function initSitesStore(store, router) { function initSitesStore(store, router) {
return { return {
state, state,
mutations, mutations,
actions: actions(store, router), actions: actions(store, router),
}; };
} }
export default initSitesStore; export default initSitesStore;

20
assets/js/store.js
View File

@ -10,19 +10,19 @@ import initActorsStore from './actors/actors';
import initTagsStore from './tags/tags'; import initTagsStore from './tags/tags';
function initStore(router) { function initStore(router) {
Vue.use(Vuex); Vue.use(Vuex);
const store = new Vuex.Store(); const store = new Vuex.Store();
store.registerModule('ui', initUiStore(store, router)); store.registerModule('ui', initUiStore(store, router));
store.registerModule('auth', initAuthStore(store, router)); store.registerModule('auth', initAuthStore(store, router));
store.registerModule('releases', initReleasesStore(store, router)); store.registerModule('releases', initReleasesStore(store, router));
store.registerModule('actors', initActorsStore(store, router)); store.registerModule('actors', initActorsStore(store, router));
store.registerModule('sites', initSitesStore(store, router)); store.registerModule('sites', initSitesStore(store, router));
store.registerModule('networks', initNetworksStore(store, router)); store.registerModule('networks', initNetworksStore(store, router));
store.registerModule('tags', initTagsStore(store, router)); store.registerModule('tags', initTagsStore(store, router));
return store; return store;
} }
export default initStore; export default initStore;

76
assets/js/tags/actions.js
View File

@ -1,15 +1,15 @@
import { graphql, get } from '../api'; import { graphql, get } from '../api';
import { import {
releaseFields, releaseFields,
} from '../fragments'; } from '../fragments';
import { curateTag } from '../curate'; import { curateTag } from '../curate';
import getDateRange from '../get-date-range'; import getDateRange from '../get-date-range';
function initTagsActions(store, _router) { function initTagsActions(store, _router) {
async function fetchTagBySlug({ _commit }, { tagSlug, limit = 100, range = 'latest' }) { async function fetchTagBySlug({ _commit }, { tagSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range); const { before, after, orderBy } = getDateRange(range);
const { tagBySlug } = await graphql(` const { tagBySlug } = await graphql(`
query Tag( query Tag(
$tagSlug:String! $tagSlug:String!
$limit:Int = 1000, $limit:Int = 1000,
@ -85,24 +85,24 @@ function initTagsActions(store, _router) {
} }
} }
`, { `, {
tagSlug, tagSlug,
limit, limit,
after, after,
before, before,
orderBy: orderBy === 'DATE_DESC' ? 'RELEASE_BY_RELEASE_ID__DATE_DESC' : 'RELEASE_BY_RELEASE_ID__DATE_ASC', orderBy: orderBy === 'DATE_DESC' ? 'RELEASE_BY_RELEASE_ID__DATE_DESC' : 'RELEASE_BY_RELEASE_ID__DATE_ASC',
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });
return curateTag(tagBySlug, store); return curateTag(tagBySlug, store);
} }
async function fetchTags({ _commit }, { async function fetchTags({ _commit }, {
limit = 100, limit = 100,
slugs = [], slugs = [],
_group, _group,
_priority, _priority,
}) { }) {
const { tags } = await graphql(` const { tags } = await graphql(`
query Tags( query Tags(
$slugs: [String!] = [], $slugs: [String!] = [],
$limit: Int = 100 $limit: Int = 100
@ -133,28 +133,28 @@ function initTagsActions(store, _router) {
} }
} }
`, { `, {
slugs, slugs,
limit, limit,
}); });
return tags.map(tag => curateTag(tag, store.state.ui.sfw)); return tags.map(tag => curateTag(tag, store.state.ui.sfw));
} }
async function fetchTagReleases({ _commit }, tagId) { async function fetchTagReleases({ _commit }, tagId) {
const releases = await get(`/tags/${tagId}/releases`, { const releases = await get(`/tags/${tagId}/releases`, {
filter: store.state.ui.filter, filter: store.state.ui.filter,
after: store.getters.after, after: store.getters.after,
before: store.getters.before, before: store.getters.before,
}); });
return releases; return releases;
} }
return { return {
fetchTagBySlug, fetchTagBySlug,
fetchTags, fetchTags,
fetchTagReleases, fetchTagReleases,
}; };
} }
export default initTagsActions; export default initTagsActions;

10
assets/js/tags/tags.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions'; import actions from './actions';
function initTagsStore(store, router) { function initTagsStore(store, router) {
return { return {
state, state,
mutations, mutations,
actions: actions(store, router), actions: actions(store, router),
}; };
} }
export default initTagsStore; export default initTagsStore;

52
assets/js/ui/actions.js
View File

@ -1,35 +1,35 @@
function initUiActions(_store, _router) { function initUiActions(_store, _router) {
function setFilter({ commit }, filter) { function setFilter({ commit }, filter) {
commit('setFilter', filter); commit('setFilter', filter);
localStorage.setItem('filter', filter); localStorage.setItem('filter', filter);
} }
function setRange({ commit }, range) { function setRange({ commit }, range) {
commit('setRange', range); commit('setRange', range);
} }
function setBatch({ commit }, batch) { function setBatch({ commit }, batch) {
commit('setBatch', batch); commit('setBatch', batch);
localStorage.setItem('batch', batch); localStorage.setItem('batch', batch);
} }
function setTheme({ commit }, theme) { function setTheme({ commit }, theme) {
commit('setTheme', theme); commit('setTheme', theme);
localStorage.setItem('theme', theme); localStorage.setItem('theme', theme);
} }
async function setSfw({ commit }, sfw) { async function setSfw({ commit }, sfw) {
commit('setSfw', sfw); commit('setSfw', sfw);
localStorage.setItem('sfw', sfw); localStorage.setItem('sfw', sfw);
} }
return { return {
setFilter, setFilter,
setRange, setRange,
setBatch, setBatch,
setSfw, setSfw,
setTheme, setTheme,
}; };
} }
export default initUiActions; export default initUiActions;

56
assets/js/ui/getters.js
View File

@ -1,47 +1,47 @@
import dayjs from 'dayjs'; import dayjs from 'dayjs';
const dateRanges = { const dateRanges = {
latest: () => ({ latest: () => ({
after: '1900-01-01', after: '1900-01-01',
before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'), before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
orderBy: 'DATE_DESC', orderBy: 'DATE_DESC',
}), }),
upcoming: () => ({ upcoming: () => ({
after: dayjs(new Date()).format('YYYY-MM-DD'), after: dayjs(new Date()).format('YYYY-MM-DD'),
before: '2100-01-01', before: '2100-01-01',
orderBy: 'DATE_ASC', orderBy: 'DATE_ASC',
}), }),
new: () => ({ new: () => ({
after: '1900-01-01', after: '1900-01-01',
before: '2100-01-01', before: '2100-01-01',
orderBy: 'CREATED_AT_DESC', orderBy: 'CREATED_AT_DESC',
}), }),
all: () => ({ all: () => ({
after: '1900-01-01', after: '1900-01-01',
before: '2100-01-01', before: '2100-01-01',
orderBy: 'DATE_DESC', orderBy: 'DATE_DESC',
}), }),
}; };
function rangeDates(state) { function rangeDates(state) {
return dateRanges[state.range](); return dateRanges[state.range]();
} }
function before(state) { function before(state) {
return dateRanges[state.range]().before; return dateRanges[state.range]().before;
} }
function after(state) { function after(state) {
return dateRanges[state.range]().after; return dateRanges[state.range]().after;
} }
function orderBy(state) { function orderBy(state) {
return dateRanges[state.range]().orderBy; return dateRanges[state.range]().orderBy;
} }
export default { export default {
rangeDates, rangeDates,
before, before,
after, after,
orderBy, orderBy,
}; };

20
assets/js/ui/mutations.js
View File

@ -1,27 +1,27 @@
function setFilter(state, filter) { function setFilter(state, filter) {
state.filter = filter; state.filter = filter;
} }
function setRange(state, range) { function setRange(state, range) {
state.range = range; state.range = range;
} }
function setBatch(state, batch) { function setBatch(state, batch) {
state.batch = batch; state.batch = batch;
} }
function setSfw(state, sfw) { function setSfw(state, sfw) {
state.sfw = sfw; state.sfw = sfw;
} }
function setTheme(state, theme) { function setTheme(state, theme) {
state.theme = theme; state.theme = theme;
} }
export default { export default {
setFilter, setFilter,
setRange, setRange,
setBatch, setBatch,
setSfw, setSfw,
setTheme, setTheme,
}; };

34
assets/js/ui/observers.js
View File

@ -1,25 +1,25 @@
function initUiObservers(store, _router) { function initUiObservers(store, _router) {
document.addEventListener('keypress', (event) => { document.addEventListener('keypress', (event) => {
if (event.target.tagName === 'INPUT') { if (event.target.tagName === 'INPUT') {
return; return;
} }
if (event.key === 's') { if (event.key === 's') {
store.dispatch('setSfw', true); store.dispatch('setSfw', true);
} }
if (event.key === 'n') { if (event.key === 'n') {
store.dispatch('setSfw', false); store.dispatch('setSfw', false);
} }
if (event.key === 'd') { if (event.key === 'd') {
store.dispatch('setTheme', 'dark'); store.dispatch('setTheme', 'dark');
} }
if (event.key === 'l') { if (event.key === 'l') {
store.dispatch('setTheme', 'light'); store.dispatch('setTheme', 'light');
} }
}); });
} }
export default initUiObservers; export default initUiObservers;

10
assets/js/ui/state.js
View File

@ -4,9 +4,9 @@ const storedSfw = localStorage.getItem('sfw');
const storedTheme = localStorage.getItem('theme'); const storedTheme = localStorage.getItem('theme');
export default { export default {
filter: storedFilter ? storedFilter.split(',') : ['gay', 'transsexual'], filter: storedFilter ? storedFilter.split(',') : ['gay', 'transsexual'],
range: 'latest', range: 'latest',
batch: storedBatch || 'all', batch: storedBatch || 'all',
sfw: storedSfw === 'true' || false, sfw: storedSfw === 'true' || false,
theme: storedTheme || 'light', theme: storedTheme || 'light',
}; };

12
assets/js/ui/ui.js
View File

@ -4,12 +4,12 @@ import getters from './getters';
import actions from './actions'; import actions from './actions';
function initUiStore(store, router) { function initUiStore(store, router) {
return { return {
state, state,
mutations, mutations,
getters, getters,
actions: actions(store, router), actions: actions(store, router),
}; };
} }
export default initUiStore; export default initUiStore;

354
config/default.js
View File

@ -1,177 +1,181 @@
module.exports = { module.exports = {
database: { database: {
host: '127.0.0.1', host: '127.0.0.1',
user: 'user', user: 'user',
password: 'password', password: 'password',
database: 'traxxx', database: 'traxxx',
}, },
web: { web: {
host: '0.0.0.0', host: '0.0.0.0',
port: 5000, port: 5000,
sfwHost: '0.0.0.0', sfwHost: '0.0.0.0',
sfwPort: 5001, sfwPort: 5001,
}, },
// include: [], // include: [],
// exclude: [], // exclude: [],
exclude: [ exclude: [
['21sextreme', [ ['21sextreme', [
// no longer updated // no longer updated
'mightymistress', 'mightymistress',
'dominatedgirls', 'dominatedgirls',
'homepornreality', 'homepornreality',
'peeandblow', 'peeandblow',
'cummingmatures', 'cummingmatures',
'mandyiskinky', 'mandyiskinky',
'speculumplays', 'speculumplays',
'creampiereality', 'creampiereality',
]], ]],
['aziani', [ ['aziani', [
'amberathome', 'amberathome',
'marycarey', 'marycarey',
'racqueldevonshire', 'racqueldevonshire',
]], ]],
['blowpass', ['sunlustxxx']], 'boobpedia',
['ddfnetwork', [ ['blowpass', ['sunlustxxx']],
'fuckinhd', ['ddfnetwork', [
'bustylover', 'fuckinhd',
]], 'bustylover',
['famedigital', [ ]],
'daringsex', ['famedigital', [
'lowartfilms', 'daringsex',
]], 'lowartfilms',
['pornpros', [ ]],
'milfhumiliation', 'freeones',
'humiliated', ['pornpros', [
'flexiblepositions', 'milfhumiliation',
'publicviolations', 'humiliated',
'amateurviolations', 'flexiblepositions',
'squirtdisgrace', 'publicviolations',
'cumdisgrace', 'amateurviolations',
'webcamhackers', 'squirtdisgrace',
'collegeteens', 'cumdisgrace',
]], 'webcamhackers',
['score', [ 'collegeteens',
'bigboobbundle', ]],
'milfbundle', ['score', [
'pornmegaload', 'bigboobbundle',
'scorelandtv', 'milfbundle',
'scoretv', 'pornmegaload',
]], 'scorelandtv',
], 'scoretv',
profiles: [ ]],
[ ['mindgeek', [
'evilangel', 'pornhub',
'famedigital', ]],
], ],
[ profiles: [
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), Burning Angel and Wicked have their own assets [
'xempire', 'evilangel',
'blowpass', 'famedigital',
], ],
[ [
// MindGeek; Brazzers and Mile High Media have their own assets // Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), Burning Angel and Wicked have their own assets
'realitykings', 'xempire',
'mofos', 'blowpass',
'digitalplayground', ],
'twistys', [
'babes', // MindGeek; Brazzers and Mile High Media have their own assets
'fakehub', 'realitykings',
'sexyhub', 'mofos',
'metrohd', 'digitalplayground',
'iconmale', 'twistys',
'men', 'babes',
'transangels', 'fakehub',
], 'sexyhub',
'wicked', 'metrohd',
'burningangel', 'iconmale',
'brazzers', 'men',
'milehighmedia', 'transangels',
[ ],
'vixen', 'wicked',
'tushy', 'burningangel',
'blacked', 'brazzers',
'tushyraw', 'milehighmedia',
'blackedraw', [
'deeper', 'vixen',
], 'tushy',
[ 'blacked',
// Nubiles 'tushyraw',
'nubiles', 'blackedraw',
'nubilesporn', 'deeper',
'deeplush', ],
'brattysis', [
'nfbusty', // Nubiles
'anilos', 'nubiles',
'hotcrazymess', 'nubilesporn',
'thatsitcomshow', 'deeplush',
], 'brattysis',
'21sextury', 'nfbusty',
'julesjordan', 'anilos',
'naughtyamerica', 'hotcrazymess',
'cherrypimps', 'thatsitcomshow',
'pimpxxx', ],
[ '21sextury',
'hussiepass', 'julesjordan',
'hushpass', 'naughtyamerica',
'interracialpass', 'cherrypimps',
'interracialpovs', 'pimpxxx',
'povpornstars', [
'seehimfuck', 'hussiepass',
'eyeontheguy', 'hushpass',
], 'interracialpass',
[ 'interracialpovs',
// Full Porn Network 'povpornstars',
'analized', 'seehimfuck',
'hergape', 'eyeontheguy',
'jamesdeen', ],
'dtfsluts', [
'analbbc', // Full Porn Network
'analviolation', 'analized',
'baddaddypov', 'hergape',
'girlfaction', 'jamesdeen',
'homemadeanalwhores', 'dtfsluts',
'mugfucked', 'analbbc',
'onlyprince', 'analviolation',
'pervertgallery', 'baddaddypov',
'povperverts', 'girlfaction',
], 'homemadeanalwhores',
'private', 'mugfucked',
'ddfnetwork', 'onlyprince',
'bangbros', 'pervertgallery',
'kellymadison', 'povperverts',
'gangbangcreampie', ],
'gloryholesecrets', 'private',
'aziani', 'ddfnetwork',
'legalporno', 'bangbros',
'score', 'kellymadison',
'boobpedia', 'gangbangcreampie',
'pornhub', 'gloryholesecrets',
'freeones', 'aziani',
'freeonesLegacy', 'legalporno',
], 'score',
proxy: { 'boobpedia',
enable: false, 'pornhub',
host: '', 'freeones',
port: 8888, ],
hostnames: [ proxy: {
'www.vixen.com', enable: false,
'www.blacked.com', host: '',
'www.blackedraw.com', port: 8888,
'www.tushy.com', hostnames: [
'www.tushyraw.com', 'www.vixen.com',
'www.deeper.com', 'www.blacked.com',
], 'www.blackedraw.com',
}, 'www.tushy.com',
fetchAfter: [1, 'week'], 'www.tushyraw.com',
nullDateLimit: 3, 'www.deeper.com',
media: { ],
path: './media', },
thumbnailSize: 320, // width for 16:9 will be exactly 576px fetchAfter: [1, 'week'],
thumbnailQuality: 100, nullDateLimit: 3,
lazySize: 90, media: {
lazyQuality: 90, path: './media',
videoQuality: [480, 360, 320, 540, 720, 1080, 2160, 270, 240, 180], thumbnailSize: 320, // width for 16:9 will be exactly 576px
limit: 25, // max number of photos per release thumbnailQuality: 100,
}, lazySize: 90,
titleSlugLength: 50, lazyQuality: 90,
videoQuality: [480, 360, 320, 540, 720, 1080, 2160, 270, 240, 180],
limit: 25, // max number of photos per release
},
titleSlugLength: 50,
}; };

1485
migrations/20190325001339_releases.js
View File

File diff suppressed because it is too large Load Diff

BIN
public/img/logos/gamma/favicon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.3 KiB

BIN
public/img/logos/mindgeek/favicon.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 2.9 KiB

BIN
public/img/logos/mindgeek/lazy/favicon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.5 KiB

BIN
public/img/logos/mindgeek/lazy/network.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.1 KiB

After

Width:  |  Height:  |  Size: 4.1 KiB

BIN
public/img/logos/mindgeek/lazy/pornhub.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.3 KiB

BIN
public/img/logos/mindgeek/lazy/transangels.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.0 KiB

After

Width:  |  Height:  |  Size: 4.0 KiB

BIN
public/img/logos/mindgeek/lazy/trueamateurs.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.9 KiB

After

Width:  |  Height:  |  Size: 4.9 KiB

BIN
public/img/logos/mindgeek/lazy/tube8vip.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.4 KiB

After

Width:  |  Height:  |  Size: 4.4 KiB

BIN
public/img/logos/mindgeek/misc/porn-hub-gay.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.0 KiB

BIN
public/img/logos/mindgeek/network.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 22 KiB

After

Width:  |  Height:  |  Size: 22 KiB

BIN
public/img/logos/mindgeek/pornhub.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 36 KiB

BIN
public/img/logos/mindgeek/thumbs/favicon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 9.1 KiB

BIN
public/img/logos/mindgeek/thumbs/network.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

BIN
public/img/logos/mindgeek/thumbs/pornhub.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.9 KiB

BIN
public/img/logos/mindgeek/thumbs/transangels.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

BIN
public/img/logos/mindgeek/thumbs/trueamateurs.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

BIN
public/img/logos/mindgeek/thumbs/tube8vip.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 13 KiB

BIN
public/img/logos/mindgeek/transangels.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 36 KiB

BIN
public/img/logos/mindgeek/trueamateurs.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 7.8 KiB

After

Width:  |  Height:  |  Size: 7.8 KiB

BIN
public/img/logos/mindgeek/tube8vip.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 5.5 KiB

After

Width:  |  Height:  |  Size: 5.5 KiB

BIN
public/img/tags/bukkake/0.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 684 KiB

BIN
public/img/tags/bukkake/lazy/0.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 8.2 KiB

BIN
public/img/tags/bukkake/thumbs/0.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 36 KiB

12233
seeds/02_sites.js
View File

File diff suppressed because it is too large Load Diff

1529
seeds/04_media.js
View File

File diff suppressed because it is too large Load Diff

3
src/.eslintrc
View File

@ -6,10 +6,11 @@
}, },
"rules": { "rules": {
"strict": 0, "strict": 0,
"indent": ["error", "tab"],
"no-tabs": "off",
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}], "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
"no-console": 0, "no-console": 0,
"no-underscore-dangle": 0, "no-underscore-dangle": 0,
"indent": "off",
"prefer-destructuring": "off", "prefer-destructuring": "off",
"template-curly-spacing": "off", "template-curly-spacing": "off",
"object-curly-newline": "off", "object-curly-newline": "off",

804
src/actors-legacy.js
View File

@ -18,522 +18,522 @@ const { curateSites } = require('./sites');
const { storeMedia, associateMedia } = require('./media'); const { storeMedia, associateMedia } = require('./media');
async function curateActor(actor) { async function curateActor(actor) {
const [aliases, avatar, photos, social] = await Promise.all([ const [aliases, avatar, photos, social] = await Promise.all([
knex('actors').where({ alias_for: actor.id }), knex('actors').where({ alias_for: actor.id }),
knex('actors_avatars') knex('actors_avatars')
.where('actor_id', actor.id) .where('actor_id', actor.id)
.join('media', 'media.id', 'actors_avatars.media_id') .join('media', 'media.id', 'actors_avatars.media_id')
.first(), .first(),
knex('actors_photos') knex('actors_photos')
.where('actor_id', actor.id) .where('actor_id', actor.id)
.join('media', 'media.id', 'actors_photos.media_id') .join('media', 'media.id', 'actors_photos.media_id')
.orderBy('index'), .orderBy('index'),
knex('actors_social') knex('actors_social')
.where('actor_id', actor.id) .where('actor_id', actor.id)
.orderBy('platform', 'desc'), .orderBy('platform', 'desc'),
]); ]);
const curatedActor = { const curatedActor = {
id: actor.id, id: actor.id,
gender: actor.gender, gender: actor.gender,
name: actor.name, name: actor.name,
description: actor.description, description: actor.description,
birthdate: actor.birthdate && new Date(actor.birthdate), birthdate: actor.birthdate && new Date(actor.birthdate),
country: actor.country_alpha2, country: actor.country_alpha2,
origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null, origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null,
residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null, residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null,
ethnicity: actor.ethnicity, ethnicity: actor.ethnicity,
height: actor.height, height: actor.height,
weight: actor.weight, weight: actor.weight,
bust: actor.bust, bust: actor.bust,
waist: actor.waist, waist: actor.waist,
hip: actor.hip, hip: actor.hip,
naturalBoobs: actor.natural_boobs, naturalBoobs: actor.natural_boobs,
aliases: aliases.map(({ name }) => name), aliases: aliases.map(({ name }) => name),
slug: actor.slug, slug: actor.slug,
avatar, avatar,
photos, photos,
hasTattoos: actor.has_tattoos, hasTattoos: actor.has_tattoos,
hasPiercings: actor.has_piercings, hasPiercings: actor.has_piercings,
tattoos: actor.tattoos, tattoos: actor.tattoos,
piercings: actor.piercings, piercings: actor.piercings,
social, social,
scrapedAt: actor.scraped_at, scrapedAt: actor.scraped_at,
}; };
if (curatedActor.birthdate) { if (curatedActor.birthdate) {
curatedActor.age = moment().diff(curatedActor.birthdate, 'years'); curatedActor.age = moment().diff(curatedActor.birthdate, 'years');
} }
if (actor.birth_city) curatedActor.origin.city = actor.birth_city; if (actor.birth_city) curatedActor.origin.city = actor.birth_city;
if (actor.birth_state) curatedActor.origin.state = actor.birth_state; if (actor.birth_state) curatedActor.origin.state = actor.birth_state;
if (actor.birth_country_alpha2) { if (actor.birth_country_alpha2) {
curatedActor.origin.country = { curatedActor.origin.country = {
alpha2: actor.birth_country_alpha2, alpha2: actor.birth_country_alpha2,
name: actor.birth_country_name, name: actor.birth_country_name,
alias: actor.birth_country_alias, alias: actor.birth_country_alias,
}; };
} }
if (actor.residence_city) curatedActor.residence.city = actor.residence_city; if (actor.residence_city) curatedActor.residence.city = actor.residence_city;
if (actor.residence_state) curatedActor.residence.state = actor.residence_state; if (actor.residence_state) curatedActor.residence.state = actor.residence_state;
if (actor.residence_country_alpha2) { if (actor.residence_country_alpha2) {
curatedActor.residence.country = { curatedActor.residence.country = {
alpha2: actor.residence_country_alpha2, alpha2: actor.residence_country_alpha2,
name: actor.residence_country_name, name: actor.residence_country_name,
alias: actor.residence_country_alias, alias: actor.residence_country_alias,
}; };
} }
return curatedActor; return curatedActor;
} }
function curateActors(releases) { function curateActors(releases) {
return Promise.all(releases.map(async release => curateActor(release))); return Promise.all(releases.map(async release => curateActor(release)));
} }
function curateActorEntry(actor, scraped, scrapeSuccess) { function curateActorEntry(actor, scraped, scrapeSuccess) {
const curatedActor = { const curatedActor = {
name: capitalize(actor.name), name: capitalize(actor.name),
slug: slugify(actor.name), slug: slugify(actor.name),
birthdate: actor.birthdate, birthdate: actor.birthdate,
description: actor.description, description: actor.description,
gender: actor.gender, gender: actor.gender,
ethnicity: actor.ethnicity, ethnicity: actor.ethnicity,
bust: actor.bust, bust: actor.bust,
waist: actor.waist, waist: actor.waist,
hip: actor.hip, hip: actor.hip,
natural_boobs: actor.naturalBoobs, natural_boobs: actor.naturalBoobs,
height: actor.height, height: actor.height,
weight: actor.weight, weight: actor.weight,
hair: actor.hair, hair: actor.hair,
eyes: actor.eyes, eyes: actor.eyes,
has_tattoos: actor.hasTattoos, has_tattoos: actor.hasTattoos,
has_piercings: actor.hasPiercings, has_piercings: actor.hasPiercings,
tattoos: actor.tattoos, tattoos: actor.tattoos,
piercings: actor.piercings, piercings: actor.piercings,
}; };
if (actor.id) { if (actor.id) {
curatedActor.id = actor.id; curatedActor.id = actor.id;
} }
if (actor.birthPlace) { if (actor.birthPlace) {
curatedActor.birth_city = actor.birthPlace.city; curatedActor.birth_city = actor.birthPlace.city;
curatedActor.birth_state = actor.birthPlace.state; curatedActor.birth_state = actor.birthPlace.state;
curatedActor.birth_country_alpha2 = actor.birthPlace.country; curatedActor.birth_country_alpha2 = actor.birthPlace.country;
} }
if (actor.residencePlace) { if (actor.residencePlace) {
curatedActor.residence_city = actor.residencePlace.city; curatedActor.residence_city = actor.residencePlace.city;
curatedActor.residence_state = actor.residencePlace.state; curatedActor.residence_state = actor.residencePlace.state;
curatedActor.residence_country_alpha2 = actor.residencePlace.country; curatedActor.residence_country_alpha2 = actor.residencePlace.country;
} }
if (scraped) { if (scraped) {
curatedActor.scraped_at = new Date(); curatedActor.scraped_at = new Date();
curatedActor.scrape_success = scrapeSuccess; curatedActor.scrape_success = scrapeSuccess;
} }
return curatedActor; return curatedActor;
} }
function curateSocialEntry(url, actorId) { function curateSocialEntry(url, actorId) {
const platforms = [ const platforms = [
// links supplied by PH often look like domain.com/domain.com/username // links supplied by PH often look like domain.com/domain.com/username
{ {
label: 'twitter', label: 'twitter',
pattern: 'http(s)\\://(*)twitter.com/:username(/)(?*)', pattern: 'http(s)\\://(*)twitter.com/:username(/)(?*)',
format: username => `https://www.twitter.com/${username}`, format: username => `https://www.twitter.com/${username}`,
}, },
{ {
label: 'youtube', label: 'youtube',
pattern: 'http(s)\\://(*)youtube.com/channel/:username(?*)', pattern: 'http(s)\\://(*)youtube.com/channel/:username(?*)',
format: username => `https://www.youtube.com/channel/${username}`, format: username => `https://www.youtube.com/channel/${username}`,
}, },
{ {
label: 'instagram', label: 'instagram',
pattern: 'http(s)\\://(*)instagram.com/:username(/)(?*)', pattern: 'http(s)\\://(*)instagram.com/:username(/)(?*)',
format: username => `https://www.instagram.com/${username}`, format: username => `https://www.instagram.com/${username}`,
}, },
{ {
label: 'snapchat', label: 'snapchat',
pattern: 'http(s)\\://(*)snapchat.com/add/:username(/)(?*)', pattern: 'http(s)\\://(*)snapchat.com/add/:username(/)(?*)',
format: username => `https://www.snapchat.com/add/${username}`, format: username => `https://www.snapchat.com/add/${username}`,
}, },
{ {
label: 'tumblr', label: 'tumblr',
pattern: 'http(s)\\://:username.tumblr.com(*)', pattern: 'http(s)\\://:username.tumblr.com(*)',
format: username => `https://${username}.tumblr.com`, format: username => `https://${username}.tumblr.com`,
}, },
{ {
label: 'onlyfans', label: 'onlyfans',
pattern: 'http(s)\\://(*)onlyfans.com/:username(/)(?*)', pattern: 'http(s)\\://(*)onlyfans.com/:username(/)(?*)',
format: username => `https://www.onlyfans.com/${username}`, format: username => `https://www.onlyfans.com/${username}`,
}, },
{ {
label: 'fancentro', label: 'fancentro',
pattern: 'http(s)\\://(*)fancentro.com/:username(/)(?*)', pattern: 'http(s)\\://(*)fancentro.com/:username(/)(?*)',
format: username => `https://www.fancentro.com/${username}`, format: username => `https://www.fancentro.com/${username}`,
}, },
{ {
label: 'modelhub', label: 'modelhub',
pattern: 'http(s)\\://(*)modelhub.com/:username(/)(?*)', pattern: 'http(s)\\://(*)modelhub.com/:username(/)(?*)',
format: username => `https://www.modelhub.com/${username}`, format: username => `https://www.modelhub.com/${username}`,
}, },
]; ];
const match = platforms.reduce((acc, platform) => { const match = platforms.reduce((acc, platform) => {
if (acc) return acc; if (acc) return acc;
const patternMatch = new UrlPattern(platform.pattern).match(url); const patternMatch = new UrlPattern(platform.pattern).match(url);
if (patternMatch) { if (patternMatch) {
return { return {
platform: platform.label, platform: platform.label,
original: url, original: url,
username: patternMatch.username, username: patternMatch.username,
url: platform.format ? platform.format(patternMatch.username) : url, url: platform.format ? platform.format(patternMatch.username) : url,
}; };
} }
return null; return null;
}, null) || { url }; }, null) || { url };
return { return {
url: match.url, url: match.url,
platform: match.platform, platform: match.platform,
actor_id: actorId, actor_id: actorId,
}; };
} }
async function curateSocialEntries(urls, actorId) { async function curateSocialEntries(urls, actorId) {
if (!urls) { if (!urls) {
return []; return [];
} }
const existingSocialLinks = await knex('actors_social').where('actor_id', actorId); const existingSocialLinks = await knex('actors_social').where('actor_id', actorId);
return urls.reduce((acc, url) => { return urls.reduce((acc, url) => {
const socialEntry = curateSocialEntry(url, actorId); const socialEntry = curateSocialEntry(url, actorId);
if (acc.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase()) || existingSocialLinks.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase())) { if (acc.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase()) || existingSocialLinks.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase())) {
// prevent duplicates // prevent duplicates
return acc; return acc;
} }
return [...acc, socialEntry]; return [...acc, socialEntry];
}, []); }, []);
} }
async function fetchActors(queryObject, limit = 100) { async function fetchActors(queryObject, limit = 100) {
const releases = await knex('actors') const releases = await knex('actors')
.select( .select(
'actors.*', 'actors.*',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias', 'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name', 'residence_countries.alias as residence_country_alias', 'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name', 'residence_countries.alias as residence_country_alias',
) )
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2') .leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('countries as residence_countries', 'actors.residence_country_alpha2', 'residence_countries.alpha2') .leftJoin('countries as residence_countries', 'actors.residence_country_alpha2', 'residence_countries.alpha2')
.orderBy(['actors.name', 'actors.gender']) .orderBy(['actors.name', 'actors.gender'])
.where(builder => whereOr(queryObject, 'actors', builder)) .where(builder => whereOr(queryObject, 'actors', builder))
.limit(limit); .limit(limit);
return curateActors(releases); return curateActors(releases);
} }
async function storeSocialLinks(urls, actorId) { async function storeSocialLinks(urls, actorId) {
const curatedSocialEntries = await curateSocialEntries(urls, actorId); const curatedSocialEntries = await curateSocialEntries(urls, actorId);
await knex('actors_social').insert(curatedSocialEntries); await knex('actors_social').insert(curatedSocialEntries);
} }
async function storeAvatars(avatars, actorId) { async function storeAvatars(avatars, actorId) {
if (!avatars || avatars.length === 0) { if (!avatars || avatars.length === 0) {
return []; return [];
} }
const avatarsBySource = await storeMedia(avatars, 'actor', 'avatar'); const avatarsBySource = await storeMedia(avatars, 'actor', 'avatar');
await associateMedia({ [actorId]: avatars }, avatarsBySource, 'actor', 'photo', 'avatar'); await associateMedia({ [actorId]: avatars }, avatarsBySource, 'actor', 'photo', 'avatar');
return avatarsBySource; return avatarsBySource;
} }
async function storeActor(actor, scraped = false, scrapeSuccess = false) { async function storeActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess); const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const [actorEntry] = await knex('actors') const [actorEntry] = await knex('actors')
.insert(curatedActor) .insert(curatedActor)
.returning('*'); .returning('*');
await storeSocialLinks(actor.social, actorEntry.id); await storeSocialLinks(actor.social, actorEntry.id);
if (actor.avatars) { if (actor.avatars) {
await storeAvatars(actor.avatars, actorEntry.id); await storeAvatars(actor.avatars, actorEntry.id);
} }
logger.info(`Added new entry for actor '${actor.name}'`); logger.info(`Added new entry for actor '${actor.name}'`);
return actorEntry; return actorEntry;
} }
async function updateActor(actor, scraped = false, scrapeSuccess = false) { async function updateActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess); const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const [actorEntry] = await knex('actors') const [actorEntry] = await knex('actors')
.where({ id: actor.id }) .where({ id: actor.id })
.update(curatedActor) .update(curatedActor)
.returning('*'); .returning('*');
await storeSocialLinks(actor.social, actor.id); await storeSocialLinks(actor.social, actor.id);
logger.info(`Updated entry for actor '${actor.name}'`); logger.info(`Updated entry for actor '${actor.name}'`);
return actorEntry; return actorEntry;
} }
async function mergeProfiles(profiles, actor) { async function mergeProfiles(profiles, actor) {
if (profiles.filter(Boolean).length === 0) { if (profiles.filter(Boolean).length === 0) {
return null; return null;
} }
const mergedProfile = profiles.reduce((prevProfile, profile) => { const mergedProfile = profiles.reduce((prevProfile, profile) => {
if (profile === null) { if (profile === null) {
return prevProfile; return prevProfile;
} }
const accProfile = { const accProfile = {
id: actor ? actor.id : null, id: actor ? actor.id : null,
name: actor ? actor.name : (prevProfile.name || profile.name), name: actor ? actor.name : (prevProfile.name || profile.name),
description: prevProfile.description || profile.description, description: prevProfile.description || profile.description,
gender: prevProfile.gender || profile.gender, gender: prevProfile.gender || profile.gender,
birthdate: !prevProfile.birthdate || Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate, birthdate: !prevProfile.birthdate || Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
birthPlace: prevProfile.birthPlace || profile.birthPlace, birthPlace: prevProfile.birthPlace || profile.birthPlace,
residencePlace: prevProfile.residencePlace || profile.residencePlace, residencePlace: prevProfile.residencePlace || profile.residencePlace,
nationality: prevProfile.nationality || profile.nationality, // used to derive country when not available nationality: prevProfile.nationality || profile.nationality, // used to derive country when not available
ethnicity: prevProfile.ethnicity || profile.ethnicity, ethnicity: prevProfile.ethnicity || profile.ethnicity,
bust: prevProfile.bust || (/\d+\w+/.test(profile.bust) ? profile.bust : null), bust: prevProfile.bust || (/\d+\w+/.test(profile.bust) ? profile.bust : null),
waist: prevProfile.waist || profile.waist, waist: prevProfile.waist || profile.waist,
hip: prevProfile.hip || profile.hip, hip: prevProfile.hip || profile.hip,
naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs, naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs,
height: prevProfile.height || profile.height, height: prevProfile.height || profile.height,
weight: prevProfile.weight || profile.weight, weight: prevProfile.weight || profile.weight,
hair: prevProfile.hair || profile.hair, hair: prevProfile.hair || profile.hair,
eyes: prevProfile.eyes || profile.eyes, eyes: prevProfile.eyes || profile.eyes,
hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings, hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings,
hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos, hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos,
piercings: prevProfile.piercings || profile.piercings, piercings: prevProfile.piercings || profile.piercings,
tattoos: prevProfile.tattoos || profile.tattoos, tattoos: prevProfile.tattoos || profile.tattoos,
social: prevProfile.social.concat(profile.social || []), social: prevProfile.social.concat(profile.social || []),
releases: prevProfile.releases.concat(profile.releases ? profile.releases : []), // don't flatten fallbacks releases: prevProfile.releases.concat(profile.releases ? profile.releases : []), // don't flatten fallbacks
}; };
if (profile.avatar) { if (profile.avatar) {
const avatar = Array.isArray(profile.avatar) const avatar = Array.isArray(profile.avatar)
? profile.avatar.map(avatarX => ({ ? profile.avatar.map(avatarX => ({
src: avatarX.src || avatarX, src: avatarX.src || avatarX,
scraper: profile.scraper, scraper: profile.scraper,
copyright: avatarX.copyright === undefined ? capitalize(profile.site?.name || profile.scraper) : profile.avatar.copyright, copyright: avatarX.copyright === undefined ? capitalize(profile.site?.name || profile.scraper) : profile.avatar.copyright,
})) }))
: { : {
src: profile.avatar.src || profile.avatar, src: profile.avatar.src || profile.avatar,
scraper: profile.scraper, scraper: profile.scraper,
copyright: profile.avatar.copyright === undefined ? capitalize(profile.site?.name || profile.scraper) : profile.avatar.copyright, copyright: profile.avatar.copyright === undefined ? capitalize(profile.site?.name || profile.scraper) : profile.avatar.copyright,
}; };
accProfile.avatars = prevProfile.avatars.concat([avatar]); // don't flatten fallbacks accProfile.avatars = prevProfile.avatars.concat([avatar]); // don't flatten fallbacks
} else { } else {
accProfile.avatars = prevProfile.avatars; accProfile.avatars = prevProfile.avatars;
} }
return accProfile; return accProfile;
}, { }, {
social: [], social: [],
avatars: [], avatars: [],
releases: [], releases: [],
}); });
const [birthPlace, residencePlace] = await Promise.all([ const [birthPlace, residencePlace] = await Promise.all([
resolvePlace(mergedProfile.birthPlace), resolvePlace(mergedProfile.birthPlace),
resolvePlace(mergedProfile.residencePlace), resolvePlace(mergedProfile.residencePlace),
]); ]);
mergedProfile.birthPlace = birthPlace; mergedProfile.birthPlace = birthPlace;
mergedProfile.residencePlace = residencePlace; mergedProfile.residencePlace = residencePlace;
if (!mergedProfile.birthPlace && mergedProfile.nationality) { if (!mergedProfile.birthPlace && mergedProfile.nationality) {
const country = await knex('countries') const country = await knex('countries')
.where('nationality', 'ilike', `%${mergedProfile.nationality}%`) .where('nationality', 'ilike', `%${mergedProfile.nationality}%`)
.orderBy('priority', 'desc') .orderBy('priority', 'desc')
.first(); .first();
mergedProfile.birthPlace = { mergedProfile.birthPlace = {
country: country.alpha2, country: country.alpha2,
}; };
} }
return mergedProfile; return mergedProfile;
} }
async function scrapeProfiles(sources, actorName, actorEntry, sitesBySlug) { async function scrapeProfiles(sources, actorName, actorEntry, sitesBySlug) {
return Promise.map(sources, async (source) => { return Promise.map(sources, async (source) => {
// const [scraperSlug, scraper] = source; // const [scraperSlug, scraper] = source;
const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] })); const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] }));
try { try {
return await profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => { return await profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
if (!scraper) { if (!scraper) {
logger.warn(`No profile profile scraper available for ${scraperSlug}`); logger.warn(`No profile profile scraper available for ${scraperSlug}`);
throw Object.assign(new Error(`No profile scraper available for ${scraperSlug}`)); throw Object.assign(new Error(`No profile scraper available for ${scraperSlug}`));
} }
logger.verbose(`Searching '${actorName}' on ${scraperSlug}`); logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
const site = sitesBySlug[scraperSlug] || null; const site = sitesBySlug[scraperSlug] || null;
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug, site, include); const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug, site, include);
if (profile && typeof profile !== 'number') { if (profile && typeof profile !== 'number') {
logger.verbose(`Found profile for '${actorName}' on ${scraperSlug}`); logger.verbose(`Found profile for '${actorName}' on ${scraperSlug}`);
return { return {
...profile, ...profile,
name: actorName, name: actorName,
scraper: scraperSlug, scraper: scraperSlug,
site, site,
releases: profile.releases?.map(release => (typeof release === 'string' releases: profile.releases?.map(release => (typeof release === 'string'
? { url: release, site } ? { url: release, site }
: { ...release, site: release.site || site } : { ...release, site: release.site || site }
)), )),
}; };
} }
logger.verbose(`No profile for '${actorName}' available on ${scraperSlug}: ${profile}`); logger.verbose(`No profile for '${actorName}' available on ${scraperSlug}: ${profile}`);
throw Object.assign(new Error(`Profile for ${actorName} not available on ${scraperSlug}`), { warn: false }); throw Object.assign(new Error(`Profile for ${actorName} not available on ${scraperSlug}`), { warn: false });
}), Promise.reject(new Error())); }), Promise.reject(new Error()));
} catch (error) { } catch (error) {
if (error.warn !== false) { if (error.warn !== false) {
logger.warn(`Error in scraper ${source}: ${error.message}`); logger.warn(`Error in scraper ${source}: ${error.message}`);
// logger.error(error.stack); // logger.error(error.stack);
} }
} }
return null; return null;
}); });
} }
async function scrapeActors(actorNames) { async function scrapeActors(actorNames) {
return Promise.map(actorNames || argv.actors, async (actorName) => { return Promise.map(actorNames || argv.actors, async (actorName) => {
try { try {
const actorSlug = slugify(actorName); const actorSlug = slugify(actorName);
const actorEntry = await knex('actors').where({ slug: actorSlug }).first(); const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors); const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
const finalSources = argv.withReleases ? sources.flat() : sources; // ignore race-to-success grouping when scenes are requested const finalSources = argv.withReleases ? sources.flat() : sources; // ignore race-to-success grouping when scenes are requested
const [siteEntries, networkEntries] = await Promise.all([ const [siteEntries, networkEntries] = await Promise.all([
knex('sites') knex('sites')
.leftJoin('networks', 'sites.network_id', 'networks.id') .leftJoin('networks', 'sites.network_id', 'networks.id')
.select( .select(
'sites.*', 'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
) )
.whereIn('sites.slug', finalSources.flat()), .whereIn('sites.slug', finalSources.flat()),
knex('networks').select('*').whereIn('slug', finalSources.flat()), knex('networks').select('*').whereIn('slug', finalSources.flat()),
]); ]);
const sites = await curateSites(siteEntries, true); const sites = await curateSites(siteEntries, true);
const networks = networkEntries.map(network => ({ ...network, isFallback: true })); const networks = networkEntries.map(network => ({ ...network, isFallback: true }));
const sitesBySlug = [].concat(networks, sites).reduce((acc, site) => ({ ...acc, [site.slug]: site }), {}); const sitesBySlug = [].concat(networks, sites).reduce((acc, site) => ({ ...acc, [site.slug]: site }), {});
const profiles = await scrapeProfiles(sources, actorName, actorEntry, sitesBySlug); const profiles = await scrapeProfiles(sources, actorName, actorEntry, sitesBySlug);
const profile = await mergeProfiles(profiles, actorEntry); const profile = await mergeProfiles(profiles, actorEntry);
if (profile === null) { if (profile === null) {
logger.warn(`Could not find profile for actor '${actorName}'`); logger.warn(`Could not find profile for actor '${actorName}'`);
if (argv.save && !actorEntry) { if (argv.save && !actorEntry) {
await storeActor({ name: actorName }, false, false); await storeActor({ name: actorName }, false, false);
} }
return null; return null;
} }
if (argv.inspect) { if (argv.inspect) {
console.log(profile); console.log(profile);
logger.info(`Found ${profile.releases.length} releases for ${actorName}`); logger.info(`Found ${profile.releases.length} releases for ${actorName}`);
} }
if (argv.save) { if (argv.save) {
if (actorEntry && profile) { if (actorEntry && profile) {
await Promise.all([ await Promise.all([
updateActor(profile, true, true), updateActor(profile, true, true),
storeAvatars(profile.avatars, actorEntry.id), storeAvatars(profile.avatars, actorEntry.id),
]); ]);
return profile; return profile;
} }
await storeActor(profile, true, true); await storeActor(profile, true, true);
} }
return profile; return profile;
} catch (error) { } catch (error) {
console.log(error); console.log(error);
logger.warn(`${actorName}: ${error}`); logger.warn(`${actorName}: ${error}`);
return null; return null;
} }
}, { }, {
concurrency: 3, concurrency: 3,
}); });
} }
async function scrapeBasicActors() { async function scrapeBasicActors() {
const basicActors = await knex('actors').where('scraped_at', null); const basicActors = await knex('actors').where('scraped_at', null);
return scrapeActors(basicActors.map(actor => actor.name)); return scrapeActors(basicActors.map(actor => actor.name));
} }
async function associateActors(mappedActors, releases) { async function associateActors(mappedActors, releases) {
const [existingActorEntries, existingAssociationEntries] = await Promise.all([ const [existingActorEntries, existingAssociationEntries] = await Promise.all([
knex('actors') knex('actors')
.whereIn('name', Object.values(mappedActors).map(actor => actor.name)) .whereIn('name', Object.values(mappedActors).map(actor => actor.name))
.orWhereIn('slug', Object.keys(mappedActors)), .orWhereIn('slug', Object.keys(mappedActors)),
knex('releases_actors').whereIn('release_id', releases.map(release => release.id)), knex('releases_actors').whereIn('release_id', releases.map(release => release.id)),
]); ]);
const associations = await Promise.map(Object.entries(mappedActors), async ([actorSlug, actor]) => { const associations = await Promise.map(Object.entries(mappedActors), async ([actorSlug, actor]) => {
try { try {
const actorEntry = existingActorEntries.find(actorX => actorX.slug === actorSlug) const actorEntry = existingActorEntries.find(actorX => actorX.slug === actorSlug)
|| await storeActor(actor); || await storeActor(actor);
// if a scene // if a scene
return Array.from(actor.releaseIds) return Array.from(actor.releaseIds)
.map(releaseId => ({ .map(releaseId => ({
release_id: releaseId, release_id: releaseId,
actor_id: actorEntry.id, actor_id: actorEntry.id,
})) }))
.filter(association => !existingAssociationEntries .filter(association => !existingAssociationEntries
// remove associations already in database // remove associations already in database
.some(associationEntry => associationEntry.actor_id === association.actor_id .some(associationEntry => associationEntry.actor_id === association.actor_id
&& associationEntry.release_id === association.release_id)); && associationEntry.release_id === association.release_id));
} catch (error) { } catch (error) {
logger.error(actor.name, error); logger.error(actor.name, error);
return null; return null;
} }
}); });
await knex('releases_actors').insert(associations.filter(association => association).flat()); await knex('releases_actors').insert(associations.filter(association => association).flat());
// basic actor scraping is failure prone, don't run together with actor association // basic actor scraping is failure prone, don't run together with actor association
// await scrapebasicactors(), // await scrapebasicactors(),
} }
module.exports = { module.exports = {
associateActors, associateActors,
fetchActors, fetchActors,
scrapeActors, scrapeActors,
scrapeBasicActors, scrapeBasicActors,
}; };

193
src/actors.js
View File

@ -1,125 +1,156 @@
'use strict'; 'use strict';
const config = require('config');
const Promise = require('bluebird');
// const logger = require('./logger')(__filename); // const logger = require('./logger')(__filename);
const knex = require('./knex'); const knex = require('./knex');
const scrapers = require('./scrapers/scrapers');
const argv = require('./argv');
const slugify = require('./utils/slugify'); const slugify = require('./utils/slugify');
const capitalize = require('./utils/capitalize'); const capitalize = require('./utils/capitalize');
function toBaseActors(actorsOrNames, release) { function toBaseActors(actorsOrNames, release) {
return actorsOrNames.map((actorOrName) => { return actorsOrNames.map((actorOrName) => {
const name = capitalize(actorOrName.name || actorOrName); const name = capitalize(actorOrName.name || actorOrName);
const slug = slugify(name); const slug = slugify(name);
const baseActor = { const baseActor = {
name, name,
slug, slug,
network: release.site.network, network: release?.site.network,
}; };
if (actorOrName.name) { if (actorOrName.name) {
return { return {
...actorOrName, ...actorOrName,
...baseActor, ...baseActor,
}; };
} }
return baseActor; return baseActor;
}); });
} }
function curateActorEntry(baseActor, batchId) { function curateActorEntry(baseActor, batchId) {
return { return {
name: baseActor.name, name: baseActor.name,
slug: baseActor.slug, slug: baseActor.slug,
network_id: null, network_id: null,
batch_id: batchId, batch_id: batchId,
}; };
} }
function curateActorEntries(baseActors, batchId) { function curateActorEntries(baseActors, batchId) {
return baseActors.map(baseActor => curateActorEntry(baseActor, batchId)); return baseActors.map(baseActor => curateActorEntry(baseActor, batchId));
} }
async function scrapeProfiles() { async function scrapeActors(actorNames) {
const baseActors = toBaseActors(actorNames);
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
const siteSlugs = sources.flat();
const [networks, sites, existingActorEntries] = await Promise.all([
knex('networks').whereIn('slug', siteSlugs),
knex('sites').whereIn('slug', siteSlugs),
knex('actors')
.select(['id', 'name', 'slug'])
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('network_id'),
]);
const existingActorEntriesBySlug = existingActorEntries.reduce((acc, actorEntry) => ({ ...acc, [actorEntry.slug]: actorEntry }), {});
const networksBySlug = networks.reduce((acc, network) => ({ ...acc, [network.slug]: { ...network, isNetwork: true } }), {});
const sitesBySlug = sites.reduce((acc, site) => ({ ...acc, [site.slug]: site }), {});
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlug[baseActor.slug]);
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
const newActorEntries = batchId && await knex('actors').insert(curatedActorEntries).returning(['id', 'name', 'slug']);
const actorEntries = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []);
console.log(actorEntries, newActorEntries, actorEntries);
} }
async function getOrCreateActors(baseActors, batchId) { async function getOrCreateActors(baseActors, batchId) {
const existingActors = await knex('actors') const existingActors = await knex('actors')
.select('id', 'alias_for', 'name', 'slug', 'network_id') .select('id', 'alias_for', 'name', 'slug', 'network_id')
.whereIn('slug', baseActors.map(baseActor => baseActor.slug)) .whereIn('slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('network_id') .whereNull('network_id')
.orWhereIn(['slug', 'network_id'], baseActors.map(baseActor => [baseActor.slug, baseActor.network.id])); .orWhereIn(['slug', 'network_id'], baseActors.map(baseActor => [baseActor.slug, baseActor.network.id]));
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug)); // const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
const existingActorSlugs = existingActors.reduce((acc, actor) => ({ const existingActorSlugs = existingActors.reduce((acc, actor) => ({
...acc, ...acc,
[actor.network_id]: { [actor.network_id]: {
...acc[actor.network_id], ...acc[actor.network_id],
[actor.slug]: true, [actor.slug]: true,
}, },
}), {}); }), {});
const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.network.id]?.[baseActor.slug] && !existingActorSlugs.null?.[baseActor.slug]); const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.network.id]?.[baseActor.slug] && !existingActorSlugs.null?.[baseActor.slug]);
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId); const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
const newActors = await knex('actors').insert(curatedActorEntries, ['id', 'alias_for', 'name', 'slug', 'network_id']); const newActors = await knex('actors').insert(curatedActorEntries, ['id', 'alias_for', 'name', 'slug', 'network_id']);
if (Array.isArray(newActors)) { if (Array.isArray(newActors)) {
return newActors.concat(existingActors); return newActors.concat(existingActors);
} }
return existingActors; return existingActors;
} }
async function associateActors(releases, batchId) { async function associateActors(releases, batchId) {
const baseActorsByReleaseId = releases.reduce((acc, release) => { const baseActorsByReleaseId = releases.reduce((acc, release) => {
if (release.actors) { if (release.actors) {
acc[release.id] = toBaseActors(release.actors, release); acc[release.id] = toBaseActors(release.actors, release);
} }
return acc; return acc;
}, {}); }, {});
const baseActors = Object.values(baseActorsByReleaseId).flat(); const baseActors = Object.values(baseActorsByReleaseId).flat();
if (baseActors.length === 0) { if (baseActors.length === 0) {
return; return;
} }
const baseActorsBySlugAndNetworkId = baseActors.reduce((acc, baseActor) => ({ const baseActorsBySlugAndNetworkId = baseActors.reduce((acc, baseActor) => ({
...acc, ...acc,
[baseActor.slug]: { [baseActor.slug]: {
...acc[baseActor.slug], ...acc[baseActor.slug],
[baseActor.network.id]: baseActor, [baseActor.network.id]: baseActor,
}, },
}), {}); }), {});
const uniqueBaseActors = Object.values(baseActorsBySlugAndNetworkId).map(baseActorsByNetworkId => Object.values(baseActorsByNetworkId)).flat(); const uniqueBaseActors = Object.values(baseActorsBySlugAndNetworkId).map(baseActorsByNetworkId => Object.values(baseActorsByNetworkId)).flat();
const actors = await getOrCreateActors(uniqueBaseActors, batchId); const actors = await getOrCreateActors(uniqueBaseActors, batchId);
console.log(actors);
const actorIdsBySlugAndNetworkId = actors.reduce((acc, actor) => ({
...acc,
[actor.network_id]: {
...acc[actor.network_id],
[actor.slug]: actor.alias_for || actor.id,
},
}), {});
console.log(actorIdsBySlugAndNetworkId); const actorIdsBySlugAndNetworkId = actors.reduce((acc, actor) => ({
...acc,
[actor.network_id]: {
...acc[actor.network_id],
[actor.slug]: actor.alias_for || actor.id,
},
}), {});
const releaseActorAssociations = Object.entries(baseActorsByReleaseId) const releaseActorAssociations = Object.entries(baseActorsByReleaseId)
.map(([releaseId, releaseActors]) => releaseActors .map(([releaseId, releaseActors]) => releaseActors
.map(releaseActor => ({ .map(releaseActor => ({
release_id: releaseId, release_id: releaseId,
actor_id: actorIdsBySlugAndNetworkId[releaseActor.network.id]?.[releaseActor.slug] || actorIdsBySlugAndNetworkId.null[releaseActor.slug], actor_id: actorIdsBySlugAndNetworkId[releaseActor.network.id]?.[releaseActor.slug] || actorIdsBySlugAndNetworkId.null[releaseActor.slug],
}))) })))
.flat(); .flat();
await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`); await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
} }
module.exports = { module.exports = {
associateActors, associateActors,
scrapeActors,
}; };

48
src/app.js
View File

@ -7,39 +7,39 @@ const knex = require('./knex');
const fetchUpdates = require('./updates'); const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep'); const { fetchScenes, fetchMovies } = require('./deep');
const { storeReleases, updateReleasesSearch } = require('./store-releases'); const { storeReleases, updateReleasesSearch } = require('./store-releases');
const { scrapeActors } = require('./actors-legacy'); const { scrapeActors } = require('./actors');
async function init() { async function init() {
if (argv.server) { if (argv.server) {
await initServer(); await initServer();
return; return;
} }
if (argv.updateSearch) { if (argv.updateSearch) {
await updateReleasesSearch(); await updateReleasesSearch();
} }
if (argv.actors) { if (argv.actors) {
await scrapeActors(argv.actors); await scrapeActors(argv.actors);
} }
const updateBaseScenes = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates(); const updateBaseScenes = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates();
const deepScenes = argv.deep const deepScenes = argv.deep
? await fetchScenes([...(argv.scenes || []), ...(updateBaseScenes || [])]) ? await fetchScenes([...(argv.scenes || []), ...(updateBaseScenes || [])])
: updateBaseScenes; : updateBaseScenes;
const sceneMovies = deepScenes && argv.sceneMovies && deepScenes.map(scene => scene.movie).filter(Boolean); const sceneMovies = deepScenes && argv.sceneMovies && deepScenes.map(scene => scene.movie).filter(Boolean);
const deepMovies = await fetchMovies([...(argv.movies || []), ...(sceneMovies || [])]); const deepMovies = await fetchMovies([...(argv.movies || []), ...(sceneMovies || [])]);
if (argv.save) { if (argv.save) {
await storeReleases([ await storeReleases([
...(deepScenes || []), ...(deepScenes || []),
...(deepMovies || []), ...(deepMovies || []),
]); ]);
} }
knex.destroy(); knex.destroy();
} }
module.exports = init; module.exports = init;

366
src/argv.js
View File

@ -4,188 +4,188 @@ const config = require('config');
const yargs = require('yargs'); const yargs = require('yargs');
const { argv } = yargs const { argv } = yargs
.command('npm start') .command('npm start')
.option('server', { .option('server', {
describe: 'Start web server', describe: 'Start web server',
type: 'boolean', type: 'boolean',
alias: 'web', alias: 'web',
}) })
.option('scrape', { .option('scrape', {
describe: 'Scrape sites and networks defined in configuration', describe: 'Scrape sites and networks defined in configuration',
type: 'boolean', type: 'boolean',
}) })
.option('networks', { .option('networks', {
describe: 'Networks to scrape (overrides configuration)', describe: 'Networks to scrape (overrides configuration)',
type: 'array', type: 'array',
alias: 'network', alias: 'network',
}) })
.option('sites', { .option('sites', {
describe: 'Sites to scrape (overrides configuration)', describe: 'Sites to scrape (overrides configuration)',
type: 'array', type: 'array',
alias: 'site', alias: 'site',
}) })
.option('actors', { .option('actors', {
describe: 'Scrape actors by name or slug', describe: 'Scrape actors by name or slug',
type: 'array', type: 'array',
alias: 'actor', alias: 'actor',
}) })
.option('actor-scenes', { .option('actor-scenes', {
describe: 'Fetch all scenes for an actor', describe: 'Fetch all scenes for an actor',
type: 'boolean', type: 'boolean',
alias: 'with-releases', alias: 'with-releases',
default: false, default: false,
}) })
.option('movie-scenes', { .option('movie-scenes', {
describe: 'Fetch all scenes for a movie', describe: 'Fetch all scenes for a movie',
type: 'boolean', type: 'boolean',
alias: 'with-releases', alias: 'with-releases',
default: false, default: false,
}) })
.option('scene-movies', { .option('scene-movies', {
describe: 'Fetch movies for scenes', describe: 'Fetch movies for scenes',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('profiles', { .option('profiles', {
describe: 'Scrape profiles for new actors after fetching scenes', describe: 'Scrape profiles for new actors after fetching scenes',
type: 'boolean', type: 'boolean',
alias: 'bios', alias: 'bios',
default: false, default: false,
}) })
.option('scene', { .option('scene', {
describe: 'Scrape scene info from URL', describe: 'Scrape scene info from URL',
type: 'array', type: 'array',
alias: 'scenes', alias: 'scenes',
}) })
.option('movie', { .option('movie', {
describe: 'Scrape movie info from URL', describe: 'Scrape movie info from URL',
type: 'array', type: 'array',
alias: 'movies', alias: 'movies',
}) })
.option('sources', { .option('sources', {
describe: 'Use these scrapers for actor data', describe: 'Use these scrapers for actor data',
type: 'array', type: 'array',
alias: 'source', alias: 'source',
}) })
.option('deep', { .option('deep', {
describe: 'Fetch details for all releases', describe: 'Fetch details for all releases',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('latest', { .option('latest', {
describe: 'Scrape latest releases if available', describe: 'Scrape latest releases if available',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('upcoming', { .option('upcoming', {
describe: 'Scrape upcoming releases if available', describe: 'Scrape upcoming releases if available',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('redownload', { .option('redownload', {
describe: 'Don\'t ignore duplicates, update existing entries', describe: 'Don\'t ignore duplicates, update existing entries',
type: 'boolean', type: 'boolean',
alias: 'force', alias: 'force',
}) })
.option('after', { .option('after', {
describe: 'Don\'t fetch scenes older than', describe: 'Don\'t fetch scenes older than',
type: 'string', type: 'string',
default: config.fetchAfter.join(' '), default: config.fetchAfter.join(' '),
}) })
.option('last', { .option('last', {
describe: 'Get the latest x releases, no matter the date range', describe: 'Get the latest x releases, no matter the date range',
type: 'number', type: 'number',
}) })
.option('null-date-limit', { .option('null-date-limit', {
describe: 'Limit amount of scenes when dates are missing.', describe: 'Limit amount of scenes when dates are missing.',
type: 'number', type: 'number',
default: config.nullDateLimit, default: config.nullDateLimit,
alias: 'limit', alias: 'limit',
}) })
.option('page', { .option('page', {
describe: 'Page to start scraping at', describe: 'Page to start scraping at',
type: 'number', type: 'number',
default: 1, default: 1,
}) })
.option('save', { .option('save', {
describe: 'Save fetched releases to database', describe: 'Save fetched releases to database',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('media', { .option('media', {
describe: 'Include any release media', describe: 'Include any release media',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('media-limit', { .option('media-limit', {
describe: 'Maximum amount of assets of each type per release', describe: 'Maximum amount of assets of each type per release',
type: 'number', type: 'number',
default: config.media.limit, default: config.media.limit,
}) })
.option('images', { .option('images', {
describe: 'Include any photos, posters or covers', describe: 'Include any photos, posters or covers',
type: 'boolean', type: 'boolean',
default: true, default: true,
alias: 'pics', alias: 'pics',
}) })
.option('videos', { .option('videos', {
describe: 'Include any trailers or teasers', describe: 'Include any trailers or teasers',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('posters', { .option('posters', {
describe: 'Include release posters', describe: 'Include release posters',
type: 'boolean', type: 'boolean',
default: true, default: true,
alias: 'poster', alias: 'poster',
}) })
.option('covers', { .option('covers', {
describe: 'Include release covers', describe: 'Include release covers',
type: 'boolean', type: 'boolean',
default: true, default: true,
alias: 'cover', alias: 'cover',
}) })
.option('photos', { .option('photos', {
describe: 'Include release photos', describe: 'Include release photos',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('trailers', { .option('trailers', {
describe: 'Include release trailers', describe: 'Include release trailers',
type: 'boolean', type: 'boolean',
default: true, default: true,
alias: 'trailer', alias: 'trailer',
}) })
.option('teasers', { .option('teasers', {
describe: 'Include release teasers', describe: 'Include release teasers',
type: 'boolean', type: 'boolean',
default: true, default: true,
alias: 'teaser', alias: 'teaser',
}) })
.option('avatars', { .option('avatars', {
describe: 'Include actor avatars', describe: 'Include actor avatars',
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('inspect', { .option('inspect', {
describe: 'Show data in console.', describe: 'Show data in console.',
type: 'boolean', type: 'boolean',
default: false, default: false,
}) })
.option('level', { .option('level', {
describe: 'Log level', describe: 'Log level',
type: 'string', type: 'string',
default: process.env.NODE_ENV === 'development' ? 'silly' : 'info', default: process.env.NODE_ENV === 'development' ? 'silly' : 'info',
}) })
.option('debug', { .option('debug', {
describe: 'Show error stack traces', describe: 'Show error stack traces',
type: 'boolean', type: 'boolean',
default: process.env.NODE_ENV === 'development', default: process.env.NODE_ENV === 'development',
}) })
.option('update-search', { .option('update-search', {
describe: 'Update search documents for all releases.', describe: 'Update search documents for all releases.',
type: 'boolean', type: 'boolean',
default: false, default: false,
}); });
module.exports = argv; module.exports = argv;

217
src/deep.js
View File

@ -11,159 +11,160 @@ const { curateSites } = require('./sites');
const { curateNetworks } = require('./networks'); const { curateNetworks } = require('./networks');
function urlToSiteSlug(url) { function urlToSiteSlug(url) {
try { try {
const slug = new URL(url) const slug = new URL(url)
.hostname .hostname
.match(/([\w-]+)\.\w+$/)?.[1]; .match(/([\w-]+)\.\w+$/)?.[1];
return slug; return slug;
} catch (error) { } catch (error) {
logger.warn(`Failed to derive site slug from '${url}': ${error.message}`); logger.warn(`Failed to derive site slug from '${url}': ${error.message}`);
return null; return null;
} }
} }
async function findSites(baseReleases) { async function findSites(baseReleases) {
const baseReleasesWithoutSite = baseReleases.filter(release => release.url && !release.site); const baseReleasesWithoutSite = baseReleases.filter(release => release.url && !release.site);
const siteSlugs = Array.from(new Set( const siteSlugs = Array.from(new Set(
baseReleasesWithoutSite baseReleasesWithoutSite
.map(baseRelease => urlToSiteSlug(baseRelease.url)) .map(baseRelease => urlToSiteSlug(baseRelease.url))
.filter(Boolean), .filter(Boolean),
)); ));
const siteEntries = await knex('sites') const siteEntries = await knex('sites')
.leftJoin('networks', 'networks.id', 'sites.network_id') .leftJoin('networks', 'networks.id', 'sites.network_id')
.select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.parameters as network_parameters', 'networks.description as network_description') .select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.parameters as network_parameters', 'networks.description as network_description')
.whereIn('sites.slug', siteSlugs); .whereIn('sites.slug', siteSlugs);
const networkEntries = await knex('networks').whereIn('slug', siteSlugs); const networkEntries = await knex('networks').whereIn('slug', siteSlugs);
const sites = await curateSites(siteEntries, true, false); const sites = await curateSites(siteEntries, true, false);
const networks = await curateNetworks(networkEntries, true, false, false); const networks = await curateNetworks(networkEntries, true, false, false);
const markedNetworks = networks.map(network => ({ ...network, isFallback: true })); const markedNetworks = networks.map(network => ({ ...network, isNetwork: true }));
const sitesBySlug = [] const sitesBySlug = []
.concat(markedNetworks, sites) .concat(markedNetworks, sites)
.reduce((accSites, site) => ({ ...accSites, [site.slug]: site }), {}); .reduce((accSites, site) => ({ ...accSites, [site.slug]: site }), {});
return sitesBySlug; return sitesBySlug;
} }
function toBaseReleases(baseReleasesOrUrls) { function toBaseReleases(baseReleasesOrUrls) {
return baseReleasesOrUrls return baseReleasesOrUrls
.map((baseReleaseOrUrl) => { .map((baseReleaseOrUrl) => {
if (baseReleaseOrUrl.url) { if (baseReleaseOrUrl.url) {
// base release with URL // base release with URL
return { return {
...baseReleaseOrUrl, ...baseReleaseOrUrl,
deep: false, deep: false,
}; };
} }
if (/^http/.test(baseReleaseOrUrl)) { if (/^http/.test(baseReleaseOrUrl)) {
// URL // URL
return { return {
url: baseReleaseOrUrl, url: baseReleaseOrUrl,
deep: false, deep: false,
}; };
} }
if (typeof baseReleaseOrUrl === 'object' && !Array.isArray(baseReleaseOrUrl)) { if (typeof baseReleaseOrUrl === 'object' && !Array.isArray(baseReleaseOrUrl)) {
// base release without URL, prepare for passthrough // base release without URL, prepare for passthrough
return { return {
...baseReleaseOrUrl, ...baseReleaseOrUrl,
deep: false, deep: false,
}; };
} }
logger.warn(`Malformed base release, discarding '${baseReleaseOrUrl}'`); logger.warn(`Malformed base release, discarding '${baseReleaseOrUrl}'`);
return null; return null;
}) })
.filter(Boolean); .filter(Boolean);
} }
async function scrapeRelease(baseRelease, sites, type = 'scene') { async function scrapeRelease(baseRelease, sites, type = 'scene') {
const site = baseRelease.site || sites[urlToSiteSlug(baseRelease.url)]; const site = baseRelease.site || sites[urlToSiteSlug(baseRelease.url)];
const siteWithFallbackNetwork = site.isNetwork ? { ...site, network: site } : site; // make site.network available, even when site is network fallback
if (!site) { if (!site) {
logger.warn(`No site available for ${baseRelease.url}`); logger.warn(`No site available for ${baseRelease.url}`);
return baseRelease; return baseRelease;
} }
if ((!baseRelease.url && !baseRelease.path) || !argv.deep) { if ((!baseRelease.url && !baseRelease.path) || !argv.deep) {
return { return {
...baseRelease, ...baseRelease,
site, site,
}; };
} }
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug]; const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
if (!scraper) { if (!scraper) {
logger.warn(`Could not find scraper for ${baseRelease.url}`); logger.warn(`Could not find scraper for ${baseRelease.url}`);
return baseRelease; return baseRelease;
} }
if ((type === 'scene' && !scraper.fetchScene) || (type === 'movie' && !scraper.fetchMovie)) { if ((type === 'scene' && !scraper.fetchScene) || (type === 'movie' && !scraper.fetchMovie)) {
logger.warn(`The '${site.name}'-scraper cannot fetch individual ${type}s`); logger.warn(`The '${site.name}'-scraper cannot fetch individual ${type}s`);
return baseRelease; return baseRelease;
} }
try { try {
logger.verbose(`Fetching ${type} ${baseRelease.url}`); logger.verbose(`Fetching ${type} ${baseRelease.url}`);
const scrapedRelease = type === 'scene' const scrapedRelease = type === 'scene'
? await scraper.fetchScene(baseRelease.url, site, baseRelease, null, include) ? await scraper.fetchScene(baseRelease.url, siteWithFallbackNetwork, baseRelease, null, include)
: await scraper.fetchMovie(baseRelease.url, site, baseRelease, null, include); : await scraper.fetchMovie(baseRelease.url, siteWithFallbackNetwork, baseRelease, null, include);
const mergedRelease = { const mergedRelease = {
...baseRelease, ...baseRelease,
...scrapedRelease, ...scrapedRelease,
deep: !!scrapedRelease, deep: !!scrapedRelease,
site, site,
}; };
if (scrapedRelease && baseRelease?.tags) { if (scrapedRelease && baseRelease?.tags) {
// accumulate all available tags // accumulate all available tags
mergedRelease.tags = baseRelease.tags.concat(scrapedRelease.tags); mergedRelease.tags = baseRelease.tags.concat(scrapedRelease.tags);
} }
return mergedRelease; return mergedRelease;
} catch (error) { } catch (error) {
logger.error(`Deep scrape failed for ${baseRelease.url}: ${error.message}`); logger.error(`Deep scrape failed for ${baseRelease.url}: ${error.message}`);
return baseRelease; return baseRelease;
} }
} }
async function scrapeReleases(baseReleases, sites, type) { async function scrapeReleases(baseReleases, sites, type) {
return Promise.map( return Promise.map(
baseReleases, baseReleases,
async baseRelease => scrapeRelease(baseRelease, sites, type), async baseRelease => scrapeRelease(baseRelease, sites, type),
{ concurrency: 10 }, { concurrency: 10 },
); );
} }
async function fetchReleases(baseReleasesOrUrls, type = 'scene') { async function fetchReleases(baseReleasesOrUrls, type = 'scene') {
const baseReleases = toBaseReleases(baseReleasesOrUrls); const baseReleases = toBaseReleases(baseReleasesOrUrls);
const sites = await findSites(baseReleases); const sites = await findSites(baseReleases);
const deepReleases = await scrapeReleases(baseReleases, sites, type); const deepReleases = await scrapeReleases(baseReleases, sites, type);
return deepReleases; return deepReleases;
} }
async function fetchScenes(baseReleasesOrUrls) { async function fetchScenes(baseReleasesOrUrls) {
return fetchReleases(baseReleasesOrUrls, 'scene'); return fetchReleases(baseReleasesOrUrls, 'scene');
} }
async function fetchMovies(baseReleasesOrUrls) { async function fetchMovies(baseReleasesOrUrls) {
return fetchReleases(baseReleasesOrUrls, 'movie'); return fetchReleases(baseReleasesOrUrls, 'movie');
} }
module.exports = { module.exports = {
fetchReleases, fetchReleases,
fetchScenes, fetchScenes,
fetchMovies, fetchMovies,
}; };

8
src/knex.js
View File

@ -4,8 +4,8 @@ const config = require('config');
const knex = require('knex'); const knex = require('knex');
module.exports = knex({ module.exports = knex({
client: 'pg', client: 'pg',
connection: config.database, connection: config.database,
// performance overhead, don't use asyncStackTraces in production // performance overhead, don't use asyncStackTraces in production
asyncStackTraces: process.env.NODE_ENV === 'development', asyncStackTraces: process.env.NODE_ENV === 'development',
}); });

48
src/logger.js
View File

@ -9,31 +9,31 @@ require('winston-daily-rotate-file');
const args = require('./argv'); const args = require('./argv');
function logger(filepath) { function logger(filepath) {
const root = filepath.match(/src\/|dist\//); const root = filepath.match(/src\/|dist\//);
const filename = filepath.slice(root.index + root[0].length) const filename = filepath.slice(root.index + root[0].length)
.replace(path.extname(filepath), ''); .replace(path.extname(filepath), '');
return winston.createLogger({ return winston.createLogger({
format: winston.format.combine( format: winston.format.combine(
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format(info => (info instanceof Error winston.format(info => (info instanceof Error
? { ...info, message: info.stack } ? { ...info, message: info.stack }
: { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(), : { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(),
winston.format.colorize(), winston.format.colorize(),
winston.format.printf(({ level, timestamp, label, message }) => `${timestamp} ${level} [${label || filename}] ${message}`), winston.format.printf(({ level, timestamp, label, message }) => `${timestamp} ${level} [${label || filename}] ${message}`),
), ),
transports: [ transports: [
new winston.transports.Console({ new winston.transports.Console({
level: args.level, level: args.level,
timestamp: true, timestamp: true,
}), }),
new winston.transports.DailyRotateFile({ new winston.transports.DailyRotateFile({
datePattern: 'YYYY-MM-DD', datePattern: 'YYYY-MM-DD',
filename: 'log/%DATE%.log', filename: 'log/%DATE%.log',
level: 'silly', level: 'silly',
}), }),
], ],
}); });
} }
module.exports = logger; module.exports = logger;

866
src/media.js
View File

File diff suppressed because it is too large Load Diff

98
src/networks.js
View File

@ -5,77 +5,77 @@ const whereOr = require('./utils/where-or');
const { fetchSites } = require('./sites'); const { fetchSites } = require('./sites');
async function curateNetwork(network, includeParameters = false, includeSites = true, includeStudios = false) { async function curateNetwork(network, includeParameters = false, includeSites = true, includeStudios = false) {
const curatedNetwork = { const curatedNetwork = {
id: network.id, id: network.id,
name: network.name, name: network.name,
url: network.url, url: network.url,
description: network.description, description: network.description,
slug: network.slug, slug: network.slug,
parameters: includeParameters ? network.parameters : null, parameters: includeParameters ? network.parameters : null,
}; };
if (includeSites) { if (includeSites) {
curatedNetwork.sites = await fetchSites({ network_id: network.id }); curatedNetwork.sites = await fetchSites({ network_id: network.id });
} }
if (includeStudios) { if (includeStudios) {
const studios = await knex('studios').where({ network_id: network.id }); const studios = await knex('studios').where({ network_id: network.id });
curatedNetwork.studios = studios.map(studio => ({ curatedNetwork.studios = studios.map(studio => ({
id: studio.id, id: studio.id,
name: studio.name, name: studio.name,
url: studio.url, url: studio.url,
description: studio.description, description: studio.description,
slug: studio.slug, slug: studio.slug,
})); }));
} }
return curatedNetwork; return curatedNetwork;
} }
function curateNetworks(releases) { function curateNetworks(releases) {
return Promise.all(releases.map(async release => curateNetwork(release))); return Promise.all(releases.map(async release => curateNetwork(release)));
} }
async function findNetworkByUrl(url) { async function findNetworkByUrl(url) {
const { hostname } = new URL(url); const { hostname } = new URL(url);
const domain = hostname.replace(/^www./, ''); const domain = hostname.replace(/^www./, '');
const network = await knex('networks') const network = await knex('networks')
.where('networks.url', 'like', `%${domain}`) .where('networks.url', 'like', `%${domain}`)
.orWhere('networks.url', url) .orWhere('networks.url', url)
.first(); .first();
if (network) { if (network) {
return curateNetwork(network, true); return curateNetwork(network, true);
} }
return null; return null;
} }
async function fetchNetworks(queryObject) { async function fetchNetworks(queryObject) {
const releases = await knex('networks') const releases = await knex('networks')
.where(builder => whereOr(queryObject, 'networks', builder)) .where(builder => whereOr(queryObject, 'networks', builder))
.limit(100); .limit(100);
return curateNetworks(releases); return curateNetworks(releases);
} }
async function fetchNetworksFromReleases() { async function fetchNetworksFromReleases() {
const releases = await knex('releases') const releases = await knex('releases')
.select('site_id', '') .select('site_id', '')
.leftJoin('sites', 'sites.id', 'releases.site_id') .leftJoin('sites', 'sites.id', 'releases.site_id')
.leftJoin('networks', 'networks.id', 'sites.network_id') .leftJoin('networks', 'networks.id', 'sites.network_id')
.groupBy('networks.id') .groupBy('networks.id')
.limit(100); .limit(100);
return curateNetworks(releases); return curateNetworks(releases);
} }
module.exports = { module.exports = {
curateNetwork, curateNetwork,
curateNetworks, curateNetworks,
fetchNetworks, fetchNetworks,
fetchNetworksFromReleases, fetchNetworksFromReleases,
findNetworkByUrl, findNetworkByUrl,
}; };

714
src/releases-legacy.js
View File

@ -11,356 +11,356 @@ const whereOr = require('./utils/where-or');
const { associateTags } = require('./tags'); const { associateTags } = require('./tags');
const { associateActors, scrapeBasicActors } = require('./actors'); const { associateActors, scrapeBasicActors } = require('./actors');
const { const {
pluckItems, pluckItems,
storeMedia, storeMedia,
associateMedia, associateMedia,
} = require('./media'); } = require('./media');
const { fetchSites } = require('./sites'); const { fetchSites } = require('./sites');
const slugify = require('./utils/slugify'); const slugify = require('./utils/slugify');
const capitalize = require('./utils/capitalize'); const capitalize = require('./utils/capitalize');
function commonQuery(queryBuilder, { function commonQuery(queryBuilder, {
filter = [], filter = [],
after = new Date(0), // January 1970 after = new Date(0), // January 1970
before = new Date(2 ** 44), // May 2109 before = new Date(2 ** 44), // May 2109
limit = 100, limit = 100,
}) { }) {
const finalFilter = [].concat(filter); // ensure filter is array const finalFilter = [].concat(filter); // ensure filter is array
queryBuilder queryBuilder
.leftJoin('sites', 'releases.site_id', 'sites.id') .leftJoin('sites', 'releases.site_id', 'sites.id')
.leftJoin('studios', 'releases.studio_id', 'studios.id') .leftJoin('studios', 'releases.studio_id', 'studios.id')
.leftJoin('networks', 'sites.network_id', 'networks.id') .leftJoin('networks', 'sites.network_id', 'networks.id')
.select( .select(
'releases.*', 'releases.*',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters', 'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url', 'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description',
) )
.whereNotExists((builder) => { .whereNotExists((builder) => {
// apply tag filters // apply tag filters
builder builder
.select('*') .select('*')
.from('tags_associated') .from('tags_associated')
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id') .leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
.whereIn('tags.slug', finalFilter) .whereIn('tags.slug', finalFilter)
.where('tags_associated.domain', 'releases') .where('tags_associated.domain', 'releases')
.whereRaw('tags_associated.target_id = releases.id'); .whereRaw('tags_associated.target_id = releases.id');
}) })
.andWhere('releases.date', '>', after) .andWhere('releases.date', '>', after)
.andWhere('releases.date', '<=', before) .andWhere('releases.date', '<=', before)
.orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }]) .orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }])
.limit(limit); .limit(limit);
} }
async function curateRelease(release) { async function curateRelease(release) {
const [actors, tags, media] = await Promise.all([ const [actors, tags, media] = await Promise.all([
knex('actors_associated') knex('actors_associated')
.select( .select(
'actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'actors.birthdate', 'actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'actors.birthdate',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias', 'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'media.thumbnail as avatar', 'media.thumbnail as avatar',
) )
.where({ release_id: release.id }) .where({ release_id: release.id })
.leftJoin('actors', 'actors.id', 'actors_associated.actor_id') .leftJoin('actors', 'actors.id', 'actors_associated.actor_id')
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2') .leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('media', (builder) => { .leftJoin('media', (builder) => {
builder builder
.on('media.target_id', 'actors.id') .on('media.target_id', 'actors.id')
.andOnVal('media.domain', 'actors') .andOnVal('media.domain', 'actors')
.andOnVal('media.index', '0'); .andOnVal('media.index', '0');
}) })
.orderBy('actors.gender'), .orderBy('actors.gender'),
knex('tags_associated') knex('tags_associated')
.select('tags.name', 'tags.slug') .select('tags.name', 'tags.slug')
.where({ .where({
domain: 'releases', domain: 'releases',
target_id: release.id, target_id: release.id,
}) })
.leftJoin('tags', 'tags.id', 'tags_associated.tag_id') .leftJoin('tags', 'tags.id', 'tags_associated.tag_id')
.orderBy('tags.priority', 'desc'), .orderBy('tags.priority', 'desc'),
knex('media') knex('media')
.where({ .where({
target_id: release.id, target_id: release.id,
domain: 'releases', domain: 'releases',
}) })
.orderBy(['role', 'index']), .orderBy(['role', 'index']),
]); ]);
const curatedRelease = { const curatedRelease = {
id: release.id, id: release.id,
type: release.type, type: release.type,
title: release.title, title: release.title,
date: release.date, date: release.date,
dateAdded: release.created_at, dateAdded: release.created_at,
description: release.description, description: release.description,
url: release.url, url: release.url,
shootId: release.shoot_id, shootId: release.shoot_id,
entryId: release.entry_id, entryId: release.entry_id,
actors: actors.map(actor => ({ actors: actors.map(actor => ({
id: actor.id, id: actor.id,
slug: actor.slug, slug: actor.slug,
name: actor.name, name: actor.name,
gender: actor.gender, gender: actor.gender,
birthdate: actor.birthdate, birthdate: actor.birthdate,
age: moment().diff(actor.birthdate, 'years'), age: moment().diff(actor.birthdate, 'years'),
ageThen: moment(release.date).diff(actor.birthdate, 'years'), ageThen: moment(release.date).diff(actor.birthdate, 'years'),
avatar: actor.avatar, avatar: actor.avatar,
origin: actor.birth_country_alpha2 origin: actor.birth_country_alpha2
? { ? {
country: { country: {
name: actor.birth_country_alias, name: actor.birth_country_alias,
alpha2: actor.birth_country_alpha2, alpha2: actor.birth_country_alpha2,
}, },
} }
: null, : null,
})), })),
director: release.director, director: release.director,
tags, tags,
duration: release.duration, duration: release.duration,
photos: media.filter(item => item.role === 'photo'), photos: media.filter(item => item.role === 'photo'),
poster: media.filter(item => item.role === 'poster')[0], poster: media.filter(item => item.role === 'poster')[0],
covers: media.filter(item => item.role === 'cover'), covers: media.filter(item => item.role === 'cover'),
trailer: media.filter(item => item.role === 'trailer')[0], trailer: media.filter(item => item.role === 'trailer')[0],
site: { site: {
id: release.site_id, id: release.site_id,
name: release.site_name, name: release.site_name,
independent: !!release.site_parameters?.independent, independent: !!release.site_parameters?.independent,
slug: release.site_slug, slug: release.site_slug,
url: release.site_url, url: release.site_url,
}, },
studio: release.studio_id studio: release.studio_id
? { ? {
id: release.studio_id, id: release.studio_id,
name: release.studio_name, name: release.studio_name,
slug: release.studio_slug, slug: release.studio_slug,
url: release.studio_url, url: release.studio_url,
} }
: null, : null,
network: { network: {
id: release.network_id, id: release.network_id,
name: release.network_name, name: release.network_name,
description: release.network_description, description: release.network_description,
slug: release.network_slug, slug: release.network_slug,
url: release.network_url, url: release.network_url,
}, },
}; };
return curatedRelease; return curatedRelease;
} }
function curateReleases(releases) { function curateReleases(releases) {
return Promise.all(releases.map(async release => curateRelease(release))); return Promise.all(releases.map(async release => curateRelease(release)));
} }
async function attachChannelSite(release) { async function attachChannelSite(release) {
if (!release.site?.isFallback && !release.channel?.force) { if (!release.site?.isFallback && !release.channel?.force) {
return release; return release;
} }
if (!release.channel) { if (!release.channel) {
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`); throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
} }
const [site] = await fetchSites({ const [site] = await fetchSites({
name: release.channel.name || release.channel, name: release.channel.name || release.channel,
slug: release.channel.slug || release.channel, slug: release.channel.slug || release.channel,
}); });
if (site) { if (site) {
return { return {
...release, ...release,
site, site,
}; };
} }
throw new Error(`Unable to match channel '${release.channel.slug || release.channel}' from generic URL: ${release.url}`); throw new Error(`Unable to match channel '${release.channel.slug || release.channel}' from generic URL: ${release.url}`);
} }
async function attachStudio(release) { async function attachStudio(release) {
if (!release.studio) { if (!release.studio) {
return release; return release;
} }
const studio = await knex('studios') const studio = await knex('studios')
.where('name', release.studio) .where('name', release.studio)
.orWhere('slug', release.studio) .orWhere('slug', release.studio)
.orWhere('url', release.studio) .orWhere('url', release.studio)
.first(); .first();
return { return {
...release, ...release,
studio, studio,
}; };
} }
async function curateReleaseEntry(release, batchId, existingRelease) { async function curateReleaseEntry(release, batchId, existingRelease) {
const slug = slugify(release.title, { const slug = slugify(release.title, {
encode: true, encode: true,
limit: config.titleSlugLength, limit: config.titleSlugLength,
}); });
const curatedRelease = { const curatedRelease = {
site_id: release.site.id, site_id: release.site.id,
studio_id: release.studio ? release.studio.id : null, studio_id: release.studio ? release.studio.id : null,
shoot_id: release.shootId || null, shoot_id: release.shootId || null,
entry_id: release.entryId || null, entry_id: release.entryId || null,
type: release.type, type: release.type,
url: release.url, url: release.url,
title: release.title, title: release.title,
slug, slug,
date: release.date, date: release.date,
description: release.description, description: release.description,
// director: release.director, // director: release.director,
duration: release.duration, duration: release.duration,
// likes: release.rating && release.rating.likes, // likes: release.rating && release.rating.likes,
// dislikes: release.rating && release.rating.dislikes, // dislikes: release.rating && release.rating.dislikes,
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars), // rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: typeof release.deep === 'boolean' ? release.deep : false, deep: typeof release.deep === 'boolean' ? release.deep : false,
deep_url: release.deepUrl, deep_url: release.deepUrl,
updated_batch_id: batchId, updated_batch_id: batchId,
...(!existingRelease && { created_batch_id: batchId }), ...(!existingRelease && { created_batch_id: batchId }),
}; };
return curatedRelease; return curatedRelease;
} }
async function fetchReleases(queryObject = {}, options = {}) { async function fetchReleases(queryObject = {}, options = {}) {
const releases = await knex('releases') const releases = await knex('releases')
.modify(commonQuery, options) .modify(commonQuery, options)
.andWhere(builder => whereOr(queryObject, 'releases', builder)); .andWhere(builder => whereOr(queryObject, 'releases', builder));
return curateReleases(releases); return curateReleases(releases);
} }
async function fetchSiteReleases(queryObject, options = {}) { async function fetchSiteReleases(queryObject, options = {}) {
const releases = await knex('releases') const releases = await knex('releases')
.modify(commonQuery, options) .modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'sites', builder)); .where(builder => whereOr(queryObject, 'sites', builder));
return curateReleases(releases); return curateReleases(releases);
} }
async function fetchNetworkReleases(queryObject, options = {}) { async function fetchNetworkReleases(queryObject, options = {}) {
const releases = await knex('releases') const releases = await knex('releases')
.modify(commonQuery, options) .modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'networks', builder)); .where(builder => whereOr(queryObject, 'networks', builder));
return curateReleases(releases); return curateReleases(releases);
} }
async function fetchActorReleases(queryObject, options = {}) { async function fetchActorReleases(queryObject, options = {}) {
const releases = await knex('actors_associated') const releases = await knex('actors_associated')
.leftJoin('releases', 'actors_associated.release_id', 'releases.id') .leftJoin('releases', 'actors_associated.release_id', 'releases.id')
.leftJoin('actors', 'actors_associated.actor_id', 'actors.id') .leftJoin('actors', 'actors_associated.actor_id', 'actors.id')
.select( .select(
'actors.name as actor_name', 'actors.name as actor_name',
) )
.modify(commonQuery, options) .modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'actors', builder)); .where(builder => whereOr(queryObject, 'actors', builder));
return curateReleases(releases); return curateReleases(releases);
} }
async function fetchTagReleases(queryObject, options = {}) { async function fetchTagReleases(queryObject, options = {}) {
const releases = await knex('tags_associated') const releases = await knex('tags_associated')
.leftJoin('releases', 'tags_associated.target_id', 'releases.id') .leftJoin('releases', 'tags_associated.target_id', 'releases.id')
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id') .leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
.select( .select(
'tags.name as tag_name', 'tags.name as tag_name',
) )
.modify(commonQuery, options) .modify(commonQuery, options)
.where('tags_associated.domain', 'releases') .where('tags_associated.domain', 'releases')
.where(builder => whereOr(queryObject, 'tags', builder)); .where(builder => whereOr(queryObject, 'tags', builder));
return curateReleases(releases); return curateReleases(releases);
} }
function accumulateActors(releases) { function accumulateActors(releases) {
return releases.reduce((acc, release) => { return releases.reduce((acc, release) => {
if (!Array.isArray(release.actors)) return acc; if (!Array.isArray(release.actors)) return acc;
release.actors.forEach((actor) => { release.actors.forEach((actor) => {
const actorName = actor.name ? actor.name.trim() : actor.trim(); const actorName = actor.name ? actor.name.trim() : actor.trim();
const actorSlug = slugify(actorName); const actorSlug = slugify(actorName);
if (!actorSlug) return; if (!actorSlug) return;
if (!acc[actorSlug]) { if (!acc[actorSlug]) {
acc[actorSlug] = { acc[actorSlug] = {
name: actorName, name: actorName,
slug: actorSlug, slug: actorSlug,
releaseIds: new Set(), releaseIds: new Set(),
avatars: [], avatars: [],
}; };
} }
acc[actorSlug].releaseIds.add(release.id); acc[actorSlug].releaseIds.add(release.id);
if (actor.name) acc[actorSlug] = { ...acc[actorSlug], ...actor }; // actor input contains profile info if (actor.name) acc[actorSlug] = { ...acc[actorSlug], ...actor }; // actor input contains profile info
if (actor.avatar) { if (actor.avatar) {
const avatar = Array.isArray(actor.avatar) const avatar = Array.isArray(actor.avatar)
? actor.avatar.map(avatarX => ({ ? actor.avatar.map(avatarX => ({
src: avatarX.src || avatarX, src: avatarX.src || avatarX,
copyright: avatarX.copyright === undefined ? capitalize(release.site?.network?.name) : avatarX.copyright, copyright: avatarX.copyright === undefined ? capitalize(release.site?.network?.name) : avatarX.copyright,
})) }))
: { : {
src: actor.avatar.src || actor.avatar, src: actor.avatar.src || actor.avatar,
copyright: actor.avatar.copyright === undefined ? capitalize(release.site?.network?.name) : actor.avatar.copyright, copyright: actor.avatar.copyright === undefined ? capitalize(release.site?.network?.name) : actor.avatar.copyright,
}; };
acc[actorSlug].avatars = acc[actorSlug].avatars.concat([avatar]); // don't flatten fallbacks acc[actorSlug].avatars = acc[actorSlug].avatars.concat([avatar]); // don't flatten fallbacks
} }
}); });
return acc; return acc;
}, {}); }, {});
} }
async function storeReleaseAssets(releases) { async function storeReleaseAssets(releases) {
if (!argv.media) { if (!argv.media) {
return; return;
} }
const releasePostersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.poster] }), {}); const releasePostersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.poster] }), {});
const releaseCoversById = releases.reduce((acc, release) => ({ ...acc, [release.id]: release.covers }), {}); const releaseCoversById = releases.reduce((acc, release) => ({ ...acc, [release.id]: release.covers }), {});
const releaseTrailersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.trailer] }), {}); const releaseTrailersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.trailer] }), {});
const releaseTeasersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.teaser] }), {}); const releaseTeasersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.teaser] }), {});
const releasePhotosById = releases.reduce((acc, release) => ({ const releasePhotosById = releases.reduce((acc, release) => ({
...acc, ...acc,
[release.id]: pluckItems(release.photos), [release.id]: pluckItems(release.photos),
}), {}); }), {});
if (argv.images && argv.posters) { if (argv.images && argv.posters) {
const posters = await storeMedia(Object.values(releasePostersById).flat(), 'release', 'poster'); const posters = await storeMedia(Object.values(releasePostersById).flat(), 'release', 'poster');
if (posters) await associateMedia(releasePostersById, posters, 'release', 'poster'); if (posters) await associateMedia(releasePostersById, posters, 'release', 'poster');
} }
if (argv.images && argv.covers) { if (argv.images && argv.covers) {
const covers = await storeMedia(Object.values(releaseCoversById).flat(), 'release', 'cover'); const covers = await storeMedia(Object.values(releaseCoversById).flat(), 'release', 'cover');
if (covers) await associateMedia(releaseCoversById, covers, 'release', 'cover'); if (covers) await associateMedia(releaseCoversById, covers, 'release', 'cover');
} }
if (argv.images && argv.photos) { if (argv.images && argv.photos) {
const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo'); const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
if (photos) await associateMedia(releasePhotosById, photos, 'release', 'photo'); if (photos) await associateMedia(releasePhotosById, photos, 'release', 'photo');
} }
if (argv.videos && argv.trailers) { if (argv.videos && argv.trailers) {
const trailers = await storeMedia(Object.values(releaseTrailersById).flat(), 'release', 'trailer'); const trailers = await storeMedia(Object.values(releaseTrailersById).flat(), 'release', 'trailer');
if (trailers) await associateMedia(releaseTrailersById, trailers, 'release', 'trailer'); if (trailers) await associateMedia(releaseTrailersById, trailers, 'release', 'trailer');
} }
if (argv.videos && argv.teasers) { if (argv.videos && argv.teasers) {
const teasers = await storeMedia(Object.values(releaseTeasersById).flat(), 'release', 'teaser'); const teasers = await storeMedia(Object.values(releaseTeasersById).flat(), 'release', 'teaser');
if (teasers) await associateMedia(releaseTeasersById, teasers, 'release', 'teaser'); if (teasers) await associateMedia(releaseTeasersById, teasers, 'release', 'teaser');
} }
} }
async function updateReleasesSearch(releaseIds) { async function updateReleasesSearch(releaseIds) {
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`); logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
const documents = await knex.raw(` const documents = await knex.raw(`
SELECT SELECT
releases.id AS release_id, releases.id AS release_id,
TO_TSVECTOR( TO_TSVECTOR(
@ -391,117 +391,117 @@ async function updateReleasesSearch(releaseIds) {
GROUP BY releases.id, sites.name, sites.slug, sites.alias, sites.url, networks.name, networks.slug, networks.url; GROUP BY releases.id, sites.name, sites.slug, sites.alias, sites.url, networks.name, networks.slug, networks.url;
`, releaseIds && [releaseIds]); `, releaseIds && [releaseIds]);
if (documents.rows?.length > 0) { if (documents.rows?.length > 0) {
const query = knex('releases_search').insert(documents.rows).toString(); const query = knex('releases_search').insert(documents.rows).toString();
await knex.raw(`${query} ON CONFLICT (release_id) DO UPDATE SET document = EXCLUDED.document`); await knex.raw(`${query} ON CONFLICT (release_id) DO UPDATE SET document = EXCLUDED.document`);
} }
} }
async function storeRelease(release, batchId) { async function storeRelease(release, batchId) {
if (!release.site) { if (!release.site) {
throw new Error(`Missing site, unable to store "${release.title}" (${release.url})`); throw new Error(`Missing site, unable to store "${release.title}" (${release.url})`);
} }
if (!release.entryId) { if (!release.entryId) {
logger.warn(`Missing entry ID, unable to store "${release.title}" (${release.url})`); logger.warn(`Missing entry ID, unable to store "${release.title}" (${release.url})`);
return null; return null;
} }
const existingRelease = await knex('releases') const existingRelease = await knex('releases')
.where({ .where({
entry_id: release.entryId, entry_id: release.entryId,
site_id: release.site.id, site_id: release.site.id,
}) })
.first(); .first();
const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease); const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
if (existingRelease && !argv.redownload) { if (existingRelease && !argv.redownload) {
return existingRelease; return existingRelease;
} }
if (existingRelease && argv.redownload) { if (existingRelease && argv.redownload) {
const [updatedRelease] = await knex('releases') const [updatedRelease] = await knex('releases')
.where('id', existingRelease.id) .where('id', existingRelease.id)
.update({ .update({
...existingRelease, ...existingRelease,
...curatedRelease, ...curatedRelease,
}) })
.returning('*'); .returning('*');
if (updatedRelease) { if (updatedRelease) {
await associateTags(release, updatedRelease.id); await associateTags(release, updatedRelease.id);
logger.info(`Updated release "${release.title}" (${existingRelease.id}, ${release.site.name})`); logger.info(`Updated release "${release.title}" (${existingRelease.id}, ${release.site.name})`);
} }
await associateTags(release, existingRelease.id); await associateTags(release, existingRelease.id);
return existingRelease; return existingRelease;
} }
const [releaseEntry] = await knex('releases') const [releaseEntry] = await knex('releases')
.insert(curatedRelease) .insert(curatedRelease)
.returning('*'); .returning('*');
await associateTags(release, releaseEntry.id); await associateTags(release, releaseEntry.id);
logger.info(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`); logger.info(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`);
return releaseEntry; return releaseEntry;
} }
async function storeReleases(releases) { async function storeReleases(releases) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id'); const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const storedReleases = await Promise.map(releases, async (release) => { const storedReleases = await Promise.map(releases, async (release) => {
try { try {
const releaseWithChannelSite = await attachChannelSite(release); const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite); const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const storedRelease = await storeRelease(releaseWithStudio, batchId); const storedRelease = await storeRelease(releaseWithStudio, batchId);
return storedRelease && { return storedRelease && {
id: storedRelease.id, id: storedRelease.id,
slug: storedRelease.slug, slug: storedRelease.slug,
...releaseWithChannelSite, ...releaseWithChannelSite,
}; };
} catch (error) { } catch (error) {
logger.error(error); logger.error(error);
return null; return null;
} }
}, { }, {
concurrency: 10, concurrency: 10,
}).filter(Boolean); }).filter(Boolean);
logger.info(`Stored ${storedReleases.length} new releases`); logger.info(`Stored ${storedReleases.length} new releases`);
const actors = accumulateActors(storedReleases); const actors = accumulateActors(storedReleases);
await associateActors(actors, storedReleases); await associateActors(actors, storedReleases);
await Promise.all([ await Promise.all([
// actors need to be stored before generating search // actors need to be stored before generating search
updateReleasesSearch(storedReleases.map(release => release.id)), updateReleasesSearch(storedReleases.map(release => release.id)),
storeReleaseAssets(storedReleases), storeReleaseAssets(storedReleases),
]); ]);
if (argv.withProfiles && Object.keys(actors).length > 0) { if (argv.withProfiles && Object.keys(actors).length > 0) {
await scrapeBasicActors(); await scrapeBasicActors();
} }
return { return {
releases: storedReleases, releases: storedReleases,
actors, actors,
}; };
} }
module.exports = { module.exports = {
fetchReleases, fetchReleases,
fetchActorReleases, fetchActorReleases,
fetchSiteReleases, fetchSiteReleases,
fetchNetworkReleases, fetchNetworkReleases,
fetchTagReleases, fetchTagReleases,
storeRelease, storeRelease,
storeReleases, storeReleases,
updateReleasesSearch, updateReleasesSearch,
}; };

12
src/releases.js
View File

@ -3,18 +3,18 @@
const knex = require('./knex'); const knex = require('./knex');
async function fetchReleases(limit = 100) { async function fetchReleases(limit = 100) {
const releases = await knex('releases').limit(limit); const releases = await knex('releases').limit(limit);
return releases; return releases;
} }
async function searchReleases(query, limit = 100) { async function searchReleases(query, limit = 100) {
const releases = await knex.raw('SELECT * FROM search_releases(?) LIMIT ?;', [query, limit]); const releases = await knex.raw('SELECT * FROM search_releases(?) LIMIT ?;', [query, limit]);
return releases.rows; return releases.rows;
} }
module.exports = { module.exports = {
fetchReleases, fetchReleases,
searchReleases, searchReleases,
}; };

199
src/scrape-releases.js
View File

@ -1,199 +0,0 @@
'use strict';
const config = require('config');
const Promise = require('bluebird');
const logger = require('./logger')(__filename);
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const knex = require('./knex');
const scrapers = require('./scrapers/scrapers');
const { findSiteByUrl } = require('./sites');
const { findNetworkByUrl } = require('./networks');
const { storeReleases } = require('./releases');
async function findSite(url, release) {
if (release?.site) return release.site;
if (!url) return null;
const site = await findSiteByUrl(url);
if (site) {
return site;
}
const network = await findNetworkByUrl(url);
if (network) {
return {
...network,
network,
isFallback: true,
};
}
return null;
}
async function scrapeRelease(source, basicRelease = null, type = 'scene', beforeFetchLatest) {
// profile scraper may return either URLs or pre-scraped scenes
const sourceIsUrlOrEmpty = typeof source === 'string' || source === undefined;
const url = sourceIsUrlOrEmpty ? source : source?.url;
const release = sourceIsUrlOrEmpty ? basicRelease : source;
const site = basicRelease?.site || await findSite(url, release);
if (!site) {
throw new Error(`Could not find site for ${url} in database`);
}
if (!argv.deep && release) {
return {
...release,
site,
};
}
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
if (!scraper) {
throw new Error(`Could not find scraper for ${url}`);
}
if ((type === 'scene' && !scraper.fetchScene) || (type === 'movie' && !scraper.fetchMovie)) {
if (release) {
logger.warn(`The '${site.name}'-scraper cannot fetch individual ${type}s`);
return null;
}
throw new Error(`The '${site.name}'-scraper cannot fetch individual ${type}s`);
}
if (!release) {
logger.info(`Scraping release from ${url}`);
}
const scrapedRelease = type === 'scene'
? await scraper.fetchScene(url, site, release, beforeFetchLatest, include)
: await scraper.fetchMovie(url, site, release, beforeFetchLatest, include);
return {
...release,
...scrapedRelease,
...(scrapedRelease && release?.tags && {
tags: release.tags.concat(scrapedRelease.tags),
}),
site,
};
}
async function accumulateMovies(releases) {
if (!argv.withMovies) return [];
const moviesByUrl = releases.reduce((acc, release) => {
if (!release.movie) return acc;
const movie = release.movie.url ? release.movie : { url: release.movie };
if (!acc[movie.url]) {
acc[movie.url] = {
...movie,
type: 'movie',
sceneIds: [],
};
}
acc[movie.url].sceneIds = acc[movie.url].sceneIds.concat(release.id);
return acc;
}, {});
const movies = await Promise.map(Object.values(moviesByUrl), async movie => scrapeRelease(movie, null, 'movie'));
const { releases: storedMovies } = await storeReleases(movies);
const movieAssociations = storedMovies.reduce((acc, movie) => acc.concat(movie.sceneIds.map(sceneId => ({
movie_id: movie.id,
scene_id: sceneId,
}))), []);
await knex('releases_movies').insert(movieAssociations);
// console.log(moviesByUrl);
return movies;
}
async function scrapeReleases(sources, type = 'scene') {
const scrapedReleases = await Promise.map(sources, async source => scrapeRelease(source, null, type), {
concurrency: 5,
}).filter(Boolean);
const curatedReleases = scrapedReleases.map(scrapedRelease => ({ ...scrapedRelease, type }));
if ((argv.scene || argv.movie) && argv.inspect) {
// only show when fetching from URL
}
if (argv.save) {
const { releases: storedReleases } = await storeReleases(curatedReleases);
await accumulateMovies(storedReleases);
if (storedReleases) {
logger.info(storedReleases.map(storedRelease => `\nhttp://${config.web.host}:${config.web.port}/scene/${storedRelease.id}/${storedRelease.slug}`).join(''));
}
return storedReleases;
}
return curatedReleases;
}
async function scrapeScenes(sources) {
return scrapeReleases(sources, 'scene');
}
async function scrapeMovies(sources) {
return scrapeReleases(sources, 'movie');
}
async function deepFetchReleases(baseReleases, beforeFetchLatest) {
const deepReleases = await Promise.map(baseReleases, async (release) => {
if (release.url || (release.path && release.site)) {
try {
const fullRelease = await scrapeRelease(release.url, release, 'scene', beforeFetchLatest);
if (fullRelease) {
return {
...release,
...fullRelease,
deep: true,
};
}
logger.warn(`Release scraper returned empty result for ${release.url}`);
return release;
} catch (error) {
logger.error(`Failed to scrape ${release.url}: ${error}`);
return {
...release,
deep: false,
};
}
}
return release;
}, {
concurrency: 2,
});
return deepReleases;
}
module.exports = {
deepFetchReleases,
scrapeMovies,
scrapeRelease,
scrapeReleases,
scrapeScenes,
};

184
src/scrape-sites.js
View File

@ -1,184 +0,0 @@
'use strict';
const Promise = require('bluebird');
const moment = require('moment');
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const logger = require('./logger')(__filename);
const knex = require('./knex');
const { fetchIncludedSites } = require('./sites');
const scrapers = require('./scrapers/scrapers');
const { deepFetchReleases } = require('./scrape-releases');
const { storeReleases } = require('./releases');
function getAfterDate() {
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
// using date
return moment
.utc(argv.after, ['YYYY-MM-DD', 'DD-MM-YYYY'])
.toDate();
}
// using time distance (e.g. "1 month")
return moment
.utc()
.subtract(...argv.after.split(' '))
.toDate();
}
async function findDuplicateReleaseIds(latestReleases, accReleases) {
const duplicateReleases = await knex('releases')
.whereIn('entry_id', latestReleases.map(({ entryId }) => entryId));
// include accumulated releases as duplicates to prevent an infinite
// loop when the next page contains the same releases as the previous
return new Set(duplicateReleases
.map(release => String(release.entry_id))
.concat(accReleases.map(release => String(release.entryId))));
}
async function scrapeUniqueReleases(scraper, site, beforeFetchLatest, accSiteReleases, afterDate = getAfterDate(), accReleases = [], page = argv.page) {
if (!argv.latest || !scraper.fetchLatest) {
return [];
}
const latestReleases = await scraper.fetchLatest(site, page, beforeFetchLatest, accSiteReleases, include);
if (!Array.isArray(latestReleases)) {
logger.warn(`Scraper returned ${latestReleases || 'null'} when fetching latest from '${site.name}' on '${site.network.name}'`);
return accReleases;
}
if (latestReleases.length === 0) {
return accReleases;
}
const latestReleasesWithSite = latestReleases.map(release => ({ ...release, site }));
const oldestReleaseOnPage = latestReleases.slice(-1)[0].date;
const duplicateReleaseIds = argv.redownload ? new Set() : await findDuplicateReleaseIds(latestReleases, accReleases);
const uniqueReleases = latestReleasesWithSite
.filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
&& (argv.last || !release.date || moment(release.date).isAfter(afterDate))); // release is older than specified date limit
logger.verbose(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
if (
uniqueReleases.length > 0
// && (oldestReleaseOnPage || page < argv.pages)
&& ((oldestReleaseOnPage
? moment(oldestReleaseOnPage).isAfter(afterDate)
: accReleases.length + uniqueReleases.length <= argv.nullDateLimit)
|| (argv.last && accReleases.length + uniqueReleases.length < argv.last))
) {
// oldest release on page is newer that specified date range, or latest count has not yet been met, fetch next page
return scrapeUniqueReleases(scraper, site, beforeFetchLatest, accSiteReleases, afterDate, accReleases.concat(uniqueReleases), page + 1);
}
if (argv.last && uniqueReleases.length >= argv.last) {
return accReleases.concat(uniqueReleases).slice(0, argv.last);
}
if (oldestReleaseOnPage) {
return accReleases.concat(uniqueReleases);
}
return accReleases.concat(uniqueReleases).slice(0, argv.nullDateLimit);
}
async function scrapeUpcomingReleases(scraper, site, beforeFetchLatest) {
if (argv.upcoming && scraper.fetchUpcoming) {
const upcomingReleases = await scraper.fetchUpcoming(site, 1, beforeFetchLatest, include);
return upcomingReleases
? upcomingReleases.map(release => ({ ...release, site, upcoming: true }))
: [];
}
return [];
}
async function scrapeSiteReleases(scraper, site, accSiteReleases) {
const beforeFetchLatest = await scraper.beforeFetchLatest?.(site, accSiteReleases);
const [newReleases, upcomingReleases] = await Promise.all([
scrapeUniqueReleases(scraper, site, beforeFetchLatest, accSiteReleases), // fetch basic release info from scene overview
scrapeUpcomingReleases(scraper, site, beforeFetchLatest, accSiteReleases), // fetch basic release info from upcoming overview
]);
if (argv.upcoming) {
logger.info(`${site.name}: ${argv.latest ? `Found ${newReleases.length}` : 'Ignoring'} latest releases,${argv.upcoming ? ' ' : ' ignoring '}${upcomingReleases.length || '0'} upcoming releases`);
}
const baseReleases = [...newReleases, ...upcomingReleases];
if (argv.deep) {
// follow URL for every release
return deepFetchReleases(baseReleases, beforeFetchLatest);
}
return baseReleases;
}
async function scrapeSite(site, network, accSiteReleases = []) {
if (site.parameters?.ignore) {
logger.warn(`Ignoring ${network.name}: ${site.name}`);
return [];
}
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
if (!scraper) {
logger.warn(`No scraper found for '${site.name}' (${site.slug})`);
return [];
}
try {
const siteReleases = await scrapeSiteReleases(scraper, site, accSiteReleases);
return siteReleases.map(release => ({ ...release, site }));
} catch (error) {
logger.error(`${site.name}: Failed to scrape releases: ${error.message}`);
return [];
}
}
async function scrapeSites() {
const networks = await fetchIncludedSites();
const scrapedNetworks = await Promise.map(networks, async (network) => {
if (network.parameters?.sequential) {
logger.info(`Scraping '${network.name}' sequentially`);
return Promise.reduce(network.sites, async (acc, site) => {
const accSiteReleases = await acc;
const siteReleases = await scrapeSite(site, network, accSiteReleases);
return accSiteReleases.concat(siteReleases);
}, Promise.resolve([]));
}
return Promise.map(network.sites, async site => scrapeSite(site, network), {
concurrency: network.parameters?.concurrency || 2,
});
},
{
// 5 networks at a time
concurrency: 5,
});
const releases = scrapedNetworks.flat(2);
if (argv.inspect) {
console.log(releases);
}
if (argv.save) {
await storeReleases(releases);
}
}
module.exports = scrapeSites;

8
src/scrapers/21naturals.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = { module.exports = {
fetchLatest: fetchApiLatest, fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile, fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming, fetchUpcoming: fetchApiUpcoming,
fetchScene, fetchScene,
}; };

8
src/scrapers/21sextreme.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = { module.exports = {
fetchLatest: fetchApiLatest, fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile, fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming, fetchUpcoming: fetchApiUpcoming,
fetchScene, fetchScene,
}; };

8
src/scrapers/21sextury.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = { module.exports = {
fetchLatest: fetchApiLatest, fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile, fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming, fetchUpcoming: fetchApiUpcoming,
fetchScene, fetchScene,
}; };

34
src/scrapers/adulttime.js
View File

@ -3,37 +3,37 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
function curateRelease(release, site) { function curateRelease(release, site) {
if (['bubblegumdungeon', 'ladygonzo'].includes(site.slug)) { if (['bubblegumdungeon', 'ladygonzo'].includes(site.slug)) {
return { return {
...release, ...release,
title: release.title.split(/:|\|/)[1].trim(), title: release.title.split(/:|\|/)[1].trim(),
}; };
} }
return release; return release;
} }
async function networkFetchScene(url, site, release) { async function networkFetchScene(url, site, release) {
const scene = await fetchScene(url, site, release); const scene = await fetchScene(url, site, release);
return curateRelease(scene, site); return curateRelease(scene, site);
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const releases = await fetchApiLatest(site, page, false); const releases = await fetchApiLatest(site, page, false);
return releases.map(release => curateRelease(release, site)); return releases.map(release => curateRelease(release, site));
} }
async function fetchUpcoming(site, page = 1) { async function fetchUpcoming(site, page = 1) {
const releases = await fetchApiUpcoming(site, page, false); const releases = await fetchApiUpcoming(site, page, false);
return releases.map(release => curateRelease(release, site)); return releases.map(release => curateRelease(release, site));
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchProfile: fetchApiProfile, fetchProfile: fetchApiProfile,
fetchScene: networkFetchScene, fetchScene: networkFetchScene,
fetchUpcoming, fetchUpcoming,
}; };

52
src/scrapers/amateurallure.js
View File

@ -3,47 +3,47 @@
const { fetchLatest, fetchScene } = require('./julesjordan'); const { fetchLatest, fetchScene } = require('./julesjordan');
function extractActors(scene) { function extractActors(scene) {
const release = scene; const release = scene;
if (!scene.actors || scene.actors.length === 0) { if (!scene.actors || scene.actors.length === 0) {
const introActorMatches = scene.title.match(/(?:presents|introduces|features|welcomes) (\w+ \w+)/i); const introActorMatches = scene.title.match(/(?:presents|introduces|features|welcomes) (\w+ \w+)/i);
const introTwoActorMatches = scene.title.match(/(?:presents|introduces|features|welcomes) (?:(\w+)|(\w+ \w+)) and (\w+ \w+)/i); const introTwoActorMatches = scene.title.match(/(?:presents|introduces|features|welcomes) (?:(\w+)|(\w+ \w+)) and (\w+ \w+)/i);
const returnActorMatches = scene.title.match(/(?:(^\w+)|(\w+ \w+))(?:,| (?:return|visit|pov|give|suck|lick|milk|love|enjoy|service|is))/i); const returnActorMatches = scene.title.match(/(?:(^\w+)|(\w+ \w+))(?:,| (?:return|visit|pov|give|suck|lick|milk|love|enjoy|service|is))/i);
const returnTwoActorMatches = scene.title.match(/(\w+ \w+) and (?:(\w+)|(\w+ \w+)) (?:return|visit|give|suck|lick|milk|love|enjoy|service|are)/i); const returnTwoActorMatches = scene.title.match(/(\w+ \w+) and (?:(\w+)|(\w+ \w+)) (?:return|visit|give|suck|lick|milk|love|enjoy|service|are)/i);
const rawActors = (introTwoActorMatches || introActorMatches || returnTwoActorMatches || returnActorMatches)?.slice(1); const rawActors = (introTwoActorMatches || introActorMatches || returnTwoActorMatches || returnActorMatches)?.slice(1);
const actors = rawActors?.filter((actor) => { const actors = rawActors?.filter((actor) => {
if (!actor) return false; if (!actor) return false;
if (/swallow|\bcum|fuck|suck|give|giving|take|takes|taking|head|teen|babe|cute|beaut|naughty|teacher|nanny|adorable|brunette|blonde|bust|audition|from|\band\b|\bto\b/i.test(actor)) return false; if (/swallow|\bcum|fuck|suck|give|giving|take|takes|taking|head|teen|babe|cute|beaut|naughty|teacher|nanny|adorable|brunette|blonde|bust|audition|from|\band\b|\bto\b/i.test(actor)) return false;
return true; return true;
}); });
if (actors) { if (actors) {
release.actors = actors; release.actors = actors;
} }
} }
if (release.actors?.length > 1 || /threesome|threeway/.test(scene.title)) { if (release.actors?.length > 1 || /threesome|threeway/.test(scene.title)) {
release.tags = scene.tags ? [...scene.tags, 'mff'] : ['mff']; release.tags = scene.tags ? [...scene.tags, 'mff'] : ['mff'];
} }
return release; return release;
} }
async function fetchLatestWrap(site, page = 1) { async function fetchLatestWrap(site, page = 1) {
const latest = await fetchLatest(site, page); const latest = await fetchLatest(site, page);
return latest.map(scene => extractActors(scene)); return latest.map(scene => extractActors(scene));
} }
async function fetchSceneWrap(url, site) { async function fetchSceneWrap(url, site) {
const scene = await fetchScene(url, site); const scene = await fetchScene(url, site);
return extractActors(scene); return extractActors(scene);
} }
module.exports = { module.exports = {
fetchLatest: fetchLatestWrap, fetchLatest: fetchLatestWrap,
fetchScene: fetchSceneWrap, fetchScene: fetchSceneWrap,
}; };

148
src/scrapers/assylum.js
View File

@ -3,7 +3,7 @@
const { get, geta, ctxa } = require('../utils/q'); const { get, geta, ctxa } = require('../utils/q');
function extractActors(actorString) { function extractActors(actorString) {
return actorString return actorString
?.replace(/.*:|\(.*\)|\d+(-|\s)year(-|\s)old|nurses?|tangled/ig, '') // remove Patient:, (date) and other nonsense ?.replace(/.*:|\(.*\)|\d+(-|\s)year(-|\s)old|nurses?|tangled/ig, '') // remove Patient:, (date) and other nonsense
.split(/\band\b|\bvs\b|\/|,|&/ig) .split(/\band\b|\bvs\b|\/|,|&/ig)
.map(actor => actor.trim()) .map(actor => actor.trim())
@ -12,120 +12,120 @@ function extractActors(actorString) {
} }
function matchActors(actorString, models) { function matchActors(actorString, models) {
return models return models
.filter(model => new RegExp(model.name, 'i') .filter(model => new RegExp(model.name, 'i')
.test(actorString)); .test(actorString));
} }
function scrapeLatest(scenes, site, models) { function scrapeLatest(scenes, site, models) {
return scenes.map(({ qu }) => { return scenes.map(({ qu }) => {
const release = {}; const release = {};
const pathname = qu.url('a.itemimg').slice(1); const pathname = qu.url('a.itemimg').slice(1);
[release.entryId] = pathname.split('/').slice(-1); [release.entryId] = pathname.split('/').slice(-1);
release.url = `${site.url}${pathname}`; release.url = `${site.url}${pathname}`;
release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true); release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
release.description = qu.q('.mas_longdescription', true); release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/); release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
const actorString = qu.q('.mas_description', true); const actorString = qu.q('.mas_description', true);
const actors = matchActors(actorString, models); const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors; if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString); else release.actors = extractActors(actorString);
const posterPath = qu.img('.itemimg img'); const posterPath = qu.img('.itemimg img');
release.poster = `${site.url}/${posterPath}`; release.poster = `${site.url}/${posterPath}`;
return release; return release;
}); });
} }
function scrapeScene({ html, qu }, url, site, models) { function scrapeScene({ html, qu }, url, site, models) {
const release = { url }; const release = { url };
[release.entryId] = url.split('/').slice(-1); [release.entryId] = url.split('/').slice(-1);
release.title = qu.q('.mas_title', true); release.title = qu.q('.mas_title', true);
release.description = qu.q('.mas_longdescription', true); release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/); release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, ''); const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
const actors = matchActors(actorString, models); const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors; if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString); else release.actors = extractActors(actorString);
release.tags = qu.all('.tags a', true); release.tags = qu.all('.tags a', true);
release.photos = qu.imgs('.stills img').map(photoPath => `${site.url}/${photoPath}`); release.photos = qu.imgs('.stills img').map(photoPath => `${site.url}/${photoPath}`);
const posterIndex = 'splash:'; const posterIndex = 'splash:';
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4); const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
if (poster) release.poster = `${site.url}/${poster}`; if (poster) release.poster = `${site.url}/${poster}`;
const trailerIndex = html.indexOf('video/mp4'); const trailerIndex = html.indexOf('video/mp4');
const trailer = html.slice(html.indexOf('/content', trailerIndex), html.indexOf('.mp4', trailerIndex) + 4); const trailer = html.slice(html.indexOf('/content', trailerIndex), html.indexOf('.mp4', trailerIndex) + 4);
if (trailer) release.trailer = { src: `${site.url}${trailer}` }; if (trailer) release.trailer = { src: `${site.url}${trailer}` };
return release; return release;
} }
function extractModels({ el }, site) { function extractModels({ el }, site) {
const models = ctxa(el, '.item'); const models = ctxa(el, '.item');
return models.map(({ qu }) => { return models.map(({ qu }) => {
const actor = { gender: 'female' }; const actor = { gender: 'female' };
const avatar = qu.q('.itemimg img'); const avatar = qu.q('.itemimg img');
actor.avatar = `${site.url}/${avatar.src}`; actor.avatar = `${site.url}/${avatar.src}`;
actor.name = avatar.alt actor.name = avatar.alt
.split(':').slice(-1)[0] .split(':').slice(-1)[0]
.replace(/xtreme girl|nurse/ig, '') .replace(/xtreme girl|nurse/ig, '')
.trim(); .trim();
const actorPath = qu.url('.itemimg'); const actorPath = qu.url('.itemimg');
actor.url = `${site.url}${actorPath.slice(1)}`; actor.url = `${site.url}${actorPath.slice(1)}`;
return actor; return actor;
}); });
} }
async function fetchModels(site, page = 1, accModels = []) { async function fetchModels(site, page = 1, accModels = []) {
const url = `${site.url}/?models/${page}`; const url = `${site.url}/?models/${page}`;
const res = await get(url); const res = await get(url);
if (res.ok) { if (res.ok) {
const models = extractModels(res.item, site); const models = extractModels(res.item, site);
const nextPage = res.item.qa('.pagenumbers', true) const nextPage = res.item.qa('.pagenumbers', true)
.map(pageX => Number(pageX)) .map(pageX => Number(pageX))
.filter(Boolean) // remove << and >> .filter(Boolean) // remove << and >>
.includes(page + 1); .includes(page + 1);
if (nextPage) { if (nextPage) {
return fetchModels(site, page + 1, accModels.concat(models)); return fetchModels(site, page + 1, accModels.concat(models));
} }
return accModels.concat(models, { name: 'Dr. Gray' }); return accModels.concat(models, { name: 'Dr. Gray' });
} }
return []; return [];
} }
async function fetchLatest(site, page = 1, models) { async function fetchLatest(site, page = 1, models) {
const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`; const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
const res = await geta(url, '.item'); const res = await geta(url, '.item');
return res.ok ? scrapeLatest(res.items, site, models) : res.status; return res.ok ? scrapeLatest(res.items, site, models) : res.status;
} }
async function fetchScene(url, site, release, beforeFetchLatest) { async function fetchScene(url, site, release, beforeFetchLatest) {
const models = beforeFetchLatest || await fetchModels(site); const models = beforeFetchLatest || await fetchModels(site);
const res = await get(url); const res = await get(url);
return res.ok ? scrapeScene(res.item, url, site, models) : res.status; return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
beforeFetchLatest: fetchModels, beforeFetchLatest: fetchModels,
}; };

164
src/scrapers/aziani.js
View File

@ -5,141 +5,141 @@ const { get, getAll, initAll, extractDate } = require('../utils/qu');
const { feetInchesToCm } = require('../utils/convert'); const { feetInchesToCm } = require('../utils/convert');
function getFallbacks(source) { function getFallbacks(source) {
return [ return [
source.replace('-1x.jpg', '-4x.jpg'), source.replace('-1x.jpg', '-4x.jpg'),
source.replace('-1x.jpg', '-3x.jpg'), source.replace('-1x.jpg', '-3x.jpg'),
source.replace('-1x.jpg', '-2x.jpg'), source.replace('-1x.jpg', '-2x.jpg'),
source, source,
]; ];
} }
function scrapeAll(scenes, site) { function scrapeAll(scenes, site) {
return scenes.map(({ qu }) => { return scenes.map(({ qu }) => {
const release = {}; const release = {};
release.entryId = qu.q('.stdimage', 'id', true).match(/set-target-(\d+)/)[1]; release.entryId = qu.q('.stdimage', 'id', true).match(/set-target-(\d+)/)[1];
release.url = qu.url('a'); release.url = qu.url('a');
release.title = qu.q('h5 a', true); release.title = qu.q('h5 a', true);
release.date = qu.date('.icon-calendar + strong', 'MM/DD/YYYY'); release.date = qu.date('.icon-calendar + strong', 'MM/DD/YYYY');
release.actors = qu.q('h3', true).replace(/featuring:\s?/i, '').split(', '); release.actors = qu.q('h3', true).replace(/featuring:\s?/i, '').split(', ');
const photoCount = qu.q('.stdimage', 'cnt'); const photoCount = qu.q('.stdimage', 'cnt');
[release.poster, ...release.photos] = Array.from({ length: Number(photoCount) }, (value, index) => { [release.poster, ...release.photos] = Array.from({ length: Number(photoCount) }, (value, index) => {
const source = qu.img('.stdimage', `src${index}_1x`, site.url); const source = qu.img('.stdimage', `src${index}_1x`, site.url);
return getFallbacks(source); return getFallbacks(source);
}); });
return release; return release;
}); });
} }
function scrapeScene({ html, qu }, url) { function scrapeScene({ html, qu }, url) {
const release = { url }; const release = { url };
release.entryId = qu.q('.stdimage', 'id', true).match(/set-target-(\d+)/)[1]; release.entryId = qu.q('.stdimage', 'id', true).match(/set-target-(\d+)/)[1];
release.title = qu.q('h2', true); release.title = qu.q('h2', true);
release.description = qu.q('p', true); release.description = qu.q('p', true);
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/); release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
release.actors = qu.all('h5:not(.video_categories) a').map(actor => ({ release.actors = qu.all('h5:not(.video_categories) a').map(actor => ({
name: qu.q(actor, null, true), name: qu.q(actor, null, true),
url: qu.url(actor, null), url: qu.url(actor, null),
})); }));
release.tags = qu.all('.video_categories a', true); release.tags = qu.all('.video_categories a', true);
release.duration = qu.dur('.video_categories + p'); release.duration = qu.dur('.video_categories + p');
const poster = qu.img('a img'); const poster = qu.img('a img');
release.poster = getFallbacks(poster); release.poster = getFallbacks(poster);
release.photos = qu.imgs('.featured-video img', 'src0_1x').map(source => getFallbacks(source)); release.photos = qu.imgs('.featured-video img', 'src0_1x').map(source => getFallbacks(source));
return release; return release;
} }
function scrapeProfile({ el, qu }) { function scrapeProfile({ el, qu }) {
const profile = {}; const profile = {};
const bio = Array.from(qu.q('.widget-content').childNodes).reduce((acc, node, index, nodes) => { const bio = Array.from(qu.q('.widget-content').childNodes).reduce((acc, node, index, nodes) => {
const nextNode = nodes[index + 1]; const nextNode = nodes[index + 1];
if (node.tagName === 'STRONG' && nextNode?.nodeType === 3) { if (node.tagName === 'STRONG' && nextNode?.nodeType === 3) {
acc[slugify(node.textContent, '_')] = nextNode.textContent.trim(); acc[slugify(node.textContent, '_')] = nextNode.textContent.trim();
} }
return acc; return acc;
}, {}); }, {});
if (bio.ethnicity) profile.ethnicity = bio.ethnicity; if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
if (bio.age) profile.age = Number(bio.age); if (bio.age) profile.age = Number(bio.age);
if (bio.height && /\d{3}/.test(bio.height)) profile.height = Number(bio.height.match(/\d+/)[0]); if (bio.height && /\d{3}/.test(bio.height)) profile.height = Number(bio.height.match(/\d+/)[0]);
if (bio.height && /\d[;']\d/.test(bio.height)) profile.height = feetInchesToCm(bio.height); if (bio.height && /\d[;']\d/.test(bio.height)) profile.height = feetInchesToCm(bio.height);
if (bio.measurements) { if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-'); const [bust, waist, hip] = bio.measurements.split('-');
if (bust && /\d+[a-zA-Z]+/.test(bust)) profile.bust = bust; if (bust && /\d+[a-zA-Z]+/.test(bust)) profile.bust = bust;
if (waist) profile.waist = Number(waist); if (waist) profile.waist = Number(waist);
if (hip) profile.hip = Number(hip); if (hip) profile.hip = Number(hip);
} }
if (bio.bust_size && !profile.bust) profile.bust = bio.bust_size.toUpperCase(); if (bio.bust_size && !profile.bust) profile.bust = bio.bust_size.toUpperCase();
if (bio.birth_location) profile.birthPlace = bio.birth_location; if (bio.birth_location) profile.birthPlace = bio.birth_location;
if (bio.status_married_or_single) profile.relationship = bio.status_married_or_single; if (bio.status_married_or_single) profile.relationship = bio.status_married_or_single;
if (bio.eye_color) profile.eyes = bio.eye_color; if (bio.eye_color) profile.eyes = bio.eye_color;
const avatar = qu.img('.tac img'); const avatar = qu.img('.tac img');
profile.avatar = getFallbacks(avatar); profile.avatar = getFallbacks(avatar);
profile.releases = scrapeAll(initAll(el, '.featured-video')); profile.releases = scrapeAll(initAll(el, '.featured-video'));
return profile; return profile;
} }
async function fetchLatest(site, page) { async function fetchLatest(site, page) {
const url = `${site.url}/tour/categories/movies_${page}_d.html`; const url = `${site.url}/tour/categories/movies_${page}_d.html`;
const res = await getAll(url, '.featured-video'); const res = await getAll(url, '.featured-video');
if (res.ok) { if (res.ok) {
return scrapeAll(res.items, site); return scrapeAll(res.items, site);
} }
return res.status; return res.status;
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
const res = await get(url, '.page-content .row'); const res = await get(url, '.page-content .row');
if (res.ok) { if (res.ok) {
return scrapeScene(res.item, url, site); return scrapeScene(res.item, url, site);
} }
return res.status; return res.status;
} }
async function fetchProfile(actorName, scraperSlug, site) { async function fetchProfile(actorName, scraperSlug, site) {
const actorSlug = slugify(actorName, ''); const actorSlug = slugify(actorName, '');
const url = `${site.url}/tour/models/${actorSlug}.html`; const url = `${site.url}/tour/models/${actorSlug}.html`;
const res = await get(url, '.page-content .row'); const res = await get(url, '.page-content .row');
if (res.ok) { if (res.ok) {
return scrapeProfile(res.item); return scrapeProfile(res.item);
} }
return res.status; return res.status;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchProfile, fetchProfile,
fetchScene, fetchScene,
}; };

8
src/scrapers/babes.js
View File

@ -3,11 +3,11 @@
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek'); const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) { async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'babes'); return fetchProfile(actorName, 'babes');
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchProfile: networkFetchProfile, fetchProfile: networkFetchProfile,
fetchScene, fetchScene,
}; };

180
src/scrapers/bamvisions.js
View File

@ -6,144 +6,144 @@ const slugify = require('../utils/slugify');
const { feetInchesToCm } = require('../utils/convert'); const { feetInchesToCm } = require('../utils/convert');
function scrapeAll(scenes, site) { function scrapeAll(scenes, site) {
return scenes.map(({ qu }) => { return scenes.map(({ qu }) => {
const release = {}; const release = {};
release.title = qu.q('h3 a', true); release.title = qu.q('h3 a', true);
release.url = qu.url('h3 a'); release.url = qu.url('h3 a');
release.date = qu.date('.item-meta li', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/); release.date = qu.date('.item-meta li', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = qu.dur('.item-meta li:nth-child(2)'); release.duration = qu.dur('.item-meta li:nth-child(2)');
release.description = qu.q('.description', true); release.description = qu.q('.description', true);
release.actors = qu.all('a[href*="/models"]', true); release.actors = qu.all('a[href*="/models"]', true);
if (/bts/i.test(release.title)) release.tags = ['behind the scenes']; if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
[release.poster, ...release.photos] = qu.all('.item-thumbs img') [release.poster, ...release.photos] = qu.all('.item-thumbs img')
.map(source => [ .map(source => [
source.getAttribute('src0_3x'), source.getAttribute('src0_3x'),
source.getAttribute('src0_2x'), source.getAttribute('src0_2x'),
source.getAttribute('src0_1x'), source.getAttribute('src0_1x'),
] ]
.filter(Boolean) .filter(Boolean)
.map(fallback => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`))); .map(fallback => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`)));
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`; release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
return release; return release;
}); });
} }
function scrapeScene({ html, qu }, url, site) { function scrapeScene({ html, qu }, url, site) {
const release = { url }; const release = { url };
release.title = qu.q('.item-episode h4 a', true); release.title = qu.q('.item-episode h4 a', true);
release.date = qu.date('.item-meta li', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/); release.date = qu.date('.item-meta li', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = qu.dur('.item-meta li:nth-child(2)'); release.duration = qu.dur('.item-meta li:nth-child(2)');
release.description = qu.q('.description', true); release.description = qu.q('.description', true);
release.actors = qu.all('.item-episode a[href*="/models"]', true); release.actors = qu.all('.item-episode a[href*="/models"]', true);
if (/bts/i.test(release.title)) release.tags = ['behind the scenes']; if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
const posterPath = html.match(/poster="(.*.jpg)"/)?.[1]; const posterPath = html.match(/poster="(.*.jpg)"/)?.[1];
const trailerPath = html.match(/video src="(.*.mp4)"/)?.[1]; const trailerPath = html.match(/video src="(.*.mp4)"/)?.[1];
if (posterPath) { if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${site.url}${posterPath}`; const poster = /^http/.test(posterPath) ? posterPath : `${site.url}${posterPath}`;
release.poster = [ release.poster = [
poster.replace('-1x', '-3x'), poster.replace('-1x', '-3x'),
poster.replace('-1x', '-2x'), poster.replace('-1x', '-2x'),
poster, poster,
]; ];
} }
if (trailerPath) { if (trailerPath) {
const trailer = /^http/.test(trailerPath) ? trailerPath : `${site.url}${trailerPath}`; const trailer = /^http/.test(trailerPath) ? trailerPath : `${site.url}${trailerPath}`;
release.trailer = { src: trailer }; release.trailer = { src: trailer };
} }
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`; release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
return release; return release;
} }
async function fetchActorReleases(actorId, site, page = 1, accScenes = []) { async function fetchActorReleases(actorId, site, page = 1, accScenes = []) {
const url = `${site.url}/sets.php?id=${actorId}&page=${page}`; const url = `${site.url}/sets.php?id=${actorId}&page=${page}`;
const res = await get(url); const res = await get(url);
if (!res.ok) return []; if (!res.ok) return [];
const quReleases = initAll(res.item.el, '.item-episode'); const quReleases = initAll(res.item.el, '.item-episode');
const releases = scrapeAll(quReleases, site); const releases = scrapeAll(quReleases, site);
const nextPage = res.item.qu.q(`a[href*="page=${page + 1}"]`); const nextPage = res.item.qu.q(`a[href*="page=${page + 1}"]`);
if (nextPage) { if (nextPage) {
return fetchActorReleases(actorId, site, page + 1, accScenes.concat(releases)); return fetchActorReleases(actorId, site, page + 1, accScenes.concat(releases));
} }
return accScenes.concat(releases); return accScenes.concat(releases);
} }
async function scrapeProfile({ qu }, site, withScenes) { async function scrapeProfile({ qu }, site, withScenes) {
const profile = {}; const profile = {};
const bio = qu.all('.stats li', true).reduce((acc, row) => { const bio = qu.all('.stats li', true).reduce((acc, row) => {
const [key, value] = row.split(':'); const [key, value] = row.split(':');
return { ...acc, [slugify(key, '_')]: value.trim() }; return { ...acc, [slugify(key, '_')]: value.trim() };
}, {}); }, {});
if (bio.height) profile.height = feetInchesToCm(bio.height); if (bio.height) profile.height = feetInchesToCm(bio.height);
if (bio.measurements) { if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-'); const [bust, waist, hip] = bio.measurements.split('-');
if (bust) profile.bust = bust; if (bust) profile.bust = bust;
if (waist) profile.waist = Number(waist); if (waist) profile.waist = Number(waist);
if (hip) profile.hip = Number(hip); if (hip) profile.hip = Number(hip);
} }
profile.avatar = [ profile.avatar = [
qu.q('.profile-pic img', 'src0_3x'), qu.q('.profile-pic img', 'src0_3x'),
qu.q('.profile-pic img', 'src0_2x'), qu.q('.profile-pic img', 'src0_2x'),
qu.q('.profile-pic img', 'src0_1x'), qu.q('.profile-pic img', 'src0_1x'),
].filter(Boolean).map(source => (/^http/.test(source) ? source : `${site.url}${source}`)); ].filter(Boolean).map(source => (/^http/.test(source) ? source : `${site.url}${source}`));
if (withScenes) { if (withScenes) {
const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1]; const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1];
if (actorId) { if (actorId) {
profile.releases = await fetchActorReleases(actorId, site); profile.releases = await fetchActorReleases(actorId, site);
} }
} }
return profile; return profile;
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const url = `${site.url}/categories/movies/${page}/latest/`; const url = `${site.url}/categories/movies/${page}/latest/`;
const res = await geta(url, '.item-episode'); const res = await geta(url, '.item-episode');
return res.ok ? scrapeAll(res.items, site) : res.status; return res.ok ? scrapeAll(res.items, site) : res.status;
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
const res = await get(url); const res = await get(url);
return res.ok ? scrapeScene(res.item, url, site) : res.status; return res.ok ? scrapeScene(res.item, url, site) : res.status;
} }
async function fetchProfile(actorName, scraperSlug, site, include) { async function fetchProfile(actorName, scraperSlug, site, include) {
const actorSlugA = slugify(actorName, ''); const actorSlugA = slugify(actorName, '');
const actorSlugB = slugify(actorName); const actorSlugB = slugify(actorName);
const resA = await get(`${site.url}/models/${actorSlugA}.html`); const resA = await get(`${site.url}/models/${actorSlugA}.html`);
const res = resA.ok ? resA : await get(`${site.url}/models/${actorSlugB}.html`); const res = resA.ok ? resA : await get(`${site.url}/models/${actorSlugB}.html`);
return res.ok ? scrapeProfile(res.item, site, include.scenes) : res.status; return res.ok ? scrapeProfile(res.item, site, include.scenes) : res.status;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchProfile, fetchProfile,
}; };

234
src/scrapers/bang.js
View File

@ -8,99 +8,99 @@ const clusterId = '617fb597b659459bafe6472470d9073a';
const authKey = 'YmFuZy1yZWFkOktqVDN0RzJacmQ1TFNRazI='; const authKey = 'YmFuZy1yZWFkOktqVDN0RzJacmQ1TFNRazI=';
const genderMap = { const genderMap = {
M: 'male', M: 'male',
F: 'female', F: 'female',
}; };
function getScreenUrl(item, scene) { function getScreenUrl(item, scene) {
return `https://i.bang.com/screenshots/${scene.dvd.id}/movie/${scene.order}/${item.screenId}.jpg`; return `https://i.bang.com/screenshots/${scene.dvd.id}/movie/${scene.order}/${item.screenId}.jpg`;
} }
function encodeId(id) { function encodeId(id) {
return Buffer return Buffer
.from(id, 'hex') .from(id, 'hex')
.toString('base64') .toString('base64')
.replace(/\+/g, '-') .replace(/\+/g, '-')
.replace(/\//g, '_') .replace(/\//g, '_')
.replace(/=/g, ','); .replace(/=/g, ',');
} }
function decodeId(id) { function decodeId(id) {
const restoredId = id const restoredId = id
.replace(/-/g, '+') .replace(/-/g, '+')
.replace(/_/g, '/') .replace(/_/g, '/')
.replace(/,/g, '='); .replace(/,/g, '=');
return Buffer return Buffer
.from(restoredId, 'base64') .from(restoredId, 'base64')
.toString('hex'); .toString('hex');
} }
function scrapeScene(scene, site) { function scrapeScene(scene, site) {
const release = { const release = {
site, site,
entryId: scene.id, entryId: scene.id,
title: scene.name, title: scene.name,
description: scene.description, description: scene.description,
tags: scene.genres.concat(scene.actions).map(genre => genre.name), tags: scene.genres.concat(scene.actions).map(genre => genre.name),
duration: scene.duration, duration: scene.duration,
}; };
const slug = slugify(release.title); const slug = slugify(release.title);
release.url = `https://www.bang.com/video/${encodeId(release.entryId)}/${slug}`; release.url = `https://www.bang.com/video/${encodeId(release.entryId)}/${slug}`;
const date = new Date(scene.releaseDate); const date = new Date(scene.releaseDate);
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: genderMap[actor.gender] })); release.actors = scene.actors.map(actor => ({ name: actor.name, gender: genderMap[actor.gender] }));
if (scene.is4k) release.tags.push('4k'); if (scene.is4k) release.tags.push('4k');
if (scene.gay) release.tags.push('gay'); if (scene.gay) release.tags.push('gay');
const defaultPoster = scene.screenshots.find(photo => photo.default === true); const defaultPoster = scene.screenshots.find(photo => photo.default === true);
const photoset = scene.screenshots.filter(photo => photo.default === false); const photoset = scene.screenshots.filter(photo => photo.default === false);
const photos = defaultPoster ? photoset : photoset.slice(1); const photos = defaultPoster ? photoset : photoset.slice(1);
const poster = defaultPoster || photoset[0]; const poster = defaultPoster || photoset[0];
release.poster = getScreenUrl(poster, scene); release.poster = getScreenUrl(poster, scene);
release.photos = photos.map(photo => getScreenUrl(photo, scene)); release.photos = photos.map(photo => getScreenUrl(photo, scene));
release.trailer = { release.trailer = {
src: `https://i.bang.com/v/${scene.dvd.id}/${scene.identifier}/preview.mp4`, src: `https://i.bang.com/v/${scene.dvd.id}/${scene.identifier}/preview.mp4`,
}; };
release.channel = scene.series.name release.channel = scene.series.name
.replace(/[! .]/g, '') .replace(/[! .]/g, '')
.replace('&', 'and'); .replace('&', 'and');
return release; return release;
} }
function scrapeLatest(scenes, site) { function scrapeLatest(scenes, site) {
return scenes.map(({ _source: scene }) => scrapeScene(scene, site)); return scenes.map(({ _source: scene }) => scrapeScene(scene, site));
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, { const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50, size: 50,
from: (page - 1) * 50, from: (page - 1) * 50,
query: { query: {
bool: { bool: {
must: [ must: [
{ {
match: { match: {
status: 'ok', status: 'ok',
}, },
}, },
{ {
range: { range: {
releaseDate: { releaseDate: {
lte: 'now', lte: 'now',
}, },
}, },
}, },
/* /*
* global fetch * global fetch
{ {
nested: { nested: {
@ -122,66 +122,66 @@ async function fetchLatest(site, page = 1) {
}, },
}, },
*/ */
{ {
nested: { nested: {
path: 'series', path: 'series',
query: { query: {
bool: { bool: {
must: [ must: [
{ {
match: { match: {
'series.id': { 'series.id': {
operator: 'AND', operator: 'AND',
query: site.parameters.siteId, query: site.parameters.siteId,
}, },
}, },
}, },
], ],
}, },
}, },
}, },
}, },
], ],
must_not: [ must_not: [
{ {
match: { match: {
type: 'trailer', type: 'trailer',
}, },
}, },
], ],
}, },
}, },
sort: [ sort: [
{ {
releaseDate: { releaseDate: {
order: 'desc', order: 'desc',
}, },
}, },
], ],
}, { }, {
encodeJSON: true, encodeJSON: true,
headers: { headers: {
Authorization: `Basic ${authKey}`, Authorization: `Basic ${authKey}`,
}, },
}); });
return scrapeLatest(res.body.hits.hits, site); return scrapeLatest(res.body.hits.hits, site);
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
const encodedId = new URL(url).pathname.split('/')[2]; const encodedId = new URL(url).pathname.split('/')[2];
const entryId = decodeId(encodedId); const entryId = decodeId(encodedId);
const res = await bhttp.get(`https://${clusterId}.us-east-1.aws.found.io/videos/video/${entryId}`, { const res = await bhttp.get(`https://${clusterId}.us-east-1.aws.found.io/videos/video/${entryId}`, {
headers: { headers: {
Authorization: `Basic ${authKey}`, Authorization: `Basic ${authKey}`,
}, },
}); });
return scrapeScene(res.body._source, site); // eslint-disable-line no-underscore-dangle return scrapeScene(res.body._source, site); // eslint-disable-line no-underscore-dangle
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
}; };

184
src/scrapers/bangbros.js
View File

@ -10,44 +10,44 @@ const slugify = require('../utils/slugify');
const { ex } = require('../utils/q'); const { ex } = require('../utils/q');
function scrape(html, site) { function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.echThumb').toArray(); const sceneElements = $('.echThumb').toArray();
return sceneElements.map((element) => { return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('.thmb_lnk'); const sceneLinkElement = $(element).find('.thmb_lnk');
const title = sceneLinkElement.attr('title'); const title = sceneLinkElement.attr('title');
const url = `https://bangbros.com${sceneLinkElement.attr('href')}`; const url = `https://bangbros.com${sceneLinkElement.attr('href')}`;
const shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1]; const shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1];
const entryId = url.split('/')[3].slice(5); const entryId = url.split('/')[3].slice(5);
const date = moment.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate(); const date = moment.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate();
const actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray(); const actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
const photoElement = $(element).find('.rollover-image'); const photoElement = $(element).find('.rollover-image');
const poster = `https:${photoElement.attr('data-original')}`; const poster = `https:${photoElement.attr('data-original')}`;
const photosUrl = photoElement.attr('data-rollover-url'); const photosUrl = photoElement.attr('data-rollover-url');
const photosMaxIndex = photoElement.attr('data-rollover-max-index'); const photosMaxIndex = photoElement.attr('data-rollover-max-index');
const photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`); const photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`);
const duration = moment.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds(); const duration = moment.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds();
const channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0]; const channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0];
return { return {
url, url,
entryId, entryId,
shootId, shootId,
title, title,
actors, actors,
date, date,
duration, duration,
poster, poster,
photos, photos,
rating: null, rating: null,
site, site,
channel, channel,
}; };
}); });
} }
/* no dates available, breaks database /* no dates available, breaks database
@ -80,63 +80,63 @@ function scrapeUpcoming(html, site) {
*/ */
function scrapeScene(html, url, _site) { function scrapeScene(html, url, _site) {
const { qu } = ex(html, '.playerSection'); const { qu } = ex(html, '.playerSection');
const release = {}; const release = {};
[release.shootId] = qu.q('.vdoTags + .vdoCast', true).match(/\w+$/); [release.shootId] = qu.q('.vdoTags + .vdoCast', true).match(/\w+$/);
[release.entryId] = url.split('/')[3].match(/\d+$/); [release.entryId] = url.split('/')[3].match(/\d+$/);
release.title = qu.q('.ps-vdoHdd h1', true); release.title = qu.q('.ps-vdoHdd h1', true);
release.description = qu.q('.vdoDesc', true); release.description = qu.q('.vdoDesc', true);
release.actors = qu.all('a[href*="/model"]', true); release.actors = qu.all('a[href*="/model"]', true);
release.tags = qu.all('.vdoTags a', true); release.tags = qu.all('.vdoTags a', true);
release.stars = Number(qu.q('div[class*="like"]', true).match(/^\d+/)[0]) / 20; release.stars = Number(qu.q('div[class*="like"]', true).match(/^\d+/)[0]) / 20;
const poster = qu.img('img#player-overlay-image'); const poster = qu.img('img#player-overlay-image');
release.poster = [ release.poster = [
poster, poster,
poster.replace('/big_trailer', '/members/450x340'), // load error fallback poster.replace('/big_trailer', '/members/450x340'), // load error fallback
]; ];
release.trailer = { src: qu.trailer() }; release.trailer = { src: qu.trailer() };
// all scenes seem to have 12 album photos available, not always included on the page // all scenes seem to have 12 album photos available, not always included on the page
const firstPhotoUrl = ex(html).qu.img('img[data-slider-index="1"]'); const firstPhotoUrl = ex(html).qu.img('img[data-slider-index="1"]');
release.photos = Array.from({ length: 12 }, (val, index) => firstPhotoUrl.replace(/big\d+/, `big${index + 1}`)); release.photos = Array.from({ length: 12 }, (val, index) => firstPhotoUrl.replace(/big\d+/, `big${index + 1}`));
const [channel] = qu.url('a[href*="/websites"]').match(/\w+$/); const [channel] = qu.url('a[href*="/websites"]').match(/\w+$/);
if (channel === 'bangcasting') release.channel = 'bangbroscasting'; if (channel === 'bangcasting') release.channel = 'bangbroscasting';
if (channel === 'remaster') release.channel = 'bangbrosremastered'; if (channel === 'remaster') release.channel = 'bangbrosremastered';
else release.channel = channel; else release.channel = channel;
return release; return release;
} }
function scrapeProfile(html) { function scrapeProfile(html) {
const { q } = ex(html); const { q } = ex(html);
const profile = {}; const profile = {};
const avatar = q('.profilePic img', 'src'); const avatar = q('.profilePic img', 'src');
if (avatar) profile.avatar = `https:${avatar}`; if (avatar) profile.avatar = `https:${avatar}`;
profile.releases = scrape(html); profile.releases = scrape(html);
return profile; return profile;
} }
function scrapeProfileSearch(html, actorName) { function scrapeProfileSearch(html, actorName) {
const { qu } = ex(html); const { qu } = ex(html);
const actorLink = qu.url(`a[title="${actorName}" i][href*="model"]`); const actorLink = qu.url(`a[title="${actorName}" i][href*="model"]`);
return actorLink ? `https://bangbros.com${actorLink}` : null; return actorLink ? `https://bangbros.com${actorLink}` : null;
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/${page}`); const res = await bhttp.get(`${site.url}/${page}`);
return scrape(res.body.toString(), site); return scrape(res.body.toString(), site);
} }
/* /*
@ -148,43 +148,43 @@ async function fetchUpcoming(site) {
*/ */
async function fetchScene(url, site, release) { async function fetchScene(url, site, release) {
if (!release?.date) { if (!release?.date) {
logger.warn(`Scraping Bang Bros scene from URL without release date: ${url}`); logger.warn(`Scraping Bang Bros scene from URL without release date: ${url}`);
} }
const { origin } = new URL(url); const { origin } = new URL(url);
const res = await bhttp.get(url); const res = await bhttp.get(url);
if (!/https?:\/\/(www.)?bangbros.com\/?$/.test(origin)) { if (!/https?:\/\/(www.)?bangbros.com\/?$/.test(origin)) {
throw new Error('Cannot fetch from this URL. Please find the scene on https://bangbros.com and try again.'); throw new Error('Cannot fetch from this URL. Please find the scene on https://bangbros.com and try again.');
} }
return scrapeScene(res.body.toString(), url, site); return scrapeScene(res.body.toString(), url, site);
} }
async function fetchProfile(actorName) { async function fetchProfile(actorName) {
const actorSlug = slugify(actorName); const actorSlug = slugify(actorName);
const url = `https://bangbros.com/search/${actorSlug}`; const url = `https://bangbros.com/search/${actorSlug}`;
const res = await bhttp.get(url); const res = await bhttp.get(url);
if (res.statusCode === 200) { if (res.statusCode === 200) {
const actorUrl = scrapeProfileSearch(res.body.toString(), actorName); const actorUrl = scrapeProfileSearch(res.body.toString(), actorName);
if (actorUrl) { if (actorUrl) {
const actorRes = await bhttp.get(actorUrl); const actorRes = await bhttp.get(actorUrl);
if (actorRes.statusCode === 200) { if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString()); return scrapeProfile(actorRes.body.toString());
} }
} }
} }
return null; return null;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchProfile, fetchProfile,
// fetchUpcoming, no dates available // fetchUpcoming, no dates available
}; };

32
src/scrapers/blowpass.js
View File

@ -5,33 +5,33 @@
const { fetchScene, fetchLatest, fetchUpcoming, fetchProfile } = require('./gamma'); const { fetchScene, fetchLatest, fetchUpcoming, fetchProfile } = require('./gamma');
async function fetchSceneWrapper(url, site, baseRelease) { async function fetchSceneWrapper(url, site, baseRelease) {
const release = await fetchScene(url, site, baseRelease); const release = await fetchScene(url, site, baseRelease);
if (site.isFallback && release.channel) { if (site.isNetwork && release.channel) {
const channelUrl = url.replace('blowpass.com', `${release.channel}.com`); const channelUrl = url.replace('blowpass.com', `${release.channel}.com`);
if (['onlyteenblowjobs', 'mommyblowsbest'].includes(release.channel)) { if (['onlyteenblowjobs', 'mommyblowsbest'].includes(release.channel)) {
release.url = channelUrl.replace(/video\/\w+\//, 'scene/'); release.url = channelUrl.replace(/video\/\w+\//, 'scene/');
return release; return release;
} }
release.url = channelUrl.replace(/video\/\w+\//, 'video/'); release.url = channelUrl.replace(/video\/\w+\//, 'video/');
} }
return release; return release;
} }
function getActorReleasesUrl(actorPath, page = 1) { function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.blowpass.com/en/videos/blowpass/latest/All-Categories/0${actorPath}/${page}`; return `https://www.blowpass.com/en/videos/blowpass/latest/All-Categories/0${actorPath}/${page}`;
} }
async function networkFetchProfile(actorName, scraperSlug, site, include) { async function networkFetchProfile(actorName, scraperSlug, site, include) {
return fetchProfile(actorName, scraperSlug, null, getActorReleasesUrl, include); return fetchProfile(actorName, scraperSlug, null, getActorReleasesUrl, include);
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchProfile: networkFetchProfile, fetchProfile: networkFetchProfile,
fetchUpcoming, fetchUpcoming,
fetchScene: fetchSceneWrapper, fetchScene: fetchSceneWrapper,
}; };

102
src/scrapers/boobpedia.js
View File

@ -5,90 +5,90 @@ const bhttp = require('bhttp');
const { ex } = require('../utils/q'); const { ex } = require('../utils/q');
function scrapeProfile(html) { function scrapeProfile(html) {
const { qu } = ex(html); /* eslint-disable-line object-curly-newline */ const { qu } = ex(html); /* eslint-disable-line object-curly-newline */
const profile = {}; const profile = {};
const bio = qu.all('.infobox tr[valign="top"]') const bio = qu.all('.infobox tr[valign="top"]')
.map(detail => qu.all(detail, 'td', true)) .map(detail => qu.all(detail, 'td', true))
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {}); .reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
/* unreliable, see: Syren De Mer /* unreliable, see: Syren De Mer
const catlinks = qa('#mw-normal-catlinks a', true); const catlinks = qa('#mw-normal-catlinks a', true);
const isTrans = catlinks.some(link => link.match(/shemale|transgender/i)); const isTrans = catlinks.some(link => link.match(/shemale|transgender/i));
profile.gender = isTrans ? 'transsexual' : 'female'; profile.gender = isTrans ? 'transsexual' : 'female';
*/ */
profile.birthdate = qu.date('.bday', 'YYYY-MM-DD'); profile.birthdate = qu.date('.bday', 'YYYY-MM-DD');
profile.description = qu.q('#mw-content-text > p', true); profile.description = qu.q('#mw-content-text > p', true);
if (bio.Born) profile.birthPlace = bio.Born.slice(bio.Born.lastIndexOf(')') + 1); if (bio.Born) profile.birthPlace = bio.Born.slice(bio.Born.lastIndexOf(')') + 1);
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity; if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Measurements) { if (bio.Measurements) {
const measurements = bio.Measurements const measurements = bio.Measurements
.match(/\d+(\w+)?-\d+-\d+/g) .match(/\d+(\w+)?-\d+-\d+/g)
?.slice(-1)[0] // allow for both '34C-25-36' and '86-64-94 cm / 34-25-37 in' ?.slice(-1)[0] // allow for both '34C-25-36' and '86-64-94 cm / 34-25-37 in'
.split('-'); .split('-');
// account for measuemrents being just e.g. '32EE' // account for measuemrents being just e.g. '32EE'
if (measurements) { if (measurements) {
const [bust, waist, hip] = measurements; const [bust, waist, hip] = measurements;
if (/[a-zA-Z]/.test(bust)) profile.bust = bust; // only use bust if cup size is included if (/[a-zA-Z]/.test(bust)) profile.bust = bust; // only use bust if cup size is included
profile.waist = Number(waist); profile.waist = Number(waist);
profile.hip = Number(hip); profile.hip = Number(hip);
} }
if (/^\d+\w+$/.test(bio.Measurements)) profile.bust = bio.Measurements; if (/^\d+\w+$/.test(bio.Measurements)) profile.bust = bio.Measurements;
} }
if (bio.Bra_cup_size) { if (bio.Bra_cup_size) {
const bust = bio.Bra_cup_size.match(/^\d+\w+/); const bust = bio.Bra_cup_size.match(/^\d+\w+/);
if (bust) [profile.bust] = bust; if (bust) [profile.bust] = bust;
} }
if (bio.Boobs === 'Enhanced') profile.naturalBoobs = false; if (bio.Boobs === 'Enhanced') profile.naturalBoobs = false;
if (bio.Boobs === 'Natural') profile.naturalBoobs = true; if (bio.Boobs === 'Natural') profile.naturalBoobs = true;
if (bio.Height) profile.height = Number(bio.Height.match(/\d+\.\d+/g).slice(-1)[0]) * 100; if (bio.Height) profile.height = Number(bio.Height.match(/\d+\.\d+/g).slice(-1)[0]) * 100;
if (bio.Weight) profile.weight = Number(bio.Weight.match(/\d+/g)[1]); if (bio.Weight) profile.weight = Number(bio.Weight.match(/\d+/g)[1]);
if (bio.Eye_color) profile.eyes = bio.Eye_color; if (bio.Eye_color) profile.eyes = bio.Eye_color;
if (bio.Hair) [profile.hair] = bio.Hair.split(','); if (bio.Hair) [profile.hair] = bio.Hair.split(',');
if (bio.Blood_group) profile.blood = bio.Blood_group; if (bio.Blood_group) profile.blood = bio.Blood_group;
if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', '); if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', ');
const avatarThumbPath = qu.img('.image img'); const avatarThumbPath = qu.img('.image img');
if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) { if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) {
const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', ''); const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', '');
profile.avatar = { profile.avatar = {
src: `http://www.boobpedia.com${avatarPath}`, src: `http://www.boobpedia.com${avatarPath}`,
copyright: null, copyright: null,
}; };
} }
profile.social = qu.urls('.infobox a.external'); profile.social = qu.urls('.infobox a.external');
return profile; return profile;
} }
async function fetchProfile(actorName) { async function fetchProfile(actorName) {
const actorSlug = actorName.replace(/\s+/, '_'); const actorSlug = actorName.replace(/\s+/, '_');
const res = await bhttp.get(`http://www.boobpedia.com/boobs/${actorSlug}`); const res = await bhttp.get(`http://www.boobpedia.com/boobs/${actorSlug}`);
if (res.statusCode === 200) { if (res.statusCode === 200) {
return scrapeProfile(res.body.toString()); return scrapeProfile(res.body.toString());
} }
return null; return null;
} }
module.exports = { module.exports = {
fetchProfile, fetchProfile,
}; };

276
src/scrapers/brazzers.js
View File

@ -11,216 +11,216 @@ const slugify = require('../utils/slugify');
const { heightToCm, lbsToKg } = require('../utils/convert'); const { heightToCm, lbsToKg } = require('../utils/convert');
const hairMap = { const hairMap = {
Blonde: 'blonde', Blonde: 'blonde',
Brunette: 'brown', Brunette: 'brown',
'Black Hair': 'black', 'Black Hair': 'black',
Redhead: 'red', Redhead: 'red',
}; };
function scrapeAll(html, site, upcoming) { function scrapeAll(html, site, upcoming) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.release-card.scene').toArray(); const sceneElements = $('.release-card.scene').toArray();
return sceneElements.reduce((acc, element) => { return sceneElements.reduce((acc, element) => {
const isUpcoming = $(element).find('.icon-upcoming.active').length === 1; const isUpcoming = $(element).find('.icon-upcoming.active').length === 1;
if ((upcoming && !isUpcoming) || (!upcoming && isUpcoming)) { if ((upcoming && !isUpcoming) || (!upcoming && isUpcoming)) {
return acc; return acc;
} }
const sceneLinkElement = $(element).find('a'); const sceneLinkElement = $(element).find('a');
const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`; const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title'); const title = sceneLinkElement.attr('title');
const entryId = url.split('/').slice(-3, -2)[0]; const entryId = url.split('/').slice(-3, -2)[0];
const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate(); const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate();
const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray(); const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
const likes = Number($(element).find('.label-rating .like-amount').text()); const likes = Number($(element).find('.label-rating .like-amount').text());
const dislikes = Number($(element).find('.label-rating .dislike-amount').text()); const dislikes = Number($(element).find('.label-rating .dislike-amount').text());
const poster = `https:${$(element).find('.card-main-img').attr('data-src')}`; const poster = `https:${$(element).find('.card-main-img').attr('data-src')}`;
const photos = $(element).find('.card-overlay .image-under').map((photoIndex, photoElement) => `https:${$(photoElement).attr('data-src')}`).toArray(); const photos = $(element).find('.card-overlay .image-under').map((photoIndex, photoElement) => `https:${$(photoElement).attr('data-src')}`).toArray();
const channel = slugify($(element).find('.collection').attr('title'), ''); const channel = slugify($(element).find('.collection').attr('title'), '');
return acc.concat({ return acc.concat({
url, url,
entryId, entryId,
title, title,
actors, actors,
date, date,
poster, poster,
photos, photos,
rating: { rating: {
likes, likes,
dislikes, dislikes,
}, },
channel, channel,
site, site,
}); });
}, []); }, []);
} }
async function scrapeScene(html, url, _site) { async function scrapeScene(html, url, _site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
const release = {}; const release = {};
const videoJson = $('script:contains("window.videoUiOptions")').html(); const videoJson = $('script:contains("window.videoUiOptions")').html();
const videoString = videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('},') + 1); const videoString = videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('},') + 1);
const videoData = JSON.parse(videoString); const videoData = JSON.parse(videoString);
[release.entryId] = url.split('/').slice(-3, -2); [release.entryId] = url.split('/').slice(-3, -2);
release.title = $('.scene-title[itemprop="name"]').text(); release.title = $('.scene-title[itemprop="name"]').text();
release.description = $('#scene-description p[itemprop="description"]') release.description = $('#scene-description p[itemprop="description"]')
.contents() .contents()
.first() .first()
.text() .text()
.trim(); .trim();
release.date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate(); release.date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate();
release.duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60; release.duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60;
const actorsFromCards = $('.featured-model .card-image a').map((actorIndex, actorElement) => { const actorsFromCards = $('.featured-model .card-image a').map((actorIndex, actorElement) => {
const avatar = `https:${$(actorElement).find('img').attr('data-src')}`; const avatar = `https:${$(actorElement).find('img').attr('data-src')}`;
return { return {
name: $(actorElement).attr('title'), name: $(actorElement).attr('title'),
avatar: [avatar.replace('medium.jpg', 'large.jpg'), avatar], avatar: [avatar.replace('medium.jpg', 'large.jpg'), avatar],
}; };
}).toArray(); }).toArray();
release.actors = actorsFromCards || $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); release.actors = actorsFromCards || $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.likes = Number($('.label-rating .like').text()); release.likes = Number($('.label-rating .like').text());
release.dislikes = Number($('.label-rating .dislike').text()); release.dislikes = Number($('.label-rating .dislike').text());
const siteElement = $('.niche-site-logo'); const siteElement = $('.niche-site-logo');
// const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`; // const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
const siteName = siteElement.attr('title'); const siteName = siteElement.attr('title');
release.channel = siteName.replace(/\s+/g, '').toLowerCase(); release.channel = siteName.replace(/\s+/g, '').toLowerCase();
release.tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); release.tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
release.photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray(); release.photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray();
const posterPath = videoData?.poster || $('meta[itemprop="thumbnailUrl"]').attr('content') || $('#trailer-player-container').attr('data-player-img'); const posterPath = videoData?.poster || $('meta[itemprop="thumbnailUrl"]').attr('content') || $('#trailer-player-container').attr('data-player-img');
if (posterPath) release.poster = `https:${posterPath}`; if (posterPath) release.poster = `https:${posterPath}`;
if (videoData) { if (videoData) {
release.trailer = Object.entries(videoData.stream_info.http.paths).map(([quality, path]) => ({ release.trailer = Object.entries(videoData.stream_info.http.paths).map(([quality, path]) => ({
src: `https:${path}`, src: `https:${path}`,
quality: Number(quality.match(/\d{3,}/)[0]), quality: Number(quality.match(/\d{3,}/)[0]),
})); }));
} }
return release; return release;
} }
function scrapeActorSearch(html, url, actorName) { function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`); const actorLink = document.querySelector(`a[title="${actorName}" i]`);
return actorLink ? actorLink.href : null; return actorLink ? actorLink.href : null;
} }
async function fetchActorReleases({ qu, html }, accReleases = []) { async function fetchActorReleases({ qu, html }, accReleases = []) {
const releases = scrapeAll(html); const releases = scrapeAll(html);
const next = qu.url('.pagination .next a'); const next = qu.url('.pagination .next a');
if (next) { if (next) {
const url = `https://www.brazzers.com${next}`; const url = `https://www.brazzers.com${next}`;
const res = await get(url); const res = await get(url);
if (res.ok) { if (res.ok) {
return fetchActorReleases(res.item, accReleases.concat(releases)); return fetchActorReleases(res.item, accReleases.concat(releases));
} }
} }
return accReleases.concat(releases); return accReleases.concat(releases);
} }
async function scrapeProfile(html, url, actorName) { async function scrapeProfile(html, url, actorName) {
const qProfile = ex(html); const qProfile = ex(html);
const { q, qa } = qProfile; const { q, qa } = qProfile;
const bioKeys = qa('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim()); const bioKeys = qa('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim());
const bioValues = qa('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim()); const bioValues = qa('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim());
const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key]: bioValues[index] }), {}); const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key]: bioValues[index] }), {});
const profile = { const profile = {
name: actorName, name: actorName,
}; };
profile.description = q('.model-profile-specs p', true); profile.description = q('.model-profile-specs p', true);
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity; if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-'); if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate(); if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate();
if (bio['Birth Location']) profile.birthPlace = bio['Birth Location']; if (bio['Birth Location']) profile.birthPlace = bio['Birth Location'];
if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase(); if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase();
if (bio.Height) profile.height = heightToCm(bio.Height); if (bio.Height) profile.height = heightToCm(bio.Height);
if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]); if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]);
if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase(); if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();
if (bio['Tits Type'] && bio['Tits Type'].match('Natural')) profile.naturalBoobs = true; if (bio['Tits Type'] && bio['Tits Type'].match('Natural')) profile.naturalBoobs = true;
if (bio['Tits Type'] && bio['Tits Type'].match('Enhanced')) profile.naturalBoobs = false; if (bio['Tits Type'] && bio['Tits Type'].match('Enhanced')) profile.naturalBoobs = false;
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true; if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true; if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
const avatarEl = q('.big-pic-model-container img'); const avatarEl = q('.big-pic-model-container img');
if (avatarEl) profile.avatar = `https:${avatarEl.src}`; if (avatarEl) profile.avatar = `https:${avatarEl.src}`;
profile.releases = await fetchActorReleases(qProfile); profile.releases = await fetchActorReleases(qProfile);
return profile; return profile;
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/page/${page}/`); const res = await bhttp.get(`${site.url}/page/${page}/`);
return scrapeAll(res.body.toString(), site, false); return scrapeAll(res.body.toString(), site, false);
} }
async function fetchUpcoming(site) { async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.url}/`); const res = await bhttp.get(`${site.url}/`);
return scrapeAll(res.body.toString(), site, true); return scrapeAll(res.body.toString(), site, true);
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
const res = await bhttp.get(url); const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site); return scrapeScene(res.body.toString(), url, site);
} }
async function fetchProfile(actorName) { async function fetchProfile(actorName) {
const searchUrl = 'https://brazzers.com/pornstars-search/'; const searchUrl = 'https://brazzers.com/pornstars-search/';
const searchRes = await bhttp.get(searchUrl, { const searchRes = await bhttp.get(searchUrl, {
headers: { headers: {
Cookie: `textSearch=${encodeURIComponent(actorName)};`, Cookie: `textSearch=${encodeURIComponent(actorName)};`,
}, },
}); });
const actorLink = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName); const actorLink = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
if (actorLink) { if (actorLink) {
const url = `https://brazzers.com${actorLink}`; const url = `https://brazzers.com${actorLink}`;
const res = await bhttp.get(url); const res = await bhttp.get(url);
return scrapeProfile(res.body.toString(), url, actorName); return scrapeProfile(res.body.toString(), url, actorName);
} }
return null; return null;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchProfile, fetchProfile,
fetchScene, fetchScene,
fetchUpcoming, fetchUpcoming,
}; };

8
src/scrapers/burningangel.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = { module.exports = {
fetchLatest: fetchApiLatest, fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile, fetchProfile: fetchApiProfile,
fetchScene, fetchScene,
fetchUpcoming: fetchApiUpcoming, fetchUpcoming: fetchApiUpcoming,
}; };

174
src/scrapers/cherrypimps.js
View File

@ -4,139 +4,139 @@ const { get, geta, ctxa, ed } = require('../utils/q');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
function scrapeAll(scenes, site) { function scrapeAll(scenes, site) {
return scenes.map(({ qu }) => { return scenes.map(({ qu }) => {
const url = qu.url('.text-thumb a'); const url = qu.url('.text-thumb a');
const { pathname } = new URL(url); const { pathname } = new URL(url);
const channelUrl = qu.url('.badge'); const channelUrl = qu.url('.badge');
if (site?.parameters?.extract && qu.q('.badge', true) !== site.name) { if (site?.parameters?.extract && qu.q('.badge', true) !== site.name) {
return null; return null;
} }
const release = {}; const release = {};
release.url = channelUrl ? `${channelUrl}${pathname}` : url; release.url = channelUrl ? `${channelUrl}${pathname}` : url;
release.entryId = pathname.match(/\/\d+/)[0].slice(1); release.entryId = pathname.match(/\/\d+/)[0].slice(1);
release.title = qu.q('.text-thumb a', true); release.title = qu.q('.text-thumb a', true);
release.date = qu.date('.date', 'YYYY-MM-DD', /\d{4}-\d{2}-\d{2}/); release.date = qu.date('.date', 'YYYY-MM-DD', /\d{4}-\d{2}-\d{2}/);
release.duration = qu.dur('.date', /(\d{2}:)?\d{2}:\d{2}/); release.duration = qu.dur('.date', /(\d{2}:)?\d{2}:\d{2}/);
release.actors = qu.all('.category a', true); release.actors = qu.all('.category a', true);
release.poster = qu.img('img.video_placeholder, .video-images img'); release.poster = qu.img('img.video_placeholder, .video-images img');
release.teaser = { src: qu.trailer() }; release.teaser = { src: qu.trailer() };
return release; return release;
}).filter(Boolean); }).filter(Boolean);
} }
function scrapeScene({ q, qd, qa }, url, _site, baseRelease) { function scrapeScene({ q, qd, qa }, url, _site, baseRelease) {
const release = { url }; const release = { url };
const { pathname } = new URL(url); const { pathname } = new URL(url);
release.entryId = pathname.match(/\/\d+/)[0].slice(1); release.entryId = pathname.match(/\/\d+/)[0].slice(1);
release.title = q('.trailer-block_title', true); release.title = q('.trailer-block_title', true);
release.description = q('.info-block:nth-child(3) .text', true); release.description = q('.info-block:nth-child(3) .text', true);
release.date = qd('.info-block_data .text', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/); release.date = qd('.info-block_data .text', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
const duration = baseRelease?.duration || Number(q('.info-block_data .text', true).match(/(\d+)\s+min/)?.[1]) * 60; const duration = baseRelease?.duration || Number(q('.info-block_data .text', true).match(/(\d+)\s+min/)?.[1]) * 60;
if (duration) release.duration = duration; if (duration) release.duration = duration;
release.actors = qa('.info-block_data a[href*="/models"]', true); release.actors = qa('.info-block_data a[href*="/models"]', true);
release.tags = qa('.info-block a[href*="/categories"]', true); release.tags = qa('.info-block a[href*="/categories"]', true);
const posterEl = q('.update_thumb'); const posterEl = q('.update_thumb');
const poster = posterEl.getAttribute('src0_3x') || posterEl.getAttribute('src0_2x') || posterEl.dataset.src; const poster = posterEl.getAttribute('src0_3x') || posterEl.getAttribute('src0_2x') || posterEl.dataset.src;
if (poster && baseRelease?.poster) release.photos = [poster]; if (poster && baseRelease?.poster) release.photos = [poster];
else if (poster) release.poster = poster; else if (poster) release.poster = poster;
return release; return release;
} }
function scrapeProfile({ q, qa, qtx }) { function scrapeProfile({ q, qa, qtx }) {
const profile = {}; const profile = {};
const keys = qa('.model-descr_line:not(.model-descr_rait) p.text span', true); const keys = qa('.model-descr_line:not(.model-descr_rait) p.text span', true);
const values = qa('.model-descr_line:not(.model-descr_rait) p.text').map(el => qtx(el)); const values = qa('.model-descr_line:not(.model-descr_rait) p.text').map(el => qtx(el));
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {}); const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)cm\)/)[1]); if (bio.height) profile.height = Number(bio.height.match(/\((\d+)cm\)/)[1]);
if (bio.weight) profile.weight = Number(bio.weight.match(/\((\d+)kg\)/)[1]); if (bio.weight) profile.weight = Number(bio.weight.match(/\((\d+)kg\)/)[1]);
if (bio.race) profile.ethnicity = bio.race; if (bio.race) profile.ethnicity = bio.race;
if (bio.date_of_birth) profile.birthdate = ed(bio.date_of_birth, 'MMMM D, YYYY'); if (bio.date_of_birth) profile.birthdate = ed(bio.date_of_birth, 'MMMM D, YYYY');
if (bio.birthplace) profile.birthPlace = bio.birthplace; if (bio.birthplace) profile.birthPlace = bio.birthplace;
if (bio.measurements) { if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-'); const [bust, waist, hip] = bio.measurements.split('-');
if (!/\?/.test(bust)) profile.bust = bust; if (!/\?/.test(bust)) profile.bust = bust;
if (!/\?/.test(waist)) profile.waist = waist; if (!/\?/.test(waist)) profile.waist = waist;
if (!/\?/.test(hip)) profile.hip = hip; if (!/\?/.test(hip)) profile.hip = hip;
} }
if (bio.hair) profile.hair = bio.hair; if (bio.hair) profile.hair = bio.hair;
if (bio.eyes) profile.eyes = bio.eyes; if (bio.eyes) profile.eyes = bio.eyes;
if (/various/i.test(bio.tattoos)) profile.hasTattoos = true; if (/various/i.test(bio.tattoos)) profile.hasTattoos = true;
else if (/none/i.test(bio.tattoos)) profile.hasTattoos = false; else if (/none/i.test(bio.tattoos)) profile.hasTattoos = false;
else if (bio.tattoos) { else if (bio.tattoos) {
profile.hasTattoos = true; profile.hasTattoos = true;
profile.tattoos = bio.tattoos; profile.tattoos = bio.tattoos;
} }
if (/various/i.test(bio.piercings)) profile.hasPiercings = true; if (/various/i.test(bio.piercings)) profile.hasPiercings = true;
else if (/none/i.test(bio.piercings)) profile.hasPiercings = false; else if (/none/i.test(bio.piercings)) profile.hasPiercings = false;
else if (bio.piercings) { else if (bio.piercings) {
profile.hasPiercings = true; profile.hasPiercings = true;
profile.piercings = bio.piercings; profile.piercings = bio.piercings;
} }
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim()); if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
const avatar = q('.model-img img'); const avatar = q('.model-img img');
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src; profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;
const releases = qa('.video-thumb'); const releases = qa('.video-thumb');
profile.releases = scrapeAll(ctxa(releases)); profile.releases = scrapeAll(ctxa(releases));
return profile; return profile;
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const url = site.parameters?.extract const url = site.parameters?.extract
? `https://cherrypimps.com/categories/movies_${page}.html` ? `https://cherrypimps.com/categories/movies_${page}.html`
: `${site.url}/categories/movies_${page}.html`; : `${site.url}/categories/movies_${page}.html`;
const res = await geta(url, 'div.video-thumb'); const res = await geta(url, 'div.video-thumb');
return res.ok ? scrapeAll(res.items, site) : res.status; return res.ok ? scrapeAll(res.items, site) : res.status;
} }
async function fetchScene(url, site, release) { async function fetchScene(url, site, release) {
const res = await get(url); const res = await get(url);
return res.ok ? scrapeScene(res.item, url, site, release) : res.status; return res.ok ? scrapeScene(res.item, url, site, release) : res.status;
} }
async function fetchProfile(actorName, scraperSlug) { async function fetchProfile(actorName, scraperSlug) {
const actorSlug = slugify(actorName); const actorSlug = slugify(actorName);
const actorSlug2 = slugify(actorName, ''); const actorSlug2 = slugify(actorName, '');
const [url, url2] = ['cherrypimps', 'wildoncam'].includes(scraperSlug) const [url, url2] = ['cherrypimps', 'wildoncam'].includes(scraperSlug)
? [`https://${scraperSlug}.com/models/${actorSlug}.html`, `https://${scraperSlug}.com/models/${actorSlug2}.html`] ? [`https://${scraperSlug}.com/models/${actorSlug}.html`, `https://${scraperSlug}.com/models/${actorSlug2}.html`]
: [`https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug}.html`, `https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug2}.html`]; : [`https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug}.html`, `https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug2}.html`];
const res = await get(url); const res = await get(url);
if (res.ok) return scrapeProfile(res.item); if (res.ok) return scrapeProfile(res.item);
const res2 = await get(url2); const res2 = await get(url2);
return res2.ok ? scrapeProfile(res2.item) : res2.status; return res2.ok ? scrapeProfile(res2.item) : res2.status;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchProfile, fetchProfile,
}; };

230
src/scrapers/ddfnetwork.js
View File

@ -7,182 +7,182 @@ const slugify = require('../utils/slugify');
/* eslint-disable newline-per-chained-call */ /* eslint-disable newline-per-chained-call */
function scrapeAll(html, site, origin) { function scrapeAll(html, site, origin) {
return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => { return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
const release = {}; const release = {};
release.title = q('a', 'title'); release.title = q('a', 'title');
release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`; release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`;
[release.entryId] = release.url.split('/').slice(-1); [release.entryId] = release.url.split('/').slice(-1);
release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime'); release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
release.actors = qa('.card-subtitle a', true).filter(Boolean); release.actors = qa('.card-subtitle a', true).filter(Boolean);
const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60; const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
if (duration) release.duration = duration; if (duration) release.duration = duration;
release.poster = q('img').dataset.src; release.poster = q('img').dataset.src;
return release; return release;
}); });
} }
async function scrapeScene(html, url, _site) { async function scrapeScene(html, url, _site) {
const { qu } = ex(html); const { qu } = ex(html);
const release = {}; const release = {};
[release.entryId] = url.split('/').slice(-1); [release.entryId] = url.split('/').slice(-1);
release.title = qu.meta('itemprop=name'); release.title = qu.meta('itemprop=name');
release.description = qu.q('.descr-box p', true); release.description = qu.q('.descr-box p', true);
release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content') release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|| qu.date('.title-border:nth-child(2) p', 'MM.DD.YYYY'); || qu.date('.title-border:nth-child(2) p', 'MM.DD.YYYY');
release.actors = qu.all('.pornstar-card > a', 'title'); release.actors = qu.all('.pornstar-card > a', 'title');
release.tags = qu.all('.tags-tab .tags a', true); release.tags = qu.all('.tags-tab .tags a', true);
release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60; release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60;
release.likes = Number(qu.q('.icon-like-red + span', true)); release.likes = Number(qu.q('.icon-like-red + span', true));
release.poster = qu.poster(); release.poster = qu.poster();
release.photos = qu.urls('.photo-slider-guest .card a'); release.photos = qu.urls('.photo-slider-guest .card a');
release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({ release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({
src: trailer.src, src: trailer.src,
quality: Number(trailer.attributes.res.value), quality: Number(trailer.attributes.res.value),
})); }));
return release; return release;
} }
async function fetchActorReleases(urls) { async function fetchActorReleases(urls) {
// DDF Network and DDF Network Stream list all scenes, exclude // DDF Network and DDF Network Stream list all scenes, exclude
const sources = urls.filter(url => !/ddfnetwork/.test(url)); const sources = urls.filter(url => !/ddfnetwork/.test(url));
const releases = await Promise.all(sources.map(async (url) => { const releases = await Promise.all(sources.map(async (url) => {
const { html } = await get(url); const { html } = await get(url);
return scrapeAll(html, null, new URL(url).origin); return scrapeAll(html, null, new URL(url).origin);
})); }));
// DDF cross-releases scenes between sites, filter duplicates by entryId // DDF cross-releases scenes between sites, filter duplicates by entryId
return Object.values(releases return Object.values(releases
.flat() .flat()
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) // sort by date so earliest scene remains .sort((releaseA, releaseB) => releaseB.date - releaseA.date) // sort by date so earliest scene remains
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {})); .reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
} }
async function scrapeProfile(html, _url, actorName) { async function scrapeProfile(html, _url, actorName) {
const { qu } = ex(html); const { qu } = ex(html);
const keys = qu.all('.about-title', true).map(key => slugify(key, '_')); const keys = qu.all('.about-title', true).map(key => slugify(key, '_'));
const values = qu.all('.about-info').map((el) => { const values = qu.all('.about-info').map((el) => {
if (el.children.length > 0) { if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', '); return Array.from(el.children, child => child.textContent.trim()).join(', ');
} }
return el.textContent.trim(); return el.textContent.trim();
}); });
const bio = keys.reduce((acc, key, index) => { const bio = keys.reduce((acc, key, index) => {
if (values[index] === '-') return acc; if (values[index] === '-') return acc;
return { return {
...acc, ...acc,
[key]: values[index], [key]: values[index],
}; };
}, {}); }, {});
const profile = { const profile = {
name: actorName, name: actorName,
}; };
profile.description = qu.q('.description-box', true); profile.description = qu.q('.description-box', true);
profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY'); profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY');
if (bio.nationality) profile.nationality = bio.nationality; if (bio.nationality) profile.nationality = bio.nationality;
if (bio.bra_size) [profile.bust] = bio.bra_size.match(/\d+\w+/); if (bio.bra_size) [profile.bust] = bio.bra_size.match(/\d+\w+/);
if (bio.waist) profile.waist = Number(bio.waist.match(/\d+/)[0]); if (bio.waist) profile.waist = Number(bio.waist.match(/\d+/)[0]);
if (bio.hips) profile.hip = Number(bio.hips.match(/\d+/)[0]); if (bio.hips) profile.hip = Number(bio.hips.match(/\d+/)[0]);
if (bio.height) profile.height = Number(bio.height.match(/\d{2,}/)[0]); if (bio.height) profile.height = Number(bio.height.match(/\d{2,}/)[0]);
if (bio.tit_style && /Enhanced/.test(bio.tit_style)) profile.naturalBoobs = false; if (bio.tit_style && /Enhanced/.test(bio.tit_style)) profile.naturalBoobs = false;
if (bio.tit_style && /Natural/.test(bio.tit_style)) profile.naturalBoobs = true; if (bio.tit_style && /Natural/.test(bio.tit_style)) profile.naturalBoobs = true;
if (bio.body_art && /Tattoo/.test(bio.body_art)) profile.hasTattoos = true; if (bio.body_art && /Tattoo/.test(bio.body_art)) profile.hasTattoos = true;
if (bio.body_art && /Piercing/.test(bio.body_art)) profile.hasPiercings = true; if (bio.body_art && /Piercing/.test(bio.body_art)) profile.hasPiercings = true;
if (bio.hair_style) profile.hair = bio.hair_style.split(',')[0].trim().toLowerCase(); if (bio.hair_style) profile.hair = bio.hair_style.split(',')[0].trim().toLowerCase();
if (bio.eye_color) profile.eyes = bio.eye_color.match(/\w+/)[0].toLowerCase(); if (bio.eye_color) profile.eyes = bio.eye_color.match(/\w+/)[0].toLowerCase();
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]); if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
const avatarEl = qu.q('.pornstar-details .card-img-top'); const avatarEl = qu.q('.pornstar-details .card-img-top');
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`; if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a')); profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a'));
return profile; return profile;
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const url = site.parameters?.native const url = site.parameters?.native
? `${site.url}/videos/search/latest/ever/allsite/-/${page}` ? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`; : `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
const res = await bhttp.get(url); const res = await bhttp.get(url);
if (res.statusCode === 200) { if (res.statusCode === 200) {
return scrapeAll(res.body.toString(), site); return scrapeAll(res.body.toString(), site);
} }
return res.statusCode; return res.statusCode;
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
// DDF's main site moved to Porn World // DDF's main site moved to Porn World
// const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`); // const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
const res = await bhttp.get(url); const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site); return scrapeScene(res.body.toString(), url, site);
} }
async function fetchProfile(actorName) { async function fetchProfile(actorName) {
const resSearch = await bhttp.post('https://ddfnetwork.com/search/ajax', const resSearch = await bhttp.post('https://ddfnetwork.com/search/ajax',
{ {
type: 'hints', type: 'hints',
word: actorName, word: actorName,
}, },
{ {
decodeJSON: true, decodeJSON: true,
headers: { headers: {
'x-requested-with': 'XMLHttpRequest', 'x-requested-with': 'XMLHttpRequest',
}, },
}); });
if (resSearch.statusCode !== 200 || Array.isArray(resSearch.body.list)) { if (resSearch.statusCode !== 200 || Array.isArray(resSearch.body.list)) {
return null; return null;
} }
if (!resSearch.body.list.pornstarsName || resSearch.body.list.pornstarsName.length === 0) { if (!resSearch.body.list.pornstarsName || resSearch.body.list.pornstarsName.length === 0) {
return null; return null;
} }
const [actor] = resSearch.body.list.pornstarsName; const [actor] = resSearch.body.list.pornstarsName;
const url = `https://ddfnetwork.com${actor.href}`; const url = `https://ddfnetwork.com${actor.href}`;
const resActor = await bhttp.get(url); const resActor = await bhttp.get(url);
if (resActor.statusCode !== 200) { if (resActor.statusCode !== 200) {
return null; return null;
} }
return scrapeProfile(resActor.body.toString(), url, actorName); return scrapeProfile(resActor.body.toString(), url, actorName);
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchProfile, fetchProfile,
fetchScene, fetchScene,
}; };

8
src/scrapers/digitalplayground.js
View File

@ -3,11 +3,11 @@
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek'); const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) { async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'digitalplayground', 'modelprofile'); return fetchProfile(actorName, 'digitalplayground', 'modelprofile');
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchProfile: networkFetchProfile, fetchProfile: networkFetchProfile,
fetchScene, fetchScene,
}; };

196
src/scrapers/dogfart.js
View File

@ -7,136 +7,136 @@ const { JSDOM } = require('jsdom');
const moment = require('moment'); const moment = require('moment');
async function getPhotos(albumUrl) { async function getPhotos(albumUrl) {
const res = await bhttp.get(albumUrl); const res = await bhttp.get(albumUrl);
const html = res.body.toString(); const html = res.body.toString();
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href; const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10); const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => { const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => {
const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`; const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
return { return {
url: pageUrl, url: pageUrl,
extract: ({ qu }) => qu.q('.scenes-module img', 'src'), extract: ({ qu }) => qu.q('.scenes-module img', 'src'),
}; };
}); });
return photoUrls; return photoUrls;
} }
function scrapeLatest(html, site) { function scrapeLatest(html, site) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const sceneElements = Array.from(document.querySelectorAll('.recent-updates')); const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));
return sceneElements.reduce((acc, element) => { return sceneElements.reduce((acc, element) => {
const siteUrl = element.querySelector('.help-block').textContent; const siteUrl = element.querySelector('.help-block').textContent;
if (`www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) { if (`www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
// different dogfart site // different dogfart site
return acc; return acc;
} }
const sceneLinkElement = element.querySelector('.thumbnail'); const sceneLinkElement = element.querySelector('.thumbnail');
const url = `https://dogfartnetwork.com${sceneLinkElement.href}`; const url = `https://dogfartnetwork.com${sceneLinkElement.href}`;
const { pathname } = new URL(url); const { pathname } = new URL(url);
const entryId = `${site.slug}_${pathname.split('/')[4]}`; const entryId = `${site.slug}_${pathname.split('/')[4]}`;
const title = element.querySelector('.scene-title').textContent; const title = element.querySelector('.scene-title').textContent;
const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim()); const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim());
const poster = `https:${element.querySelector('img').src}`; const poster = `https:${element.querySelector('img').src}`;
const teaser = sceneLinkElement.dataset.preview_clip_url; const teaser = sceneLinkElement.dataset.preview_clip_url;
return [ return [
...acc, ...acc,
{ {
url, url,
entryId, entryId,
title, title,
actors, actors,
poster, poster,
teaser: { teaser: {
src: teaser, src: teaser,
}, },
site, site,
}, },
]; ];
}, []); }, []);
} }
async function scrapeScene(html, url, site) { async function scrapeScene(html, url, site) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const title = document.querySelector('.description-title').textContent; const title = document.querySelector('.description-title').textContent;
const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent); const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
const metaDescription = document.querySelector('meta[itemprop="description"]').content; const metaDescription = document.querySelector('meta[itemprop="description"]').content;
const description = metaDescription const description = metaDescription
? metaDescription.content ? metaDescription.content
: document.querySelector('.description') : document.querySelector('.description')
.textContent .textContent
.replace(/[ \t\n]{2,}/g, ' ') .replace(/[ \t\n]{2,}/g, ' ')
.replace('...read more', '') .replace('...read more', '')
.trim(); .trim();
const channel = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase(); const channel = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
const { origin, pathname } = new URL(url); const { origin, pathname } = new URL(url);
const entryId = `${channel}_${pathname.split('/').slice(-2)[0]}`; const entryId = `${channel}_${pathname.split('/').slice(-2)[0]}`;
const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content); const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
const duration = moment const duration = moment
.duration(`00:${document .duration(`00:${document
.querySelectorAll('.extra-info p')[1] .querySelectorAll('.extra-info p')[1]
.textContent .textContent
.match(/\d+:\d+$/)[0]}`) .match(/\d+:\d+$/)[0]}`)
.asSeconds(); .asSeconds();
const trailerElement = document.querySelector('.html5-video'); const trailerElement = document.querySelector('.html5-video');
const poster = `https:${trailerElement.dataset.poster}`; const poster = `https:${trailerElement.dataset.poster}`;
const { trailer } = trailerElement.dataset; const { trailer } = trailerElement.dataset;
const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href; const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href;
const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url); const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2); const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2);
const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent); const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
return { return {
entryId, entryId,
url: `${origin}${pathname}`, url: `${origin}${pathname}`,
title, title,
description, description,
actors, actors,
date, date,
duration, duration,
poster, poster,
photos, photos,
trailer: { trailer: {
src: trailer, src: trailer,
}, },
tags, tags,
rating: { rating: {
stars, stars,
}, },
site, site,
channel, channel,
}; };
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`); const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
return scrapeLatest(res.body.toString(), site); return scrapeLatest(res.body.toString(), site);
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
const res = await bhttp.get(url); const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site); return scrapeScene(res.body.toString(), url, site);
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
}; };

8
src/scrapers/evilangel.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = { module.exports = {
fetchLatest: fetchApiLatest, fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile, fetchProfile: fetchApiProfile,
fetchScene, fetchScene,
fetchUpcoming: fetchApiUpcoming, fetchUpcoming: fetchApiUpcoming,
}; };

8
src/scrapers/fakehub.js
View File

@ -3,11 +3,11 @@
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek'); const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) { async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'fakehub', 'modelprofile'); return fetchProfile(actorName, 'fakehub', 'modelprofile');
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchProfile: networkFetchProfile, fetchProfile: networkFetchProfile,
fetchScene, fetchScene,
}; };

134
src/scrapers/famedigital.js
View File

@ -1,115 +1,115 @@
'use strict'; 'use strict';
const { const {
fetchLatest, fetchLatest,
fetchApiLatest, fetchApiLatest,
fetchUpcoming, fetchUpcoming,
fetchApiUpcoming, fetchApiUpcoming,
fetchScene, fetchScene,
fetchProfile, fetchProfile,
fetchApiProfile, fetchApiProfile,
scrapeAll, scrapeAll,
} = require('./gamma'); } = require('./gamma');
const { get } = require('../utils/q'); const { get } = require('../utils/q');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
function extractLowArtActors(release) { function extractLowArtActors(release) {
const actors = release.title const actors = release.title
.replace(/solo/i, '') .replace(/solo/i, '')
.split(/,|\band\b/ig) .split(/,|\band\b/ig)
.map(actor => actor.trim()); .map(actor => actor.trim());
return { return {
...release, ...release,
actors, actors,
}; };
} }
async function networkFetchLatest(site, page = 1) { async function networkFetchLatest(site, page = 1) {
if (site.parameters?.api) return fetchApiLatest(site, page, false); if (site.parameters?.api) return fetchApiLatest(site, page, false);
const releases = await fetchLatest(site, page); const releases = await fetchLatest(site, page);
if (site.slug === 'lowartfilms') { if (site.slug === 'lowartfilms') {
return releases.map(release => extractLowArtActors(release)); return releases.map(release => extractLowArtActors(release));
} }
return releases; return releases;
} }
async function networkFetchScene(url, site) { async function networkFetchScene(url, site) {
const release = await fetchScene(url, site); const release = await fetchScene(url, site);
if (site.slug === 'lowartfilms') { if (site.slug === 'lowartfilms') {
return extractLowArtActors(release); return extractLowArtActors(release);
} }
return release; return release;
} }
async function networkFetchUpcoming(site, page = 1) { async function networkFetchUpcoming(site, page = 1) {
if (site.parameters?.api) return fetchApiUpcoming(site, page, true); if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
return fetchUpcoming(site, page); return fetchUpcoming(site, page);
} }
function getActorReleasesUrl(actorPath, page = 1) { function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`; return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
} }
async function fetchClassicProfile(actorName, siteSlug) { async function fetchClassicProfile(actorName, siteSlug) {
const actorSlug = slugify(actorName); const actorSlug = slugify(actorName);
const url = `https://${siteSlug}.com/en/pornstars`; const url = `https://${siteSlug}.com/en/pornstars`;
const pornstarsRes = await get(url); const pornstarsRes = await get(url);
if (!pornstarsRes.ok) return null; if (!pornstarsRes.ok) return null;
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]') const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find(el => slugify(el.textContent) === actorSlug) .find(el => slugify(el.textContent) === actorSlug)
?.value; ?.value;
if (actorPath) { if (actorPath) {
const actorUrl = `https://${siteSlug}.com${actorPath}`; const actorUrl = `https://${siteSlug}.com${actorPath}`;
const res = await get(actorUrl); const res = await get(actorUrl);
if (res.ok) { if (res.ok) {
const releases = scrapeAll(res.item, null, `https://www.${siteSlug}.com`, false); const releases = scrapeAll(res.item, null, `https://www.${siteSlug}.com`, false);
return { releases }; return { releases };
} }
} }
return null; return null;
} }
async function networkFetchProfile(actorName, scraperSlug, site, include) { async function networkFetchProfile(actorName, scraperSlug, site, include) {
// not all Fame Digital sites offer Gamma actors // not all Fame Digital sites offer Gamma actors
const [devils, rocco, peter, silvia] = await Promise.all([ const [devils, rocco, peter, silvia] = await Promise.all([
fetchApiProfile(actorName, 'devilsfilm', true), fetchApiProfile(actorName, 'devilsfilm', true),
fetchApiProfile(actorName, 'roccosiffredi'), fetchApiProfile(actorName, 'roccosiffredi'),
include.scenes ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl, include) : [], include.scenes ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl, include) : [],
include.scenes ? fetchClassicProfile(actorName, 'silviasaint') : [], include.scenes ? fetchClassicProfile(actorName, 'silviasaint') : [],
include.scenes ? fetchClassicProfile(actorName, 'silverstonedvd') : [], include.scenes ? fetchClassicProfile(actorName, 'silverstonedvd') : [],
]); ]);
if (devils || rocco || peter) { if (devils || rocco || peter) {
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []); const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []);
return { return {
...peter, ...peter,
...rocco, ...rocco,
...devils, ...devils,
releases, releases,
}; };
} }
return null; return null;
} }
module.exports = { module.exports = {
fetchLatest: networkFetchLatest, fetchLatest: networkFetchLatest,
fetchProfile: networkFetchProfile, fetchProfile: networkFetchProfile,
fetchScene: networkFetchScene, fetchScene: networkFetchScene,
fetchUpcoming: networkFetchUpcoming, fetchUpcoming: networkFetchUpcoming,
}; };

6
src/scrapers/fantasymassage.js
View File

@ -4,7 +4,7 @@ const { fetchLatest, fetchUpcoming, fetchScene } = require('./gamma');
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchUpcoming, fetchUpcoming,
}; };

106
src/scrapers/freeones.js
View File

@ -5,89 +5,89 @@ const { JSDOM } = require('jsdom');
const moment = require('moment'); const moment = require('moment');
function scrapeProfile(html, actorName) { function scrapeProfile(html, actorName) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const profile = { name: actorName }; const profile = { name: actorName };
const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), el => decodeURI(el.href)).reduce((acc, item) => { const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), el => decodeURI(el.href)).reduce((acc, item) => {
const keyMatch = item.match(/\[\w+\]/); const keyMatch = item.match(/\[\w+\]/);
if (keyMatch) { if (keyMatch) {
const key = keyMatch[0].slice(1, -1); const key = keyMatch[0].slice(1, -1);
const [, value] = item.split('='); const [, value] = item.split('=');
// both hip and waist link to 'waist', assume biggest value is hip // both hip and waist link to 'waist', assume biggest value is hip
if (key === 'waist' && acc.waist) { if (key === 'waist' && acc.waist) {
if (acc.waist > value) { if (acc.waist > value) {
acc.hip = acc.waist; acc.hip = acc.waist;
acc.waist = value; acc.waist = value;
return acc; return acc;
} }
acc.hip = value; acc.hip = value;
return acc; return acc;
} }
acc[key] = value; acc[key] = value;
} }
return acc; return acc;
}, {}); }, {});
if (bio.dateOfBirth) profile.birthdate = moment.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate(); if (bio.dateOfBirth) profile.birthdate = moment.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
if (profile.placeOfBirth || bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`; if (profile.placeOfBirth || bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`;
profile.eyes = bio.eyeColor; profile.eyes = bio.eyeColor;
profile.hair = bio.hairColor; profile.hair = bio.hairColor;
profile.ethnicity = bio.ethnicity; profile.ethnicity = bio.ethnicity;
profile.bust = bio.bra; profile.bust = bio.bra;
if (bio.waist) profile.waist = Number(bio.waist.split(',')[0]); if (bio.waist) profile.waist = Number(bio.waist.split(',')[0]);
if (bio.hip) profile.hip = Number(bio.hip.split(',')[0]); if (bio.hip) profile.hip = Number(bio.hip.split(',')[0]);
if (bio.height) profile.height = Number(bio.height.split(',')[0]); if (bio.height) profile.height = Number(bio.height.split(',')[0]);
if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]); if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]);
profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), el => el.href); profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), el => el.href);
const avatar = document.querySelector('.profile-image-large img').src; const avatar = document.querySelector('.profile-image-large img').src;
if (!avatar.match('placeholder')) profile.avatar = { src: avatar, copyright: null }; if (!avatar.match('placeholder')) profile.avatar = { src: avatar, copyright: null };
return profile; return profile;
} }
function scrapeSearch(html) { function scrapeSearch(html) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
return document.querySelector('a.image-link')?.href || null; return document.querySelector('a.image-link')?.href || null;
} }
async function fetchProfile(actorName) { async function fetchProfile(actorName) {
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-'); const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
const res = await bhttp.get(`https://freeones.nl/${actorSlug}/profile`); const res = await bhttp.get(`https://freeones.nl/${actorSlug}/profile`);
if (res.statusCode === 200) { if (res.statusCode === 200) {
return scrapeProfile(res.body.toString(), actorName); return scrapeProfile(res.body.toString(), actorName);
} }
const searchRes = await bhttp.get(`https://freeones.nl/babes?q=${actorName}`); const searchRes = await bhttp.get(`https://freeones.nl/babes?q=${actorName}`);
const actorPath = scrapeSearch(searchRes.body.toString()); const actorPath = scrapeSearch(searchRes.body.toString());
if (actorPath) { if (actorPath) {
const actorRes = await bhttp.get(`https://freeones.nl${actorPath}/profile`); const actorRes = await bhttp.get(`https://freeones.nl${actorPath}/profile`);
if (actorRes.statusCode === 200) { if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString(), actorName); return scrapeProfile(actorRes.body.toString(), actorName);
} }
return null; return null;
} }
return null; return null;
} }
module.exports = { module.exports = {
fetchProfile, fetchProfile,
}; };

170
src/scrapers/freeones_legacy.js
View File

@ -6,135 +6,135 @@ const { JSDOM } = require('jsdom');
const moment = require('moment'); const moment = require('moment');
async function scrapeProfileFrontpage(html, url, name) { async function scrapeProfileFrontpage(html, url, name) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const bioEl = document.querySelector('.dashboard-bio-list'); const bioEl = document.querySelector('.dashboard-bio-list');
const bioUrl = `https:${document.querySelector('.seemore a').href}`; const bioUrl = `https:${document.querySelector('.seemore a').href}`;
const keys = Array.from(bioEl.querySelectorAll('dt'), el => el.textContent.trim()); const keys = Array.from(bioEl.querySelectorAll('dt'), el => el.textContent.trim());
const values = Array.from(bioEl.querySelectorAll('dd'), el => el.textContent.trim()); const values = Array.from(bioEl.querySelectorAll('dd'), el => el.textContent.trim());
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {}); const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
const profile = { const profile = {
name, name,
gender: 'female', gender: 'female',
}; };
const birthdateString = bio['Date of Birth:']; const birthdateString = bio['Date of Birth:'];
const measurementsString = bio['Measurements:']; const measurementsString = bio['Measurements:'];
const birthCityString = bio['Place of Birth:']; const birthCityString = bio['Place of Birth:'];
const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString; const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString;
const birthCountryString = bio['Country of Origin:']; const birthCountryString = bio['Country of Origin:'];
const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString; const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString;
const piercingsString = bio['Piercings:']; const piercingsString = bio['Piercings:'];
const tattoosString = bio['Tattoos:']; const tattoosString = bio['Tattoos:'];
if (birthdateString && birthdateString !== 'Unknown (add)') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate(); if (birthdateString && birthdateString !== 'Unknown (add)') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate();
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement)); if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
if (bio['Fake Boobs:']) profile.naturalBoobs = bio['Fake Boobs:'] === 'No'; if (bio['Fake Boobs:']) profile.naturalBoobs = bio['Fake Boobs:'] === 'No';
profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`; profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`;
profile.hair = bio['Hair Color:'].toLowerCase(); profile.hair = bio['Hair Color:'].toLowerCase();
profile.eyes = bio['Eye Color:'].toLowerCase(); profile.eyes = bio['Eye Color:'].toLowerCase();
if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None'); if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None'); if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString; if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString;
if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString; if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString;
profile.social = Array.from(bioEl.querySelectorAll('.dashboard-socialmedia a'), el => el.href); profile.social = Array.from(bioEl.querySelectorAll('.dashboard-socialmedia a'), el => el.href);
return { return {
profile, profile,
url: bioUrl, url: bioUrl,
}; };
} }
async function scrapeProfileBio(html, frontpageProfile, url, name) { async function scrapeProfileBio(html, frontpageProfile, url, name) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const bioEl = document.querySelector('#biographyTable'); const bioEl = document.querySelector('#biographyTable');
const keys = Array.from(bioEl.querySelectorAll('td:nth-child(1)'), el => el.textContent.trim()); const keys = Array.from(bioEl.querySelectorAll('td:nth-child(1)'), el => el.textContent.trim());
const values = Array.from(bioEl.querySelectorAll('td:nth-child(2)'), el => el.textContent.trim()); const values = Array.from(bioEl.querySelectorAll('td:nth-child(2)'), el => el.textContent.trim());
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {}); const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
const profile = { const profile = {
...frontpageProfile, ...frontpageProfile,
name, name,
gender: 'female', gender: 'female',
}; };
const birthdateString = bio['Date of Birth:']; const birthdateString = bio['Date of Birth:'];
const measurementsString = bio['Measurements:']; const measurementsString = bio['Measurements:'];
const birthCityString = bio['Place of Birth:']; const birthCityString = bio['Place of Birth:'];
const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString; const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString;
const birthCountryString = bio['Country of Origin:']; const birthCountryString = bio['Country of Origin:'];
const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString; const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString;
const piercingsString = bio['Piercings:']; const piercingsString = bio['Piercings:'];
const tattoosString = bio['Tattoos:']; const tattoosString = bio['Tattoos:'];
if (birthdateString && birthdateString !== 'Unknown') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate(); if (birthdateString && birthdateString !== 'Unknown') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate();
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement)); if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
if (bio['Fake boobs']) profile.naturalBoobs = bio['Fake boobs:'] === 'No'; if (bio['Fake boobs']) profile.naturalBoobs = bio['Fake boobs:'] === 'No';
profile.ethnicity = bio['Ethnicity:']; profile.ethnicity = bio['Ethnicity:'];
profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`; profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`;
profile.hair = bio['Hair Color:'].toLowerCase(); profile.hair = bio['Hair Color:'].toLowerCase();
profile.eyes = bio['Eye Color:'].toLowerCase(); profile.eyes = bio['Eye Color:'].toLowerCase();
profile.height = Number(bio['Height:'].match(/\d+/)[0]); profile.height = Number(bio['Height:'].match(/\d+/)[0]);
profile.weight = Number(bio['Weight:'].match(/\d+/)[0]); profile.weight = Number(bio['Weight:'].match(/\d+/)[0]);
if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None'); if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None'); if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString; if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString;
if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString; if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString;
profile.social = Array.from(bioEl.querySelectorAll('#socialmedia a'), el => el.href); profile.social = Array.from(bioEl.querySelectorAll('#socialmedia a'), el => el.href);
return profile; return profile;
} }
async function fetchProfile(actorName) { async function fetchProfile(actorName) {
const slug = actorName.replace(' ', '_'); const slug = actorName.replace(' ', '_');
const frontpageUrl = `https://www.freeones.com/html/v_links/${slug}`; const frontpageUrl = `https://www.freeones.com/html/v_links/${slug}`;
const resFrontpage = await bhttp.get(frontpageUrl); const resFrontpage = await bhttp.get(frontpageUrl);
if (resFrontpage.statusCode === 200) { if (resFrontpage.statusCode === 200) {
const { url, bio } = await scrapeProfileFrontpage(resFrontpage.body.toString(), frontpageUrl, actorName); const { url, bio } = await scrapeProfileFrontpage(resFrontpage.body.toString(), frontpageUrl, actorName);
const resBio = await bhttp.get(url); const resBio = await bhttp.get(url);
return scrapeProfileBio(resBio.body.toString(), bio, url, actorName); return scrapeProfileBio(resBio.body.toString(), bio, url, actorName);
} }
// apparently some actors are appended 'Babe' as their surname... // apparently some actors are appended 'Babe' as their surname...
const fallbackSlug = `${slug}_Babe`; const fallbackSlug = `${slug}_Babe`;
const fallbackUrl = `https://www.freeones.com/html/s_links/${fallbackSlug}`; const fallbackUrl = `https://www.freeones.com/html/s_links/${fallbackSlug}`;
const resFallback = await bhttp.get(fallbackUrl); const resFallback = await bhttp.get(fallbackUrl);
if (resFallback.statusCode === 200) { if (resFallback.statusCode === 200) {
const { url, profile } = await scrapeProfileFrontpage(resFallback.body.toString(), fallbackUrl, actorName); const { url, profile } = await scrapeProfileFrontpage(resFallback.body.toString(), fallbackUrl, actorName);
const resBio = await bhttp.get(url); const resBio = await bhttp.get(url);
return scrapeProfileBio(resBio.body.toString(), profile, url, actorName); return scrapeProfileBio(resBio.body.toString(), profile, url, actorName);
} }
return null; return null;
} }
module.exports = { module.exports = {
fetchProfile, fetchProfile,
}; };

104
src/scrapers/fullpornnetwork.js
View File

@ -4,93 +4,93 @@ const { get, geta, ctxa } = require('../utils/q');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
function scrapeAll(scenes) { function scrapeAll(scenes) {
return scenes.map(({ el, qu }) => { return scenes.map(({ el, qu }) => {
const release = {}; const release = {};
release.entryId = el.dataset.setid || qu.q('.update_thumb', 'id').match(/\w+-\w+-(\d+)-\d+/)[1]; release.entryId = el.dataset.setid || qu.q('.update_thumb', 'id').match(/\w+-\w+-(\d+)-\d+/)[1];
release.url = qu.url('.title'); release.url = qu.url('.title');
release.title = qu.q('.title', true); release.title = qu.q('.title', true);
release.description = qu.q('.title', 'title'); release.description = qu.q('.title', 'title');
release.date = qu.date('.video-data > span:last-child', 'YYYY-MM-DD'); release.date = qu.date('.video-data > span:last-child', 'YYYY-MM-DD');
release.duration = qu.dur('.video-data > span'); release.duration = qu.dur('.video-data > span');
release.actors = qu.all('.update_models a', true); release.actors = qu.all('.update_models a', true);
const poster = qu.q('.update_thumb', 'src0_1x'); const poster = qu.q('.update_thumb', 'src0_1x');
release.poster = [ release.poster = [
poster.replace('-1x', '-2x'), poster.replace('-1x', '-2x'),
poster, poster,
]; ];
return release; return release;
}); });
} }
function scrapeScene({ q, qa, qd, qtx }, url, _site) { function scrapeScene({ q, qa, qd, qtx }, url, _site) {
const release = { url }; const release = { url };
release.entryId = q('#image_parent img', 'id').match(/\w+-\w+-(\d+)-\d+/)[1]; release.entryId = q('#image_parent img', 'id').match(/\w+-\w+-(\d+)-\d+/)[1];
release.title = q('.trailer_title', true); release.title = q('.trailer_title', true);
release.description = qtx('.text p'); release.description = qtx('.text p');
release.date = qd('span[data-dateadded]', 'YYYY-MM-DD', null, 'data-dateadded'); release.date = qd('span[data-dateadded]', 'YYYY-MM-DD', null, 'data-dateadded');
release.actors = qa('.update_models a', true); release.actors = qa('.update_models a', true);
release.tags = qa('.video-info a[href*="/categories"]', true); release.tags = qa('.video-info a[href*="/categories"]', true);
const poster = q('#image_parent img', 'src0_1x'); const poster = q('#image_parent img', 'src0_1x');
release.poster = [ release.poster = [
poster.replace('-1x', '-2x'), poster.replace('-1x', '-2x'),
poster, poster,
]; ];
return release; return release;
} }
function scrapeProfile({ el, q, qtx }) { function scrapeProfile({ el, q, qtx }) {
const profile = {}; const profile = {};
const description = qtx('.model-bio'); const description = qtx('.model-bio');
if (description) profile.description = description; if (description) profile.description = description;
profile.avatar = [ profile.avatar = [
q('.model-image img', 'src0_2x'), q('.model-image img', 'src0_2x'),
q('.model-image img', 'src0_1x'), q('.model-image img', 'src0_1x'),
]; ];
profile.releases = scrapeAll(ctxa(el, '.update')); profile.releases = scrapeAll(ctxa(el, '.update'));
return profile; return profile;
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const url = `${site.url}/categories/movies_${page}_d.html`; const url = `${site.url}/categories/movies_${page}_d.html`;
const res = await geta(url, '.latest-updates .update'); const res = await geta(url, '.latest-updates .update');
return res.ok ? scrapeAll(res.items, site) : res.status; return res.ok ? scrapeAll(res.items, site) : res.status;
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
const res = await get(url, '.content-wrapper'); const res = await get(url, '.content-wrapper');
return res.ok ? scrapeScene(res.item, url, site) : res.status; return res.ok ? scrapeScene(res.item, url, site) : res.status;
} }
async function fetchProfile(actorName, scraperSlug) { async function fetchProfile(actorName, scraperSlug) {
const actorSlug = slugify(actorName, ''); const actorSlug = slugify(actorName, '');
const url = scraperSlug === 'povperverts' const url = scraperSlug === 'povperverts'
? `https://povperverts.net/models/${actorSlug}.html` ? `https://povperverts.net/models/${actorSlug}.html`
: `https://${scraperSlug}.com/models/${actorSlug}.html`; : `https://${scraperSlug}.com/models/${actorSlug}.html`;
const res = await get(url); const res = await get(url);
return res.ok ? scrapeProfile(res.item, actorName) : res.status; return res.ok ? scrapeProfile(res.item, actorName) : res.status;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchProfile, fetchProfile,
}; };

Some files were not shown because too many files have changed in this diff Show More