DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 35 Pull Requests 1 Releases Wiki Activity

Switched to tabs. Adding missing actor entries when scraping actors, with batch ID.

This commit is contained in:
ThePendulum 2020-05-14 04:26:05 +02:00
parent f1eb29c713
commit 11eb66f834
178 changed files with 16594 additions and 16929 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.editorconfig
View File

@ -5,7 +5,7 @@ root = true
[*]
end_of_line = lf
insert_final_newline = true
indent_style = space
indent_style = tab
indent_size = 4
# Matches multiple files with brace expansion notation

5
.eslintrc
View File

@ -7,13 +7,14 @@
"sourceType": "module"
},
"rules": {
"indent": ["error", "tab"],
"no-tabs": "off",
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
"no-console": 0,
"indent": "off",
"template-curly-spacing": "off",
"max-len": 0,
"vue/no-v-html": 0,
"vue/html-indent": ["error", 4],
"vue/html-indent": ["error", "tab"],
"vue/multiline-html-element-content-newline": 0,
"vue/singleline-html-element-content-newline": 0,
"no-param-reassign": ["error", {

494
assets/components/actors/actor.vue
View File

@ -1,248 +1,248 @@
<template>
<div
v-if="actor"
class="content actor"
>
<FilterBar :fetch-releases="fetchActor" />
<div
v-if="actor"
class="content actor"
>
<FilterBar :fetch-releases="fetchActor" />
<div class="actor-header">
<h2 class="header-name">
<span v-if="actor.network">{{ actor.name }} ({{ actor.network.name }})</span>
<span v-else="">{{ actor.name }}</span>
<div class="actor-header">
<h2 class="header-name">
<span v-if="actor.network">{{ actor.name }} ({{ actor.network.name }})</span>
<span v-else="">{{ actor.name }}</span>
<Gender
:gender="actor.gender"
class="header-gender"
/>
</h2>
<Gender
:gender="actor.gender"
class="header-gender"
/>
</h2>
<li
v-if="actor.aliases.length"
class="bio-item"
>
<dfn class="bio-label">Also known as</dfn>
<span>{{ actor.aliases.join(', ') }}</span>
</li>
<li
v-if="actor.aliases.length"
class="bio-item"
>
<dfn class="bio-label">Also known as</dfn>
<span>{{ actor.aliases.join(', ') }}</span>
</li>
<Social
v-if="actor.social && actor.social.length > 0"
:actor="actor"
class="header-social"
/>
</div>
<Social
v-if="actor.social && actor.social.length > 0"
:actor="actor"
class="header-social"
/>
</div>
<div class="actor-inner">
<div
class="profile"
:class="{ expanded, 'with-avatar': !!actor.avatar }"
>
<a
v-if="actor.avatar"
:href="`/media/${actor.avatar.path}`"
target="_blank"
rel="noopener noreferrer"
class="avatar-link"
>
<img
:src="`/media/${actor.avatar.thumbnail}`"
:title="actor.avatar.copyright && `© ${actor.avatar.copyright}`"
class="avatar"
>
</a>
<div class="actor-inner">
<div
class="profile"
:class="{ expanded, 'with-avatar': !!actor.avatar }"
>
<a
v-if="actor.avatar"
:href="`/media/${actor.avatar.path}`"
target="_blank"
rel="noopener noreferrer"
class="avatar-link"
>
<img
:src="`/media/${actor.avatar.thumbnail}`"
:title="actor.avatar.copyright && `© ${actor.avatar.copyright}`"
class="avatar"
>
</a>
<span
v-show="expanded"
class="expand collapse-header noselect"
@click="expanded = false"
><Icon icon="arrow-up3" /></span>
<span
v-show="expanded"
class="expand collapse-header noselect"
@click="expanded = false"
><Icon icon="arrow-up3" /></span>
<ul class="bio nolist">
<li
v-if="actor.birthdate"
class="bio-item"
>
<dfn class="bio-label"><Icon icon="cake" />Birthdate</dfn>
<ul class="bio nolist">
<li
v-if="actor.birthdate"
class="bio-item"
>
<dfn class="bio-label"><Icon icon="cake" />Birthdate</dfn>
<span
v-if="actor.birthdate"
class="birthdate"
>{{ formatDate(actor.birthdate, 'MMMM D, YYYY') }}<span class="age">{{ actor.age }}</span></span>
</li>
<span
v-if="actor.birthdate"
class="birthdate"
>{{ formatDate(actor.birthdate, 'MMMM D, YYYY') }}<span class="age">{{ actor.age }}</span></span>
</li>
<li
v-if="actor.origin"
class="bio-item birth"
>
<dfn class="bio-label"><Icon icon="home2" />Born in</dfn>
<li
v-if="actor.origin"
class="bio-item birth"
>
<dfn class="bio-label"><Icon icon="home2" />Born in</dfn>
<span>
<span
v-if="actor.origin.city"
class="city hideable"
>{{ actor.origin.city }}</span><span
v-if="actor.origin.state && (!actor.origin.city || (actor.origin.country && actor.origin.country.alpha2 === 'US'))"
class="state hideable"
>{{ actor.origin.city ? `, ${actor.origin.state}` : actor.origin.state }}</span>
<span>
<span
v-if="actor.origin.city"
class="city hideable"
>{{ actor.origin.city }}</span><span
v-if="actor.origin.state && (!actor.origin.city || (actor.origin.country && actor.origin.country.alpha2 === 'US'))"
class="state hideable"
>{{ actor.origin.city ? `, ${actor.origin.state}` : actor.origin.state }}</span>
<span
v-if="actor.origin.country"
class="country birthcountry"
>
<img
class="flag"
:src="`/img/flags/svg-simple/${actor.origin.country.alpha2.toLowerCase()}.svg`"
>{{ actor.origin.country.alias || actor.origin.country.name }}
</span>
</span>
</li>
<span
v-if="actor.origin.country"
class="country birthcountry"
>
<img
class="flag"
:src="`/img/flags/svg-simple/${actor.origin.country.alpha2.toLowerCase()}.svg`"
>{{ actor.origin.country.alias || actor.origin.country.name }}
</span>
</span>
</li>
<li
v-if="actor.residence"
class="bio-item residence"
>
<dfn class="bio-label"><Icon icon="location" />Lives in</dfn>
<li
v-if="actor.residence"
class="bio-item residence"
>
<dfn class="bio-label"><Icon icon="location" />Lives in</dfn>
<span>
<span
v-if="actor.residence.city"
class="city hideable"
>{{ actor.residence.city }}</span><span
v-if="actor.residence.state && actor.residence.country && actor.residence.country.alpha2 === 'US'"
class="state hideable"
>{{ actor.residence.city ? `, ${actor.residence.state}` : actor.residence.state }}</span>
<span>
<span
v-if="actor.residence.city"
class="city hideable"
>{{ actor.residence.city }}</span><span
v-if="actor.residence.state && actor.residence.country && actor.residence.country.alpha2 === 'US'"
class="state hideable"
>{{ actor.residence.city ? `, ${actor.residence.state}` : actor.residence.state }}</span>
<span
v-if="actor.residence.country"
class="country"
>
<img
class="flag"
:src="`/img/flags/${actor.residence.country.alpha2.toLowerCase()}.png`"
>{{ actor.residence.country.alias || actor.residence.country.name }}
</span>
</span>
</li>
<span
v-if="actor.residence.country"
class="country"
>
<img
class="flag"
:src="`/img/flags/${actor.residence.country.alpha2.toLowerCase()}.png`"
>{{ actor.residence.country.alias || actor.residence.country.name }}
</span>
</span>
</li>
<li
v-if="actor.ethnicity"
class="bio-item ethnicity hideable"
>
<dfn class="bio-label"><Icon icon="earth2" />Ethnicity</dfn>
<span>{{ actor.ethnicity }}</span>
</li>
<li
v-if="actor.ethnicity"
class="bio-item ethnicity hideable"
>
<dfn class="bio-label"><Icon icon="earth2" />Ethnicity</dfn>
<span>{{ actor.ethnicity }}</span>
</li>
<li
v-if="actor.bust || actor.waist || actor.hip"
title="bust-waist-hip"
class="bio-item"
>
<dfn class="bio-label"><Icon icon="ruler" />Figure</dfn>
<span>
<Icon
v-if="actor.naturalBoobs === false"
v-tooltip="'Boobs enhanced'"
icon="magic-wand"
class="enhanced"
/>{{ actor.bust || '??' }}-{{ actor.waist || '??' }}-{{ actor.hip || '??' }}
</span>
</li>
<li
v-if="actor.bust || actor.waist || actor.hip"
title="bust-waist-hip"
class="bio-item"
>
<dfn class="bio-label"><Icon icon="ruler" />Figure</dfn>
<span>
<Icon
v-if="actor.naturalBoobs === false"
v-tooltip="'Boobs enhanced'"
icon="magic-wand"
class="enhanced"
/>{{ actor.bust || '??' }}-{{ actor.waist || '??' }}-{{ actor.hip || '??' }}
</span>
</li>
<li
v-if="actor.height"
class="bio-item height"
>
<dfn class="bio-label"><Icon icon="height" />Height</dfn>
<span>
<span class="height-metric">{{ actor.height.metric }} cm</span>
<span class="height-imperial">{{ actor.height.imperial }}</span>
</span>
</li>
<li
v-if="actor.height"
class="bio-item height"
>
<dfn class="bio-label"><Icon icon="height" />Height</dfn>
<span>
<span class="height-metric">{{ actor.height.metric }} cm</span>
<span class="height-imperial">{{ actor.height.imperial }}</span>
</span>
</li>
<li
v-if="actor.weight"
class="bio-item weight hideable"
>
<dfn class="bio-label"><Icon icon="scale" />Weight</dfn>
<li
v-if="actor.weight"
class="bio-item weight hideable"
>
<dfn class="bio-label"><Icon icon="scale" />Weight</dfn>
<span>
<span class="weight-metric">{{ actor.weight.metric }} kg</span>
<span class="weight-imperial">{{ actor.weight.imperial }} lbs</span>
</span>
</li>
<span>
<span class="weight-metric">{{ actor.weight.metric }} kg</span>
<span class="weight-imperial">{{ actor.weight.imperial }} lbs</span>
</span>
</li>
<li
v-if="actor.hasTattoos"
class="bio-item tattoos hideable"
>
<dfn class="bio-label"><Icon icon="flower" />Tattoos</dfn>
<li
v-if="actor.hasTattoos"
class="bio-item tattoos hideable"
>
<dfn class="bio-label"><Icon icon="flower" />Tattoos</dfn>
<span
v-if="actor.tattoos"
v-tooltip="actor.tattoos"
class="bio-value"
>{{ actor.tattoos }}</span>
<span v-else>Yes</span>
</li>
<span
v-if="actor.tattoos"
v-tooltip="actor.tattoos"
class="bio-value"
>{{ actor.tattoos }}</span>
<span v-else>Yes</span>
</li>
<li
v-if="actor.hasPiercings"
class="bio-item piercings hideable"
>
<dfn class="bio-label"><Icon icon="trophy4" />Piercings</dfn>
<li
v-if="actor.hasPiercings"
class="bio-item piercings hideable"
>
<dfn class="bio-label"><Icon icon="trophy4" />Piercings</dfn>
<span
v-if="actor.piercings"
v-tooltip="actor.piercings"
class="bio-value"
>{{ actor.piercings }}</span>
<span v-else>Yes</span>
</li>
<span
v-if="actor.piercings"
v-tooltip="actor.piercings"
class="bio-value"
>{{ actor.piercings }}</span>
<span v-else>Yes</span>
</li>
<li class="bio-item scraped hideable">Updated {{ formatDate(actor.scrapedAt, 'YYYY-MM-DD HH:mm') }}, ID: {{ actor.id }}</li>
</ul>
<li class="bio-item scraped hideable">Updated {{ formatDate(actor.updatedAt, 'YYYY-MM-DD HH:mm') }}, ID: {{ actor.id }}</li>
</ul>
<span
v-show="!expanded"
class="expand expand-header collapse-header noselect"
@click="expanded = true"
><Icon icon="arrow-down3" /></span>
<span
v-show="!expanded"
class="expand expand-header collapse-header noselect"
@click="expanded = true"
><Icon icon="arrow-down3" /></span>
<p
v-if="actor.description"
class="description"
>{{ actor.description }}</p>
<p
v-if="actor.description"
class="description"
>{{ actor.description }}</p>
<Social
v-if="actor.social && actor.social.length > 0"
:actor="actor"
class="profile-social"
/>
<Social
v-if="actor.social && actor.social.length > 0"
:actor="actor"
class="profile-social"
/>
<span
v-show="expanded"
class="expand expand-header collapse-header noselect"
@click="expanded = false"
><Icon icon="arrow-up3" /></span>
</div>
<span
v-show="expanded"
class="expand expand-header collapse-header noselect"
@click="expanded = false"
><Icon icon="arrow-up3" /></span>
</div>
<div class="actor-content">
<div
v-if="actor.avatar || (actor.photos && actor.photos.length > 0)"
class="photos-container"
>
<Photos :actor="actor" />
<div class="actor-content">
<div
v-if="actor.avatar || (actor.photos && actor.photos.length > 0)"
class="photos-container"
>
<Photos :actor="actor" />
<Photos
:actor="actor"
:class="{ expanded }"
class="compact"
/>
</div>
<Photos
:actor="actor"
:class="{ expanded }"
class="compact"
/>
</div>
<Releases :releases="actor.releases" />
</div>
</div>
</div>
<Releases :releases="actor.releases" />
</div>
</div>
</div>
</template>
<script>
@ -253,52 +253,52 @@ import Gender from './gender.vue';
import Social from './social.vue';
async function fetchActor() {
this.actor = await this.$store.dispatch('fetchActorBySlug', {
actorSlug: this.$route.params.actorSlug,
range: this.$route.params.range,
});
this.actor = await this.$store.dispatch('fetchActorBySlug', {
actorSlug: this.$route.params.actorSlug,
range: this.$route.params.range,
});
}
async function route() {
await this.fetchActor();
await this.fetchActor();
}
function scrollPhotos(event) {
event.currentTarget.scrollLeft += event.deltaY; // eslint-disable-line no-param-reassign
event.currentTarget.scrollLeft += event.deltaY; // eslint-disable-line no-param-reassign
}
async function mounted() {
await this.fetchActor();
await this.fetchActor();
if (this.actor) {
this.pageTitle = this.actor.name;
}
if (this.actor) {
this.pageTitle = this.actor.name;
}
}
export default {
components: {
FilterBar,
Photos,
Releases,
Gender,
Social,
},
data() {
return {
actor: null,
releases: null,
pageTitle: null,
expanded: false,
};
},
watch: {
$route: route,
},
mounted,
methods: {
fetchActor,
scrollPhotos,
},
components: {
FilterBar,
Photos,
Releases,
Gender,
Social,
},
data() {
return {
actor: null,
releases: null,
pageTitle: null,
expanded: false,
};
},
watch: {
$route: route,
},
mounted,
methods: {
fetchActor,
scrollPhotos,
},
};
</script>
@ -426,7 +426,7 @@ export default {
}
.bio-value {
margin: 0 0 0 2rem;
margin: 0 0 0 2rem;
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;

296
assets/components/networks/network.vue
View File

@ -1,124 +1,124 @@
<template>
<div
v-if="network"
class="content"
>
<FilterBar :fetch-releases="fetchNetwork" />
<div
v-if="network"
class="content"
>
<FilterBar :fetch-releases="fetchNetwork" />
<div
class="network"
:class="{ nosites: sites.length === 0 && networks.length === 0 }"
>
<div
v-show="sites.length > 0 || networks.length > 0"
class="sidebar"
:class="{ expanded }"
>
<a
v-tooltip.bottom="`Go to ${network.url}`"
:href="network.url"
target="_blank"
rel="noopener noreferrer"
class="title"
>
<img
:src="`/img/logos/${network.slug}/network.png`"
class="logo"
>
</a>
<div
class="network"
:class="{ nosites: sites.length === 0 && networks.length === 0 }"
>
<div
v-show="sites.length > 0 || networks.length > 0"
class="sidebar"
:class="{ expanded }"
>
<a
v-tooltip.bottom="`Go to ${network.url}`"
:href="network.url"
target="_blank"
rel="noopener noreferrer"
class="title"
>
<img
:src="`/img/logos/${network.slug}/thumbs/network.png`"
class="logo"
>
</a>
<p
v-if="network.description"
class="description"
>{{ network.description }}</p>
<p
v-if="network.description"
class="description"
>{{ network.description }}</p>
<Sites
v-if="sites.length"
:sites="sites"
:class="{ expanded }"
/>
<Sites
v-if="sites.length"
:sites="sites"
:class="{ expanded }"
/>
<div
v-if="networks.length > 0"
class="networks"
>
<Network
v-for="childNetwork in networks"
:key="`network-${childNetwork.id}`"
:network="childNetwork"
/>
</div>
<div
v-if="networks.length > 0"
class="networks"
>
<Network
v-for="childNetwork in networks"
:key="`network-${childNetwork.id}`"
:network="childNetwork"
/>
</div>
<Network
v-if="network.parent"
:network="network.parent"
class="parent"
/>
</div>
<Network
v-if="network.parent"
:network="network.parent"
class="parent"
/>
</div>
<template v-if="sites.length > 0 || networks.length > 0">
<span
v-show="!expanded"
class="expand expand-sidebar noselect"
@click="expanded = true"
><Icon icon="arrow-right3" /></span>
<template v-if="sites.length > 0 || networks.length > 0">
<span
v-show="!expanded"
class="expand expand-sidebar noselect"
@click="expanded = true"
><Icon icon="arrow-right3" /></span>
<span
v-show="expanded"
class="expand expand-sidebar noselect"
@click="expanded = false"
><Icon icon="arrow-left3" /></span>
</template>
<span
v-show="expanded"
class="expand expand-sidebar noselect"
@click="expanded = false"
><Icon icon="arrow-left3" /></span>
</template>
<div
class="header"
:class="{ hideable: sites.length > 0 || networks.length > 0 }"
>
<a
v-tooltip.bottom="`Go to ${network.url}`"
:href="network.url"
target="_blank"
rel="noopener noreferrer"
class="title"
>
<img
:src="`/img/logos/${network.slug}/network.png`"
class="logo"
>
</a>
</div>
<div
class="header"
:class="{ hideable: sites.length > 0 || networks.length > 0 }"
>
<a
v-tooltip.bottom="`Go to ${network.url}`"
:href="network.url"
target="_blank"
rel="noopener noreferrer"
class="title"
>
<img
:src="`/img/logos/${network.slug}/thumbs/network.png`"
class="logo"
>
</a>
</div>
<div class="content-inner">
<template v-if="sites.length > 0 || networks.length > 0">
<span
v-show="expanded"
class="expand collapse-header noselect"
@click="expanded = false"
><Icon icon="arrow-up3" /></span>
<div class="content-inner">
<template v-if="sites.length > 0 || networks.length > 0">
<span
v-show="expanded"
class="expand collapse-header noselect"
@click="expanded = false"
><Icon icon="arrow-up3" /></span>
<Sites
:sites="sites"
:class="{ expanded }"
class="compact"
/>
<Sites
:sites="sites"
:class="{ expanded }"
class="compact"
/>
<span
v-show="!expanded"
class="expand expand-header noselect"
@click="expanded = true"
><Icon icon="arrow-down3" /></span>
<span
v-show="!expanded"
class="expand expand-header noselect"
@click="expanded = true"
><Icon icon="arrow-down3" /></span>
<span
v-show="expanded"
class="expand expand-header noselect"
@click="expanded = false"
><Icon icon="arrow-up3" /></span>
</template>
<span
v-show="expanded"
class="expand expand-header noselect"
@click="expanded = false"
><Icon icon="arrow-up3" /></span>
</template>
<Releases :releases="releases" />
</div>
</div>
</div>
<Releases :releases="releases" />
</div>
</div>
</div>
</template>
<script>
@ -128,59 +128,59 @@ import Sites from '../sites/sites.vue';
import Network from '../tile/network.vue';
async function fetchNetwork() {
this.network = await this.$store.dispatch('fetchNetworkBySlug', {
networkSlug: this.$route.params.networkSlug,
range: this.$route.params.range,
});
this.network = await this.$store.dispatch('fetchNetworkBySlug', {
networkSlug: this.$route.params.networkSlug,
range: this.$route.params.range,
});
if (this.network.studios) {
this.studios = this.network.studios.map(studio => ({
...studio,
network: this.network,
}));
}
if (this.network.studios) {
this.studios = this.network.studios.map(studio => ({
...studio,
network: this.network,
}));
}
this.networks = this.network.networks;
this.sites = this.network.sites
.filter(site => !site.independent);
this.networks = this.network.networks;
this.sites = this.network.sites
.filter(site => !site.independent);
this.releases = this.network.releases;
this.releases = this.network.releases;
}
async function route() {
await this.fetchNetwork();
await this.fetchNetwork();
}
async function mounted() {
await this.fetchNetwork();
this.pageTitle = this.network.name;
await this.fetchNetwork();
this.pageTitle = this.network.name;
}
export default {
components: {
FilterBar,
Releases,
Sites,
Network,
},
data() {
return {
network: null,
sites: [],
networks: [],
studios: [],
releases: [],
pageTitle: null,
expanded: false,
};
},
watch: {
$route: route,
},
mounted,
methods: {
fetchNetwork,
},
components: {
FilterBar,
Releases,
Sites,
Network,
},
data() {
return {
network: null,
sites: [],
networks: [],
studios: [],
releases: [],
pageTitle: null,
expanded: false,
};
},
watch: {
$route: route,
},
mounted,
methods: {
fetchNetwork,
},
};
</script>

34
assets/components/tile/site.vue
View File

@ -1,25 +1,25 @@
<template>
<a
:href="`/site/${site.slug}`"
:title="site.name"
class="tile"
>
<img
:src="`/img/logos/${site.network.slug}/${site.slug}.png`"
:alt="site.name"
class="logo"
>
</a>
<a
:href="`/site/${site.slug}`"
:title="site.name"
class="tile"
>
<img
:src="`/img/logos/${site.network.slug}/thumbs/${site.slug}.png`"
:alt="site.name"
class="logo"
>
</a>
</template>
<script>
export default {
props: {
site: {
type: Object,
default: null,
},
},
props: {
site: {
type: Object,
default: null,
},
},
};
</script>

156
assets/js/actors/actions.js
View File

@ -1,59 +1,61 @@
import { graphql, get } from '../api';
import {
releasePosterFragment,
releaseActorsFragment,
releaseTagsFragment,
releasePosterFragment,
releaseActorsFragment,
releaseTagsFragment,
} from '../fragments';
import { curateRelease } from '../curate';
import getDateRange from '../get-date-range';
function curateActor(actor) {
if (!actor) {
return null;
}
if (!actor) {
return null;
}
const curatedActor = {
...actor,
height: actor.heightMetric && {
metric: actor.heightMetric,
imperial: actor.heightImperial,
},
weight: actor.weightMetric && {
metric: actor.weightMetric,
imperial: actor.weightImperial,
},
origin: actor.birthCountry && {
city: actor.birthCity,
state: actor.birthState,
country: actor.birthCountry,
},
residence: actor.residenceCountry && {
city: actor.residenceCity,
state: actor.residenceState,
country: actor.residenceCountry,
},
};
const curatedActor = {
...actor,
height: actor.heightMetric && {
metric: actor.heightMetric,
imperial: actor.heightImperial,
},
weight: actor.weightMetric && {
metric: actor.weightMetric,
imperial: actor.weightImperial,
},
origin: actor.birthCountry && {
city: actor.birthCity,
state: actor.birthState,
country: actor.birthCountry,
},
residence: actor.residenceCountry && {
city: actor.residenceCity,
state: actor.residenceState,
country: actor.residenceCountry,
},
scrapedAt: new Date(actor.createdAt),
updatedAt: new Date(actor.updatedAt),
};
if (actor.avatar) {
curatedActor.avatar = actor.avatar.media;
}
if (actor.avatar) {
curatedActor.avatar = actor.avatar.media;
}
if (actor.releases) {
curatedActor.releases = actor.releases.map(release => curateRelease(release.release));
}
if (actor.releases) {
curatedActor.releases = actor.releases.map(release => curateRelease(release.release));
}
if (actor.photos) {
curatedActor.photos = actor.photos.map(photo => photo.media);
}
if (actor.photos) {
curatedActor.photos = actor.photos.map(photo => photo.media);
}
return curatedActor;
return curatedActor;
}
function initActorActions(store, _router) {
async function fetchActorBySlug({ _commit }, { actorSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
async function fetchActorBySlug({ _commit }, { actorSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
const { actors: [actor] } = await graphql(`
const { actors: [actor] } = await graphql(`
query Actor(
$actorSlug: String!
$limit:Int = 1000,
@ -90,6 +92,8 @@ function initActorActions(store, _router) {
tattoos
piercings
description
createdAt
updatedAt
network {
id
name
@ -184,27 +188,27 @@ function initActorActions(store, _router) {
}
}
`, {
actorSlug,
limit,
after,
before,
orderBy: orderBy === 'DATE_DESC' ? 'RELEASE_BY_RELEASE_ID__DATE_DESC' : 'RELEASE_BY_RELEASE_ID__DATE_ASC',
exclude: store.state.ui.filter,
});
actorSlug,
limit,
after,
before,
orderBy: orderBy === 'DATE_DESC' ? 'RELEASE_BY_RELEASE_ID__DATE_DESC' : 'RELEASE_BY_RELEASE_ID__DATE_ASC',
exclude: store.state.ui.filter,
});
return curateActor(actor);
}
return curateActor(actor);
}
async function fetchActors({ _commit }, {
limit = 100,
letter,
gender,
}) {
const genderFilter = gender === null
? 'isNull: true'
: `equalTo: "${gender}"`;
async function fetchActors({ _commit }, {
limit = 100,
letter,
gender,
}) {
const genderFilter = gender === null
? 'isNull: true'
: `equalTo: "${gender}"`;
const { actors } = await graphql(`
const { actors } = await graphql(`
query Actors(
$limit: Int,
$letter: String! = "",
@ -249,28 +253,28 @@ function initActorActions(store, _router) {
}
}
`, {
limit,
letter,
});
limit,
letter,
});
return actors.map(actor => curateActor(actor));
}
return actors.map(actor => curateActor(actor));
}
async function fetchActorReleases({ _commit }, actorId) {
const releases = await get(`/actors/${actorId}/releases`, {
filter: store.state.ui.filter,
after: store.getters.after,
before: store.getters.before,
});
async function fetchActorReleases({ _commit }, actorId) {
const releases = await get(`/actors/${actorId}/releases`, {
filter: store.state.ui.filter,
after: store.getters.after,
before: store.getters.before,
});
return releases;
}
return releases;
}
return {
fetchActorBySlug,
fetchActors,
fetchActorReleases,
};
return {
fetchActorBySlug,
fetchActors,
fetchActorReleases,
};
}
export default initActorActions;

10
assets/js/actors/actors.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions';
function initActorsStore(store, router) {
return {
state,
mutations,
actions: actions(store, router),
};
return {
state,
mutations,
actions: actions(store, router),
};
}
export default initActorsStore;

94
assets/js/api.js
View File

@ -1,71 +1,71 @@
import config from 'config';
async function get(endpoint, query = {}) {
const curatedQuery = Object.entries(query).reduce((acc, [key, value]) => (value ? { ...acc, [key]: value } : acc), {}); // remove empty values
const q = new URLSearchParams(curatedQuery).toString();
const curatedQuery = Object.entries(query).reduce((acc, [key, value]) => (value ? { ...acc, [key]: value } : acc), {}); // remove empty values
const q = new URLSearchParams(curatedQuery).toString();
const res = await fetch(`${config.api.url}${endpoint}?${q}`, {
method: 'GET',
mode: 'cors',
credentials: 'same-origin',
});
const res = await fetch(`${config.api.url}${endpoint}?${q}`, {
method: 'GET',
mode: 'cors',
credentials: 'same-origin',
});
if (res.ok) {
return res.json();
}
if (res.ok) {
return res.json();
}
const errorMsg = await res.text();
const errorMsg = await res.text();
throw new Error(errorMsg);
throw new Error(errorMsg);
}
async function post(endpoint, data) {
const res = await fetch(`${config.api.url}${endpoint}`, {
method: 'POST',
mode: 'cors',
headers: {
'Content-Type': 'application/json',
},
credentials: 'same-origin',
body: JSON.stringify(data),
});
const res = await fetch(`${config.api.url}${endpoint}`, {
method: 'POST',
mode: 'cors',
headers: {
'Content-Type': 'application/json',
},
credentials: 'same-origin',
body: JSON.stringify(data),
});
if (res.ok) {
return res.json();
}
if (res.ok) {
return res.json();
}
const errorMsg = await res.text();
const errorMsg = await res.text();
throw new Error(errorMsg);
throw new Error(errorMsg);
}
async function graphql(query, variables = null) {
const res = await fetch('/graphql', {
method: 'POST',
mode: 'cors',
headers: {
'Content-Type': 'application/json',
},
credentials: 'same-origin',
body: JSON.stringify({
query,
variables,
}),
});
const res = await fetch('/graphql', {
method: 'POST',
mode: 'cors',
headers: {
'Content-Type': 'application/json',
},
credentials: 'same-origin',
body: JSON.stringify({
query,
variables,
}),
});
if (res.ok) {
const { data } = await res.json();
if (res.ok) {
const { data } = await res.json();
return data;
}
return data;
}
const errorMsg = await res.text();
const errorMsg = await res.text();
throw new Error(errorMsg);
throw new Error(errorMsg);
}
export {
get,
post,
graphql,
get,
post,
graphql,
};

10
assets/js/auth/auth.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions';
function initAuthStore(store, router) {
return {
state,
mutations,
actions: actions(store, router),
};
return {
state,
mutations,
actions: actions(store, router),
};
}
export default initAuthStore;

4
assets/js/auth/state.js
View File

@ -1,4 +1,4 @@
export default {
authenticated: false,
user: null,
authenticated: false,
user: null,
};

16
assets/js/config/default.js
View File

@ -1,10 +1,10 @@
export default {
api: {
url: `${window.location.origin}/api`,
},
filename: {
pattern: '{site.name} - {title} ({actors.$n.name}, {date} {shootId})',
separator: ', ',
date: 'DD-MM-YYYY',
},
api: {
url: `${window.location.origin}/api`,
},
filename: {
pattern: '{site.name} - {title} ({actors.$n.name}, {date} {shootId})',
separator: ', ',
date: 'DD-MM-YYYY',
},
};

128
assets/js/curate.js
View File

@ -1,94 +1,94 @@
import dayjs from 'dayjs';
function curateActor(actor, release) {
const curatedActor = {
...actor,
origin: actor.originCountry && {
country: actor.originCountry,
},
};
const curatedActor = {
...actor,
origin: actor.originCountry && {
country: actor.originCountry,
},
};
if (actor.avatar) curatedActor.avatar = actor.avatar.media;
if (actor.avatar) curatedActor.avatar = actor.avatar.media;
if (release && release.date && curatedActor.birthdate) {
curatedActor.ageThen = dayjs(release.date).diff(actor.birthdate, 'year');
}
if (release && release.date && curatedActor.birthdate) {
curatedActor.ageThen = dayjs(release.date).diff(actor.birthdate, 'year');
}
return curatedActor;
return curatedActor;
}
function curateRelease(release) {
const curatedRelease = {
...release,
actors: [],
poster: release.poster && release.poster.media,
tags: release.tags ? release.tags.map(({ tag }) => tag) : [],
};
const curatedRelease = {
...release,
actors: [],
poster: release.poster && release.poster.media,
tags: release.tags ? release.tags.map(({ tag }) => tag) : [],
};
if (release.site) curatedRelease.network = release.site.network;
if (release.scenes) curatedRelease.scenes = release.scenes.map(({ scene }) => curateRelease(scene));
if (release.movies) curatedRelease.movies = release.movies.map(({ movie }) => curateRelease(movie));
if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media);
if (release.trailer) curatedRelease.trailer = release.trailer.media;
if (release.teaser) curatedRelease.teaser = release.teaser.media;
if (release.actors) curatedRelease.actors = release.actors.map(({ actor }) => curateActor(actor, curatedRelease));
if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.map(({ tag }) => tag);
if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.map(({ actor }) => curateActor(actor, curatedRelease));
if (release.site) curatedRelease.network = release.site.network;
if (release.scenes) curatedRelease.scenes = release.scenes.map(({ scene }) => curateRelease(scene));
if (release.movies) curatedRelease.movies = release.movies.map(({ movie }) => curateRelease(movie));
if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media);
if (release.trailer) curatedRelease.trailer = release.trailer.media;
if (release.teaser) curatedRelease.teaser = release.teaser.media;
if (release.actors) curatedRelease.actors = release.actors.map(({ actor }) => curateActor(actor, curatedRelease));
if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.map(({ tag }) => tag);
if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.map(({ actor }) => curateActor(actor, curatedRelease));
return curatedRelease;
return curatedRelease;
}
function curateSite(site, network) {
const curatedSite = {
id: site.id,
name: site.name,
slug: site.slug,
url: site.url,
independent: site.independent,
};
const curatedSite = {
id: site.id,
name: site.name,
slug: site.slug,
url: site.url,
independent: site.independent,
};
if (site.releases) curatedSite.releases = site.releases.map(release => curateRelease(release));
if (site.network || network) curatedSite.network = site.network || network;
if (site.tags) curatedSite.tags = site.tags.map(({ tag }) => tag);
if (site.releases) curatedSite.releases = site.releases.map(release => curateRelease(release));
if (site.network || network) curatedSite.network = site.network || network;
if (site.tags) curatedSite.tags = site.tags.map(({ tag }) => tag);
return curatedSite;
return curatedSite;
}
function curateNetwork(network, releases) {
const curatedNetwork = {
id: network.id,
name: network.name,
slug: network.slug,
url: network.url,
networks: [],
};
const curatedNetwork = {
id: network.id,
name: network.name,
slug: network.slug,
url: network.url,
networks: [],
};
if (network.parent) curatedNetwork.parent = curateNetwork(network.parent);
if (network.sites) curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork));
if (network.networks) curatedNetwork.networks = network.networks.map(subNetwork => curateNetwork(subNetwork));
if (network.studios) curatedNetwork.studios = network.studios;
if (releases) curatedNetwork.releases = releases.map(release => curateRelease(release));
if (network.parent) curatedNetwork.parent = curateNetwork(network.parent);
if (network.sites) curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork));
if (network.networks) curatedNetwork.networks = network.networks.map(subNetwork => curateNetwork(subNetwork));
if (network.studios) curatedNetwork.studios = network.studios;
if (releases) curatedNetwork.releases = releases.map(release => curateRelease(release));
return curatedNetwork;
return curatedNetwork;
}
function curateTag(tag) {
const curatedTag = {
...tag,
};
const curatedTag = {
...tag,
};
if (tag.releases) curatedTag.releases = tag.releases.map(({ release }) => curateRelease(release));
if (tag.photos) curatedTag.photos = tag.photos.map(({ media }) => media);
if (tag.poster) curatedTag.poster = tag.poster.media;
if (tag.releases) curatedTag.releases = tag.releases.map(({ release }) => curateRelease(release));
if (tag.photos) curatedTag.photos = tag.photos.map(({ media }) => media);
if (tag.poster) curatedTag.poster = tag.poster.media;
return curatedTag;
return curatedTag;
}
export {
curateActor,
curateRelease,
curateSite,
curateNetwork,
curateTag,
curateActor,
curateRelease,
curateSite,
curateNetwork,
curateTag,
};

20
assets/js/fragments.js
View File

@ -278,14 +278,14 @@ const releaseFragment = `
`;
export {
releaseActorsFragment,
releaseFields,
releaseTagsFragment,
releasePosterFragment,
releasePhotosFragment,
releaseTrailerFragment,
releasesFragment,
releaseFragment,
siteFragment,
sitesFragment,
releaseActorsFragment,
releaseFields,
releaseTagsFragment,
releasePosterFragment,
releasePhotosFragment,
releaseTrailerFragment,
releasesFragment,
releaseFragment,
siteFragment,
sitesFragment,
};

42
assets/js/get-date-range.js
View File

@ -1,30 +1,30 @@
import dayjs from 'dayjs';
const dateRanges = {
latest: () => ({
after: '1900-01-01',
before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
orderBy: 'DATE_DESC',
}),
upcoming: () => ({
after: dayjs(new Date()).format('YYYY-MM-DD'),
before: '2100-01-01',
orderBy: 'DATE_ASC',
}),
new: () => ({
after: '1900-01-01',
before: '2100-01-01',
orderBy: 'CREATED_AT_DESC',
}),
all: () => ({
after: '1900-01-01',
before: '2100-01-01',
orderBy: 'DATE_DESC',
}),
latest: () => ({
after: '1900-01-01',
before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
orderBy: 'DATE_DESC',
}),
upcoming: () => ({
after: dayjs(new Date()).format('YYYY-MM-DD'),
before: '2100-01-01',
orderBy: 'DATE_ASC',
}),
new: () => ({
after: '1900-01-01',
before: '2100-01-01',
orderBy: 'CREATED_AT_DESC',
}),
all: () => ({
after: '1900-01-01',
before: '2100-01-01',
orderBy: 'DATE_DESC',
}),
};
function getDateRange(range) {
return dateRanges[range]();
return dateRanges[range]();
}
export default getDateRange;

72
assets/js/main.js
View File

@ -14,48 +14,48 @@ import Container from '../components/container/container.vue';
import Icon from '../components/icon/icon.vue';
function init() {
const store = initStore(router);
const store = initStore(router);
initUiObservers(store, router);
initUiObservers(store, router);
if (window.env.sfw) {
store.dispatch('setSfw', true);
}
if (window.env.sfw) {
store.dispatch('setSfw', true);
}
Vue.mixin({
components: {
Icon,
},
watch: {
pageTitle(title) {
if (title) {
document.title = `traxxx - ${title}`;
return;
}
Vue.mixin({
components: {
Icon,
},
watch: {
pageTitle(title) {
if (title) {
document.title = `traxxx - ${title}`;
return;
}
document.title = 'traxxx';
},
},
methods: {
formatDate: (date, format) => dayjs(date).format(format),
isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB),
isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB),
},
});
document.title = 'traxxx';
},
},
methods: {
formatDate: (date, format) => dayjs(date).format(format),
isAfter: (dateA, dateB) => dayjs(dateA).isAfter(dateB),
isBefore: (dateA, dateB) => dayjs(dateA).isBefore(dateB),
},
});
Vue.use(VTooltip);
Vue.use(VueLazyLoad, {
throttleWait: 0,
});
Vue.use(VTooltip);
Vue.use(VueLazyLoad, {
throttleWait: 0,
});
new Vue({ // eslint-disable-line no-new
el: '#container',
store,
router,
render(createElement) {
return createElement(Container);
},
});
new Vue({ // eslint-disable-line no-new
el: '#container',
store,
router,
render(createElement) {
return createElement(Container);
},
});
}
init();

44
assets/js/networks/actions.js
View File

@ -4,10 +4,10 @@ import { curateNetwork } from '../curate';
import getDateRange from '../get-date-range';
function initNetworksActions(store, _router) {
async function fetchNetworkBySlug({ _commit }, { networkSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
async function fetchNetworkBySlug({ _commit }, { networkSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
const { network, releases } = await graphql(`
const { network, releases } = await graphql(`
query Network(
$networkSlug: String!
$limit:Int = 1000,
@ -107,21 +107,21 @@ function initNetworksActions(store, _router) {
}
}
`, {
networkSlug,
limit,
after,
before,
orderBy,
afterTime: store.getters.after,
beforeTime: store.getters.before,
exclude: store.state.ui.filter,
});
networkSlug,
limit,
after,
before,
orderBy,
afterTime: store.getters.after,
beforeTime: store.getters.before,
exclude: store.state.ui.filter,
});
return curateNetwork(network, releases);
}
return curateNetwork(network, releases);
}
async function fetchNetworks({ _commit }) {
const { networks } = await graphql(`
async function fetchNetworks({ _commit }) {
const { networks } = await graphql(`
query Networks {
networks(orderBy: NAME_ASC) {
id
@ -133,13 +133,13 @@ function initNetworksActions(store, _router) {
}
`);
return networks.map(network => curateNetwork(network));
}
return networks.map(network => curateNetwork(network));
}
return {
fetchNetworkBySlug,
fetchNetworks,
};
return {
fetchNetworkBySlug,
fetchNetworks,
};
}
export default initNetworksActions;

10
assets/js/networks/networks.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions';
function initNetworksStore(store, router) {
return {
state,
mutations,
actions: actions(store, router),
};
return {
state,
mutations,
actions: actions(store, router),
};
}
export default initNetworksStore;

62
assets/js/releases/actions.js
View File

@ -4,10 +4,10 @@ import { curateRelease } from '../curate';
import getDateRange from '../get-date-range';
function initReleasesActions(store, _router) {
async function fetchReleases({ _commit }, { limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
async function fetchReleases({ _commit }, { limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
const { releases } = await graphql(`
const { releases } = await graphql(`
query Releases(
$limit:Int = 1000,
$after:Date = "1900-01-01",
@ -18,18 +18,18 @@ function initReleasesActions(store, _router) {
${releasesFragment}
}
`, {
limit,
after,
before,
orderBy,
exclude: store.state.ui.filter,
});
limit,
after,
before,
orderBy,
exclude: store.state.ui.filter,
});
return releases.map(release => curateRelease(release));
}
return releases.map(release => curateRelease(release));
}
async function searchReleases({ _commit }, { query, limit = 20 }) {
const res = await graphql(`
async function searchReleases({ _commit }, { query, limit = 20 }) {
const res = await graphql(`
query SearchReleases(
$query: String!
$limit: Int = 20
@ -88,34 +88,34 @@ function initReleasesActions(store, _router) {
}
}
`, {
query,
limit,
});
query,
limit,
});
if (!res) return [];
if (!res) return [];
return res.releases.map(release => curateRelease(release));
}
return res.releases.map(release => curateRelease(release));
}
async function fetchReleaseById({ _commit }, releaseId) {
// const release = await get(`/releases/${releaseId}`);
async function fetchReleaseById({ _commit }, releaseId) {
// const release = await get(`/releases/${releaseId}`);
const { release } = await graphql(`
const { release } = await graphql(`
query Release($releaseId:Int!) {
${releaseFragment}
}
`, {
releaseId: Number(releaseId),
});
releaseId: Number(releaseId),
});
return curateRelease(release);
}
return curateRelease(release);
}
return {
fetchReleases,
fetchReleaseById,
searchReleases,
};
return {
fetchReleases,
fetchReleaseById,
searchReleases,
};
}
export default initReleasesActions;

8
assets/js/releases/mutations.js
View File

@ -1,14 +1,14 @@
import Vue from 'vue';
function setCache(state, { target, releases }) {
Vue.set(state.cache, target, releases);
Vue.set(state.cache, target, releases);
}
function deleteCache(state, target) {
Vue.delete(state.cache, target);
Vue.delete(state.cache, target);
}
export default {
setCache,
deleteCache,
setCache,
deleteCache,
};

10
assets/js/releases/releases.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions';
function initReleasesStore(store, router) {
return {
state,
mutations,
actions: actions(store, router),
};
return {
state,
mutations,
actions: actions(store, router),
};
}
export default initReleasesStore;

2
assets/js/releases/state.js
View File

@ -1,3 +1,3 @@
export default {
cache: {},
cache: {},
};

260
assets/js/router.js
View File

@ -16,139 +16,139 @@ import NotFound from '../components/errors/404.vue';
Vue.use(VueRouter);
const routes = [
{
path: '/',
redirect: {
name: 'latest',
},
},
{
path: '/home',
redirect: {
name: 'latest',
},
},
{
path: '/latest',
component: Home,
name: 'latest',
},
{
path: '/upcoming',
component: Home,
name: 'upcoming',
},
{
path: '/new',
component: Home,
name: 'new',
},
{
path: '/scene/:releaseId/:releaseSlug?',
component: Release,
name: 'scene',
},
{
path: '/movie/:releaseId/:releaseSlug?',
component: Release,
name: 'movie',
},
{
path: '/actor/:actorSlug',
name: 'actor',
redirect: from => ({
name: 'actorRange',
params: {
...from.params,
range: 'latest',
},
}),
},
{
path: '/actor/:actorSlug/:range',
component: Actor,
name: 'actorRange',
},
{
path: '/site/:siteSlug',
component: Site,
name: 'site',
redirect: from => ({
name: 'siteRange',
params: {
...from.params,
range: 'latest',
},
}),
},
{
path: '/site/:siteSlug/:range',
component: Site,
name: 'siteRange',
},
{
path: '/network/:networkSlug',
component: Network,
name: 'network',
redirect: from => ({
name: 'networkRange',
params: {
...from.params,
range: 'latest',
},
}),
},
{
path: '/network/:networkSlug/:range',
component: Network,
name: 'networkRange',
},
{
path: '/tag/:tagSlug',
component: Tag,
name: 'tag',
redirect: from => ({
name: 'tagRange',
params: {
...from.params,
range: 'latest',
},
}),
},
{
path: '/tag/:tagSlug/:range',
component: Tag,
name: 'tagRange',
},
{
path: '/actors/:gender?/:letter?',
component: Actors,
name: 'actors',
},
{
path: '/networks',
component: Networks,
name: 'networks',
},
{
path: '/tags',
component: Tags,
name: 'tags',
},
{
path: '/search',
component: Search,
name: 'search',
},
{
path: '*',
component: NotFound,
},
{
path: '/',
redirect: {
name: 'latest',
},
},
{
path: '/home',
redirect: {
name: 'latest',
},
},
{
path: '/latest',
component: Home,
name: 'latest',
},
{
path: '/upcoming',
component: Home,
name: 'upcoming',
},
{
path: '/new',
component: Home,
name: 'new',
},
{
path: '/scene/:releaseId/:releaseSlug?',
component: Release,
name: 'scene',
},
{
path: '/movie/:releaseId/:releaseSlug?',
component: Release,
name: 'movie',
},
{
path: '/actor/:actorSlug',
name: 'actor',
redirect: from => ({
name: 'actorRange',
params: {
...from.params,
range: 'latest',
},
}),
},
{
path: '/actor/:actorSlug/:range',
component: Actor,
name: 'actorRange',
},
{
path: '/site/:siteSlug',
component: Site,
name: 'site',
redirect: from => ({
name: 'siteRange',
params: {
...from.params,
range: 'latest',
},
}),
},
{
path: '/site/:siteSlug/:range',
component: Site,
name: 'siteRange',
},
{
path: '/network/:networkSlug',
component: Network,
name: 'network',
redirect: from => ({
name: 'networkRange',
params: {
...from.params,
range: 'latest',
},
}),
},
{
path: '/network/:networkSlug/:range',
component: Network,
name: 'networkRange',
},
{
path: '/tag/:tagSlug',
component: Tag,
name: 'tag',
redirect: from => ({
name: 'tagRange',
params: {
...from.params,
range: 'latest',
},
}),
},
{
path: '/tag/:tagSlug/:range',
component: Tag,
name: 'tagRange',
},
{
path: '/actors/:gender?/:letter?',
component: Actors,
name: 'actors',
},
{
path: '/networks',
component: Networks,
name: 'networks',
},
{
path: '/tags',
component: Tags,
name: 'tags',
},
{
path: '/search',
component: Search,
name: 'search',
},
{
path: '*',
component: NotFound,
},
];
const router = new VueRouter({
mode: 'history',
routes,
mode: 'history',
routes,
});
export default router;

66
assets/js/sites/actions.js
View File

@ -4,10 +4,10 @@ import { curateSite } from '../curate';
import getDateRange from '../get-date-range';
function initSitesActions(store, _router) {
async function fetchSiteBySlug({ _commit }, { siteSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
async function fetchSiteBySlug({ _commit }, { siteSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
const { site } = await graphql(`
const { site } = await graphql(`
query Site(
$siteSlug: String!,
$limit:Int = 100,
@ -37,20 +37,20 @@ function initSitesActions(store, _router) {
}
}
`, {
siteSlug,
limit,
after,
before,
orderBy,
isNew: store.getters.isNew,
exclude: store.state.ui.filter,
});
siteSlug,
limit,
after,
before,
orderBy,
isNew: store.getters.isNew,
exclude: store.state.ui.filter,
});
return curateSite(site);
}
return curateSite(site);
}
async function fetchSites({ _commit }, { limit = 100 }) {
const { sites } = await graphql(`
async function fetchSites({ _commit }, { limit = 100 }) {
const { sites } = await graphql(`
query Sites(
$actorSlug: String!
$limit:Int = 100,
@ -64,16 +64,16 @@ function initSitesActions(store, _router) {
}
}
`, {
limit,
after: store.getters.after,
before: store.getters.before,
});
limit,
after: store.getters.after,
before: store.getters.before,
});
return sites;
}
return sites;
}
async function searchSites({ _commit }, { query, limit = 20 }) {
const { sites } = await graphql(`
async function searchSites({ _commit }, { query, limit = 20 }) {
const { sites } = await graphql(`
query SearchSites(
$query: String!
$limit:Int = 20,
@ -93,18 +93,18 @@ function initSitesActions(store, _router) {
}
}
`, {
query,
limit,
});
query,
limit,
});
return sites;
}
return sites;
}
return {
fetchSiteBySlug,
fetchSites,
searchSites,
};
return {
fetchSiteBySlug,
fetchSites,
searchSites,
};
}
export default initSitesActions;

10
assets/js/sites/sites.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions';
function initSitesStore(store, router) {
return {
state,
mutations,
actions: actions(store, router),
};
return {
state,
mutations,
actions: actions(store, router),
};
}
export default initSitesStore;

20
assets/js/store.js
View File

@ -10,19 +10,19 @@ import initActorsStore from './actors/actors';
import initTagsStore from './tags/tags';
function initStore(router) {
Vue.use(Vuex);
Vue.use(Vuex);
const store = new Vuex.Store();
const store = new Vuex.Store();
store.registerModule('ui', initUiStore(store, router));
store.registerModule('auth', initAuthStore(store, router));
store.registerModule('releases', initReleasesStore(store, router));
store.registerModule('actors', initActorsStore(store, router));
store.registerModule('sites', initSitesStore(store, router));
store.registerModule('networks', initNetworksStore(store, router));
store.registerModule('tags', initTagsStore(store, router));
store.registerModule('ui', initUiStore(store, router));
store.registerModule('auth', initAuthStore(store, router));
store.registerModule('releases', initReleasesStore(store, router));
store.registerModule('actors', initActorsStore(store, router));
store.registerModule('sites', initSitesStore(store, router));
store.registerModule('networks', initNetworksStore(store, router));
store.registerModule('tags', initTagsStore(store, router));
return store;
return store;
}
export default initStore;

76
assets/js/tags/actions.js
View File

@ -1,15 +1,15 @@
import { graphql, get } from '../api';
import {
releaseFields,
releaseFields,
} from '../fragments';
import { curateTag } from '../curate';
import getDateRange from '../get-date-range';
function initTagsActions(store, _router) {
async function fetchTagBySlug({ _commit }, { tagSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
async function fetchTagBySlug({ _commit }, { tagSlug, limit = 100, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
const { tagBySlug } = await graphql(`
const { tagBySlug } = await graphql(`
query Tag(
$tagSlug:String!
$limit:Int = 1000,
@ -85,24 +85,24 @@ function initTagsActions(store, _router) {
}
}
`, {
tagSlug,
limit,
after,
before,
orderBy: orderBy === 'DATE_DESC' ? 'RELEASE_BY_RELEASE_ID__DATE_DESC' : 'RELEASE_BY_RELEASE_ID__DATE_ASC',
exclude: store.state.ui.filter,
});
tagSlug,
limit,
after,
before,
orderBy: orderBy === 'DATE_DESC' ? 'RELEASE_BY_RELEASE_ID__DATE_DESC' : 'RELEASE_BY_RELEASE_ID__DATE_ASC',
exclude: store.state.ui.filter,
});
return curateTag(tagBySlug, store);
}
return curateTag(tagBySlug, store);
}
async function fetchTags({ _commit }, {
limit = 100,
slugs = [],
_group,
_priority,
}) {
const { tags } = await graphql(`
async function fetchTags({ _commit }, {
limit = 100,
slugs = [],
_group,
_priority,
}) {
const { tags } = await graphql(`
query Tags(
$slugs: [String!] = [],
$limit: Int = 100
@ -133,28 +133,28 @@ function initTagsActions(store, _router) {
}
}
`, {
slugs,
limit,
});
slugs,
limit,
});
return tags.map(tag => curateTag(tag, store.state.ui.sfw));
}
return tags.map(tag => curateTag(tag, store.state.ui.sfw));
}
async function fetchTagReleases({ _commit }, tagId) {
const releases = await get(`/tags/${tagId}/releases`, {
filter: store.state.ui.filter,
after: store.getters.after,
before: store.getters.before,
});
async function fetchTagReleases({ _commit }, tagId) {
const releases = await get(`/tags/${tagId}/releases`, {
filter: store.state.ui.filter,
after: store.getters.after,
before: store.getters.before,
});
return releases;
}
return releases;
}
return {
fetchTagBySlug,
fetchTags,
fetchTagReleases,
};
return {
fetchTagBySlug,
fetchTags,
fetchTagReleases,
};
}
export default initTagsActions;

10
assets/js/tags/tags.js
View File

@ -3,11 +3,11 @@ import mutations from './mutations';
import actions from './actions';
function initTagsStore(store, router) {
return {
state,
mutations,
actions: actions(store, router),
};
return {
state,
mutations,
actions: actions(store, router),
};
}
export default initTagsStore;

52
assets/js/ui/actions.js
View File

@ -1,35 +1,35 @@
function initUiActions(_store, _router) {
function setFilter({ commit }, filter) {
commit('setFilter', filter);
localStorage.setItem('filter', filter);
}
function setFilter({ commit }, filter) {
commit('setFilter', filter);
localStorage.setItem('filter', filter);
}
function setRange({ commit }, range) {
commit('setRange', range);
}
function setRange({ commit }, range) {
commit('setRange', range);
}
function setBatch({ commit }, batch) {
commit('setBatch', batch);
localStorage.setItem('batch', batch);
}
function setBatch({ commit }, batch) {
commit('setBatch', batch);
localStorage.setItem('batch', batch);
}
function setTheme({ commit }, theme) {
commit('setTheme', theme);
localStorage.setItem('theme', theme);
}
function setTheme({ commit }, theme) {
commit('setTheme', theme);
localStorage.setItem('theme', theme);
}
async function setSfw({ commit }, sfw) {
commit('setSfw', sfw);
localStorage.setItem('sfw', sfw);
}
async function setSfw({ commit }, sfw) {
commit('setSfw', sfw);
localStorage.setItem('sfw', sfw);
}
return {
setFilter,
setRange,
setBatch,
setSfw,
setTheme,
};
return {
setFilter,
setRange,
setBatch,
setSfw,
setTheme,
};
}
export default initUiActions;

56
assets/js/ui/getters.js
View File

@ -1,47 +1,47 @@
import dayjs from 'dayjs';
const dateRanges = {
latest: () => ({
after: '1900-01-01',
before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
orderBy: 'DATE_DESC',
}),
upcoming: () => ({
after: dayjs(new Date()).format('YYYY-MM-DD'),
before: '2100-01-01',
orderBy: 'DATE_ASC',
}),
new: () => ({
after: '1900-01-01',
before: '2100-01-01',
orderBy: 'CREATED_AT_DESC',
}),
all: () => ({
after: '1900-01-01',
before: '2100-01-01',
orderBy: 'DATE_DESC',
}),
latest: () => ({
after: '1900-01-01',
before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
orderBy: 'DATE_DESC',
}),
upcoming: () => ({
after: dayjs(new Date()).format('YYYY-MM-DD'),
before: '2100-01-01',
orderBy: 'DATE_ASC',
}),
new: () => ({
after: '1900-01-01',
before: '2100-01-01',
orderBy: 'CREATED_AT_DESC',
}),
all: () => ({
after: '1900-01-01',
before: '2100-01-01',
orderBy: 'DATE_DESC',
}),
};
function rangeDates(state) {
return dateRanges[state.range]();
return dateRanges[state.range]();
}
function before(state) {
return dateRanges[state.range]().before;
return dateRanges[state.range]().before;
}
function after(state) {
return dateRanges[state.range]().after;
return dateRanges[state.range]().after;
}
function orderBy(state) {
return dateRanges[state.range]().orderBy;
return dateRanges[state.range]().orderBy;
}
export default {
rangeDates,
before,
after,
orderBy,
rangeDates,
before,
after,
orderBy,
};

20
assets/js/ui/mutations.js
View File

@ -1,27 +1,27 @@
function setFilter(state, filter) {
state.filter = filter;
state.filter = filter;
}
function setRange(state, range) {
state.range = range;
state.range = range;
}
function setBatch(state, batch) {
state.batch = batch;
state.batch = batch;
}
function setSfw(state, sfw) {
state.sfw = sfw;
state.sfw = sfw;
}
function setTheme(state, theme) {
state.theme = theme;
state.theme = theme;
}
export default {
setFilter,
setRange,
setBatch,
setSfw,
setTheme,
setFilter,
setRange,
setBatch,
setSfw,
setTheme,
};

34
assets/js/ui/observers.js
View File

@ -1,25 +1,25 @@
function initUiObservers(store, _router) {
document.addEventListener('keypress', (event) => {
if (event.target.tagName === 'INPUT') {
return;
}
document.addEventListener('keypress', (event) => {
if (event.target.tagName === 'INPUT') {
return;
}
if (event.key === 's') {
store.dispatch('setSfw', true);
}
if (event.key === 's') {
store.dispatch('setSfw', true);
}
if (event.key === 'n') {
store.dispatch('setSfw', false);
}
if (event.key === 'n') {
store.dispatch('setSfw', false);
}
if (event.key === 'd') {
store.dispatch('setTheme', 'dark');
}
if (event.key === 'd') {
store.dispatch('setTheme', 'dark');
}
if (event.key === 'l') {
store.dispatch('setTheme', 'light');
}
});
if (event.key === 'l') {
store.dispatch('setTheme', 'light');
}
});
}
export default initUiObservers;

10
assets/js/ui/state.js
View File

@ -4,9 +4,9 @@ const storedSfw = localStorage.getItem('sfw');
const storedTheme = localStorage.getItem('theme');
export default {
filter: storedFilter ? storedFilter.split(',') : ['gay', 'transsexual'],
range: 'latest',
batch: storedBatch || 'all',
sfw: storedSfw === 'true' || false,
theme: storedTheme || 'light',
filter: storedFilter ? storedFilter.split(',') : ['gay', 'transsexual'],
range: 'latest',
batch: storedBatch || 'all',
sfw: storedSfw === 'true' || false,
theme: storedTheme || 'light',
};

12
assets/js/ui/ui.js
View File

@ -4,12 +4,12 @@ import getters from './getters';
import actions from './actions';
function initUiStore(store, router) {
return {
state,
mutations,
getters,
actions: actions(store, router),
};
return {
state,
mutations,
getters,
actions: actions(store, router),
};
}
export default initUiStore;

354
config/default.js
View File

@ -1,177 +1,181 @@
module.exports = {
database: {
host: '127.0.0.1',
user: 'user',
password: 'password',
database: 'traxxx',
},
web: {
host: '0.0.0.0',
port: 5000,
sfwHost: '0.0.0.0',
sfwPort: 5001,
},
// include: [],
// exclude: [],
exclude: [
['21sextreme', [
// no longer updated
'mightymistress',
'dominatedgirls',
'homepornreality',
'peeandblow',
'cummingmatures',
'mandyiskinky',
'speculumplays',
'creampiereality',
]],
['aziani', [
'amberathome',
'marycarey',
'racqueldevonshire',
]],
['blowpass', ['sunlustxxx']],
['ddfnetwork', [
'fuckinhd',
'bustylover',
]],
['famedigital', [
'daringsex',
'lowartfilms',
]],
['pornpros', [
'milfhumiliation',
'humiliated',
'flexiblepositions',
'publicviolations',
'amateurviolations',
'squirtdisgrace',
'cumdisgrace',
'webcamhackers',
'collegeteens',
]],
['score', [
'bigboobbundle',
'milfbundle',
'pornmegaload',
'scorelandtv',
'scoretv',
]],
],
profiles: [
[
'evilangel',
'famedigital',
],
[
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), Burning Angel and Wicked have their own assets
'xempire',
'blowpass',
],
[
// MindGeek; Brazzers and Mile High Media have their own assets
'realitykings',
'mofos',
'digitalplayground',
'twistys',
'babes',
'fakehub',
'sexyhub',
'metrohd',
'iconmale',
'men',
'transangels',
],
'wicked',
'burningangel',
'brazzers',
'milehighmedia',
[
'vixen',
'tushy',
'blacked',
'tushyraw',
'blackedraw',
'deeper',
],
[
// Nubiles
'nubiles',
'nubilesporn',
'deeplush',
'brattysis',
'nfbusty',
'anilos',
'hotcrazymess',
'thatsitcomshow',
],
'21sextury',
'julesjordan',
'naughtyamerica',
'cherrypimps',
'pimpxxx',
[
'hussiepass',
'hushpass',
'interracialpass',
'interracialpovs',
'povpornstars',
'seehimfuck',
'eyeontheguy',
],
[
// Full Porn Network
'analized',
'hergape',
'jamesdeen',
'dtfsluts',
'analbbc',
'analviolation',
'baddaddypov',
'girlfaction',
'homemadeanalwhores',
'mugfucked',
'onlyprince',
'pervertgallery',
'povperverts',
],
'private',
'ddfnetwork',
'bangbros',
'kellymadison',
'gangbangcreampie',
'gloryholesecrets',
'aziani',
'legalporno',
'score',
'boobpedia',
'pornhub',
'freeones',
'freeonesLegacy',
],
proxy: {
enable: false,
host: '',
port: 8888,
hostnames: [
'www.vixen.com',
'www.blacked.com',
'www.blackedraw.com',
'www.tushy.com',
'www.tushyraw.com',
'www.deeper.com',
],
},
fetchAfter: [1, 'week'],
nullDateLimit: 3,
media: {
path: './media',
thumbnailSize: 320, // width for 16:9 will be exactly 576px
thumbnailQuality: 100,
lazySize: 90,
lazyQuality: 90,
videoQuality: [480, 360, 320, 540, 720, 1080, 2160, 270, 240, 180],
limit: 25, // max number of photos per release
},
titleSlugLength: 50,
database: {
host: '127.0.0.1',
user: 'user',
password: 'password',
database: 'traxxx',
},
web: {
host: '0.0.0.0',
port: 5000,
sfwHost: '0.0.0.0',
sfwPort: 5001,
},
// include: [],
// exclude: [],
exclude: [
['21sextreme', [
// no longer updated
'mightymistress',
'dominatedgirls',
'homepornreality',
'peeandblow',
'cummingmatures',
'mandyiskinky',
'speculumplays',
'creampiereality',
]],
['aziani', [
'amberathome',
'marycarey',
'racqueldevonshire',
]],
'boobpedia',
['blowpass', ['sunlustxxx']],
['ddfnetwork', [
'fuckinhd',
'bustylover',
]],
['famedigital', [
'daringsex',
'lowartfilms',
]],
'freeones',
['pornpros', [
'milfhumiliation',
'humiliated',
'flexiblepositions',
'publicviolations',
'amateurviolations',
'squirtdisgrace',
'cumdisgrace',
'webcamhackers',
'collegeteens',
]],
['score', [
'bigboobbundle',
'milfbundle',
'pornmegaload',
'scorelandtv',
'scoretv',
]],
['mindgeek', [
'pornhub',
]],
],
profiles: [
[
'evilangel',
'famedigital',
],
[
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), Burning Angel and Wicked have their own assets
'xempire',
'blowpass',
],
[
// MindGeek; Brazzers and Mile High Media have their own assets
'realitykings',
'mofos',
'digitalplayground',
'twistys',
'babes',
'fakehub',
'sexyhub',
'metrohd',
'iconmale',
'men',
'transangels',
],
'wicked',
'burningangel',
'brazzers',
'milehighmedia',
[
'vixen',
'tushy',
'blacked',
'tushyraw',
'blackedraw',
'deeper',
],
[
// Nubiles
'nubiles',
'nubilesporn',
'deeplush',
'brattysis',
'nfbusty',
'anilos',
'hotcrazymess',
'thatsitcomshow',
],
'21sextury',
'julesjordan',
'naughtyamerica',
'cherrypimps',
'pimpxxx',
[
'hussiepass',
'hushpass',
'interracialpass',
'interracialpovs',
'povpornstars',
'seehimfuck',
'eyeontheguy',
],
[
// Full Porn Network
'analized',
'hergape',
'jamesdeen',
'dtfsluts',
'analbbc',
'analviolation',
'baddaddypov',
'girlfaction',
'homemadeanalwhores',
'mugfucked',
'onlyprince',
'pervertgallery',
'povperverts',
],
'private',
'ddfnetwork',
'bangbros',
'kellymadison',
'gangbangcreampie',
'gloryholesecrets',
'aziani',
'legalporno',
'score',
'boobpedia',
'pornhub',
'freeones',
],
proxy: {
enable: false,
host: '',
port: 8888,
hostnames: [
'www.vixen.com',
'www.blacked.com',
'www.blackedraw.com',
'www.tushy.com',
'www.tushyraw.com',
'www.deeper.com',
],
},
fetchAfter: [1, 'week'],
nullDateLimit: 3,
media: {
path: './media',
thumbnailSize: 320, // width for 16:9 will be exactly 576px
thumbnailQuality: 100,
lazySize: 90,
lazyQuality: 90,
videoQuality: [480, 360, 320, 540, 720, 1080, 2160, 270, 240, 180],
limit: 25, // max number of photos per release
},
titleSlugLength: 50,
};

1485
migrations/20190325001339_releases.js
View File

File diff suppressed because it is too large Load Diff

BIN
public/img/logos/gamma/favicon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.3 KiB

BIN
public/img/logos/mindgeek/favicon.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 2.9 KiB

BIN
public/img/logos/mindgeek/lazy/favicon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.5 KiB

BIN
public/img/logos/mindgeek/lazy/network.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.1 KiB

After

Width:  |  Height:  |  Size: 4.1 KiB

BIN
public/img/logos/mindgeek/lazy/pornhub.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.3 KiB

BIN
public/img/logos/mindgeek/lazy/transangels.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.0 KiB

After

Width:  |  Height:  |  Size: 4.0 KiB

BIN
public/img/logos/mindgeek/lazy/trueamateurs.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.9 KiB

After

Width:  |  Height:  |  Size: 4.9 KiB

BIN
public/img/logos/mindgeek/lazy/tube8vip.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.4 KiB

After

Width:  |  Height:  |  Size: 4.4 KiB

BIN
public/img/logos/mindgeek/misc/porn-hub-gay.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.0 KiB

BIN
public/img/logos/mindgeek/network.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 22 KiB

After

Width:  |  Height:  |  Size: 22 KiB

BIN
public/img/logos/mindgeek/pornhub.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 36 KiB

BIN
public/img/logos/mindgeek/thumbs/favicon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 9.1 KiB

BIN
public/img/logos/mindgeek/thumbs/network.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

BIN
public/img/logos/mindgeek/thumbs/pornhub.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.9 KiB

BIN
public/img/logos/mindgeek/thumbs/transangels.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

BIN
public/img/logos/mindgeek/thumbs/trueamateurs.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

BIN
public/img/logos/mindgeek/thumbs/tube8vip.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 13 KiB

BIN
public/img/logos/mindgeek/transangels.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 36 KiB

BIN
public/img/logos/mindgeek/trueamateurs.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 7.8 KiB

After

Width:  |  Height:  |  Size: 7.8 KiB

BIN
public/img/logos/mindgeek/tube8vip.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 5.5 KiB

After

Width:  |  Height:  |  Size: 5.5 KiB

BIN
public/img/tags/bukkake/0.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 684 KiB

BIN
public/img/tags/bukkake/lazy/0.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 8.2 KiB

BIN
public/img/tags/bukkake/thumbs/0.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 36 KiB

12233
seeds/02_sites.js
View File

File diff suppressed because it is too large Load Diff

1529
seeds/04_media.js
View File

File diff suppressed because it is too large Load Diff

3
src/.eslintrc
View File

@ -6,10 +6,11 @@
},
"rules": {
"strict": 0,
"indent": ["error", "tab"],
"no-tabs": "off",
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
"no-console": 0,
"no-underscore-dangle": 0,
"indent": "off",
"prefer-destructuring": "off",
"template-curly-spacing": "off",
"object-curly-newline": "off",

804
src/actors-legacy.js
View File

@ -18,522 +18,522 @@ const { curateSites } = require('./sites');
const { storeMedia, associateMedia } = require('./media');
async function curateActor(actor) {
const [aliases, avatar, photos, social] = await Promise.all([
knex('actors').where({ alias_for: actor.id }),
knex('actors_avatars')
.where('actor_id', actor.id)
.join('media', 'media.id', 'actors_avatars.media_id')
.first(),
knex('actors_photos')
.where('actor_id', actor.id)
.join('media', 'media.id', 'actors_photos.media_id')
.orderBy('index'),
knex('actors_social')
.where('actor_id', actor.id)
.orderBy('platform', 'desc'),
]);
const [aliases, avatar, photos, social] = await Promise.all([
knex('actors').where({ alias_for: actor.id }),
knex('actors_avatars')
.where('actor_id', actor.id)
.join('media', 'media.id', 'actors_avatars.media_id')
.first(),
knex('actors_photos')
.where('actor_id', actor.id)
.join('media', 'media.id', 'actors_photos.media_id')
.orderBy('index'),
knex('actors_social')
.where('actor_id', actor.id)
.orderBy('platform', 'desc'),
]);
const curatedActor = {
id: actor.id,
gender: actor.gender,
name: actor.name,
description: actor.description,
birthdate: actor.birthdate && new Date(actor.birthdate),
country: actor.country_alpha2,
origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null,
residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null,
ethnicity: actor.ethnicity,
height: actor.height,
weight: actor.weight,
bust: actor.bust,
waist: actor.waist,
hip: actor.hip,
naturalBoobs: actor.natural_boobs,
aliases: aliases.map(({ name }) => name),
slug: actor.slug,
avatar,
photos,
hasTattoos: actor.has_tattoos,
hasPiercings: actor.has_piercings,
tattoos: actor.tattoos,
piercings: actor.piercings,
social,
scrapedAt: actor.scraped_at,
};
const curatedActor = {
id: actor.id,
gender: actor.gender,
name: actor.name,
description: actor.description,
birthdate: actor.birthdate && new Date(actor.birthdate),
country: actor.country_alpha2,
origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null,
residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null,
ethnicity: actor.ethnicity,
height: actor.height,
weight: actor.weight,
bust: actor.bust,
waist: actor.waist,
hip: actor.hip,
naturalBoobs: actor.natural_boobs,
aliases: aliases.map(({ name }) => name),
slug: actor.slug,
avatar,
photos,
hasTattoos: actor.has_tattoos,
hasPiercings: actor.has_piercings,
tattoos: actor.tattoos,
piercings: actor.piercings,
social,
scrapedAt: actor.scraped_at,
};
if (curatedActor.birthdate) {
curatedActor.age = moment().diff(curatedActor.birthdate, 'years');
}
if (curatedActor.birthdate) {
curatedActor.age = moment().diff(curatedActor.birthdate, 'years');
}
if (actor.birth_city) curatedActor.origin.city = actor.birth_city;
if (actor.birth_state) curatedActor.origin.state = actor.birth_state;
if (actor.birth_city) curatedActor.origin.city = actor.birth_city;
if (actor.birth_state) curatedActor.origin.state = actor.birth_state;
if (actor.birth_country_alpha2) {
curatedActor.origin.country = {
alpha2: actor.birth_country_alpha2,
name: actor.birth_country_name,
alias: actor.birth_country_alias,
};
}
if (actor.birth_country_alpha2) {
curatedActor.origin.country = {
alpha2: actor.birth_country_alpha2,
name: actor.birth_country_name,
alias: actor.birth_country_alias,
};
}
if (actor.residence_city) curatedActor.residence.city = actor.residence_city;
if (actor.residence_state) curatedActor.residence.state = actor.residence_state;
if (actor.residence_city) curatedActor.residence.city = actor.residence_city;
if (actor.residence_state) curatedActor.residence.state = actor.residence_state;
if (actor.residence_country_alpha2) {
curatedActor.residence.country = {
alpha2: actor.residence_country_alpha2,
name: actor.residence_country_name,
alias: actor.residence_country_alias,
};
}
if (actor.residence_country_alpha2) {
curatedActor.residence.country = {
alpha2: actor.residence_country_alpha2,
name: actor.residence_country_name,
alias: actor.residence_country_alias,
};
}
return curatedActor;
return curatedActor;
}
function curateActors(releases) {
return Promise.all(releases.map(async release => curateActor(release)));
return Promise.all(releases.map(async release => curateActor(release)));
}
function curateActorEntry(actor, scraped, scrapeSuccess) {
const curatedActor = {
name: capitalize(actor.name),
slug: slugify(actor.name),
birthdate: actor.birthdate,
description: actor.description,
gender: actor.gender,
ethnicity: actor.ethnicity,
bust: actor.bust,
waist: actor.waist,
hip: actor.hip,
natural_boobs: actor.naturalBoobs,
height: actor.height,
weight: actor.weight,
hair: actor.hair,
eyes: actor.eyes,
has_tattoos: actor.hasTattoos,
has_piercings: actor.hasPiercings,
tattoos: actor.tattoos,
piercings: actor.piercings,
};
const curatedActor = {
name: capitalize(actor.name),
slug: slugify(actor.name),
birthdate: actor.birthdate,
description: actor.description,
gender: actor.gender,
ethnicity: actor.ethnicity,
bust: actor.bust,
waist: actor.waist,
hip: actor.hip,
natural_boobs: actor.naturalBoobs,
height: actor.height,
weight: actor.weight,
hair: actor.hair,
eyes: actor.eyes,
has_tattoos: actor.hasTattoos,
has_piercings: actor.hasPiercings,
tattoos: actor.tattoos,
piercings: actor.piercings,
};
if (actor.id) {
curatedActor.id = actor.id;
}
if (actor.id) {
curatedActor.id = actor.id;
}
if (actor.birthPlace) {
curatedActor.birth_city = actor.birthPlace.city;
curatedActor.birth_state = actor.birthPlace.state;
curatedActor.birth_country_alpha2 = actor.birthPlace.country;
}
if (actor.birthPlace) {
curatedActor.birth_city = actor.birthPlace.city;
curatedActor.birth_state = actor.birthPlace.state;
curatedActor.birth_country_alpha2 = actor.birthPlace.country;
}
if (actor.residencePlace) {
curatedActor.residence_city = actor.residencePlace.city;
curatedActor.residence_state = actor.residencePlace.state;
curatedActor.residence_country_alpha2 = actor.residencePlace.country;
}
if (actor.residencePlace) {
curatedActor.residence_city = actor.residencePlace.city;
curatedActor.residence_state = actor.residencePlace.state;
curatedActor.residence_country_alpha2 = actor.residencePlace.country;
}
if (scraped) {
curatedActor.scraped_at = new Date();
curatedActor.scrape_success = scrapeSuccess;
}
if (scraped) {
curatedActor.scraped_at = new Date();
curatedActor.scrape_success = scrapeSuccess;
}
return curatedActor;
return curatedActor;
}
function curateSocialEntry(url, actorId) {
const platforms = [
// links supplied by PH often look like domain.com/domain.com/username
{
label: 'twitter',
pattern: 'http(s)\\://(*)twitter.com/:username(/)(?*)',
format: username => `https://www.twitter.com/${username}`,
},
{
label: 'youtube',
pattern: 'http(s)\\://(*)youtube.com/channel/:username(?*)',
format: username => `https://www.youtube.com/channel/${username}`,
},
{
label: 'instagram',
pattern: 'http(s)\\://(*)instagram.com/:username(/)(?*)',
format: username => `https://www.instagram.com/${username}`,
},
{
label: 'snapchat',
pattern: 'http(s)\\://(*)snapchat.com/add/:username(/)(?*)',
format: username => `https://www.snapchat.com/add/${username}`,
},
{
label: 'tumblr',
pattern: 'http(s)\\://:username.tumblr.com(*)',
format: username => `https://${username}.tumblr.com`,
},
{
label: 'onlyfans',
pattern: 'http(s)\\://(*)onlyfans.com/:username(/)(?*)',
format: username => `https://www.onlyfans.com/${username}`,
},
{
label: 'fancentro',
pattern: 'http(s)\\://(*)fancentro.com/:username(/)(?*)',
format: username => `https://www.fancentro.com/${username}`,
},
{
label: 'modelhub',
pattern: 'http(s)\\://(*)modelhub.com/:username(/)(?*)',
format: username => `https://www.modelhub.com/${username}`,
},
];
const platforms = [
// links supplied by PH often look like domain.com/domain.com/username
{
label: 'twitter',
pattern: 'http(s)\\://(*)twitter.com/:username(/)(?*)',
format: username => `https://www.twitter.com/${username}`,
},
{
label: 'youtube',
pattern: 'http(s)\\://(*)youtube.com/channel/:username(?*)',
format: username => `https://www.youtube.com/channel/${username}`,
},
{
label: 'instagram',
pattern: 'http(s)\\://(*)instagram.com/:username(/)(?*)',
format: username => `https://www.instagram.com/${username}`,
},
{
label: 'snapchat',
pattern: 'http(s)\\://(*)snapchat.com/add/:username(/)(?*)',
format: username => `https://www.snapchat.com/add/${username}`,
},
{
label: 'tumblr',
pattern: 'http(s)\\://:username.tumblr.com(*)',
format: username => `https://${username}.tumblr.com`,
},
{
label: 'onlyfans',
pattern: 'http(s)\\://(*)onlyfans.com/:username(/)(?*)',
format: username => `https://www.onlyfans.com/${username}`,
},
{
label: 'fancentro',
pattern: 'http(s)\\://(*)fancentro.com/:username(/)(?*)',
format: username => `https://www.fancentro.com/${username}`,
},
{
label: 'modelhub',
pattern: 'http(s)\\://(*)modelhub.com/:username(/)(?*)',
format: username => `https://www.modelhub.com/${username}`,
},
];
const match = platforms.reduce((acc, platform) => {
if (acc) return acc;
const match = platforms.reduce((acc, platform) => {
if (acc) return acc;
const patternMatch = new UrlPattern(platform.pattern).match(url);
const patternMatch = new UrlPattern(platform.pattern).match(url);
if (patternMatch) {
return {
platform: platform.label,
original: url,
username: patternMatch.username,
url: platform.format ? platform.format(patternMatch.username) : url,
};
}
if (patternMatch) {
return {
platform: platform.label,
original: url,
username: patternMatch.username,
url: platform.format ? platform.format(patternMatch.username) : url,
};
}
return null;
}, null) || { url };
return null;
}, null) || { url };
return {
url: match.url,
platform: match.platform,
actor_id: actorId,
};
return {
url: match.url,
platform: match.platform,
actor_id: actorId,
};
}
async function curateSocialEntries(urls, actorId) {
if (!urls) {
return [];
}
if (!urls) {
return [];
}
const existingSocialLinks = await knex('actors_social').where('actor_id', actorId);
const existingSocialLinks = await knex('actors_social').where('actor_id', actorId);
return urls.reduce((acc, url) => {
const socialEntry = curateSocialEntry(url, actorId);
return urls.reduce((acc, url) => {
const socialEntry = curateSocialEntry(url, actorId);
if (acc.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase()) || existingSocialLinks.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase())) {
// prevent duplicates
return acc;
}
if (acc.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase()) || existingSocialLinks.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase())) {
// prevent duplicates
return acc;
}
return [...acc, socialEntry];
}, []);
return [...acc, socialEntry];
}, []);
}
async function fetchActors(queryObject, limit = 100) {
const releases = await knex('actors')
.select(
'actors.*',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name', 'residence_countries.alias as residence_country_alias',
)
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('countries as residence_countries', 'actors.residence_country_alpha2', 'residence_countries.alpha2')
.orderBy(['actors.name', 'actors.gender'])
.where(builder => whereOr(queryObject, 'actors', builder))
.limit(limit);
const releases = await knex('actors')
.select(
'actors.*',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name', 'residence_countries.alias as residence_country_alias',
)
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('countries as residence_countries', 'actors.residence_country_alpha2', 'residence_countries.alpha2')
.orderBy(['actors.name', 'actors.gender'])
.where(builder => whereOr(queryObject, 'actors', builder))
.limit(limit);
return curateActors(releases);
return curateActors(releases);
}
async function storeSocialLinks(urls, actorId) {
const curatedSocialEntries = await curateSocialEntries(urls, actorId);
const curatedSocialEntries = await curateSocialEntries(urls, actorId);
await knex('actors_social').insert(curatedSocialEntries);
await knex('actors_social').insert(curatedSocialEntries);
}
async function storeAvatars(avatars, actorId) {
if (!avatars || avatars.length === 0) {
return [];
}
if (!avatars || avatars.length === 0) {
return [];
}
const avatarsBySource = await storeMedia(avatars, 'actor', 'avatar');
await associateMedia({ [actorId]: avatars }, avatarsBySource, 'actor', 'photo', 'avatar');
const avatarsBySource = await storeMedia(avatars, 'actor', 'avatar');
await associateMedia({ [actorId]: avatars }, avatarsBySource, 'actor', 'photo', 'avatar');
return avatarsBySource;
return avatarsBySource;
}
async function storeActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const [actorEntry] = await knex('actors')
.insert(curatedActor)
.returning('*');
const [actorEntry] = await knex('actors')
.insert(curatedActor)
.returning('*');
await storeSocialLinks(actor.social, actorEntry.id);
await storeSocialLinks(actor.social, actorEntry.id);
if (actor.avatars) {
await storeAvatars(actor.avatars, actorEntry.id);
}
if (actor.avatars) {
await storeAvatars(actor.avatars, actorEntry.id);
}
logger.info(`Added new entry for actor '${actor.name}'`);
logger.info(`Added new entry for actor '${actor.name}'`);
return actorEntry;
return actorEntry;
}
async function updateActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const [actorEntry] = await knex('actors')
.where({ id: actor.id })
.update(curatedActor)
.returning('*');
const [actorEntry] = await knex('actors')
.where({ id: actor.id })
.update(curatedActor)
.returning('*');
await storeSocialLinks(actor.social, actor.id);
await storeSocialLinks(actor.social, actor.id);
logger.info(`Updated entry for actor '${actor.name}'`);
logger.info(`Updated entry for actor '${actor.name}'`);
return actorEntry;
return actorEntry;
}
async function mergeProfiles(profiles, actor) {
if (profiles.filter(Boolean).length === 0) {
return null;
}
if (profiles.filter(Boolean).length === 0) {
return null;
}
const mergedProfile = profiles.reduce((prevProfile, profile) => {
if (profile === null) {
return prevProfile;
}
const mergedProfile = profiles.reduce((prevProfile, profile) => {
if (profile === null) {
return prevProfile;
}
const accProfile = {
id: actor ? actor.id : null,
name: actor ? actor.name : (prevProfile.name || profile.name),
description: prevProfile.description || profile.description,
gender: prevProfile.gender || profile.gender,
birthdate: !prevProfile.birthdate || Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
birthPlace: prevProfile.birthPlace || profile.birthPlace,
residencePlace: prevProfile.residencePlace || profile.residencePlace,
nationality: prevProfile.nationality || profile.nationality, // used to derive country when not available
ethnicity: prevProfile.ethnicity || profile.ethnicity,
bust: prevProfile.bust || (/\d+\w+/.test(profile.bust) ? profile.bust : null),
waist: prevProfile.waist || profile.waist,
hip: prevProfile.hip || profile.hip,
naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs,
height: prevProfile.height || profile.height,
weight: prevProfile.weight || profile.weight,
hair: prevProfile.hair || profile.hair,
eyes: prevProfile.eyes || profile.eyes,
hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings,
hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos,
piercings: prevProfile.piercings || profile.piercings,
tattoos: prevProfile.tattoos || profile.tattoos,
social: prevProfile.social.concat(profile.social || []),
releases: prevProfile.releases.concat(profile.releases ? profile.releases : []), // don't flatten fallbacks
};
const accProfile = {
id: actor ? actor.id : null,
name: actor ? actor.name : (prevProfile.name || profile.name),
description: prevProfile.description || profile.description,
gender: prevProfile.gender || profile.gender,
birthdate: !prevProfile.birthdate || Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
birthPlace: prevProfile.birthPlace || profile.birthPlace,
residencePlace: prevProfile.residencePlace || profile.residencePlace,
nationality: prevProfile.nationality || profile.nationality, // used to derive country when not available
ethnicity: prevProfile.ethnicity || profile.ethnicity,
bust: prevProfile.bust || (/\d+\w+/.test(profile.bust) ? profile.bust : null),
waist: prevProfile.waist || profile.waist,
hip: prevProfile.hip || profile.hip,
naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs,
height: prevProfile.height || profile.height,
weight: prevProfile.weight || profile.weight,
hair: prevProfile.hair || profile.hair,
eyes: prevProfile.eyes || profile.eyes,
hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings,
hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos,
piercings: prevProfile.piercings || profile.piercings,
tattoos: prevProfile.tattoos || profile.tattoos,
social: prevProfile.social.concat(profile.social || []),
releases: prevProfile.releases.concat(profile.releases ? profile.releases : []), // don't flatten fallbacks
};
if (profile.avatar) {
const avatar = Array.isArray(profile.avatar)
? profile.avatar.map(avatarX => ({
src: avatarX.src || avatarX,
scraper: profile.scraper,
copyright: avatarX.copyright === undefined ? capitalize(profile.site?.name || profile.scraper) : profile.avatar.copyright,
}))
: {
src: profile.avatar.src || profile.avatar,
scraper: profile.scraper,
copyright: profile.avatar.copyright === undefined ? capitalize(profile.site?.name || profile.scraper) : profile.avatar.copyright,
};
if (profile.avatar) {
const avatar = Array.isArray(profile.avatar)
? profile.avatar.map(avatarX => ({
src: avatarX.src || avatarX,
scraper: profile.scraper,
copyright: avatarX.copyright === undefined ? capitalize(profile.site?.name || profile.scraper) : profile.avatar.copyright,
}))
: {
src: profile.avatar.src || profile.avatar,
scraper: profile.scraper,
copyright: profile.avatar.copyright === undefined ? capitalize(profile.site?.name || profile.scraper) : profile.avatar.copyright,
};
accProfile.avatars = prevProfile.avatars.concat([avatar]); // don't flatten fallbacks
} else {
accProfile.avatars = prevProfile.avatars;
}
accProfile.avatars = prevProfile.avatars.concat([avatar]); // don't flatten fallbacks
} else {
accProfile.avatars = prevProfile.avatars;
}
return accProfile;
}, {
social: [],
avatars: [],
releases: [],
});
return accProfile;
}, {
social: [],
avatars: [],
releases: [],
});
const [birthPlace, residencePlace] = await Promise.all([
resolvePlace(mergedProfile.birthPlace),
resolvePlace(mergedProfile.residencePlace),
]);
const [birthPlace, residencePlace] = await Promise.all([
resolvePlace(mergedProfile.birthPlace),
resolvePlace(mergedProfile.residencePlace),
]);
mergedProfile.birthPlace = birthPlace;
mergedProfile.residencePlace = residencePlace;
mergedProfile.birthPlace = birthPlace;
mergedProfile.residencePlace = residencePlace;
if (!mergedProfile.birthPlace && mergedProfile.nationality) {
const country = await knex('countries')
.where('nationality', 'ilike', `%${mergedProfile.nationality}%`)
.orderBy('priority', 'desc')
.first();
if (!mergedProfile.birthPlace && mergedProfile.nationality) {
const country = await knex('countries')
.where('nationality', 'ilike', `%${mergedProfile.nationality}%`)
.orderBy('priority', 'desc')
.first();
mergedProfile.birthPlace = {
country: country.alpha2,
};
}
mergedProfile.birthPlace = {
country: country.alpha2,
};
}
return mergedProfile;
return mergedProfile;
}
async function scrapeProfiles(sources, actorName, actorEntry, sitesBySlug) {
return Promise.map(sources, async (source) => {
// const [scraperSlug, scraper] = source;
const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] }));
return Promise.map(sources, async (source) => {
// const [scraperSlug, scraper] = source;
const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] }));
try {
return await profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
if (!scraper) {
logger.warn(`No profile profile scraper available for ${scraperSlug}`);
throw Object.assign(new Error(`No profile scraper available for ${scraperSlug}`));
}
try {
return await profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
if (!scraper) {
logger.warn(`No profile profile scraper available for ${scraperSlug}`);
throw Object.assign(new Error(`No profile scraper available for ${scraperSlug}`));
}
logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
const site = sitesBySlug[scraperSlug] || null;
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug, site, include);
const site = sitesBySlug[scraperSlug] || null;
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug, site, include);
if (profile && typeof profile !== 'number') {
logger.verbose(`Found profile for '${actorName}' on ${scraperSlug}`);
if (profile && typeof profile !== 'number') {
logger.verbose(`Found profile for '${actorName}' on ${scraperSlug}`);
return {
...profile,
name: actorName,
scraper: scraperSlug,
site,
releases: profile.releases?.map(release => (typeof release === 'string'
? { url: release, site }
: { ...release, site: release.site || site }
)),
};
}
return {
...profile,
name: actorName,
scraper: scraperSlug,
site,
releases: profile.releases?.map(release => (typeof release === 'string'
? { url: release, site }
: { ...release, site: release.site || site }
)),
};
}
logger.verbose(`No profile for '${actorName}' available on ${scraperSlug}: ${profile}`);
throw Object.assign(new Error(`Profile for ${actorName} not available on ${scraperSlug}`), { warn: false });
}), Promise.reject(new Error()));
} catch (error) {
if (error.warn !== false) {
logger.warn(`Error in scraper ${source}: ${error.message}`);
// logger.error(error.stack);
}
}
logger.verbose(`No profile for '${actorName}' available on ${scraperSlug}: ${profile}`);
throw Object.assign(new Error(`Profile for ${actorName} not available on ${scraperSlug}`), { warn: false });
}), Promise.reject(new Error()));
} catch (error) {
if (error.warn !== false) {
logger.warn(`Error in scraper ${source}: ${error.message}`);
// logger.error(error.stack);
}
}
return null;
});
return null;
});
}
async function scrapeActors(actorNames) {
return Promise.map(actorNames || argv.actors, async (actorName) => {
try {
const actorSlug = slugify(actorName);
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
return Promise.map(actorNames || argv.actors, async (actorName) => {
try {
const actorSlug = slugify(actorName);
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
const finalSources = argv.withReleases ? sources.flat() : sources; // ignore race-to-success grouping when scenes are requested
const finalSources = argv.withReleases ? sources.flat() : sources; // ignore race-to-success grouping when scenes are requested
const [siteEntries, networkEntries] = await Promise.all([
knex('sites')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.whereIn('sites.slug', finalSources.flat()),
knex('networks').select('*').whereIn('slug', finalSources.flat()),
]);
const [siteEntries, networkEntries] = await Promise.all([
knex('sites')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.whereIn('sites.slug', finalSources.flat()),
knex('networks').select('*').whereIn('slug', finalSources.flat()),
]);
const sites = await curateSites(siteEntries, true);
const networks = networkEntries.map(network => ({ ...network, isFallback: true }));
const sitesBySlug = [].concat(networks, sites).reduce((acc, site) => ({ ...acc, [site.slug]: site }), {});
const sites = await curateSites(siteEntries, true);
const networks = networkEntries.map(network => ({ ...network, isFallback: true }));
const sitesBySlug = [].concat(networks, sites).reduce((acc, site) => ({ ...acc, [site.slug]: site }), {});
const profiles = await scrapeProfiles(sources, actorName, actorEntry, sitesBySlug);
const profile = await mergeProfiles(profiles, actorEntry);
const profiles = await scrapeProfiles(sources, actorName, actorEntry, sitesBySlug);
const profile = await mergeProfiles(profiles, actorEntry);
if (profile === null) {
logger.warn(`Could not find profile for actor '${actorName}'`);
if (profile === null) {
logger.warn(`Could not find profile for actor '${actorName}'`);
if (argv.save && !actorEntry) {
await storeActor({ name: actorName }, false, false);
}
if (argv.save && !actorEntry) {
await storeActor({ name: actorName }, false, false);
}
return null;
}
return null;
}
if (argv.inspect) {
console.log(profile);
logger.info(`Found ${profile.releases.length} releases for ${actorName}`);
}
if (argv.inspect) {
console.log(profile);
logger.info(`Found ${profile.releases.length} releases for ${actorName}`);
}
if (argv.save) {
if (actorEntry && profile) {
await Promise.all([
updateActor(profile, true, true),
storeAvatars(profile.avatars, actorEntry.id),
]);
if (argv.save) {
if (actorEntry && profile) {
await Promise.all([
updateActor(profile, true, true),
storeAvatars(profile.avatars, actorEntry.id),
]);
return profile;
}
return profile;
}
await storeActor(profile, true, true);
}
await storeActor(profile, true, true);
}
return profile;
} catch (error) {
console.log(error);
logger.warn(`${actorName}: ${error}`);
return profile;
} catch (error) {
console.log(error);
logger.warn(`${actorName}: ${error}`);
return null;
}
}, {
concurrency: 3,
});
return null;
}
}, {
concurrency: 3,
});
}
async function scrapeBasicActors() {
const basicActors = await knex('actors').where('scraped_at', null);
const basicActors = await knex('actors').where('scraped_at', null);
return scrapeActors(basicActors.map(actor => actor.name));
return scrapeActors(basicActors.map(actor => actor.name));
}
async function associateActors(mappedActors, releases) {
const [existingActorEntries, existingAssociationEntries] = await Promise.all([
knex('actors')
.whereIn('name', Object.values(mappedActors).map(actor => actor.name))
.orWhereIn('slug', Object.keys(mappedActors)),
knex('releases_actors').whereIn('release_id', releases.map(release => release.id)),
]);
const [existingActorEntries, existingAssociationEntries] = await Promise.all([
knex('actors')
.whereIn('name', Object.values(mappedActors).map(actor => actor.name))
.orWhereIn('slug', Object.keys(mappedActors)),
knex('releases_actors').whereIn('release_id', releases.map(release => release.id)),
]);
const associations = await Promise.map(Object.entries(mappedActors), async ([actorSlug, actor]) => {
try {
const actorEntry = existingActorEntries.find(actorX => actorX.slug === actorSlug)
const associations = await Promise.map(Object.entries(mappedActors), async ([actorSlug, actor]) => {
try {
const actorEntry = existingActorEntries.find(actorX => actorX.slug === actorSlug)
|| await storeActor(actor);
// if a scene
return Array.from(actor.releaseIds)
.map(releaseId => ({
release_id: releaseId,
actor_id: actorEntry.id,
}))
.filter(association => !existingAssociationEntries
// remove associations already in database
.some(associationEntry => associationEntry.actor_id === association.actor_id
// if a scene
return Array.from(actor.releaseIds)
.map(releaseId => ({
release_id: releaseId,
actor_id: actorEntry.id,
}))
.filter(association => !existingAssociationEntries
// remove associations already in database
.some(associationEntry => associationEntry.actor_id === association.actor_id
&& associationEntry.release_id === association.release_id));
} catch (error) {
logger.error(actor.name, error);
return null;
}
});
} catch (error) {
logger.error(actor.name, error);
return null;
}
});
await knex('releases_actors').insert(associations.filter(association => association).flat());
await knex('releases_actors').insert(associations.filter(association => association).flat());
// basic actor scraping is failure prone, don't run together with actor association
// await scrapebasicactors(),
// basic actor scraping is failure prone, don't run together with actor association
// await scrapebasicactors(),
}
module.exports = {
associateActors,
fetchActors,
scrapeActors,
scrapeBasicActors,
associateActors,
fetchActors,
scrapeActors,
scrapeBasicActors,
};

193
src/actors.js
View File

@ -1,125 +1,156 @@
'use strict';
const config = require('config');
const Promise = require('bluebird');
// const logger = require('./logger')(__filename);
const knex = require('./knex');
const scrapers = require('./scrapers/scrapers');
const argv = require('./argv');
const slugify = require('./utils/slugify');
const capitalize = require('./utils/capitalize');
function toBaseActors(actorsOrNames, release) {
return actorsOrNames.map((actorOrName) => {
const name = capitalize(actorOrName.name || actorOrName);
const slug = slugify(name);
return actorsOrNames.map((actorOrName) => {
const name = capitalize(actorOrName.name || actorOrName);
const slug = slugify(name);
const baseActor = {
name,
slug,
network: release.site.network,
};
const baseActor = {
name,
slug,
network: release?.site.network,
};
if (actorOrName.name) {
return {
...actorOrName,
...baseActor,
};
}
if (actorOrName.name) {
return {
...actorOrName,
...baseActor,
};
}
return baseActor;
});
return baseActor;
});
}
function curateActorEntry(baseActor, batchId) {
return {
name: baseActor.name,
slug: baseActor.slug,
network_id: null,
batch_id: batchId,
};
return {
name: baseActor.name,
slug: baseActor.slug,
network_id: null,
batch_id: batchId,
};
}
function curateActorEntries(baseActors, batchId) {
return baseActors.map(baseActor => curateActorEntry(baseActor, batchId));
return baseActors.map(baseActor => curateActorEntry(baseActor, batchId));
}
async function scrapeProfiles() {
async function scrapeActors(actorNames) {
const baseActors = toBaseActors(actorNames);
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
const siteSlugs = sources.flat();
const [networks, sites, existingActorEntries] = await Promise.all([
knex('networks').whereIn('slug', siteSlugs),
knex('sites').whereIn('slug', siteSlugs),
knex('actors')
.select(['id', 'name', 'slug'])
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('network_id'),
]);
const existingActorEntriesBySlug = existingActorEntries.reduce((acc, actorEntry) => ({ ...acc, [actorEntry.slug]: actorEntry }), {});
const networksBySlug = networks.reduce((acc, network) => ({ ...acc, [network.slug]: { ...network, isNetwork: true } }), {});
const sitesBySlug = sites.reduce((acc, site) => ({ ...acc, [site.slug]: site }), {});
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlug[baseActor.slug]);
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
const newActorEntries = batchId && await knex('actors').insert(curatedActorEntries).returning(['id', 'name', 'slug']);
const actorEntries = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []);
console.log(actorEntries, newActorEntries, actorEntries);
}
async function getOrCreateActors(baseActors, batchId) {
const existingActors = await knex('actors')
.select('id', 'alias_for', 'name', 'slug', 'network_id')
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('network_id')
.orWhereIn(['slug', 'network_id'], baseActors.map(baseActor => [baseActor.slug, baseActor.network.id]));
const existingActors = await knex('actors')
.select('id', 'alias_for', 'name', 'slug', 'network_id')
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('network_id')
.orWhereIn(['slug', 'network_id'], baseActors.map(baseActor => [baseActor.slug, baseActor.network.id]));
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
const existingActorSlugs = existingActors.reduce((acc, actor) => ({
...acc,
[actor.network_id]: {
...acc[actor.network_id],
[actor.slug]: true,
},
}), {});
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
const existingActorSlugs = existingActors.reduce((acc, actor) => ({
...acc,
[actor.network_id]: {
...acc[actor.network_id],
[actor.slug]: true,
},
}), {});
const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.network.id]?.[baseActor.slug] && !existingActorSlugs.null?.[baseActor.slug]);
const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.network.id]?.[baseActor.slug] && !existingActorSlugs.null?.[baseActor.slug]);
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
const newActors = await knex('actors').insert(curatedActorEntries, ['id', 'alias_for', 'name', 'slug', 'network_id']);
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
const newActors = await knex('actors').insert(curatedActorEntries, ['id', 'alias_for', 'name', 'slug', 'network_id']);
if (Array.isArray(newActors)) {
return newActors.concat(existingActors);
}
if (Array.isArray(newActors)) {
return newActors.concat(existingActors);
}
return existingActors;
return existingActors;
}
async function associateActors(releases, batchId) {
const baseActorsByReleaseId = releases.reduce((acc, release) => {
if (release.actors) {
acc[release.id] = toBaseActors(release.actors, release);
}
const baseActorsByReleaseId = releases.reduce((acc, release) => {
if (release.actors) {
acc[release.id] = toBaseActors(release.actors, release);
}
return acc;
}, {});
return acc;
}, {});
const baseActors = Object.values(baseActorsByReleaseId).flat();
const baseActors = Object.values(baseActorsByReleaseId).flat();
if (baseActors.length === 0) {
return;
}
if (baseActors.length === 0) {
return;
}
const baseActorsBySlugAndNetworkId = baseActors.reduce((acc, baseActor) => ({
...acc,
[baseActor.slug]: {
...acc[baseActor.slug],
[baseActor.network.id]: baseActor,
},
}), {});
const baseActorsBySlugAndNetworkId = baseActors.reduce((acc, baseActor) => ({
...acc,
[baseActor.slug]: {
...acc[baseActor.slug],
[baseActor.network.id]: baseActor,
},
}), {});
const uniqueBaseActors = Object.values(baseActorsBySlugAndNetworkId).map(baseActorsByNetworkId => Object.values(baseActorsByNetworkId)).flat();
const uniqueBaseActors = Object.values(baseActorsBySlugAndNetworkId).map(baseActorsByNetworkId => Object.values(baseActorsByNetworkId)).flat();
const actors = await getOrCreateActors(uniqueBaseActors, batchId);
console.log(actors);
const actorIdsBySlugAndNetworkId = actors.reduce((acc, actor) => ({
...acc,
[actor.network_id]: {
...acc[actor.network_id],
[actor.slug]: actor.alias_for || actor.id,
},
}), {});
const actors = await getOrCreateActors(uniqueBaseActors, batchId);
console.log(actorIdsBySlugAndNetworkId);
const actorIdsBySlugAndNetworkId = actors.reduce((acc, actor) => ({
...acc,
[actor.network_id]: {
...acc[actor.network_id],
[actor.slug]: actor.alias_for || actor.id,
},
}), {});
const releaseActorAssociations = Object.entries(baseActorsByReleaseId)
.map(([releaseId, releaseActors]) => releaseActors
.map(releaseActor => ({
release_id: releaseId,
actor_id: actorIdsBySlugAndNetworkId[releaseActor.network.id]?.[releaseActor.slug] || actorIdsBySlugAndNetworkId.null[releaseActor.slug],
})))
.flat();
const releaseActorAssociations = Object.entries(baseActorsByReleaseId)
.map(([releaseId, releaseActors]) => releaseActors
.map(releaseActor => ({
release_id: releaseId,
actor_id: actorIdsBySlugAndNetworkId[releaseActor.network.id]?.[releaseActor.slug] || actorIdsBySlugAndNetworkId.null[releaseActor.slug],
})))
.flat();
await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
}
module.exports = {
associateActors,
associateActors,
scrapeActors,
};

48
src/app.js
View File

@ -7,39 +7,39 @@ const knex = require('./knex');
const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep');
const { storeReleases, updateReleasesSearch } = require('./store-releases');
const { scrapeActors } = require('./actors-legacy');
const { scrapeActors } = require('./actors');
async function init() {
if (argv.server) {
await initServer();
return;
}
if (argv.server) {
await initServer();
return;
}
if (argv.updateSearch) {
await updateReleasesSearch();
}
if (argv.updateSearch) {
await updateReleasesSearch();
}
if (argv.actors) {
await scrapeActors(argv.actors);
}
if (argv.actors) {
await scrapeActors(argv.actors);
}
const updateBaseScenes = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates();
const updateBaseScenes = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates();
const deepScenes = argv.deep
? await fetchScenes([...(argv.scenes || []), ...(updateBaseScenes || [])])
: updateBaseScenes;
const deepScenes = argv.deep
? await fetchScenes([...(argv.scenes || []), ...(updateBaseScenes || [])])
: updateBaseScenes;
const sceneMovies = deepScenes && argv.sceneMovies && deepScenes.map(scene => scene.movie).filter(Boolean);
const deepMovies = await fetchMovies([...(argv.movies || []), ...(sceneMovies || [])]);
const sceneMovies = deepScenes && argv.sceneMovies && deepScenes.map(scene => scene.movie).filter(Boolean);
const deepMovies = await fetchMovies([...(argv.movies || []), ...(sceneMovies || [])]);
if (argv.save) {
await storeReleases([
...(deepScenes || []),
...(deepMovies || []),
]);
}
if (argv.save) {
await storeReleases([
...(deepScenes || []),
...(deepMovies || []),
]);
}
knex.destroy();
knex.destroy();
}
module.exports = init;

366
src/argv.js
View File

@ -4,188 +4,188 @@ const config = require('config');
const yargs = require('yargs');
const { argv } = yargs
.command('npm start')
.option('server', {
describe: 'Start web server',
type: 'boolean',
alias: 'web',
})
.option('scrape', {
describe: 'Scrape sites and networks defined in configuration',
type: 'boolean',
})
.option('networks', {
describe: 'Networks to scrape (overrides configuration)',
type: 'array',
alias: 'network',
})
.option('sites', {
describe: 'Sites to scrape (overrides configuration)',
type: 'array',
alias: 'site',
})
.option('actors', {
describe: 'Scrape actors by name or slug',
type: 'array',
alias: 'actor',
})
.option('actor-scenes', {
describe: 'Fetch all scenes for an actor',
type: 'boolean',
alias: 'with-releases',
default: false,
})
.option('movie-scenes', {
describe: 'Fetch all scenes for a movie',
type: 'boolean',
alias: 'with-releases',
default: false,
})
.option('scene-movies', {
describe: 'Fetch movies for scenes',
type: 'boolean',
default: true,
})
.option('profiles', {
describe: 'Scrape profiles for new actors after fetching scenes',
type: 'boolean',
alias: 'bios',
default: false,
})
.option('scene', {
describe: 'Scrape scene info from URL',
type: 'array',
alias: 'scenes',
})
.option('movie', {
describe: 'Scrape movie info from URL',
type: 'array',
alias: 'movies',
})
.option('sources', {
describe: 'Use these scrapers for actor data',
type: 'array',
alias: 'source',
})
.option('deep', {
describe: 'Fetch details for all releases',
type: 'boolean',
default: true,
})
.option('latest', {
describe: 'Scrape latest releases if available',
type: 'boolean',
default: true,
})
.option('upcoming', {
describe: 'Scrape upcoming releases if available',
type: 'boolean',
default: true,
})
.option('redownload', {
describe: 'Don\'t ignore duplicates, update existing entries',
type: 'boolean',
alias: 'force',
})
.option('after', {
describe: 'Don\'t fetch scenes older than',
type: 'string',
default: config.fetchAfter.join(' '),
})
.option('last', {
describe: 'Get the latest x releases, no matter the date range',
type: 'number',
})
.option('null-date-limit', {
describe: 'Limit amount of scenes when dates are missing.',
type: 'number',
default: config.nullDateLimit,
alias: 'limit',
})
.option('page', {
describe: 'Page to start scraping at',
type: 'number',
default: 1,
})
.option('save', {
describe: 'Save fetched releases to database',
type: 'boolean',
default: true,
})
.option('media', {
describe: 'Include any release media',
type: 'boolean',
default: true,
})
.option('media-limit', {
describe: 'Maximum amount of assets of each type per release',
type: 'number',
default: config.media.limit,
})
.option('images', {
describe: 'Include any photos, posters or covers',
type: 'boolean',
default: true,
alias: 'pics',
})
.option('videos', {
describe: 'Include any trailers or teasers',
type: 'boolean',
default: true,
})
.option('posters', {
describe: 'Include release posters',
type: 'boolean',
default: true,
alias: 'poster',
})
.option('covers', {
describe: 'Include release covers',
type: 'boolean',
default: true,
alias: 'cover',
})
.option('photos', {
describe: 'Include release photos',
type: 'boolean',
default: true,
})
.option('trailers', {
describe: 'Include release trailers',
type: 'boolean',
default: true,
alias: 'trailer',
})
.option('teasers', {
describe: 'Include release teasers',
type: 'boolean',
default: true,
alias: 'teaser',
})
.option('avatars', {
describe: 'Include actor avatars',
type: 'boolean',
default: true,
})
.option('inspect', {
describe: 'Show data in console.',
type: 'boolean',
default: false,
})
.option('level', {
describe: 'Log level',
type: 'string',
default: process.env.NODE_ENV === 'development' ? 'silly' : 'info',
})
.option('debug', {
describe: 'Show error stack traces',
type: 'boolean',
default: process.env.NODE_ENV === 'development',
})
.option('update-search', {
describe: 'Update search documents for all releases.',
type: 'boolean',
default: false,
});
.command('npm start')
.option('server', {
describe: 'Start web server',
type: 'boolean',
alias: 'web',
})
.option('scrape', {
describe: 'Scrape sites and networks defined in configuration',
type: 'boolean',
})
.option('networks', {
describe: 'Networks to scrape (overrides configuration)',
type: 'array',
alias: 'network',
})
.option('sites', {
describe: 'Sites to scrape (overrides configuration)',
type: 'array',
alias: 'site',
})
.option('actors', {
describe: 'Scrape actors by name or slug',
type: 'array',
alias: 'actor',
})
.option('actor-scenes', {
describe: 'Fetch all scenes for an actor',
type: 'boolean',
alias: 'with-releases',
default: false,
})
.option('movie-scenes', {
describe: 'Fetch all scenes for a movie',
type: 'boolean',
alias: 'with-releases',
default: false,
})
.option('scene-movies', {
describe: 'Fetch movies for scenes',
type: 'boolean',
default: true,
})
.option('profiles', {
describe: 'Scrape profiles for new actors after fetching scenes',
type: 'boolean',
alias: 'bios',
default: false,
})
.option('scene', {
describe: 'Scrape scene info from URL',
type: 'array',
alias: 'scenes',
})
.option('movie', {
describe: 'Scrape movie info from URL',
type: 'array',
alias: 'movies',
})
.option('sources', {
describe: 'Use these scrapers for actor data',
type: 'array',
alias: 'source',
})
.option('deep', {
describe: 'Fetch details for all releases',
type: 'boolean',
default: true,
})
.option('latest', {
describe: 'Scrape latest releases if available',
type: 'boolean',
default: true,
})
.option('upcoming', {
describe: 'Scrape upcoming releases if available',
type: 'boolean',
default: true,
})
.option('redownload', {
describe: 'Don\'t ignore duplicates, update existing entries',
type: 'boolean',
alias: 'force',
})
.option('after', {
describe: 'Don\'t fetch scenes older than',
type: 'string',
default: config.fetchAfter.join(' '),
})
.option('last', {
describe: 'Get the latest x releases, no matter the date range',
type: 'number',
})
.option('null-date-limit', {
describe: 'Limit amount of scenes when dates are missing.',
type: 'number',
default: config.nullDateLimit,
alias: 'limit',
})
.option('page', {
describe: 'Page to start scraping at',
type: 'number',
default: 1,
})
.option('save', {
describe: 'Save fetched releases to database',
type: 'boolean',
default: true,
})
.option('media', {
describe: 'Include any release media',
type: 'boolean',
default: true,
})
.option('media-limit', {
describe: 'Maximum amount of assets of each type per release',
type: 'number',
default: config.media.limit,
})
.option('images', {
describe: 'Include any photos, posters or covers',
type: 'boolean',
default: true,
alias: 'pics',
})
.option('videos', {
describe: 'Include any trailers or teasers',
type: 'boolean',
default: true,
})
.option('posters', {
describe: 'Include release posters',
type: 'boolean',
default: true,
alias: 'poster',
})
.option('covers', {
describe: 'Include release covers',
type: 'boolean',
default: true,
alias: 'cover',
})
.option('photos', {
describe: 'Include release photos',
type: 'boolean',
default: true,
})
.option('trailers', {
describe: 'Include release trailers',
type: 'boolean',
default: true,
alias: 'trailer',
})
.option('teasers', {
describe: 'Include release teasers',
type: 'boolean',
default: true,
alias: 'teaser',
})
.option('avatars', {
describe: 'Include actor avatars',
type: 'boolean',
default: true,
})
.option('inspect', {
describe: 'Show data in console.',
type: 'boolean',
default: false,
})
.option('level', {
describe: 'Log level',
type: 'string',
default: process.env.NODE_ENV === 'development' ? 'silly' : 'info',
})
.option('debug', {
describe: 'Show error stack traces',
type: 'boolean',
default: process.env.NODE_ENV === 'development',
})
.option('update-search', {
describe: 'Update search documents for all releases.',
type: 'boolean',
default: false,
});
module.exports = argv;

217
src/deep.js
View File

@ -11,159 +11,160 @@ const { curateSites } = require('./sites');
const { curateNetworks } = require('./networks');
function urlToSiteSlug(url) {
try {
const slug = new URL(url)
.hostname
.match(/([\w-]+)\.\w+$/)?.[1];
try {
const slug = new URL(url)
.hostname
.match(/([\w-]+)\.\w+$/)?.[1];
return slug;
} catch (error) {
logger.warn(`Failed to derive site slug from '${url}': ${error.message}`);
return slug;
} catch (error) {
logger.warn(`Failed to derive site slug from '${url}': ${error.message}`);
return null;
}
return null;
}
}
async function findSites(baseReleases) {
const baseReleasesWithoutSite = baseReleases.filter(release => release.url && !release.site);
const baseReleasesWithoutSite = baseReleases.filter(release => release.url && !release.site);
const siteSlugs = Array.from(new Set(
baseReleasesWithoutSite
.map(baseRelease => urlToSiteSlug(baseRelease.url))
.filter(Boolean),
));
const siteSlugs = Array.from(new Set(
baseReleasesWithoutSite
.map(baseRelease => urlToSiteSlug(baseRelease.url))
.filter(Boolean),
));
const siteEntries = await knex('sites')
.leftJoin('networks', 'networks.id', 'sites.network_id')
.select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.parameters as network_parameters', 'networks.description as network_description')
.whereIn('sites.slug', siteSlugs);
const siteEntries = await knex('sites')
.leftJoin('networks', 'networks.id', 'sites.network_id')
.select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.parameters as network_parameters', 'networks.description as network_description')
.whereIn('sites.slug', siteSlugs);
const networkEntries = await knex('networks').whereIn('slug', siteSlugs);
const networkEntries = await knex('networks').whereIn('slug', siteSlugs);
const sites = await curateSites(siteEntries, true, false);
const networks = await curateNetworks(networkEntries, true, false, false);
const markedNetworks = networks.map(network => ({ ...network, isFallback: true }));
const sites = await curateSites(siteEntries, true, false);
const networks = await curateNetworks(networkEntries, true, false, false);
const markedNetworks = networks.map(network => ({ ...network, isNetwork: true }));
const sitesBySlug = []
.concat(markedNetworks, sites)
.reduce((accSites, site) => ({ ...accSites, [site.slug]: site }), {});
const sitesBySlug = []
.concat(markedNetworks, sites)
.reduce((accSites, site) => ({ ...accSites, [site.slug]: site }), {});
return sitesBySlug;
return sitesBySlug;
}
function toBaseReleases(baseReleasesOrUrls) {
return baseReleasesOrUrls
.map((baseReleaseOrUrl) => {
if (baseReleaseOrUrl.url) {
// base release with URL
return {
...baseReleaseOrUrl,
deep: false,
};
}
return baseReleasesOrUrls
.map((baseReleaseOrUrl) => {
if (baseReleaseOrUrl.url) {
// base release with URL
return {
...baseReleaseOrUrl,
deep: false,
};
}
if (/^http/.test(baseReleaseOrUrl)) {
// URL
return {
url: baseReleaseOrUrl,
deep: false,
};
}
if (/^http/.test(baseReleaseOrUrl)) {
// URL
return {
url: baseReleaseOrUrl,
deep: false,
};
}
if (typeof baseReleaseOrUrl === 'object' && !Array.isArray(baseReleaseOrUrl)) {
// base release without URL, prepare for passthrough
return {
...baseReleaseOrUrl,
deep: false,
};
}
if (typeof baseReleaseOrUrl === 'object' && !Array.isArray(baseReleaseOrUrl)) {
// base release without URL, prepare for passthrough
return {
...baseReleaseOrUrl,
deep: false,
};
}
logger.warn(`Malformed base release, discarding '${baseReleaseOrUrl}'`);
return null;
})
.filter(Boolean);
logger.warn(`Malformed base release, discarding '${baseReleaseOrUrl}'`);
return null;
})
.filter(Boolean);
}
async function scrapeRelease(baseRelease, sites, type = 'scene') {
const site = baseRelease.site || sites[urlToSiteSlug(baseRelease.url)];
const site = baseRelease.site || sites[urlToSiteSlug(baseRelease.url)];
const siteWithFallbackNetwork = site.isNetwork ? { ...site, network: site } : site; // make site.network available, even when site is network fallback
if (!site) {
logger.warn(`No site available for ${baseRelease.url}`);
return baseRelease;
}
if (!site) {
logger.warn(`No site available for ${baseRelease.url}`);
return baseRelease;
}
if ((!baseRelease.url && !baseRelease.path) || !argv.deep) {
return {
...baseRelease,
site,
};
}
if ((!baseRelease.url && !baseRelease.path) || !argv.deep) {
return {
...baseRelease,
site,
};
}
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
if (!scraper) {
logger.warn(`Could not find scraper for ${baseRelease.url}`);
return baseRelease;
}
if (!scraper) {
logger.warn(`Could not find scraper for ${baseRelease.url}`);
return baseRelease;
}
if ((type === 'scene' && !scraper.fetchScene) || (type === 'movie' && !scraper.fetchMovie)) {
logger.warn(`The '${site.name}'-scraper cannot fetch individual ${type}s`);
return baseRelease;
}
if ((type === 'scene' && !scraper.fetchScene) || (type === 'movie' && !scraper.fetchMovie)) {
logger.warn(`The '${site.name}'-scraper cannot fetch individual ${type}s`);
return baseRelease;
}
try {
logger.verbose(`Fetching ${type} ${baseRelease.url}`);
try {
logger.verbose(`Fetching ${type} ${baseRelease.url}`);
const scrapedRelease = type === 'scene'
? await scraper.fetchScene(baseRelease.url, site, baseRelease, null, include)
: await scraper.fetchMovie(baseRelease.url, site, baseRelease, null, include);
const scrapedRelease = type === 'scene'
? await scraper.fetchScene(baseRelease.url, siteWithFallbackNetwork, baseRelease, null, include)
: await scraper.fetchMovie(baseRelease.url, siteWithFallbackNetwork, baseRelease, null, include);
const mergedRelease = {
...baseRelease,
...scrapedRelease,
deep: !!scrapedRelease,
site,
};
const mergedRelease = {
...baseRelease,
...scrapedRelease,
deep: !!scrapedRelease,
site,
};
if (scrapedRelease && baseRelease?.tags) {
// accumulate all available tags
mergedRelease.tags = baseRelease.tags.concat(scrapedRelease.tags);
}
if (scrapedRelease && baseRelease?.tags) {
// accumulate all available tags
mergedRelease.tags = baseRelease.tags.concat(scrapedRelease.tags);
}
return mergedRelease;
} catch (error) {
logger.error(`Deep scrape failed for ${baseRelease.url}: ${error.message}`);
return baseRelease;
}
return mergedRelease;
} catch (error) {
logger.error(`Deep scrape failed for ${baseRelease.url}: ${error.message}`);
return baseRelease;
}
}
async function scrapeReleases(baseReleases, sites, type) {
return Promise.map(
baseReleases,
async baseRelease => scrapeRelease(baseRelease, sites, type),
{ concurrency: 10 },
);
return Promise.map(
baseReleases,
async baseRelease => scrapeRelease(baseRelease, sites, type),
{ concurrency: 10 },
);
}
async function fetchReleases(baseReleasesOrUrls, type = 'scene') {
const baseReleases = toBaseReleases(baseReleasesOrUrls);
const sites = await findSites(baseReleases);
const baseReleases = toBaseReleases(baseReleasesOrUrls);
const sites = await findSites(baseReleases);
const deepReleases = await scrapeReleases(baseReleases, sites, type);
const deepReleases = await scrapeReleases(baseReleases, sites, type);
return deepReleases;
return deepReleases;
}
async function fetchScenes(baseReleasesOrUrls) {
return fetchReleases(baseReleasesOrUrls, 'scene');
return fetchReleases(baseReleasesOrUrls, 'scene');
}
async function fetchMovies(baseReleasesOrUrls) {
return fetchReleases(baseReleasesOrUrls, 'movie');
return fetchReleases(baseReleasesOrUrls, 'movie');
}
module.exports = {
fetchReleases,
fetchScenes,
fetchMovies,
fetchReleases,
fetchScenes,
fetchMovies,
};

8
src/knex.js
View File

@ -4,8 +4,8 @@ const config = require('config');
const knex = require('knex');
module.exports = knex({
client: 'pg',
connection: config.database,
// performance overhead, don't use asyncStackTraces in production
asyncStackTraces: process.env.NODE_ENV === 'development',
client: 'pg',
connection: config.database,
// performance overhead, don't use asyncStackTraces in production
asyncStackTraces: process.env.NODE_ENV === 'development',
});

48
src/logger.js
View File

@ -9,31 +9,31 @@ require('winston-daily-rotate-file');
const args = require('./argv');
function logger(filepath) {
const root = filepath.match(/src\/|dist\//);
const filename = filepath.slice(root.index + root[0].length)
.replace(path.extname(filepath), '');
const root = filepath.match(/src\/|dist\//);
const filename = filepath.slice(root.index + root[0].length)
.replace(path.extname(filepath), '');
return winston.createLogger({
format: winston.format.combine(
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format(info => (info instanceof Error
? { ...info, message: info.stack }
: { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(),
winston.format.colorize(),
winston.format.printf(({ level, timestamp, label, message }) => `${timestamp} ${level} [${label || filename}] ${message}`),
),
transports: [
new winston.transports.Console({
level: args.level,
timestamp: true,
}),
new winston.transports.DailyRotateFile({
datePattern: 'YYYY-MM-DD',
filename: 'log/%DATE%.log',
level: 'silly',
}),
],
});
return winston.createLogger({
format: winston.format.combine(
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format(info => (info instanceof Error
? { ...info, message: info.stack }
: { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(),
winston.format.colorize(),
winston.format.printf(({ level, timestamp, label, message }) => `${timestamp} ${level} [${label || filename}] ${message}`),
),
transports: [
new winston.transports.Console({
level: args.level,
timestamp: true,
}),
new winston.transports.DailyRotateFile({
datePattern: 'YYYY-MM-DD',
filename: 'log/%DATE%.log',
level: 'silly',
}),
],
});
}
module.exports = logger;

866
src/media.js
View File

File diff suppressed because it is too large Load Diff

98
src/networks.js
View File

@ -5,77 +5,77 @@ const whereOr = require('./utils/where-or');
const { fetchSites } = require('./sites');
async function curateNetwork(network, includeParameters = false, includeSites = true, includeStudios = false) {
const curatedNetwork = {
id: network.id,
name: network.name,
url: network.url,
description: network.description,
slug: network.slug,
parameters: includeParameters ? network.parameters : null,
};
const curatedNetwork = {
id: network.id,
name: network.name,
url: network.url,
description: network.description,
slug: network.slug,
parameters: includeParameters ? network.parameters : null,
};
if (includeSites) {
curatedNetwork.sites = await fetchSites({ network_id: network.id });
}
if (includeSites) {
curatedNetwork.sites = await fetchSites({ network_id: network.id });
}
if (includeStudios) {
const studios = await knex('studios').where({ network_id: network.id });
if (includeStudios) {
const studios = await knex('studios').where({ network_id: network.id });
curatedNetwork.studios = studios.map(studio => ({
id: studio.id,
name: studio.name,
url: studio.url,
description: studio.description,
slug: studio.slug,
}));
}
curatedNetwork.studios = studios.map(studio => ({
id: studio.id,
name: studio.name,
url: studio.url,
description: studio.description,
slug: studio.slug,
}));
}
return curatedNetwork;
return curatedNetwork;
}
function curateNetworks(releases) {
return Promise.all(releases.map(async release => curateNetwork(release)));
return Promise.all(releases.map(async release => curateNetwork(release)));
}
async function findNetworkByUrl(url) {
const { hostname } = new URL(url);
const domain = hostname.replace(/^www./, '');
const { hostname } = new URL(url);
const domain = hostname.replace(/^www./, '');
const network = await knex('networks')
.where('networks.url', 'like', `%${domain}`)
.orWhere('networks.url', url)
.first();
const network = await knex('networks')
.where('networks.url', 'like', `%${domain}`)
.orWhere('networks.url', url)
.first();
if (network) {
return curateNetwork(network, true);
}
if (network) {
return curateNetwork(network, true);
}
return null;
return null;
}
async function fetchNetworks(queryObject) {
const releases = await knex('networks')
.where(builder => whereOr(queryObject, 'networks', builder))
.limit(100);
const releases = await knex('networks')
.where(builder => whereOr(queryObject, 'networks', builder))
.limit(100);
return curateNetworks(releases);
return curateNetworks(releases);
}
async function fetchNetworksFromReleases() {
const releases = await knex('releases')
.select('site_id', '')
.leftJoin('sites', 'sites.id', 'releases.site_id')
.leftJoin('networks', 'networks.id', 'sites.network_id')
.groupBy('networks.id')
.limit(100);
const releases = await knex('releases')
.select('site_id', '')
.leftJoin('sites', 'sites.id', 'releases.site_id')
.leftJoin('networks', 'networks.id', 'sites.network_id')
.groupBy('networks.id')
.limit(100);
return curateNetworks(releases);
return curateNetworks(releases);
}
module.exports = {
curateNetwork,
curateNetworks,
fetchNetworks,
fetchNetworksFromReleases,
findNetworkByUrl,
curateNetwork,
curateNetworks,
fetchNetworks,
fetchNetworksFromReleases,
findNetworkByUrl,
};

714
src/releases-legacy.js
View File

@ -11,356 +11,356 @@ const whereOr = require('./utils/where-or');
const { associateTags } = require('./tags');
const { associateActors, scrapeBasicActors } = require('./actors');
const {
pluckItems,
storeMedia,
associateMedia,
pluckItems,
storeMedia,
associateMedia,
} = require('./media');
const { fetchSites } = require('./sites');
const slugify = require('./utils/slugify');
const capitalize = require('./utils/capitalize');
function commonQuery(queryBuilder, {
filter = [],
after = new Date(0), // January 1970
before = new Date(2 ** 44), // May 2109
limit = 100,
filter = [],
after = new Date(0), // January 1970
before = new Date(2 ** 44), // May 2109
limit = 100,
}) {
const finalFilter = [].concat(filter); // ensure filter is array
const finalFilter = [].concat(filter); // ensure filter is array
queryBuilder
.leftJoin('sites', 'releases.site_id', 'sites.id')
.leftJoin('studios', 'releases.studio_id', 'studios.id')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.select(
'releases.*',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description',
)
.whereNotExists((builder) => {
// apply tag filters
builder
.select('*')
.from('tags_associated')
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
.whereIn('tags.slug', finalFilter)
.where('tags_associated.domain', 'releases')
.whereRaw('tags_associated.target_id = releases.id');
})
.andWhere('releases.date', '>', after)
.andWhere('releases.date', '<=', before)
.orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }])
.limit(limit);
queryBuilder
.leftJoin('sites', 'releases.site_id', 'sites.id')
.leftJoin('studios', 'releases.studio_id', 'studios.id')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.select(
'releases.*',
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description',
)
.whereNotExists((builder) => {
// apply tag filters
builder
.select('*')
.from('tags_associated')
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
.whereIn('tags.slug', finalFilter)
.where('tags_associated.domain', 'releases')
.whereRaw('tags_associated.target_id = releases.id');
})
.andWhere('releases.date', '>', after)
.andWhere('releases.date', '<=', before)
.orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }])
.limit(limit);
}
async function curateRelease(release) {
const [actors, tags, media] = await Promise.all([
knex('actors_associated')
.select(
'actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'actors.birthdate',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'media.thumbnail as avatar',
)
.where({ release_id: release.id })
.leftJoin('actors', 'actors.id', 'actors_associated.actor_id')
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('media', (builder) => {
builder
.on('media.target_id', 'actors.id')
.andOnVal('media.domain', 'actors')
.andOnVal('media.index', '0');
})
.orderBy('actors.gender'),
knex('tags_associated')
.select('tags.name', 'tags.slug')
.where({
domain: 'releases',
target_id: release.id,
})
.leftJoin('tags', 'tags.id', 'tags_associated.tag_id')
.orderBy('tags.priority', 'desc'),
knex('media')
.where({
target_id: release.id,
domain: 'releases',
})
.orderBy(['role', 'index']),
]);
const [actors, tags, media] = await Promise.all([
knex('actors_associated')
.select(
'actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'actors.birthdate',
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
'media.thumbnail as avatar',
)
.where({ release_id: release.id })
.leftJoin('actors', 'actors.id', 'actors_associated.actor_id')
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
.leftJoin('media', (builder) => {
builder
.on('media.target_id', 'actors.id')
.andOnVal('media.domain', 'actors')
.andOnVal('media.index', '0');
})
.orderBy('actors.gender'),
knex('tags_associated')
.select('tags.name', 'tags.slug')
.where({
domain: 'releases',
target_id: release.id,
})
.leftJoin('tags', 'tags.id', 'tags_associated.tag_id')
.orderBy('tags.priority', 'desc'),
knex('media')
.where({
target_id: release.id,
domain: 'releases',
})
.orderBy(['role', 'index']),
]);
const curatedRelease = {
id: release.id,
type: release.type,
title: release.title,
date: release.date,
dateAdded: release.created_at,
description: release.description,
url: release.url,
shootId: release.shoot_id,
entryId: release.entry_id,
actors: actors.map(actor => ({
id: actor.id,
slug: actor.slug,
name: actor.name,
gender: actor.gender,
birthdate: actor.birthdate,
age: moment().diff(actor.birthdate, 'years'),
ageThen: moment(release.date).diff(actor.birthdate, 'years'),
avatar: actor.avatar,
origin: actor.birth_country_alpha2
? {
country: {
name: actor.birth_country_alias,
alpha2: actor.birth_country_alpha2,
},
}
: null,
})),
director: release.director,
tags,
duration: release.duration,
photos: media.filter(item => item.role === 'photo'),
poster: media.filter(item => item.role === 'poster')[0],
covers: media.filter(item => item.role === 'cover'),
trailer: media.filter(item => item.role === 'trailer')[0],
site: {
id: release.site_id,
name: release.site_name,
independent: !!release.site_parameters?.independent,
slug: release.site_slug,
url: release.site_url,
},
studio: release.studio_id
? {
id: release.studio_id,
name: release.studio_name,
slug: release.studio_slug,
url: release.studio_url,
}
: null,
network: {
id: release.network_id,
name: release.network_name,
description: release.network_description,
slug: release.network_slug,
url: release.network_url,
},
};
const curatedRelease = {
id: release.id,
type: release.type,
title: release.title,
date: release.date,
dateAdded: release.created_at,
description: release.description,
url: release.url,
shootId: release.shoot_id,
entryId: release.entry_id,
actors: actors.map(actor => ({
id: actor.id,
slug: actor.slug,
name: actor.name,
gender: actor.gender,
birthdate: actor.birthdate,
age: moment().diff(actor.birthdate, 'years'),
ageThen: moment(release.date).diff(actor.birthdate, 'years'),
avatar: actor.avatar,
origin: actor.birth_country_alpha2
? {
country: {
name: actor.birth_country_alias,
alpha2: actor.birth_country_alpha2,
},
}
: null,
})),
director: release.director,
tags,
duration: release.duration,
photos: media.filter(item => item.role === 'photo'),
poster: media.filter(item => item.role === 'poster')[0],
covers: media.filter(item => item.role === 'cover'),
trailer: media.filter(item => item.role === 'trailer')[0],
site: {
id: release.site_id,
name: release.site_name,
independent: !!release.site_parameters?.independent,
slug: release.site_slug,
url: release.site_url,
},
studio: release.studio_id
? {
id: release.studio_id,
name: release.studio_name,
slug: release.studio_slug,
url: release.studio_url,
}
: null,
network: {
id: release.network_id,
name: release.network_name,
description: release.network_description,
slug: release.network_slug,
url: release.network_url,
},
};
return curatedRelease;
return curatedRelease;
}
function curateReleases(releases) {
return Promise.all(releases.map(async release => curateRelease(release)));
return Promise.all(releases.map(async release => curateRelease(release)));
}
async function attachChannelSite(release) {
if (!release.site?.isFallback && !release.channel?.force) {
return release;
}
if (!release.site?.isFallback && !release.channel?.force) {
return release;
}
if (!release.channel) {
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
}
if (!release.channel) {
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
}
const [site] = await fetchSites({
name: release.channel.name || release.channel,
slug: release.channel.slug || release.channel,
});
const [site] = await fetchSites({
name: release.channel.name || release.channel,
slug: release.channel.slug || release.channel,
});
if (site) {
return {
...release,
site,
};
}
if (site) {
return {
...release,
site,
};
}
throw new Error(`Unable to match channel '${release.channel.slug || release.channel}' from generic URL: ${release.url}`);
throw new Error(`Unable to match channel '${release.channel.slug || release.channel}' from generic URL: ${release.url}`);
}
async function attachStudio(release) {
if (!release.studio) {
return release;
}
if (!release.studio) {
return release;
}
const studio = await knex('studios')
.where('name', release.studio)
.orWhere('slug', release.studio)
.orWhere('url', release.studio)
.first();
const studio = await knex('studios')
.where('name', release.studio)
.orWhere('slug', release.studio)
.orWhere('url', release.studio)
.first();
return {
...release,
studio,
};
return {
...release,
studio,
};
}
async function curateReleaseEntry(release, batchId, existingRelease) {
const slug = slugify(release.title, {
encode: true,
limit: config.titleSlugLength,
});
const slug = slugify(release.title, {
encode: true,
limit: config.titleSlugLength,
});
const curatedRelease = {
site_id: release.site.id,
studio_id: release.studio ? release.studio.id : null,
shoot_id: release.shootId || null,
entry_id: release.entryId || null,
type: release.type,
url: release.url,
title: release.title,
slug,
date: release.date,
description: release.description,
// director: release.director,
duration: release.duration,
// likes: release.rating && release.rating.likes,
// dislikes: release.rating && release.rating.dislikes,
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: typeof release.deep === 'boolean' ? release.deep : false,
deep_url: release.deepUrl,
updated_batch_id: batchId,
...(!existingRelease && { created_batch_id: batchId }),
};
const curatedRelease = {
site_id: release.site.id,
studio_id: release.studio ? release.studio.id : null,
shoot_id: release.shootId || null,
entry_id: release.entryId || null,
type: release.type,
url: release.url,
title: release.title,
slug,
date: release.date,
description: release.description,
// director: release.director,
duration: release.duration,
// likes: release.rating && release.rating.likes,
// dislikes: release.rating && release.rating.dislikes,
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: typeof release.deep === 'boolean' ? release.deep : false,
deep_url: release.deepUrl,
updated_batch_id: batchId,
...(!existingRelease && { created_batch_id: batchId }),
};
return curatedRelease;
return curatedRelease;
}
async function fetchReleases(queryObject = {}, options = {}) {
const releases = await knex('releases')
.modify(commonQuery, options)
.andWhere(builder => whereOr(queryObject, 'releases', builder));
const releases = await knex('releases')
.modify(commonQuery, options)
.andWhere(builder => whereOr(queryObject, 'releases', builder));
return curateReleases(releases);
return curateReleases(releases);
}
async function fetchSiteReleases(queryObject, options = {}) {
const releases = await knex('releases')
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'sites', builder));
const releases = await knex('releases')
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'sites', builder));
return curateReleases(releases);
return curateReleases(releases);
}
async function fetchNetworkReleases(queryObject, options = {}) {
const releases = await knex('releases')
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'networks', builder));
const releases = await knex('releases')
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'networks', builder));
return curateReleases(releases);
return curateReleases(releases);
}
async function fetchActorReleases(queryObject, options = {}) {
const releases = await knex('actors_associated')
.leftJoin('releases', 'actors_associated.release_id', 'releases.id')
.leftJoin('actors', 'actors_associated.actor_id', 'actors.id')
.select(
'actors.name as actor_name',
)
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'actors', builder));
const releases = await knex('actors_associated')
.leftJoin('releases', 'actors_associated.release_id', 'releases.id')
.leftJoin('actors', 'actors_associated.actor_id', 'actors.id')
.select(
'actors.name as actor_name',
)
.modify(commonQuery, options)
.where(builder => whereOr(queryObject, 'actors', builder));
return curateReleases(releases);
return curateReleases(releases);
}
async function fetchTagReleases(queryObject, options = {}) {
const releases = await knex('tags_associated')
.leftJoin('releases', 'tags_associated.target_id', 'releases.id')
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
.select(
'tags.name as tag_name',
)
.modify(commonQuery, options)
.where('tags_associated.domain', 'releases')
.where(builder => whereOr(queryObject, 'tags', builder));
const releases = await knex('tags_associated')
.leftJoin('releases', 'tags_associated.target_id', 'releases.id')
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
.select(
'tags.name as tag_name',
)
.modify(commonQuery, options)
.where('tags_associated.domain', 'releases')
.where(builder => whereOr(queryObject, 'tags', builder));
return curateReleases(releases);
return curateReleases(releases);
}
function accumulateActors(releases) {
return releases.reduce((acc, release) => {
if (!Array.isArray(release.actors)) return acc;
return releases.reduce((acc, release) => {
if (!Array.isArray(release.actors)) return acc;
release.actors.forEach((actor) => {
const actorName = actor.name ? actor.name.trim() : actor.trim();
const actorSlug = slugify(actorName);
release.actors.forEach((actor) => {
const actorName = actor.name ? actor.name.trim() : actor.trim();
const actorSlug = slugify(actorName);
if (!actorSlug) return;
if (!actorSlug) return;
if (!acc[actorSlug]) {
acc[actorSlug] = {
name: actorName,
slug: actorSlug,
releaseIds: new Set(),
avatars: [],
};
}
if (!acc[actorSlug]) {
acc[actorSlug] = {
name: actorName,
slug: actorSlug,
releaseIds: new Set(),
avatars: [],
};
}
acc[actorSlug].releaseIds.add(release.id);
acc[actorSlug].releaseIds.add(release.id);
if (actor.name) acc[actorSlug] = { ...acc[actorSlug], ...actor }; // actor input contains profile info
if (actor.avatar) {
const avatar = Array.isArray(actor.avatar)
? actor.avatar.map(avatarX => ({
src: avatarX.src || avatarX,
copyright: avatarX.copyright === undefined ? capitalize(release.site?.network?.name) : avatarX.copyright,
}))
: {
src: actor.avatar.src || actor.avatar,
copyright: actor.avatar.copyright === undefined ? capitalize(release.site?.network?.name) : actor.avatar.copyright,
};
if (actor.name) acc[actorSlug] = { ...acc[actorSlug], ...actor }; // actor input contains profile info
if (actor.avatar) {
const avatar = Array.isArray(actor.avatar)
? actor.avatar.map(avatarX => ({
src: avatarX.src || avatarX,
copyright: avatarX.copyright === undefined ? capitalize(release.site?.network?.name) : avatarX.copyright,
}))
: {
src: actor.avatar.src || actor.avatar,
copyright: actor.avatar.copyright === undefined ? capitalize(release.site?.network?.name) : actor.avatar.copyright,
};
acc[actorSlug].avatars = acc[actorSlug].avatars.concat([avatar]); // don't flatten fallbacks
}
});
acc[actorSlug].avatars = acc[actorSlug].avatars.concat([avatar]); // don't flatten fallbacks
}
});
return acc;
}, {});
return acc;
}, {});
}
async function storeReleaseAssets(releases) {
if (!argv.media) {
return;
}
if (!argv.media) {
return;
}
const releasePostersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.poster] }), {});
const releaseCoversById = releases.reduce((acc, release) => ({ ...acc, [release.id]: release.covers }), {});
const releaseTrailersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.trailer] }), {});
const releaseTeasersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.teaser] }), {});
const releasePhotosById = releases.reduce((acc, release) => ({
...acc,
[release.id]: pluckItems(release.photos),
}), {});
const releasePostersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.poster] }), {});
const releaseCoversById = releases.reduce((acc, release) => ({ ...acc, [release.id]: release.covers }), {});
const releaseTrailersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.trailer] }), {});
const releaseTeasersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.teaser] }), {});
const releasePhotosById = releases.reduce((acc, release) => ({
...acc,
[release.id]: pluckItems(release.photos),
}), {});
if (argv.images && argv.posters) {
const posters = await storeMedia(Object.values(releasePostersById).flat(), 'release', 'poster');
if (posters) await associateMedia(releasePostersById, posters, 'release', 'poster');
}
if (argv.images && argv.posters) {
const posters = await storeMedia(Object.values(releasePostersById).flat(), 'release', 'poster');
if (posters) await associateMedia(releasePostersById, posters, 'release', 'poster');
}
if (argv.images && argv.covers) {
const covers = await storeMedia(Object.values(releaseCoversById).flat(), 'release', 'cover');
if (covers) await associateMedia(releaseCoversById, covers, 'release', 'cover');
}
if (argv.images && argv.covers) {
const covers = await storeMedia(Object.values(releaseCoversById).flat(), 'release', 'cover');
if (covers) await associateMedia(releaseCoversById, covers, 'release', 'cover');
}
if (argv.images && argv.photos) {
const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
if (photos) await associateMedia(releasePhotosById, photos, 'release', 'photo');
}
if (argv.images && argv.photos) {
const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
if (photos) await associateMedia(releasePhotosById, photos, 'release', 'photo');
}
if (argv.videos && argv.trailers) {
const trailers = await storeMedia(Object.values(releaseTrailersById).flat(), 'release', 'trailer');
if (trailers) await associateMedia(releaseTrailersById, trailers, 'release', 'trailer');
}
if (argv.videos && argv.trailers) {
const trailers = await storeMedia(Object.values(releaseTrailersById).flat(), 'release', 'trailer');
if (trailers) await associateMedia(releaseTrailersById, trailers, 'release', 'trailer');
}
if (argv.videos && argv.teasers) {
const teasers = await storeMedia(Object.values(releaseTeasersById).flat(), 'release', 'teaser');
if (teasers) await associateMedia(releaseTeasersById, teasers, 'release', 'teaser');
}
if (argv.videos && argv.teasers) {
const teasers = await storeMedia(Object.values(releaseTeasersById).flat(), 'release', 'teaser');
if (teasers) await associateMedia(releaseTeasersById, teasers, 'release', 'teaser');
}
}
async function updateReleasesSearch(releaseIds) {
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
const documents = await knex.raw(`
const documents = await knex.raw(`
SELECT
releases.id AS release_id,
TO_TSVECTOR(
@ -391,117 +391,117 @@ async function updateReleasesSearch(releaseIds) {
GROUP BY releases.id, sites.name, sites.slug, sites.alias, sites.url, networks.name, networks.slug, networks.url;
`, releaseIds && [releaseIds]);
if (documents.rows?.length > 0) {
const query = knex('releases_search').insert(documents.rows).toString();
await knex.raw(`${query} ON CONFLICT (release_id) DO UPDATE SET document = EXCLUDED.document`);
}
if (documents.rows?.length > 0) {
const query = knex('releases_search').insert(documents.rows).toString();
await knex.raw(`${query} ON CONFLICT (release_id) DO UPDATE SET document = EXCLUDED.document`);
}
}
async function storeRelease(release, batchId) {
if (!release.site) {
throw new Error(`Missing site, unable to store "${release.title}" (${release.url})`);
}
if (!release.site) {
throw new Error(`Missing site, unable to store "${release.title}" (${release.url})`);
}
if (!release.entryId) {
logger.warn(`Missing entry ID, unable to store "${release.title}" (${release.url})`);
return null;
}
if (!release.entryId) {
logger.warn(`Missing entry ID, unable to store "${release.title}" (${release.url})`);
return null;
}
const existingRelease = await knex('releases')
.where({
entry_id: release.entryId,
site_id: release.site.id,
})
.first();
const existingRelease = await knex('releases')
.where({
entry_id: release.entryId,
site_id: release.site.id,
})
.first();
const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
if (existingRelease && !argv.redownload) {
return existingRelease;
}
if (existingRelease && !argv.redownload) {
return existingRelease;
}
if (existingRelease && argv.redownload) {
const [updatedRelease] = await knex('releases')
.where('id', existingRelease.id)
.update({
...existingRelease,
...curatedRelease,
})
.returning('*');
if (existingRelease && argv.redownload) {
const [updatedRelease] = await knex('releases')
.where('id', existingRelease.id)
.update({
...existingRelease,
...curatedRelease,
})
.returning('*');
if (updatedRelease) {
await associateTags(release, updatedRelease.id);
logger.info(`Updated release "${release.title}" (${existingRelease.id}, ${release.site.name})`);
}
if (updatedRelease) {
await associateTags(release, updatedRelease.id);
logger.info(`Updated release "${release.title}" (${existingRelease.id}, ${release.site.name})`);
}
await associateTags(release, existingRelease.id);
await associateTags(release, existingRelease.id);
return existingRelease;
}
return existingRelease;
}
const [releaseEntry] = await knex('releases')
.insert(curatedRelease)
.returning('*');
const [releaseEntry] = await knex('releases')
.insert(curatedRelease)
.returning('*');
await associateTags(release, releaseEntry.id);
await associateTags(release, releaseEntry.id);
logger.info(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`);
logger.info(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`);
return releaseEntry;
return releaseEntry;
}
async function storeReleases(releases) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const storedReleases = await Promise.map(releases, async (release) => {
try {
const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const storedRelease = await storeRelease(releaseWithStudio, batchId);
const storedReleases = await Promise.map(releases, async (release) => {
try {
const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const storedRelease = await storeRelease(releaseWithStudio, batchId);
return storedRelease && {
id: storedRelease.id,
slug: storedRelease.slug,
...releaseWithChannelSite,
};
} catch (error) {
logger.error(error);
return storedRelease && {
id: storedRelease.id,
slug: storedRelease.slug,
...releaseWithChannelSite,
};
} catch (error) {
logger.error(error);
return null;
}
}, {
concurrency: 10,
}).filter(Boolean);
return null;
}
}, {
concurrency: 10,
}).filter(Boolean);
logger.info(`Stored ${storedReleases.length} new releases`);
logger.info(`Stored ${storedReleases.length} new releases`);
const actors = accumulateActors(storedReleases);
const actors = accumulateActors(storedReleases);
await associateActors(actors, storedReleases);
await associateActors(actors, storedReleases);
await Promise.all([
// actors need to be stored before generating search
updateReleasesSearch(storedReleases.map(release => release.id)),
storeReleaseAssets(storedReleases),
]);
await Promise.all([
// actors need to be stored before generating search
updateReleasesSearch(storedReleases.map(release => release.id)),
storeReleaseAssets(storedReleases),
]);
if (argv.withProfiles && Object.keys(actors).length > 0) {
await scrapeBasicActors();
}
if (argv.withProfiles && Object.keys(actors).length > 0) {
await scrapeBasicActors();
}
return {
releases: storedReleases,
actors,
};
return {
releases: storedReleases,
actors,
};
}
module.exports = {
fetchReleases,
fetchActorReleases,
fetchSiteReleases,
fetchNetworkReleases,
fetchTagReleases,
storeRelease,
storeReleases,
updateReleasesSearch,
fetchReleases,
fetchActorReleases,
fetchSiteReleases,
fetchNetworkReleases,
fetchTagReleases,
storeRelease,
storeReleases,
updateReleasesSearch,
};

12
src/releases.js
View File

@ -3,18 +3,18 @@
const knex = require('./knex');
async function fetchReleases(limit = 100) {
const releases = await knex('releases').limit(limit);
const releases = await knex('releases').limit(limit);
return releases;
return releases;
}
async function searchReleases(query, limit = 100) {
const releases = await knex.raw('SELECT * FROM search_releases(?) LIMIT ?;', [query, limit]);
const releases = await knex.raw('SELECT * FROM search_releases(?) LIMIT ?;', [query, limit]);
return releases.rows;
return releases.rows;
}
module.exports = {
fetchReleases,
searchReleases,
fetchReleases,
searchReleases,
};

199
src/scrape-releases.js
View File

@ -1,199 +0,0 @@
'use strict';
const config = require('config');
const Promise = require('bluebird');
const logger = require('./logger')(__filename);
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const knex = require('./knex');
const scrapers = require('./scrapers/scrapers');
const { findSiteByUrl } = require('./sites');
const { findNetworkByUrl } = require('./networks');
const { storeReleases } = require('./releases');
async function findSite(url, release) {
if (release?.site) return release.site;
if (!url) return null;
const site = await findSiteByUrl(url);
if (site) {
return site;
}
const network = await findNetworkByUrl(url);
if (network) {
return {
...network,
network,
isFallback: true,
};
}
return null;
}
async function scrapeRelease(source, basicRelease = null, type = 'scene', beforeFetchLatest) {
// profile scraper may return either URLs or pre-scraped scenes
const sourceIsUrlOrEmpty = typeof source === 'string' || source === undefined;
const url = sourceIsUrlOrEmpty ? source : source?.url;
const release = sourceIsUrlOrEmpty ? basicRelease : source;
const site = basicRelease?.site || await findSite(url, release);
if (!site) {
throw new Error(`Could not find site for ${url} in database`);
}
if (!argv.deep && release) {
return {
...release,
site,
};
}
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
if (!scraper) {
throw new Error(`Could not find scraper for ${url}`);
}
if ((type === 'scene' && !scraper.fetchScene) || (type === 'movie' && !scraper.fetchMovie)) {
if (release) {
logger.warn(`The '${site.name}'-scraper cannot fetch individual ${type}s`);
return null;
}
throw new Error(`The '${site.name}'-scraper cannot fetch individual ${type}s`);
}
if (!release) {
logger.info(`Scraping release from ${url}`);
}
const scrapedRelease = type === 'scene'
? await scraper.fetchScene(url, site, release, beforeFetchLatest, include)
: await scraper.fetchMovie(url, site, release, beforeFetchLatest, include);
return {
...release,
...scrapedRelease,
...(scrapedRelease && release?.tags && {
tags: release.tags.concat(scrapedRelease.tags),
}),
site,
};
}
async function accumulateMovies(releases) {
if (!argv.withMovies) return [];
const moviesByUrl = releases.reduce((acc, release) => {
if (!release.movie) return acc;
const movie = release.movie.url ? release.movie : { url: release.movie };
if (!acc[movie.url]) {
acc[movie.url] = {
...movie,
type: 'movie',
sceneIds: [],
};
}
acc[movie.url].sceneIds = acc[movie.url].sceneIds.concat(release.id);
return acc;
}, {});
const movies = await Promise.map(Object.values(moviesByUrl), async movie => scrapeRelease(movie, null, 'movie'));
const { releases: storedMovies } = await storeReleases(movies);
const movieAssociations = storedMovies.reduce((acc, movie) => acc.concat(movie.sceneIds.map(sceneId => ({
movie_id: movie.id,
scene_id: sceneId,
}))), []);
await knex('releases_movies').insert(movieAssociations);
// console.log(moviesByUrl);
return movies;
}
async function scrapeReleases(sources, type = 'scene') {
const scrapedReleases = await Promise.map(sources, async source => scrapeRelease(source, null, type), {
concurrency: 5,
}).filter(Boolean);
const curatedReleases = scrapedReleases.map(scrapedRelease => ({ ...scrapedRelease, type }));
if ((argv.scene || argv.movie) && argv.inspect) {
// only show when fetching from URL
}
if (argv.save) {
const { releases: storedReleases } = await storeReleases(curatedReleases);
await accumulateMovies(storedReleases);
if (storedReleases) {
logger.info(storedReleases.map(storedRelease => `\nhttp://${config.web.host}:${config.web.port}/scene/${storedRelease.id}/${storedRelease.slug}`).join(''));
}
return storedReleases;
}
return curatedReleases;
}
async function scrapeScenes(sources) {
return scrapeReleases(sources, 'scene');
}
async function scrapeMovies(sources) {
return scrapeReleases(sources, 'movie');
}
async function deepFetchReleases(baseReleases, beforeFetchLatest) {
const deepReleases = await Promise.map(baseReleases, async (release) => {
if (release.url || (release.path && release.site)) {
try {
const fullRelease = await scrapeRelease(release.url, release, 'scene', beforeFetchLatest);
if (fullRelease) {
return {
...release,
...fullRelease,
deep: true,
};
}
logger.warn(`Release scraper returned empty result for ${release.url}`);
return release;
} catch (error) {
logger.error(`Failed to scrape ${release.url}: ${error}`);
return {
...release,
deep: false,
};
}
}
return release;
}, {
concurrency: 2,
});
return deepReleases;
}
module.exports = {
deepFetchReleases,
scrapeMovies,
scrapeRelease,
scrapeReleases,
scrapeScenes,
};

184
src/scrape-sites.js
View File

@ -1,184 +0,0 @@
'use strict';
const Promise = require('bluebird');
const moment = require('moment');
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const logger = require('./logger')(__filename);
const knex = require('./knex');
const { fetchIncludedSites } = require('./sites');
const scrapers = require('./scrapers/scrapers');
const { deepFetchReleases } = require('./scrape-releases');
const { storeReleases } = require('./releases');
function getAfterDate() {
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
// using date
return moment
.utc(argv.after, ['YYYY-MM-DD', 'DD-MM-YYYY'])
.toDate();
}
// using time distance (e.g. "1 month")
return moment
.utc()
.subtract(...argv.after.split(' '))
.toDate();
}
async function findDuplicateReleaseIds(latestReleases, accReleases) {
const duplicateReleases = await knex('releases')
.whereIn('entry_id', latestReleases.map(({ entryId }) => entryId));
// include accumulated releases as duplicates to prevent an infinite
// loop when the next page contains the same releases as the previous
return new Set(duplicateReleases
.map(release => String(release.entry_id))
.concat(accReleases.map(release => String(release.entryId))));
}
async function scrapeUniqueReleases(scraper, site, beforeFetchLatest, accSiteReleases, afterDate = getAfterDate(), accReleases = [], page = argv.page) {
if (!argv.latest || !scraper.fetchLatest) {
return [];
}
const latestReleases = await scraper.fetchLatest(site, page, beforeFetchLatest, accSiteReleases, include);
if (!Array.isArray(latestReleases)) {
logger.warn(`Scraper returned ${latestReleases || 'null'} when fetching latest from '${site.name}' on '${site.network.name}'`);
return accReleases;
}
if (latestReleases.length === 0) {
return accReleases;
}
const latestReleasesWithSite = latestReleases.map(release => ({ ...release, site }));
const oldestReleaseOnPage = latestReleases.slice(-1)[0].date;
const duplicateReleaseIds = argv.redownload ? new Set() : await findDuplicateReleaseIds(latestReleases, accReleases);
const uniqueReleases = latestReleasesWithSite
.filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
&& (argv.last || !release.date || moment(release.date).isAfter(afterDate))); // release is older than specified date limit
logger.verbose(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
if (
uniqueReleases.length > 0
// && (oldestReleaseOnPage || page < argv.pages)
&& ((oldestReleaseOnPage
? moment(oldestReleaseOnPage).isAfter(afterDate)
: accReleases.length + uniqueReleases.length <= argv.nullDateLimit)
|| (argv.last && accReleases.length + uniqueReleases.length < argv.last))
) {
// oldest release on page is newer that specified date range, or latest count has not yet been met, fetch next page
return scrapeUniqueReleases(scraper, site, beforeFetchLatest, accSiteReleases, afterDate, accReleases.concat(uniqueReleases), page + 1);
}
if (argv.last && uniqueReleases.length >= argv.last) {
return accReleases.concat(uniqueReleases).slice(0, argv.last);
}
if (oldestReleaseOnPage) {
return accReleases.concat(uniqueReleases);
}
return accReleases.concat(uniqueReleases).slice(0, argv.nullDateLimit);
}
async function scrapeUpcomingReleases(scraper, site, beforeFetchLatest) {
if (argv.upcoming && scraper.fetchUpcoming) {
const upcomingReleases = await scraper.fetchUpcoming(site, 1, beforeFetchLatest, include);
return upcomingReleases
? upcomingReleases.map(release => ({ ...release, site, upcoming: true }))
: [];
}
return [];
}
async function scrapeSiteReleases(scraper, site, accSiteReleases) {
const beforeFetchLatest = await scraper.beforeFetchLatest?.(site, accSiteReleases);
const [newReleases, upcomingReleases] = await Promise.all([
scrapeUniqueReleases(scraper, site, beforeFetchLatest, accSiteReleases), // fetch basic release info from scene overview
scrapeUpcomingReleases(scraper, site, beforeFetchLatest, accSiteReleases), // fetch basic release info from upcoming overview
]);
if (argv.upcoming) {
logger.info(`${site.name}: ${argv.latest ? `Found ${newReleases.length}` : 'Ignoring'} latest releases,${argv.upcoming ? ' ' : ' ignoring '}${upcomingReleases.length || '0'} upcoming releases`);
}
const baseReleases = [...newReleases, ...upcomingReleases];
if (argv.deep) {
// follow URL for every release
return deepFetchReleases(baseReleases, beforeFetchLatest);
}
return baseReleases;
}
async function scrapeSite(site, network, accSiteReleases = []) {
if (site.parameters?.ignore) {
logger.warn(`Ignoring ${network.name}: ${site.name}`);
return [];
}
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
if (!scraper) {
logger.warn(`No scraper found for '${site.name}' (${site.slug})`);
return [];
}
try {
const siteReleases = await scrapeSiteReleases(scraper, site, accSiteReleases);
return siteReleases.map(release => ({ ...release, site }));
} catch (error) {
logger.error(`${site.name}: Failed to scrape releases: ${error.message}`);
return [];
}
}
async function scrapeSites() {
const networks = await fetchIncludedSites();
const scrapedNetworks = await Promise.map(networks, async (network) => {
if (network.parameters?.sequential) {
logger.info(`Scraping '${network.name}' sequentially`);
return Promise.reduce(network.sites, async (acc, site) => {
const accSiteReleases = await acc;
const siteReleases = await scrapeSite(site, network, accSiteReleases);
return accSiteReleases.concat(siteReleases);
}, Promise.resolve([]));
}
return Promise.map(network.sites, async site => scrapeSite(site, network), {
concurrency: network.parameters?.concurrency || 2,
});
},
{
// 5 networks at a time
concurrency: 5,
});
const releases = scrapedNetworks.flat(2);
if (argv.inspect) {
console.log(releases);
}
if (argv.save) {
await storeReleases(releases);
}
}
module.exports = scrapeSites;

8
src/scrapers/21naturals.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = {
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming,
fetchScene,
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming,
fetchScene,
};

8
src/scrapers/21sextreme.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = {
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming,
fetchScene,
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming,
fetchScene,
};

8
src/scrapers/21sextury.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = {
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming,
fetchScene,
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming,
fetchScene,
};

34
src/scrapers/adulttime.js
View File

@ -3,37 +3,37 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
function curateRelease(release, site) {
if (['bubblegumdungeon', 'ladygonzo'].includes(site.slug)) {
return {
...release,
title: release.title.split(/:|\|/)[1].trim(),
};
}
if (['bubblegumdungeon', 'ladygonzo'].includes(site.slug)) {
return {
...release,
title: release.title.split(/:|\|/)[1].trim(),
};
}
return release;
return release;
}
async function networkFetchScene(url, site, release) {
const scene = await fetchScene(url, site, release);
const scene = await fetchScene(url, site, release);
return curateRelease(scene, site);
return curateRelease(scene, site);
}
async function fetchLatest(site, page = 1) {
const releases = await fetchApiLatest(site, page, false);
const releases = await fetchApiLatest(site, page, false);
return releases.map(release => curateRelease(release, site));
return releases.map(release => curateRelease(release, site));
}
async function fetchUpcoming(site, page = 1) {
const releases = await fetchApiUpcoming(site, page, false);
const releases = await fetchApiUpcoming(site, page, false);
return releases.map(release => curateRelease(release, site));
return releases.map(release => curateRelease(release, site));
}
module.exports = {
fetchLatest,
fetchProfile: fetchApiProfile,
fetchScene: networkFetchScene,
fetchUpcoming,
fetchLatest,
fetchProfile: fetchApiProfile,
fetchScene: networkFetchScene,
fetchUpcoming,
};

52
src/scrapers/amateurallure.js
View File

@ -3,47 +3,47 @@
const { fetchLatest, fetchScene } = require('./julesjordan');
function extractActors(scene) {
const release = scene;
const release = scene;
if (!scene.actors || scene.actors.length === 0) {
const introActorMatches = scene.title.match(/(?:presents|introduces|features|welcomes) (\w+ \w+)/i);
const introTwoActorMatches = scene.title.match(/(?:presents|introduces|features|welcomes) (?:(\w+)|(\w+ \w+)) and (\w+ \w+)/i);
const returnActorMatches = scene.title.match(/(?:(^\w+)|(\w+ \w+))(?:,| (?:return|visit|pov|give|suck|lick|milk|love|enjoy|service|is))/i);
const returnTwoActorMatches = scene.title.match(/(\w+ \w+) and (?:(\w+)|(\w+ \w+)) (?:return|visit|give|suck|lick|milk|love|enjoy|service|are)/i);
if (!scene.actors || scene.actors.length === 0) {
const introActorMatches = scene.title.match(/(?:presents|introduces|features|welcomes) (\w+ \w+)/i);
const introTwoActorMatches = scene.title.match(/(?:presents|introduces|features|welcomes) (?:(\w+)|(\w+ \w+)) and (\w+ \w+)/i);
const returnActorMatches = scene.title.match(/(?:(^\w+)|(\w+ \w+))(?:,| (?:return|visit|pov|give|suck|lick|milk|love|enjoy|service|is))/i);
const returnTwoActorMatches = scene.title.match(/(\w+ \w+) and (?:(\w+)|(\w+ \w+)) (?:return|visit|give|suck|lick|milk|love|enjoy|service|are)/i);
const rawActors = (introTwoActorMatches || introActorMatches || returnTwoActorMatches || returnActorMatches)?.slice(1);
const actors = rawActors?.filter((actor) => {
if (!actor) return false;
if (/swallow|\bcum|fuck|suck|give|giving|take|takes|taking|head|teen|babe|cute|beaut|naughty|teacher|nanny|adorable|brunette|blonde|bust|audition|from|\band\b|\bto\b/i.test(actor)) return false;
const rawActors = (introTwoActorMatches || introActorMatches || returnTwoActorMatches || returnActorMatches)?.slice(1);
const actors = rawActors?.filter((actor) => {
if (!actor) return false;
if (/swallow|\bcum|fuck|suck|give|giving|take|takes|taking|head|teen|babe|cute|beaut|naughty|teacher|nanny|adorable|brunette|blonde|bust|audition|from|\band\b|\bto\b/i.test(actor)) return false;
return true;
});
return true;
});
if (actors) {
release.actors = actors;
}
}
if (actors) {
release.actors = actors;
}
}
if (release.actors?.length > 1 || /threesome|threeway/.test(scene.title)) {
release.tags = scene.tags ? [...scene.tags, 'mff'] : ['mff'];
}
if (release.actors?.length > 1 || /threesome|threeway/.test(scene.title)) {
release.tags = scene.tags ? [...scene.tags, 'mff'] : ['mff'];
}
return release;
return release;
}
async function fetchLatestWrap(site, page = 1) {
const latest = await fetchLatest(site, page);
const latest = await fetchLatest(site, page);
return latest.map(scene => extractActors(scene));
return latest.map(scene => extractActors(scene));
}
async function fetchSceneWrap(url, site) {
const scene = await fetchScene(url, site);
const scene = await fetchScene(url, site);
return extractActors(scene);
return extractActors(scene);
}
module.exports = {
fetchLatest: fetchLatestWrap,
fetchScene: fetchSceneWrap,
fetchLatest: fetchLatestWrap,
fetchScene: fetchSceneWrap,
};

148
src/scrapers/assylum.js
View File

@ -3,7 +3,7 @@
const { get, geta, ctxa } = require('../utils/q');
function extractActors(actorString) {
return actorString
return actorString
?.replace(/.*:|\(.*\)|\d+(-|\s)year(-|\s)old|nurses?|tangled/ig, '') // remove Patient:, (date) and other nonsense
.split(/\band\b|\bvs\b|\/|,|&/ig)
.map(actor => actor.trim())
@ -12,120 +12,120 @@ function extractActors(actorString) {
}
function matchActors(actorString, models) {
return models
.filter(model => new RegExp(model.name, 'i')
.test(actorString));
return models
.filter(model => new RegExp(model.name, 'i')
.test(actorString));
}
function scrapeLatest(scenes, site, models) {
return scenes.map(({ qu }) => {
const release = {};
return scenes.map(({ qu }) => {
const release = {};
const pathname = qu.url('a.itemimg').slice(1);
[release.entryId] = pathname.split('/').slice(-1);
release.url = `${site.url}${pathname}`;
const pathname = qu.url('a.itemimg').slice(1);
[release.entryId] = pathname.split('/').slice(-1);
release.url = `${site.url}${pathname}`;
release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
const actorString = qu.q('.mas_description', true);
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
const actorString = qu.q('.mas_description', true);
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
const posterPath = qu.img('.itemimg img');
release.poster = `${site.url}/${posterPath}`;
const posterPath = qu.img('.itemimg img');
release.poster = `${site.url}/${posterPath}`;
return release;
});
return release;
});
}
function scrapeScene({ html, qu }, url, site, models) {
const release = { url };
const release = { url };
[release.entryId] = url.split('/').slice(-1);
release.title = qu.q('.mas_title', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
[release.entryId] = url.split('/').slice(-1);
release.title = qu.q('.mas_title', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
release.tags = qu.all('.tags a', true);
release.tags = qu.all('.tags a', true);
release.photos = qu.imgs('.stills img').map(photoPath => `${site.url}/${photoPath}`);
release.photos = qu.imgs('.stills img').map(photoPath => `${site.url}/${photoPath}`);
const posterIndex = 'splash:';
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
if (poster) release.poster = `${site.url}/${poster}`;
const posterIndex = 'splash:';
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
if (poster) release.poster = `${site.url}/${poster}`;
const trailerIndex = html.indexOf('video/mp4');
const trailer = html.slice(html.indexOf('/content', trailerIndex), html.indexOf('.mp4', trailerIndex) + 4);
if (trailer) release.trailer = { src: `${site.url}${trailer}` };
const trailerIndex = html.indexOf('video/mp4');
const trailer = html.slice(html.indexOf('/content', trailerIndex), html.indexOf('.mp4', trailerIndex) + 4);
if (trailer) release.trailer = { src: `${site.url}${trailer}` };
return release;
return release;
}
function extractModels({ el }, site) {
const models = ctxa(el, '.item');
const models = ctxa(el, '.item');
return models.map(({ qu }) => {
const actor = { gender: 'female' };
return models.map(({ qu }) => {
const actor = { gender: 'female' };
const avatar = qu.q('.itemimg img');
actor.avatar = `${site.url}/${avatar.src}`;
actor.name = avatar.alt
.split(':').slice(-1)[0]
.replace(/xtreme girl|nurse/ig, '')
.trim();
const avatar = qu.q('.itemimg img');
actor.avatar = `${site.url}/${avatar.src}`;
actor.name = avatar.alt
.split(':').slice(-1)[0]
.replace(/xtreme girl|nurse/ig, '')
.trim();
const actorPath = qu.url('.itemimg');
actor.url = `${site.url}${actorPath.slice(1)}`;
const actorPath = qu.url('.itemimg');
actor.url = `${site.url}${actorPath.slice(1)}`;
return actor;
});
return actor;
});
}
async function fetchModels(site, page = 1, accModels = []) {
const url = `${site.url}/?models/${page}`;
const res = await get(url);
const url = `${site.url}/?models/${page}`;
const res = await get(url);
if (res.ok) {
const models = extractModels(res.item, site);
const nextPage = res.item.qa('.pagenumbers', true)
.map(pageX => Number(pageX))
.filter(Boolean) // remove << and >>
.includes(page + 1);
if (res.ok) {
const models = extractModels(res.item, site);
const nextPage = res.item.qa('.pagenumbers', true)
.map(pageX => Number(pageX))
.filter(Boolean) // remove << and >>
.includes(page + 1);
if (nextPage) {
return fetchModels(site, page + 1, accModels.concat(models));
}
if (nextPage) {
return fetchModels(site, page + 1, accModels.concat(models));
}
return accModels.concat(models, { name: 'Dr. Gray' });
}
return accModels.concat(models, { name: 'Dr. Gray' });
}
return [];
return [];
}
async function fetchLatest(site, page = 1, models) {
const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
const res = await geta(url, '.item');
const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
const res = await geta(url, '.item');
return res.ok ? scrapeLatest(res.items, site, models) : res.status;
return res.ok ? scrapeLatest(res.items, site, models) : res.status;
}
async function fetchScene(url, site, release, beforeFetchLatest) {
const models = beforeFetchLatest || await fetchModels(site);
const res = await get(url);
const models = beforeFetchLatest || await fetchModels(site);
const res = await get(url);
return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
}
module.exports = {
fetchLatest,
fetchScene,
beforeFetchLatest: fetchModels,
fetchLatest,
fetchScene,
beforeFetchLatest: fetchModels,
};

164
src/scrapers/aziani.js
View File

@ -5,141 +5,141 @@ const { get, getAll, initAll, extractDate } = require('../utils/qu');
const { feetInchesToCm } = require('../utils/convert');
function getFallbacks(source) {
return [
source.replace('-1x.jpg', '-4x.jpg'),
source.replace('-1x.jpg', '-3x.jpg'),
source.replace('-1x.jpg', '-2x.jpg'),
source,
];
return [
source.replace('-1x.jpg', '-4x.jpg'),
source.replace('-1x.jpg', '-3x.jpg'),
source.replace('-1x.jpg', '-2x.jpg'),
source,
];
}
function scrapeAll(scenes, site) {
return scenes.map(({ qu }) => {
const release = {};
return scenes.map(({ qu }) => {
const release = {};
release.entryId = qu.q('.stdimage', 'id', true).match(/set-target-(\d+)/)[1];
release.url = qu.url('a');
release.entryId = qu.q('.stdimage', 'id', true).match(/set-target-(\d+)/)[1];
release.url = qu.url('a');
release.title = qu.q('h5 a', true);
release.date = qu.date('.icon-calendar + strong', 'MM/DD/YYYY');
release.title = qu.q('h5 a', true);
release.date = qu.date('.icon-calendar + strong', 'MM/DD/YYYY');
release.actors = qu.q('h3', true).replace(/featuring:\s?/i, '').split(', ');
release.actors = qu.q('h3', true).replace(/featuring:\s?/i, '').split(', ');
const photoCount = qu.q('.stdimage', 'cnt');
[release.poster, ...release.photos] = Array.from({ length: Number(photoCount) }, (value, index) => {
const source = qu.img('.stdimage', `src${index}_1x`, site.url);
const photoCount = qu.q('.stdimage', 'cnt');
[release.poster, ...release.photos] = Array.from({ length: Number(photoCount) }, (value, index) => {
const source = qu.img('.stdimage', `src${index}_1x`, site.url);
return getFallbacks(source);
});
return getFallbacks(source);
});
return release;
});
return release;
});
}
function scrapeScene({ html, qu }, url) {
const release = { url };
const release = { url };
release.entryId = qu.q('.stdimage', 'id', true).match(/set-target-(\d+)/)[1];
release.entryId = qu.q('.stdimage', 'id', true).match(/set-target-(\d+)/)[1];
release.title = qu.q('h2', true);
release.description = qu.q('p', true);
release.title = qu.q('h2', true);
release.description = qu.q('p', true);
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
release.actors = qu.all('h5:not(.video_categories) a').map(actor => ({
name: qu.q(actor, null, true),
url: qu.url(actor, null),
}));
release.actors = qu.all('h5:not(.video_categories) a').map(actor => ({
name: qu.q(actor, null, true),
url: qu.url(actor, null),
}));
release.tags = qu.all('.video_categories a', true);
release.tags = qu.all('.video_categories a', true);
release.duration = qu.dur('.video_categories + p');
release.duration = qu.dur('.video_categories + p');
const poster = qu.img('a img');
const poster = qu.img('a img');
release.poster = getFallbacks(poster);
release.photos = qu.imgs('.featured-video img', 'src0_1x').map(source => getFallbacks(source));
release.poster = getFallbacks(poster);
release.photos = qu.imgs('.featured-video img', 'src0_1x').map(source => getFallbacks(source));
return release;
return release;
}
function scrapeProfile({ el, qu }) {
const profile = {};
const profile = {};
const bio = Array.from(qu.q('.widget-content').childNodes).reduce((acc, node, index, nodes) => {
const nextNode = nodes[index + 1];
const bio = Array.from(qu.q('.widget-content').childNodes).reduce((acc, node, index, nodes) => {
const nextNode = nodes[index + 1];
if (node.tagName === 'STRONG' && nextNode?.nodeType === 3) {
acc[slugify(node.textContent, '_')] = nextNode.textContent.trim();
}
if (node.tagName === 'STRONG' && nextNode?.nodeType === 3) {
acc[slugify(node.textContent, '_')] = nextNode.textContent.trim();
}
return acc;
}, {});
return acc;
}, {});
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
if (bio.age) profile.age = Number(bio.age);
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
if (bio.age) profile.age = Number(bio.age);
if (bio.height && /\d{3}/.test(bio.height)) profile.height = Number(bio.height.match(/\d+/)[0]);
if (bio.height && /\d[;']\d/.test(bio.height)) profile.height = feetInchesToCm(bio.height);
if (bio.height && /\d{3}/.test(bio.height)) profile.height = Number(bio.height.match(/\d+/)[0]);
if (bio.height && /\d[;']\d/.test(bio.height)) profile.height = feetInchesToCm(bio.height);
if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-');
if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-');
if (bust && /\d+[a-zA-Z]+/.test(bust)) profile.bust = bust;
if (waist) profile.waist = Number(waist);
if (hip) profile.hip = Number(hip);
}
if (bust && /\d+[a-zA-Z]+/.test(bust)) profile.bust = bust;
if (waist) profile.waist = Number(waist);
if (hip) profile.hip = Number(hip);
}
if (bio.bust_size && !profile.bust) profile.bust = bio.bust_size.toUpperCase();
if (bio.bust_size && !profile.bust) profile.bust = bio.bust_size.toUpperCase();
if (bio.birth_location) profile.birthPlace = bio.birth_location;
if (bio.status_married_or_single) profile.relationship = bio.status_married_or_single;
if (bio.birth_location) profile.birthPlace = bio.birth_location;
if (bio.status_married_or_single) profile.relationship = bio.status_married_or_single;
if (bio.eye_color) profile.eyes = bio.eye_color;
if (bio.eye_color) profile.eyes = bio.eye_color;
const avatar = qu.img('.tac img');
profile.avatar = getFallbacks(avatar);
const avatar = qu.img('.tac img');
profile.avatar = getFallbacks(avatar);
profile.releases = scrapeAll(initAll(el, '.featured-video'));
profile.releases = scrapeAll(initAll(el, '.featured-video'));
return profile;
return profile;
}
async function fetchLatest(site, page) {
const url = `${site.url}/tour/categories/movies_${page}_d.html`;
const res = await getAll(url, '.featured-video');
const url = `${site.url}/tour/categories/movies_${page}_d.html`;
const res = await getAll(url, '.featured-video');
if (res.ok) {
return scrapeAll(res.items, site);
}
if (res.ok) {
return scrapeAll(res.items, site);
}
return res.status;
return res.status;
}
async function fetchScene(url, site) {
const res = await get(url, '.page-content .row');
const res = await get(url, '.page-content .row');
if (res.ok) {
return scrapeScene(res.item, url, site);
}
if (res.ok) {
return scrapeScene(res.item, url, site);
}
return res.status;
return res.status;
}
async function fetchProfile(actorName, scraperSlug, site) {
const actorSlug = slugify(actorName, '');
const url = `${site.url}/tour/models/${actorSlug}.html`;
const res = await get(url, '.page-content .row');
const actorSlug = slugify(actorName, '');
const url = `${site.url}/tour/models/${actorSlug}.html`;
const res = await get(url, '.page-content .row');
if (res.ok) {
return scrapeProfile(res.item);
}
if (res.ok) {
return scrapeProfile(res.item);
}
return res.status;
return res.status;
}
module.exports = {
fetchLatest,
fetchProfile,
fetchScene,
fetchLatest,
fetchProfile,
fetchScene,
};

8
src/scrapers/babes.js
View File

@ -3,11 +3,11 @@
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'babes');
return fetchProfile(actorName, 'babes');
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
};

180
src/scrapers/bamvisions.js
View File

@ -6,144 +6,144 @@ const slugify = require('../utils/slugify');
const { feetInchesToCm } = require('../utils/convert');
function scrapeAll(scenes, site) {
return scenes.map(({ qu }) => {
const release = {};
return scenes.map(({ qu }) => {
const release = {};
release.title = qu.q('h3 a', true);
release.url = qu.url('h3 a');
release.title = qu.q('h3 a', true);
release.url = qu.url('h3 a');
release.date = qu.date('.item-meta li', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = qu.dur('.item-meta li:nth-child(2)');
release.description = qu.q('.description', true);
release.date = qu.date('.item-meta li', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = qu.dur('.item-meta li:nth-child(2)');
release.description = qu.q('.description', true);
release.actors = qu.all('a[href*="/models"]', true);
if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
release.actors = qu.all('a[href*="/models"]', true);
if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
[release.poster, ...release.photos] = qu.all('.item-thumbs img')
.map(source => [
source.getAttribute('src0_3x'),
source.getAttribute('src0_2x'),
source.getAttribute('src0_1x'),
]
.filter(Boolean)
.map(fallback => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`)));
[release.poster, ...release.photos] = qu.all('.item-thumbs img')
.map(source => [
source.getAttribute('src0_3x'),
source.getAttribute('src0_2x'),
source.getAttribute('src0_1x'),
]
.filter(Boolean)
.map(fallback => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`)));
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
return release;
});
return release;
});
}
function scrapeScene({ html, qu }, url, site) {
const release = { url };
const release = { url };
release.title = qu.q('.item-episode h4 a', true);
release.date = qu.date('.item-meta li', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = qu.dur('.item-meta li:nth-child(2)');
release.description = qu.q('.description', true);
release.title = qu.q('.item-episode h4 a', true);
release.date = qu.date('.item-meta li', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = qu.dur('.item-meta li:nth-child(2)');
release.description = qu.q('.description', true);
release.actors = qu.all('.item-episode a[href*="/models"]', true);
if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
release.actors = qu.all('.item-episode a[href*="/models"]', true);
if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
const posterPath = html.match(/poster="(.*.jpg)"/)?.[1];
const trailerPath = html.match(/video src="(.*.mp4)"/)?.[1];
const posterPath = html.match(/poster="(.*.jpg)"/)?.[1];
const trailerPath = html.match(/video src="(.*.mp4)"/)?.[1];
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${site.url}${posterPath}`;
release.poster = [
poster.replace('-1x', '-3x'),
poster.replace('-1x', '-2x'),
poster,
];
}
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${site.url}${posterPath}`;
release.poster = [
poster.replace('-1x', '-3x'),
poster.replace('-1x', '-2x'),
poster,
];
}
if (trailerPath) {
const trailer = /^http/.test(trailerPath) ? trailerPath : `${site.url}${trailerPath}`;
release.trailer = { src: trailer };
}
if (trailerPath) {
const trailer = /^http/.test(trailerPath) ? trailerPath : `${site.url}${trailerPath}`;
release.trailer = { src: trailer };
}
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
return release;
return release;
}
async function fetchActorReleases(actorId, site, page = 1, accScenes = []) {
const url = `${site.url}/sets.php?id=${actorId}&page=${page}`;
const res = await get(url);
const url = `${site.url}/sets.php?id=${actorId}&page=${page}`;
const res = await get(url);
if (!res.ok) return [];
if (!res.ok) return [];
const quReleases = initAll(res.item.el, '.item-episode');
const releases = scrapeAll(quReleases, site);
const quReleases = initAll(res.item.el, '.item-episode');
const releases = scrapeAll(quReleases, site);
const nextPage = res.item.qu.q(`a[href*="page=${page + 1}"]`);
const nextPage = res.item.qu.q(`a[href*="page=${page + 1}"]`);
if (nextPage) {
return fetchActorReleases(actorId, site, page + 1, accScenes.concat(releases));
}
if (nextPage) {
return fetchActorReleases(actorId, site, page + 1, accScenes.concat(releases));
}
return accScenes.concat(releases);
return accScenes.concat(releases);
}
async function scrapeProfile({ qu }, site, withScenes) {
const profile = {};
const profile = {};
const bio = qu.all('.stats li', true).reduce((acc, row) => {
const [key, value] = row.split(':');
return { ...acc, [slugify(key, '_')]: value.trim() };
}, {});
const bio = qu.all('.stats li', true).reduce((acc, row) => {
const [key, value] = row.split(':');
return { ...acc, [slugify(key, '_')]: value.trim() };
}, {});
if (bio.height) profile.height = feetInchesToCm(bio.height);
if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-');
if (bio.height) profile.height = feetInchesToCm(bio.height);
if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-');
if (bust) profile.bust = bust;
if (waist) profile.waist = Number(waist);
if (hip) profile.hip = Number(hip);
}
if (bust) profile.bust = bust;
if (waist) profile.waist = Number(waist);
if (hip) profile.hip = Number(hip);
}
profile.avatar = [
qu.q('.profile-pic img', 'src0_3x'),
qu.q('.profile-pic img', 'src0_2x'),
qu.q('.profile-pic img', 'src0_1x'),
].filter(Boolean).map(source => (/^http/.test(source) ? source : `${site.url}${source}`));
profile.avatar = [
qu.q('.profile-pic img', 'src0_3x'),
qu.q('.profile-pic img', 'src0_2x'),
qu.q('.profile-pic img', 'src0_1x'),
].filter(Boolean).map(source => (/^http/.test(source) ? source : `${site.url}${source}`));
if (withScenes) {
const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1];
if (withScenes) {
const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1];
if (actorId) {
profile.releases = await fetchActorReleases(actorId, site);
}
}
if (actorId) {
profile.releases = await fetchActorReleases(actorId, site);
}
}
return profile;
return profile;
}
async function fetchLatest(site, page = 1) {
const url = `${site.url}/categories/movies/${page}/latest/`;
const res = await geta(url, '.item-episode');
const url = `${site.url}/categories/movies/${page}/latest/`;
const res = await geta(url, '.item-episode');
return res.ok ? scrapeAll(res.items, site) : res.status;
return res.ok ? scrapeAll(res.items, site) : res.status;
}
async function fetchScene(url, site) {
const res = await get(url);
const res = await get(url);
return res.ok ? scrapeScene(res.item, url, site) : res.status;
return res.ok ? scrapeScene(res.item, url, site) : res.status;
}
async function fetchProfile(actorName, scraperSlug, site, include) {
const actorSlugA = slugify(actorName, '');
const actorSlugB = slugify(actorName);
const actorSlugA = slugify(actorName, '');
const actorSlugB = slugify(actorName);
const resA = await get(`${site.url}/models/${actorSlugA}.html`);
const res = resA.ok ? resA : await get(`${site.url}/models/${actorSlugB}.html`);
const resA = await get(`${site.url}/models/${actorSlugA}.html`);
const res = resA.ok ? resA : await get(`${site.url}/models/${actorSlugB}.html`);
return res.ok ? scrapeProfile(res.item, site, include.scenes) : res.status;
return res.ok ? scrapeProfile(res.item, site, include.scenes) : res.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
fetchLatest,
fetchScene,
fetchProfile,
};

234
src/scrapers/bang.js
View File

@ -8,99 +8,99 @@ const clusterId = '617fb597b659459bafe6472470d9073a';
const authKey = 'YmFuZy1yZWFkOktqVDN0RzJacmQ1TFNRazI=';
const genderMap = {
M: 'male',
F: 'female',
M: 'male',
F: 'female',
};
function getScreenUrl(item, scene) {
return `https://i.bang.com/screenshots/${scene.dvd.id}/movie/${scene.order}/${item.screenId}.jpg`;
return `https://i.bang.com/screenshots/${scene.dvd.id}/movie/${scene.order}/${item.screenId}.jpg`;
}
function encodeId(id) {
return Buffer
.from(id, 'hex')
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=/g, ',');
return Buffer
.from(id, 'hex')
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=/g, ',');
}
function decodeId(id) {
const restoredId = id
.replace(/-/g, '+')
.replace(/_/g, '/')
.replace(/,/g, '=');
const restoredId = id
.replace(/-/g, '+')
.replace(/_/g, '/')
.replace(/,/g, '=');
return Buffer
.from(restoredId, 'base64')
.toString('hex');
return Buffer
.from(restoredId, 'base64')
.toString('hex');
}
function scrapeScene(scene, site) {
const release = {
site,
entryId: scene.id,
title: scene.name,
description: scene.description,
tags: scene.genres.concat(scene.actions).map(genre => genre.name),
duration: scene.duration,
};
const release = {
site,
entryId: scene.id,
title: scene.name,
description: scene.description,
tags: scene.genres.concat(scene.actions).map(genre => genre.name),
duration: scene.duration,
};
const slug = slugify(release.title);
release.url = `https://www.bang.com/video/${encodeId(release.entryId)}/${slug}`;
const slug = slugify(release.title);
release.url = `https://www.bang.com/video/${encodeId(release.entryId)}/${slug}`;
const date = new Date(scene.releaseDate);
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
const date = new Date(scene.releaseDate);
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: genderMap[actor.gender] }));
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: genderMap[actor.gender] }));
if (scene.is4k) release.tags.push('4k');
if (scene.gay) release.tags.push('gay');
if (scene.is4k) release.tags.push('4k');
if (scene.gay) release.tags.push('gay');
const defaultPoster = scene.screenshots.find(photo => photo.default === true);
const photoset = scene.screenshots.filter(photo => photo.default === false);
const defaultPoster = scene.screenshots.find(photo => photo.default === true);
const photoset = scene.screenshots.filter(photo => photo.default === false);
const photos = defaultPoster ? photoset : photoset.slice(1);
const poster = defaultPoster || photoset[0];
const photos = defaultPoster ? photoset : photoset.slice(1);
const poster = defaultPoster || photoset[0];
release.poster = getScreenUrl(poster, scene);
release.photos = photos.map(photo => getScreenUrl(photo, scene));
release.poster = getScreenUrl(poster, scene);
release.photos = photos.map(photo => getScreenUrl(photo, scene));
release.trailer = {
src: `https://i.bang.com/v/${scene.dvd.id}/${scene.identifier}/preview.mp4`,
};
release.trailer = {
src: `https://i.bang.com/v/${scene.dvd.id}/${scene.identifier}/preview.mp4`,
};
release.channel = scene.series.name
.replace(/[! .]/g, '')
.replace('&', 'and');
release.channel = scene.series.name
.replace(/[! .]/g, '')
.replace('&', 'and');
return release;
return release;
}
function scrapeLatest(scenes, site) {
return scenes.map(({ _source: scene }) => scrapeScene(scene, site));
return scenes.map(({ _source: scene }) => scrapeScene(scene, site));
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50,
from: (page - 1) * 50,
query: {
bool: {
must: [
{
match: {
status: 'ok',
},
},
{
range: {
releaseDate: {
lte: 'now',
},
},
},
/*
const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50,
from: (page - 1) * 50,
query: {
bool: {
must: [
{
match: {
status: 'ok',
},
},
{
range: {
releaseDate: {
lte: 'now',
},
},
},
/*
* global fetch
{
nested: {
@ -122,66 +122,66 @@ async function fetchLatest(site, page = 1) {
},
},
*/
{
nested: {
path: 'series',
query: {
bool: {
must: [
{
match: {
'series.id': {
operator: 'AND',
query: site.parameters.siteId,
},
},
},
],
},
},
},
},
],
must_not: [
{
match: {
type: 'trailer',
},
},
],
},
},
sort: [
{
releaseDate: {
order: 'desc',
},
},
],
}, {
encodeJSON: true,
headers: {
Authorization: `Basic ${authKey}`,
},
});
{
nested: {
path: 'series',
query: {
bool: {
must: [
{
match: {
'series.id': {
operator: 'AND',
query: site.parameters.siteId,
},
},
},
],
},
},
},
},
],
must_not: [
{
match: {
type: 'trailer',
},
},
],
},
},
sort: [
{
releaseDate: {
order: 'desc',
},
},
],
}, {
encodeJSON: true,
headers: {
Authorization: `Basic ${authKey}`,
},
});
return scrapeLatest(res.body.hits.hits, site);
return scrapeLatest(res.body.hits.hits, site);
}
async function fetchScene(url, site) {
const encodedId = new URL(url).pathname.split('/')[2];
const entryId = decodeId(encodedId);
const encodedId = new URL(url).pathname.split('/')[2];
const entryId = decodeId(encodedId);
const res = await bhttp.get(`https://${clusterId}.us-east-1.aws.found.io/videos/video/${entryId}`, {
headers: {
Authorization: `Basic ${authKey}`,
},
});
const res = await bhttp.get(`https://${clusterId}.us-east-1.aws.found.io/videos/video/${entryId}`, {
headers: {
Authorization: `Basic ${authKey}`,
},
});
return scrapeScene(res.body._source, site); // eslint-disable-line no-underscore-dangle
return scrapeScene(res.body._source, site); // eslint-disable-line no-underscore-dangle
}
module.exports = {
fetchLatest,
fetchScene,
fetchLatest,
fetchScene,
};

184
src/scrapers/bangbros.js
View File

@ -10,44 +10,44 @@ const slugify = require('../utils/slugify');
const { ex } = require('../utils/q');
function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.echThumb').toArray();
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.echThumb').toArray();
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('.thmb_lnk');
const title = sceneLinkElement.attr('title');
const url = `https://bangbros.com${sceneLinkElement.attr('href')}`;
const shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1];
const entryId = url.split('/')[3].slice(5);
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('.thmb_lnk');
const title = sceneLinkElement.attr('title');
const url = `https://bangbros.com${sceneLinkElement.attr('href')}`;
const shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1];
const entryId = url.split('/')[3].slice(5);
const date = moment.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate();
const actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
const date = moment.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate();
const actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
const photoElement = $(element).find('.rollover-image');
const poster = `https:${photoElement.attr('data-original')}`;
const photoElement = $(element).find('.rollover-image');
const poster = `https:${photoElement.attr('data-original')}`;
const photosUrl = photoElement.attr('data-rollover-url');
const photosMaxIndex = photoElement.attr('data-rollover-max-index');
const photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`);
const photosUrl = photoElement.attr('data-rollover-url');
const photosMaxIndex = photoElement.attr('data-rollover-max-index');
const photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`);
const duration = moment.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds();
const channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0];
const duration = moment.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds();
const channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0];
return {
url,
entryId,
shootId,
title,
actors,
date,
duration,
poster,
photos,
rating: null,
site,
channel,
};
});
return {
url,
entryId,
shootId,
title,
actors,
date,
duration,
poster,
photos,
rating: null,
site,
channel,
};
});
}
/* no dates available, breaks database
@ -80,63 +80,63 @@ function scrapeUpcoming(html, site) {
*/
function scrapeScene(html, url, _site) {
const { qu } = ex(html, '.playerSection');
const release = {};
const { qu } = ex(html, '.playerSection');
const release = {};
[release.shootId] = qu.q('.vdoTags + .vdoCast', true).match(/\w+$/);
[release.entryId] = url.split('/')[3].match(/\d+$/);
release.title = qu.q('.ps-vdoHdd h1', true);
release.description = qu.q('.vdoDesc', true);
[release.shootId] = qu.q('.vdoTags + .vdoCast', true).match(/\w+$/);
[release.entryId] = url.split('/')[3].match(/\d+$/);
release.title = qu.q('.ps-vdoHdd h1', true);
release.description = qu.q('.vdoDesc', true);
release.actors = qu.all('a[href*="/model"]', true);
release.tags = qu.all('.vdoTags a', true);
release.actors = qu.all('a[href*="/model"]', true);
release.tags = qu.all('.vdoTags a', true);
release.stars = Number(qu.q('div[class*="like"]', true).match(/^\d+/)[0]) / 20;
release.stars = Number(qu.q('div[class*="like"]', true).match(/^\d+/)[0]) / 20;
const poster = qu.img('img#player-overlay-image');
release.poster = [
poster,
poster.replace('/big_trailer', '/members/450x340'), // load error fallback
];
const poster = qu.img('img#player-overlay-image');
release.poster = [
poster,
poster.replace('/big_trailer', '/members/450x340'), // load error fallback
];
release.trailer = { src: qu.trailer() };
release.trailer = { src: qu.trailer() };
// all scenes seem to have 12 album photos available, not always included on the page
const firstPhotoUrl = ex(html).qu.img('img[data-slider-index="1"]');
release.photos = Array.from({ length: 12 }, (val, index) => firstPhotoUrl.replace(/big\d+/, `big${index + 1}`));
// all scenes seem to have 12 album photos available, not always included on the page
const firstPhotoUrl = ex(html).qu.img('img[data-slider-index="1"]');
release.photos = Array.from({ length: 12 }, (val, index) => firstPhotoUrl.replace(/big\d+/, `big${index + 1}`));
const [channel] = qu.url('a[href*="/websites"]').match(/\w+$/);
const [channel] = qu.url('a[href*="/websites"]').match(/\w+$/);
if (channel === 'bangcasting') release.channel = 'bangbroscasting';
if (channel === 'remaster') release.channel = 'bangbrosremastered';
else release.channel = channel;
if (channel === 'bangcasting') release.channel = 'bangbroscasting';
if (channel === 'remaster') release.channel = 'bangbrosremastered';
else release.channel = channel;
return release;
return release;
}
function scrapeProfile(html) {
const { q } = ex(html);
const profile = {};
const { q } = ex(html);
const profile = {};
const avatar = q('.profilePic img', 'src');
if (avatar) profile.avatar = `https:${avatar}`;
const avatar = q('.profilePic img', 'src');
if (avatar) profile.avatar = `https:${avatar}`;
profile.releases = scrape(html);
profile.releases = scrape(html);
return profile;
return profile;
}
function scrapeProfileSearch(html, actorName) {
const { qu } = ex(html);
const actorLink = qu.url(`a[title="${actorName}" i][href*="model"]`);
const { qu } = ex(html);
const actorLink = qu.url(`a[title="${actorName}" i][href*="model"]`);
return actorLink ? `https://bangbros.com${actorLink}` : null;
return actorLink ? `https://bangbros.com${actorLink}` : null;
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/${page}`);
const res = await bhttp.get(`${site.url}/${page}`);
return scrape(res.body.toString(), site);
return scrape(res.body.toString(), site);
}
/*
@ -148,43 +148,43 @@ async function fetchUpcoming(site) {
*/
async function fetchScene(url, site, release) {
if (!release?.date) {
logger.warn(`Scraping Bang Bros scene from URL without release date: ${url}`);
}
if (!release?.date) {
logger.warn(`Scraping Bang Bros scene from URL without release date: ${url}`);
}
const { origin } = new URL(url);
const res = await bhttp.get(url);
const { origin } = new URL(url);
const res = await bhttp.get(url);
if (!/https?:\/\/(www.)?bangbros.com\/?$/.test(origin)) {
throw new Error('Cannot fetch from this URL. Please find the scene on https://bangbros.com and try again.');
}
if (!/https?:\/\/(www.)?bangbros.com\/?$/.test(origin)) {
throw new Error('Cannot fetch from this URL. Please find the scene on https://bangbros.com and try again.');
}
return scrapeScene(res.body.toString(), url, site);
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName) {
const actorSlug = slugify(actorName);
const url = `https://bangbros.com/search/${actorSlug}`;
const res = await bhttp.get(url);
const actorSlug = slugify(actorName);
const url = `https://bangbros.com/search/${actorSlug}`;
const res = await bhttp.get(url);
if (res.statusCode === 200) {
const actorUrl = scrapeProfileSearch(res.body.toString(), actorName);
if (res.statusCode === 200) {
const actorUrl = scrapeProfileSearch(res.body.toString(), actorName);
if (actorUrl) {
const actorRes = await bhttp.get(actorUrl);
if (actorUrl) {
const actorRes = await bhttp.get(actorUrl);
if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString());
}
}
}
if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString());
}
}
}
return null;
return null;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
// fetchUpcoming, no dates available
fetchLatest,
fetchScene,
fetchProfile,
// fetchUpcoming, no dates available
};

32
src/scrapers/blowpass.js
View File

@ -5,33 +5,33 @@
const { fetchScene, fetchLatest, fetchUpcoming, fetchProfile } = require('./gamma');
async function fetchSceneWrapper(url, site, baseRelease) {
const release = await fetchScene(url, site, baseRelease);
const release = await fetchScene(url, site, baseRelease);
if (site.isFallback && release.channel) {
const channelUrl = url.replace('blowpass.com', `${release.channel}.com`);
if (site.isNetwork && release.channel) {
const channelUrl = url.replace('blowpass.com', `${release.channel}.com`);
if (['onlyteenblowjobs', 'mommyblowsbest'].includes(release.channel)) {
release.url = channelUrl.replace(/video\/\w+\//, 'scene/');
return release;
}
if (['onlyteenblowjobs', 'mommyblowsbest'].includes(release.channel)) {
release.url = channelUrl.replace(/video\/\w+\//, 'scene/');
return release;
}
release.url = channelUrl.replace(/video\/\w+\//, 'video/');
}
release.url = channelUrl.replace(/video\/\w+\//, 'video/');
}
return release;
return release;
}
function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.blowpass.com/en/videos/blowpass/latest/All-Categories/0${actorPath}/${page}`;
return `https://www.blowpass.com/en/videos/blowpass/latest/All-Categories/0${actorPath}/${page}`;
}
async function networkFetchProfile(actorName, scraperSlug, site, include) {
return fetchProfile(actorName, scraperSlug, null, getActorReleasesUrl, include);
return fetchProfile(actorName, scraperSlug, null, getActorReleasesUrl, include);
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchUpcoming,
fetchScene: fetchSceneWrapper,
fetchLatest,
fetchProfile: networkFetchProfile,
fetchUpcoming,
fetchScene: fetchSceneWrapper,
};

102
src/scrapers/boobpedia.js
View File

@ -5,90 +5,90 @@ const bhttp = require('bhttp');
const { ex } = require('../utils/q');
function scrapeProfile(html) {
const { qu } = ex(html); /* eslint-disable-line object-curly-newline */
const profile = {};
const { qu } = ex(html); /* eslint-disable-line object-curly-newline */
const profile = {};
const bio = qu.all('.infobox tr[valign="top"]')
.map(detail => qu.all(detail, 'td', true))
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
const bio = qu.all('.infobox tr[valign="top"]')
.map(detail => qu.all(detail, 'td', true))
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
/* unreliable, see: Syren De Mer
/* unreliable, see: Syren De Mer
const catlinks = qa('#mw-normal-catlinks a', true);
const isTrans = catlinks.some(link => link.match(/shemale|transgender/i));
profile.gender = isTrans ? 'transsexual' : 'female';
*/
profile.birthdate = qu.date('.bday', 'YYYY-MM-DD');
profile.birthdate = qu.date('.bday', 'YYYY-MM-DD');
profile.description = qu.q('#mw-content-text > p', true);
profile.description = qu.q('#mw-content-text > p', true);
if (bio.Born) profile.birthPlace = bio.Born.slice(bio.Born.lastIndexOf(')') + 1);
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Born) profile.birthPlace = bio.Born.slice(bio.Born.lastIndexOf(')') + 1);
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Measurements) {
const measurements = bio.Measurements
.match(/\d+(\w+)?-\d+-\d+/g)
if (bio.Measurements) {
const measurements = bio.Measurements
.match(/\d+(\w+)?-\d+-\d+/g)
?.slice(-1)[0] // allow for both '34C-25-36' and '86-64-94 cm / 34-25-37 in'
.split('-');
// account for measuemrents being just e.g. '32EE'
if (measurements) {
const [bust, waist, hip] = measurements;
// account for measuemrents being just e.g. '32EE'
if (measurements) {
const [bust, waist, hip] = measurements;
if (/[a-zA-Z]/.test(bust)) profile.bust = bust; // only use bust if cup size is included
if (/[a-zA-Z]/.test(bust)) profile.bust = bust; // only use bust if cup size is included
profile.waist = Number(waist);
profile.hip = Number(hip);
}
profile.waist = Number(waist);
profile.hip = Number(hip);
}
if (/^\d+\w+$/.test(bio.Measurements)) profile.bust = bio.Measurements;
}
if (/^\d+\w+$/.test(bio.Measurements)) profile.bust = bio.Measurements;
}
if (bio.Bra_cup_size) {
const bust = bio.Bra_cup_size.match(/^\d+\w+/);
if (bust) [profile.bust] = bust;
}
if (bio.Bra_cup_size) {
const bust = bio.Bra_cup_size.match(/^\d+\w+/);
if (bust) [profile.bust] = bust;
}
if (bio.Boobs === 'Enhanced') profile.naturalBoobs = false;
if (bio.Boobs === 'Natural') profile.naturalBoobs = true;
if (bio.Boobs === 'Enhanced') profile.naturalBoobs = false;
if (bio.Boobs === 'Natural') profile.naturalBoobs = true;
if (bio.Height) profile.height = Number(bio.Height.match(/\d+\.\d+/g).slice(-1)[0]) * 100;
if (bio.Weight) profile.weight = Number(bio.Weight.match(/\d+/g)[1]);
if (bio.Height) profile.height = Number(bio.Height.match(/\d+\.\d+/g).slice(-1)[0]) * 100;
if (bio.Weight) profile.weight = Number(bio.Weight.match(/\d+/g)[1]);
if (bio.Eye_color) profile.eyes = bio.Eye_color;
if (bio.Hair) [profile.hair] = bio.Hair.split(',');
if (bio.Eye_color) profile.eyes = bio.Eye_color;
if (bio.Hair) [profile.hair] = bio.Hair.split(',');
if (bio.Blood_group) profile.blood = bio.Blood_group;
if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', ');
if (bio.Blood_group) profile.blood = bio.Blood_group;
if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', ');
const avatarThumbPath = qu.img('.image img');
const avatarThumbPath = qu.img('.image img');
if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) {
const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', '');
if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) {
const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', '');
profile.avatar = {
src: `http://www.boobpedia.com${avatarPath}`,
copyright: null,
};
}
profile.avatar = {
src: `http://www.boobpedia.com${avatarPath}`,
copyright: null,
};
}
profile.social = qu.urls('.infobox a.external');
profile.social = qu.urls('.infobox a.external');
return profile;
return profile;
}
async function fetchProfile(actorName) {
const actorSlug = actorName.replace(/\s+/, '_');
const res = await bhttp.get(`http://www.boobpedia.com/boobs/${actorSlug}`);
const actorSlug = actorName.replace(/\s+/, '_');
const res = await bhttp.get(`http://www.boobpedia.com/boobs/${actorSlug}`);
if (res.statusCode === 200) {
return scrapeProfile(res.body.toString());
}
if (res.statusCode === 200) {
return scrapeProfile(res.body.toString());
}
return null;
return null;
}
module.exports = {
fetchProfile,
fetchProfile,
};

276
src/scrapers/brazzers.js
View File

@ -11,216 +11,216 @@ const slugify = require('../utils/slugify');
const { heightToCm, lbsToKg } = require('../utils/convert');
const hairMap = {
Blonde: 'blonde',
Brunette: 'brown',
'Black Hair': 'black',
Redhead: 'red',
Blonde: 'blonde',
Brunette: 'brown',
'Black Hair': 'black',
Redhead: 'red',
};
function scrapeAll(html, site, upcoming) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.release-card.scene').toArray();
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.release-card.scene').toArray();
return sceneElements.reduce((acc, element) => {
const isUpcoming = $(element).find('.icon-upcoming.active').length === 1;
return sceneElements.reduce((acc, element) => {
const isUpcoming = $(element).find('.icon-upcoming.active').length === 1;
if ((upcoming && !isUpcoming) || (!upcoming && isUpcoming)) {
return acc;
}
if ((upcoming && !isUpcoming) || (!upcoming && isUpcoming)) {
return acc;
}
const sceneLinkElement = $(element).find('a');
const sceneLinkElement = $(element).find('a');
const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title');
const entryId = url.split('/').slice(-3, -2)[0];
const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title');
const entryId = url.split('/').slice(-3, -2)[0];
const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate();
const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate();
const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
const likes = Number($(element).find('.label-rating .like-amount').text());
const dislikes = Number($(element).find('.label-rating .dislike-amount').text());
const likes = Number($(element).find('.label-rating .like-amount').text());
const dislikes = Number($(element).find('.label-rating .dislike-amount').text());
const poster = `https:${$(element).find('.card-main-img').attr('data-src')}`;
const photos = $(element).find('.card-overlay .image-under').map((photoIndex, photoElement) => `https:${$(photoElement).attr('data-src')}`).toArray();
const poster = `https:${$(element).find('.card-main-img').attr('data-src')}`;
const photos = $(element).find('.card-overlay .image-under').map((photoIndex, photoElement) => `https:${$(photoElement).attr('data-src')}`).toArray();
const channel = slugify($(element).find('.collection').attr('title'), '');
const channel = slugify($(element).find('.collection').attr('title'), '');
return acc.concat({
url,
entryId,
title,
actors,
date,
poster,
photos,
rating: {
likes,
dislikes,
},
channel,
site,
});
}, []);
return acc.concat({
url,
entryId,
title,
actors,
date,
poster,
photos,
rating: {
likes,
dislikes,
},
channel,
site,
});
}, []);
}
async function scrapeScene(html, url, _site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const release = {};
const $ = cheerio.load(html, { normalizeWhitespace: true });
const release = {};
const videoJson = $('script:contains("window.videoUiOptions")').html();
const videoString = videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('},') + 1);
const videoData = JSON.parse(videoString);
const videoJson = $('script:contains("window.videoUiOptions")').html();
const videoString = videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('},') + 1);
const videoData = JSON.parse(videoString);
[release.entryId] = url.split('/').slice(-3, -2);
release.title = $('.scene-title[itemprop="name"]').text();
[release.entryId] = url.split('/').slice(-3, -2);
release.title = $('.scene-title[itemprop="name"]').text();
release.description = $('#scene-description p[itemprop="description"]')
.contents()
.first()
.text()
.trim();
release.description = $('#scene-description p[itemprop="description"]')
.contents()
.first()
.text()
.trim();
release.date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate();
release.duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60;
release.date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate();
release.duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60;
const actorsFromCards = $('.featured-model .card-image a').map((actorIndex, actorElement) => {
const avatar = `https:${$(actorElement).find('img').attr('data-src')}`;
const actorsFromCards = $('.featured-model .card-image a').map((actorIndex, actorElement) => {
const avatar = `https:${$(actorElement).find('img').attr('data-src')}`;
return {
name: $(actorElement).attr('title'),
avatar: [avatar.replace('medium.jpg', 'large.jpg'), avatar],
};
}).toArray();
return {
name: $(actorElement).attr('title'),
avatar: [avatar.replace('medium.jpg', 'large.jpg'), avatar],
};
}).toArray();
release.actors = actorsFromCards || $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.actors = actorsFromCards || $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.likes = Number($('.label-rating .like').text());
release.dislikes = Number($('.label-rating .dislike').text());
release.likes = Number($('.label-rating .like').text());
release.dislikes = Number($('.label-rating .dislike').text());
const siteElement = $('.niche-site-logo');
// const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
const siteName = siteElement.attr('title');
release.channel = siteName.replace(/\s+/g, '').toLowerCase();
const siteElement = $('.niche-site-logo');
// const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
const siteName = siteElement.attr('title');
release.channel = siteName.replace(/\s+/g, '').toLowerCase();
release.tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
release.photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray();
release.tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
release.photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray();
const posterPath = videoData?.poster || $('meta[itemprop="thumbnailUrl"]').attr('content') || $('#trailer-player-container').attr('data-player-img');
if (posterPath) release.poster = `https:${posterPath}`;
const posterPath = videoData?.poster || $('meta[itemprop="thumbnailUrl"]').attr('content') || $('#trailer-player-container').attr('data-player-img');
if (posterPath) release.poster = `https:${posterPath}`;
if (videoData) {
release.trailer = Object.entries(videoData.stream_info.http.paths).map(([quality, path]) => ({
src: `https:${path}`,
quality: Number(quality.match(/\d{3,}/)[0]),
}));
}
if (videoData) {
release.trailer = Object.entries(videoData.stream_info.http.paths).map(([quality, path]) => ({
src: `https:${path}`,
quality: Number(quality.match(/\d{3,}/)[0]),
}));
}
return release;
return release;
}
function scrapeActorSearch(html, url, actorName) {
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
const { document } = new JSDOM(html).window;
const actorLink = document.querySelector(`a[title="${actorName}" i]`);
return actorLink ? actorLink.href : null;
return actorLink ? actorLink.href : null;
}
async function fetchActorReleases({ qu, html }, accReleases = []) {
const releases = scrapeAll(html);
const next = qu.url('.pagination .next a');
const releases = scrapeAll(html);
const next = qu.url('.pagination .next a');
if (next) {
const url = `https://www.brazzers.com${next}`;
const res = await get(url);
if (next) {
const url = `https://www.brazzers.com${next}`;
const res = await get(url);
if (res.ok) {
return fetchActorReleases(res.item, accReleases.concat(releases));
}
}
if (res.ok) {
return fetchActorReleases(res.item, accReleases.concat(releases));
}
}
return accReleases.concat(releases);
return accReleases.concat(releases);
}
async function scrapeProfile(html, url, actorName) {
const qProfile = ex(html);
const { q, qa } = qProfile;
const qProfile = ex(html);
const { q, qa } = qProfile;
const bioKeys = qa('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim());
const bioValues = qa('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim());
const bioKeys = qa('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim());
const bioValues = qa('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim());
const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key]: bioValues[index] }), {});
const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key]: bioValues[index] }), {});
const profile = {
name: actorName,
};
const profile = {
name: actorName,
};
profile.description = q('.model-profile-specs p', true);
profile.description = q('.model-profile-specs p', true);
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate();
if (bio['Birth Location']) profile.birthPlace = bio['Birth Location'];
if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase();
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate();
if (bio['Birth Location']) profile.birthPlace = bio['Birth Location'];
if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase();
if (bio.Height) profile.height = heightToCm(bio.Height);
if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]);
if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();
if (bio.Height) profile.height = heightToCm(bio.Height);
if (bio.Weight) profile.weight = lbsToKg(bio.Weight.match(/\d+/)[0]);
if (bio['Hair Color']) profile.hair = hairMap[bio['Hair Color']] || bio['Hair Color'].toLowerCase();
if (bio['Tits Type'] && bio['Tits Type'].match('Natural')) profile.naturalBoobs = true;
if (bio['Tits Type'] && bio['Tits Type'].match('Enhanced')) profile.naturalBoobs = false;
if (bio['Tits Type'] && bio['Tits Type'].match('Natural')) profile.naturalBoobs = true;
if (bio['Tits Type'] && bio['Tits Type'].match('Enhanced')) profile.naturalBoobs = false;
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
const avatarEl = q('.big-pic-model-container img');
if (avatarEl) profile.avatar = `https:${avatarEl.src}`;
const avatarEl = q('.big-pic-model-container img');
if (avatarEl) profile.avatar = `https:${avatarEl.src}`;
profile.releases = await fetchActorReleases(qProfile);
profile.releases = await fetchActorReleases(qProfile);
return profile;
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/page/${page}/`);
const res = await bhttp.get(`${site.url}/page/${page}/`);
return scrapeAll(res.body.toString(), site, false);
return scrapeAll(res.body.toString(), site, false);
}
async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.url}/`);
const res = await bhttp.get(`${site.url}/`);
return scrapeAll(res.body.toString(), site, true);
return scrapeAll(res.body.toString(), site, true);
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site);
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName) {
const searchUrl = 'https://brazzers.com/pornstars-search/';
const searchRes = await bhttp.get(searchUrl, {
headers: {
Cookie: `textSearch=${encodeURIComponent(actorName)};`,
},
});
const searchUrl = 'https://brazzers.com/pornstars-search/';
const searchRes = await bhttp.get(searchUrl, {
headers: {
Cookie: `textSearch=${encodeURIComponent(actorName)};`,
},
});
const actorLink = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
const actorLink = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName);
if (actorLink) {
const url = `https://brazzers.com${actorLink}`;
const res = await bhttp.get(url);
if (actorLink) {
const url = `https://brazzers.com${actorLink}`;
const res = await bhttp.get(url);
return scrapeProfile(res.body.toString(), url, actorName);
}
return scrapeProfile(res.body.toString(), url, actorName);
}
return null;
return null;
}
module.exports = {
fetchLatest,
fetchProfile,
fetchScene,
fetchUpcoming,
fetchLatest,
fetchProfile,
fetchScene,
fetchUpcoming,
};

8
src/scrapers/burningangel.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = {
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchScene,
fetchUpcoming: fetchApiUpcoming,
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchScene,
fetchUpcoming: fetchApiUpcoming,
};

174
src/scrapers/cherrypimps.js
View File

@ -4,139 +4,139 @@ const { get, geta, ctxa, ed } = require('../utils/q');
const slugify = require('../utils/slugify');
function scrapeAll(scenes, site) {
return scenes.map(({ qu }) => {
const url = qu.url('.text-thumb a');
const { pathname } = new URL(url);
const channelUrl = qu.url('.badge');
return scenes.map(({ qu }) => {
const url = qu.url('.text-thumb a');
const { pathname } = new URL(url);
const channelUrl = qu.url('.badge');
if (site?.parameters?.extract && qu.q('.badge', true) !== site.name) {
return null;
}
if (site?.parameters?.extract && qu.q('.badge', true) !== site.name) {
return null;
}
const release = {};
const release = {};
release.url = channelUrl ? `${channelUrl}${pathname}` : url;
release.entryId = pathname.match(/\/\d+/)[0].slice(1);
release.title = qu.q('.text-thumb a', true);
release.url = channelUrl ? `${channelUrl}${pathname}` : url;
release.entryId = pathname.match(/\/\d+/)[0].slice(1);
release.title = qu.q('.text-thumb a', true);
release.date = qu.date('.date', 'YYYY-MM-DD', /\d{4}-\d{2}-\d{2}/);
release.duration = qu.dur('.date', /(\d{2}:)?\d{2}:\d{2}/);
release.date = qu.date('.date', 'YYYY-MM-DD', /\d{4}-\d{2}-\d{2}/);
release.duration = qu.dur('.date', /(\d{2}:)?\d{2}:\d{2}/);
release.actors = qu.all('.category a', true);
release.actors = qu.all('.category a', true);
release.poster = qu.img('img.video_placeholder, .video-images img');
release.teaser = { src: qu.trailer() };
release.poster = qu.img('img.video_placeholder, .video-images img');
release.teaser = { src: qu.trailer() };
return release;
}).filter(Boolean);
return release;
}).filter(Boolean);
}
function scrapeScene({ q, qd, qa }, url, _site, baseRelease) {
const release = { url };
const release = { url };
const { pathname } = new URL(url);
release.entryId = pathname.match(/\/\d+/)[0].slice(1);
const { pathname } = new URL(url);
release.entryId = pathname.match(/\/\d+/)[0].slice(1);
release.title = q('.trailer-block_title', true);
release.description = q('.info-block:nth-child(3) .text', true);
release.date = qd('.info-block_data .text', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.title = q('.trailer-block_title', true);
release.description = q('.info-block:nth-child(3) .text', true);
release.date = qd('.info-block_data .text', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
const duration = baseRelease?.duration || Number(q('.info-block_data .text', true).match(/(\d+)\s+min/)?.[1]) * 60;
if (duration) release.duration = duration;
const duration = baseRelease?.duration || Number(q('.info-block_data .text', true).match(/(\d+)\s+min/)?.[1]) * 60;
if (duration) release.duration = duration;
release.actors = qa('.info-block_data a[href*="/models"]', true);
release.tags = qa('.info-block a[href*="/categories"]', true);
release.actors = qa('.info-block_data a[href*="/models"]', true);
release.tags = qa('.info-block a[href*="/categories"]', true);
const posterEl = q('.update_thumb');
const poster = posterEl.getAttribute('src0_3x') || posterEl.getAttribute('src0_2x') || posterEl.dataset.src;
const posterEl = q('.update_thumb');
const poster = posterEl.getAttribute('src0_3x') || posterEl.getAttribute('src0_2x') || posterEl.dataset.src;
if (poster && baseRelease?.poster) release.photos = [poster];
else if (poster) release.poster = poster;
if (poster && baseRelease?.poster) release.photos = [poster];
else if (poster) release.poster = poster;
return release;
return release;
}
function scrapeProfile({ q, qa, qtx }) {
const profile = {};
const profile = {};
const keys = qa('.model-descr_line:not(.model-descr_rait) p.text span', true);
const values = qa('.model-descr_line:not(.model-descr_rait) p.text').map(el => qtx(el));
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
const keys = qa('.model-descr_line:not(.model-descr_rait) p.text span', true);
const values = qa('.model-descr_line:not(.model-descr_rait) p.text').map(el => qtx(el));
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)cm\)/)[1]);
if (bio.weight) profile.weight = Number(bio.weight.match(/\((\d+)kg\)/)[1]);
if (bio.race) profile.ethnicity = bio.race;
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)cm\)/)[1]);
if (bio.weight) profile.weight = Number(bio.weight.match(/\((\d+)kg\)/)[1]);
if (bio.race) profile.ethnicity = bio.race;
if (bio.date_of_birth) profile.birthdate = ed(bio.date_of_birth, 'MMMM D, YYYY');
if (bio.birthplace) profile.birthPlace = bio.birthplace;
if (bio.date_of_birth) profile.birthdate = ed(bio.date_of_birth, 'MMMM D, YYYY');
if (bio.birthplace) profile.birthPlace = bio.birthplace;
if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-');
if (!/\?/.test(bust)) profile.bust = bust;
if (!/\?/.test(waist)) profile.waist = waist;
if (!/\?/.test(hip)) profile.hip = hip;
}
if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-');
if (!/\?/.test(bust)) profile.bust = bust;
if (!/\?/.test(waist)) profile.waist = waist;
if (!/\?/.test(hip)) profile.hip = hip;
}
if (bio.hair) profile.hair = bio.hair;
if (bio.eyes) profile.eyes = bio.eyes;
if (bio.hair) profile.hair = bio.hair;
if (bio.eyes) profile.eyes = bio.eyes;
if (/various/i.test(bio.tattoos)) profile.hasTattoos = true;
else if (/none/i.test(bio.tattoos)) profile.hasTattoos = false;
else if (bio.tattoos) {
profile.hasTattoos = true;
profile.tattoos = bio.tattoos;
}
if (/various/i.test(bio.tattoos)) profile.hasTattoos = true;
else if (/none/i.test(bio.tattoos)) profile.hasTattoos = false;
else if (bio.tattoos) {
profile.hasTattoos = true;
profile.tattoos = bio.tattoos;
}
if (/various/i.test(bio.piercings)) profile.hasPiercings = true;
else if (/none/i.test(bio.piercings)) profile.hasPiercings = false;
else if (bio.piercings) {
profile.hasPiercings = true;
profile.piercings = bio.piercings;
}
if (/various/i.test(bio.piercings)) profile.hasPiercings = true;
else if (/none/i.test(bio.piercings)) profile.hasPiercings = false;
else if (bio.piercings) {
profile.hasPiercings = true;
profile.piercings = bio.piercings;
}
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
const avatar = q('.model-img img');
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;
const avatar = q('.model-img img');
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;
const releases = qa('.video-thumb');
profile.releases = scrapeAll(ctxa(releases));
const releases = qa('.video-thumb');
profile.releases = scrapeAll(ctxa(releases));
return profile;
return profile;
}
async function fetchLatest(site, page = 1) {
const url = site.parameters?.extract
? `https://cherrypimps.com/categories/movies_${page}.html`
: `${site.url}/categories/movies_${page}.html`;
const res = await geta(url, 'div.video-thumb');
const url = site.parameters?.extract
? `https://cherrypimps.com/categories/movies_${page}.html`
: `${site.url}/categories/movies_${page}.html`;
const res = await geta(url, 'div.video-thumb');
return res.ok ? scrapeAll(res.items, site) : res.status;
return res.ok ? scrapeAll(res.items, site) : res.status;
}
async function fetchScene(url, site, release) {
const res = await get(url);
const res = await get(url);
return res.ok ? scrapeScene(res.item, url, site, release) : res.status;
return res.ok ? scrapeScene(res.item, url, site, release) : res.status;
}
async function fetchProfile(actorName, scraperSlug) {
const actorSlug = slugify(actorName);
const actorSlug2 = slugify(actorName, '');
const actorSlug = slugify(actorName);
const actorSlug2 = slugify(actorName, '');
const [url, url2] = ['cherrypimps', 'wildoncam'].includes(scraperSlug)
? [`https://${scraperSlug}.com/models/${actorSlug}.html`, `https://${scraperSlug}.com/models/${actorSlug2}.html`]
: [`https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug}.html`, `https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug2}.html`];
const [url, url2] = ['cherrypimps', 'wildoncam'].includes(scraperSlug)
? [`https://${scraperSlug}.com/models/${actorSlug}.html`, `https://${scraperSlug}.com/models/${actorSlug2}.html`]
: [`https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug}.html`, `https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug2}.html`];
const res = await get(url);
if (res.ok) return scrapeProfile(res.item);
const res = await get(url);
if (res.ok) return scrapeProfile(res.item);
const res2 = await get(url2);
return res2.ok ? scrapeProfile(res2.item) : res2.status;
const res2 = await get(url2);
return res2.ok ? scrapeProfile(res2.item) : res2.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
fetchLatest,
fetchScene,
fetchProfile,
};

230
src/scrapers/ddfnetwork.js
View File

@ -7,182 +7,182 @@ const slugify = require('../utils/slugify');
/* eslint-disable newline-per-chained-call */
function scrapeAll(html, site, origin) {
return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
const release = {};
return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
const release = {};
release.title = q('a', 'title');
release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`;
[release.entryId] = release.url.split('/').slice(-1);
release.title = q('a', 'title');
release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`;
[release.entryId] = release.url.split('/').slice(-1);
release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
release.actors = qa('.card-subtitle a', true).filter(Boolean);
release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
release.actors = qa('.card-subtitle a', true).filter(Boolean);
const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
if (duration) release.duration = duration;
const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
if (duration) release.duration = duration;
release.poster = q('img').dataset.src;
release.poster = q('img').dataset.src;
return release;
});
return release;
});
}
async function scrapeScene(html, url, _site) {
const { qu } = ex(html);
const release = {};
const { qu } = ex(html);
const release = {};
[release.entryId] = url.split('/').slice(-1);
[release.entryId] = url.split('/').slice(-1);
release.title = qu.meta('itemprop=name');
release.description = qu.q('.descr-box p', true);
release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
release.title = qu.meta('itemprop=name');
release.description = qu.q('.descr-box p', true);
release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|| qu.date('.title-border:nth-child(2) p', 'MM.DD.YYYY');
release.actors = qu.all('.pornstar-card > a', 'title');
release.tags = qu.all('.tags-tab .tags a', true);
release.actors = qu.all('.pornstar-card > a', 'title');
release.tags = qu.all('.tags-tab .tags a', true);
release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60;
release.likes = Number(qu.q('.icon-like-red + span', true));
release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60;
release.likes = Number(qu.q('.icon-like-red + span', true));
release.poster = qu.poster();
release.photos = qu.urls('.photo-slider-guest .card a');
release.poster = qu.poster();
release.photos = qu.urls('.photo-slider-guest .card a');
release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({
src: trailer.src,
quality: Number(trailer.attributes.res.value),
}));
release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({
src: trailer.src,
quality: Number(trailer.attributes.res.value),
}));
return release;
return release;
}
async function fetchActorReleases(urls) {
// DDF Network and DDF Network Stream list all scenes, exclude
const sources = urls.filter(url => !/ddfnetwork/.test(url));
// DDF Network and DDF Network Stream list all scenes, exclude
const sources = urls.filter(url => !/ddfnetwork/.test(url));
const releases = await Promise.all(sources.map(async (url) => {
const { html } = await get(url);
const releases = await Promise.all(sources.map(async (url) => {
const { html } = await get(url);
return scrapeAll(html, null, new URL(url).origin);
}));
return scrapeAll(html, null, new URL(url).origin);
}));
// DDF cross-releases scenes between sites, filter duplicates by entryId
return Object.values(releases
.flat()
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) // sort by date so earliest scene remains
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
// DDF cross-releases scenes between sites, filter duplicates by entryId
return Object.values(releases
.flat()
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) // sort by date so earliest scene remains
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
}
async function scrapeProfile(html, _url, actorName) {
const { qu } = ex(html);
const { qu } = ex(html);
const keys = qu.all('.about-title', true).map(key => slugify(key, '_'));
const values = qu.all('.about-info').map((el) => {
if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', ');
}
const keys = qu.all('.about-title', true).map(key => slugify(key, '_'));
const values = qu.all('.about-info').map((el) => {
if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', ');
}
return el.textContent.trim();
});
return el.textContent.trim();
});
const bio = keys.reduce((acc, key, index) => {
if (values[index] === '-') return acc;
const bio = keys.reduce((acc, key, index) => {
if (values[index] === '-') return acc;
return {
...acc,
[key]: values[index],
};
}, {});
return {
...acc,
[key]: values[index],
};
}, {});
const profile = {
name: actorName,
};
const profile = {
name: actorName,
};
profile.description = qu.q('.description-box', true);
profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY');
profile.description = qu.q('.description-box', true);
profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY');
if (bio.nationality) profile.nationality = bio.nationality;
if (bio.nationality) profile.nationality = bio.nationality;
if (bio.bra_size) [profile.bust] = bio.bra_size.match(/\d+\w+/);
if (bio.waist) profile.waist = Number(bio.waist.match(/\d+/)[0]);
if (bio.hips) profile.hip = Number(bio.hips.match(/\d+/)[0]);
if (bio.bra_size) [profile.bust] = bio.bra_size.match(/\d+\w+/);
if (bio.waist) profile.waist = Number(bio.waist.match(/\d+/)[0]);
if (bio.hips) profile.hip = Number(bio.hips.match(/\d+/)[0]);
if (bio.height) profile.height = Number(bio.height.match(/\d{2,}/)[0]);
if (bio.height) profile.height = Number(bio.height.match(/\d{2,}/)[0]);
if (bio.tit_style && /Enhanced/.test(bio.tit_style)) profile.naturalBoobs = false;
if (bio.tit_style && /Natural/.test(bio.tit_style)) profile.naturalBoobs = true;
if (bio.tit_style && /Enhanced/.test(bio.tit_style)) profile.naturalBoobs = false;
if (bio.tit_style && /Natural/.test(bio.tit_style)) profile.naturalBoobs = true;
if (bio.body_art && /Tattoo/.test(bio.body_art)) profile.hasTattoos = true;
if (bio.body_art && /Piercing/.test(bio.body_art)) profile.hasPiercings = true;
if (bio.body_art && /Tattoo/.test(bio.body_art)) profile.hasTattoos = true;
if (bio.body_art && /Piercing/.test(bio.body_art)) profile.hasPiercings = true;
if (bio.hair_style) profile.hair = bio.hair_style.split(',')[0].trim().toLowerCase();
if (bio.eye_color) profile.eyes = bio.eye_color.match(/\w+/)[0].toLowerCase();
if (bio.hair_style) profile.hair = bio.hair_style.split(',')[0].trim().toLowerCase();
if (bio.eye_color) profile.eyes = bio.eye_color.match(/\w+/)[0].toLowerCase();
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
const avatarEl = qu.q('.pornstar-details .card-img-top');
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
const avatarEl = qu.q('.pornstar-details .card-img-top');
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a'));
profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a'));
return profile;
return profile;
}
async function fetchLatest(site, page = 1) {
const url = site.parameters?.native
? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
const url = site.parameters?.native
? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
const res = await bhttp.get(url);
const res = await bhttp.get(url);
if (res.statusCode === 200) {
return scrapeAll(res.body.toString(), site);
}
if (res.statusCode === 200) {
return scrapeAll(res.body.toString(), site);
}
return res.statusCode;
return res.statusCode;
}
async function fetchScene(url, site) {
// DDF's main site moved to Porn World
// const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
const res = await bhttp.get(url);
// DDF's main site moved to Porn World
// const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site);
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName) {
const resSearch = await bhttp.post('https://ddfnetwork.com/search/ajax',
{
type: 'hints',
word: actorName,
},
{
decodeJSON: true,
headers: {
'x-requested-with': 'XMLHttpRequest',
},
});
const resSearch = await bhttp.post('https://ddfnetwork.com/search/ajax',
{
type: 'hints',
word: actorName,
},
{
decodeJSON: true,
headers: {
'x-requested-with': 'XMLHttpRequest',
},
});
if (resSearch.statusCode !== 200 || Array.isArray(resSearch.body.list)) {
return null;
}
if (resSearch.statusCode !== 200 || Array.isArray(resSearch.body.list)) {
return null;
}
if (!resSearch.body.list.pornstarsName || resSearch.body.list.pornstarsName.length === 0) {
return null;
}
if (!resSearch.body.list.pornstarsName || resSearch.body.list.pornstarsName.length === 0) {
return null;
}
const [actor] = resSearch.body.list.pornstarsName;
const url = `https://ddfnetwork.com${actor.href}`;
const [actor] = resSearch.body.list.pornstarsName;
const url = `https://ddfnetwork.com${actor.href}`;
const resActor = await bhttp.get(url);
const resActor = await bhttp.get(url);
if (resActor.statusCode !== 200) {
return null;
}
if (resActor.statusCode !== 200) {
return null;
}
return scrapeProfile(resActor.body.toString(), url, actorName);
return scrapeProfile(resActor.body.toString(), url, actorName);
}
module.exports = {
fetchLatest,
fetchProfile,
fetchScene,
fetchLatest,
fetchProfile,
fetchScene,
};

8
src/scrapers/digitalplayground.js
View File

@ -3,11 +3,11 @@
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'digitalplayground', 'modelprofile');
return fetchProfile(actorName, 'digitalplayground', 'modelprofile');
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
};

196
src/scrapers/dogfart.js
View File

@ -7,136 +7,136 @@ const { JSDOM } = require('jsdom');
const moment = require('moment');
async function getPhotos(albumUrl) {
const res = await bhttp.get(albumUrl);
const html = res.body.toString();
const { document } = new JSDOM(html).window;
const res = await bhttp.get(albumUrl);
const html = res.body.toString();
const { document } = new JSDOM(html).window;
const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => {
const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => {
const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
return {
url: pageUrl,
extract: ({ qu }) => qu.q('.scenes-module img', 'src'),
};
});
return {
url: pageUrl,
extract: ({ qu }) => qu.q('.scenes-module img', 'src'),
};
});
return photoUrls;
return photoUrls;
}
function scrapeLatest(html, site) {
const { document } = new JSDOM(html).window;
const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));
const { document } = new JSDOM(html).window;
const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));
return sceneElements.reduce((acc, element) => {
const siteUrl = element.querySelector('.help-block').textContent;
return sceneElements.reduce((acc, element) => {
const siteUrl = element.querySelector('.help-block').textContent;
if (`www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
// different dogfart site
return acc;
}
if (`www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
// different dogfart site
return acc;
}
const sceneLinkElement = element.querySelector('.thumbnail');
const url = `https://dogfartnetwork.com${sceneLinkElement.href}`;
const { pathname } = new URL(url);
const entryId = `${site.slug}_${pathname.split('/')[4]}`;
const sceneLinkElement = element.querySelector('.thumbnail');
const url = `https://dogfartnetwork.com${sceneLinkElement.href}`;
const { pathname } = new URL(url);
const entryId = `${site.slug}_${pathname.split('/')[4]}`;
const title = element.querySelector('.scene-title').textContent;
const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim());
const title = element.querySelector('.scene-title').textContent;
const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim());
const poster = `https:${element.querySelector('img').src}`;
const teaser = sceneLinkElement.dataset.preview_clip_url;
const poster = `https:${element.querySelector('img').src}`;
const teaser = sceneLinkElement.dataset.preview_clip_url;
return [
...acc,
{
url,
entryId,
title,
actors,
poster,
teaser: {
src: teaser,
},
site,
},
];
}, []);
return [
...acc,
{
url,
entryId,
title,
actors,
poster,
teaser: {
src: teaser,
},
site,
},
];
}, []);
}
async function scrapeScene(html, url, site) {
const { document } = new JSDOM(html).window;
const { document } = new JSDOM(html).window;
const title = document.querySelector('.description-title').textContent;
const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
const metaDescription = document.querySelector('meta[itemprop="description"]').content;
const description = metaDescription
? metaDescription.content
: document.querySelector('.description')
.textContent
.replace(/[ \t\n]{2,}/g, ' ')
.replace('...read more', '')
.trim();
const title = document.querySelector('.description-title').textContent;
const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
const metaDescription = document.querySelector('meta[itemprop="description"]').content;
const description = metaDescription
? metaDescription.content
: document.querySelector('.description')
.textContent
.replace(/[ \t\n]{2,}/g, ' ')
.replace('...read more', '')
.trim();
const channel = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
const { origin, pathname } = new URL(url);
const entryId = `${channel}_${pathname.split('/').slice(-2)[0]}`;
const channel = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
const { origin, pathname } = new URL(url);
const entryId = `${channel}_${pathname.split('/').slice(-2)[0]}`;
const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
const duration = moment
.duration(`00:${document
.querySelectorAll('.extra-info p')[1]
.textContent
.match(/\d+:\d+$/)[0]}`)
.asSeconds();
const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
const duration = moment
.duration(`00:${document
.querySelectorAll('.extra-info p')[1]
.textContent
.match(/\d+:\d+$/)[0]}`)
.asSeconds();
const trailerElement = document.querySelector('.html5-video');
const poster = `https:${trailerElement.dataset.poster}`;
const { trailer } = trailerElement.dataset;
const trailerElement = document.querySelector('.html5-video');
const poster = `https:${trailerElement.dataset.poster}`;
const { trailer } = trailerElement.dataset;
const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href;
const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href;
const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2);
const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2);
const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
return {
entryId,
url: `${origin}${pathname}`,
title,
description,
actors,
date,
duration,
poster,
photos,
trailer: {
src: trailer,
},
tags,
rating: {
stars,
},
site,
channel,
};
return {
entryId,
url: `${origin}${pathname}`,
title,
description,
actors,
date,
duration,
poster,
photos,
trailer: {
src: trailer,
},
tags,
rating: {
stars,
},
site,
channel,
};
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
return scrapeLatest(res.body.toString(), site);
return scrapeLatest(res.body.toString(), site);
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site);
return scrapeScene(res.body.toString(), url, site);
}
module.exports = {
fetchLatest,
fetchScene,
fetchLatest,
fetchScene,
};

8
src/scrapers/evilangel.js
View File

@ -3,8 +3,8 @@
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = {
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchScene,
fetchUpcoming: fetchApiUpcoming,
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchScene,
fetchUpcoming: fetchApiUpcoming,
};

8
src/scrapers/fakehub.js
View File

@ -3,11 +3,11 @@
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
async function networkFetchProfile(actorName) {
return fetchProfile(actorName, 'fakehub', 'modelprofile');
return fetchProfile(actorName, 'fakehub', 'modelprofile');
}
module.exports = {
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
fetchLatest,
fetchProfile: networkFetchProfile,
fetchScene,
};

134
src/scrapers/famedigital.js
View File

@ -1,115 +1,115 @@
'use strict';
const {
fetchLatest,
fetchApiLatest,
fetchUpcoming,
fetchApiUpcoming,
fetchScene,
fetchProfile,
fetchApiProfile,
scrapeAll,
fetchLatest,
fetchApiLatest,
fetchUpcoming,
fetchApiUpcoming,
fetchScene,
fetchProfile,
fetchApiProfile,
scrapeAll,
} = require('./gamma');
const { get } = require('../utils/q');
const slugify = require('../utils/slugify');
function extractLowArtActors(release) {
const actors = release.title
.replace(/solo/i, '')
.split(/,|\band\b/ig)
.map(actor => actor.trim());
const actors = release.title
.replace(/solo/i, '')
.split(/,|\band\b/ig)
.map(actor => actor.trim());
return {
...release,
actors,
};
return {
...release,
actors,
};
}
async function networkFetchLatest(site, page = 1) {
if (site.parameters?.api) return fetchApiLatest(site, page, false);
if (site.parameters?.api) return fetchApiLatest(site, page, false);
const releases = await fetchLatest(site, page);
const releases = await fetchLatest(site, page);
if (site.slug === 'lowartfilms') {
return releases.map(release => extractLowArtActors(release));
}
if (site.slug === 'lowartfilms') {
return releases.map(release => extractLowArtActors(release));
}
return releases;
return releases;
}
async function networkFetchScene(url, site) {
const release = await fetchScene(url, site);
const release = await fetchScene(url, site);
if (site.slug === 'lowartfilms') {
return extractLowArtActors(release);
}
if (site.slug === 'lowartfilms') {
return extractLowArtActors(release);
}
return release;
return release;
}
async function networkFetchUpcoming(site, page = 1) {
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
return fetchUpcoming(site, page);
return fetchUpcoming(site, page);
}
function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
}
async function fetchClassicProfile(actorName, siteSlug) {
const actorSlug = slugify(actorName);
const actorSlug = slugify(actorName);
const url = `https://${siteSlug}.com/en/pornstars`;
const pornstarsRes = await get(url);
const url = `https://${siteSlug}.com/en/pornstars`;
const pornstarsRes = await get(url);
if (!pornstarsRes.ok) return null;
if (!pornstarsRes.ok) return null;
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find(el => slugify(el.textContent) === actorSlug)
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find(el => slugify(el.textContent) === actorSlug)
?.value;
if (actorPath) {
const actorUrl = `https://${siteSlug}.com${actorPath}`;
const res = await get(actorUrl);
if (actorPath) {
const actorUrl = `https://${siteSlug}.com${actorPath}`;
const res = await get(actorUrl);
if (res.ok) {
const releases = scrapeAll(res.item, null, `https://www.${siteSlug}.com`, false);
if (res.ok) {
const releases = scrapeAll(res.item, null, `https://www.${siteSlug}.com`, false);
return { releases };
}
}
return { releases };
}
}
return null;
return null;
}
async function networkFetchProfile(actorName, scraperSlug, site, include) {
// not all Fame Digital sites offer Gamma actors
const [devils, rocco, peter, silvia] = await Promise.all([
fetchApiProfile(actorName, 'devilsfilm', true),
fetchApiProfile(actorName, 'roccosiffredi'),
include.scenes ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl, include) : [],
include.scenes ? fetchClassicProfile(actorName, 'silviasaint') : [],
include.scenes ? fetchClassicProfile(actorName, 'silverstonedvd') : [],
]);
// not all Fame Digital sites offer Gamma actors
const [devils, rocco, peter, silvia] = await Promise.all([
fetchApiProfile(actorName, 'devilsfilm', true),
fetchApiProfile(actorName, 'roccosiffredi'),
include.scenes ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl, include) : [],
include.scenes ? fetchClassicProfile(actorName, 'silviasaint') : [],
include.scenes ? fetchClassicProfile(actorName, 'silverstonedvd') : [],
]);
if (devils || rocco || peter) {
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []);
if (devils || rocco || peter) {
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []);
return {
...peter,
...rocco,
...devils,
releases,
};
}
return {
...peter,
...rocco,
...devils,
releases,
};
}
return null;
return null;
}
module.exports = {
fetchLatest: networkFetchLatest,
fetchProfile: networkFetchProfile,
fetchScene: networkFetchScene,
fetchUpcoming: networkFetchUpcoming,
fetchLatest: networkFetchLatest,
fetchProfile: networkFetchProfile,
fetchScene: networkFetchScene,
fetchUpcoming: networkFetchUpcoming,
};

6
src/scrapers/fantasymassage.js
View File

@ -4,7 +4,7 @@ const { fetchLatest, fetchUpcoming, fetchScene } = require('./gamma');
module.exports = {
fetchLatest,
fetchScene,
fetchUpcoming,
fetchLatest,
fetchScene,
fetchUpcoming,
};

106
src/scrapers/freeones.js
View File

@ -5,89 +5,89 @@ const { JSDOM } = require('jsdom');
const moment = require('moment');
function scrapeProfile(html, actorName) {
const { document } = new JSDOM(html).window;
const profile = { name: actorName };
const { document } = new JSDOM(html).window;
const profile = { name: actorName };
const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), el => decodeURI(el.href)).reduce((acc, item) => {
const keyMatch = item.match(/\[\w+\]/);
const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), el => decodeURI(el.href)).reduce((acc, item) => {
const keyMatch = item.match(/\[\w+\]/);
if (keyMatch) {
const key = keyMatch[0].slice(1, -1);
const [, value] = item.split('=');
if (keyMatch) {
const key = keyMatch[0].slice(1, -1);
const [, value] = item.split('=');
// both hip and waist link to 'waist', assume biggest value is hip
if (key === 'waist' && acc.waist) {
if (acc.waist > value) {
acc.hip = acc.waist;
acc.waist = value;
// both hip and waist link to 'waist', assume biggest value is hip
if (key === 'waist' && acc.waist) {
if (acc.waist > value) {
acc.hip = acc.waist;
acc.waist = value;
return acc;
}
return acc;
}
acc.hip = value;
acc.hip = value;
return acc;
}
return acc;
}
acc[key] = value;
}
acc[key] = value;
}
return acc;
}, {});
return acc;
}, {});
if (bio.dateOfBirth) profile.birthdate = moment.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
if (bio.dateOfBirth) profile.birthdate = moment.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
if (profile.placeOfBirth || bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`;
profile.eyes = bio.eyeColor;
profile.hair = bio.hairColor;
profile.ethnicity = bio.ethnicity;
if (profile.placeOfBirth || bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`;
profile.eyes = bio.eyeColor;
profile.hair = bio.hairColor;
profile.ethnicity = bio.ethnicity;
profile.bust = bio.bra;
if (bio.waist) profile.waist = Number(bio.waist.split(',')[0]);
if (bio.hip) profile.hip = Number(bio.hip.split(',')[0]);
profile.bust = bio.bra;
if (bio.waist) profile.waist = Number(bio.waist.split(',')[0]);
if (bio.hip) profile.hip = Number(bio.hip.split(',')[0]);
if (bio.height) profile.height = Number(bio.height.split(',')[0]);
if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]);
if (bio.height) profile.height = Number(bio.height.split(',')[0]);
if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]);
profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), el => el.href);
profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), el => el.href);
const avatar = document.querySelector('.profile-image-large img').src;
if (!avatar.match('placeholder')) profile.avatar = { src: avatar, copyright: null };
const avatar = document.querySelector('.profile-image-large img').src;
if (!avatar.match('placeholder')) profile.avatar = { src: avatar, copyright: null };
return profile;
return profile;
}
function scrapeSearch(html) {
const { document } = new JSDOM(html).window;
const { document } = new JSDOM(html).window;
return document.querySelector('a.image-link')?.href || null;
return document.querySelector('a.image-link')?.href || null;
}
async function fetchProfile(actorName) {
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
const res = await bhttp.get(`https://freeones.nl/${actorSlug}/profile`);
const res = await bhttp.get(`https://freeones.nl/${actorSlug}/profile`);
if (res.statusCode === 200) {
return scrapeProfile(res.body.toString(), actorName);
}
if (res.statusCode === 200) {
return scrapeProfile(res.body.toString(), actorName);
}
const searchRes = await bhttp.get(`https://freeones.nl/babes?q=${actorName}`);
const actorPath = scrapeSearch(searchRes.body.toString());
const searchRes = await bhttp.get(`https://freeones.nl/babes?q=${actorName}`);
const actorPath = scrapeSearch(searchRes.body.toString());
if (actorPath) {
const actorRes = await bhttp.get(`https://freeones.nl${actorPath}/profile`);
if (actorPath) {
const actorRes = await bhttp.get(`https://freeones.nl${actorPath}/profile`);
if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString(), actorName);
}
if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString(), actorName);
}
return null;
}
return null;
}
return null;
return null;
}
module.exports = {
fetchProfile,
fetchProfile,
};

170
src/scrapers/freeones_legacy.js
View File

@ -6,135 +6,135 @@ const { JSDOM } = require('jsdom');
const moment = require('moment');
async function scrapeProfileFrontpage(html, url, name) {
const { document } = new JSDOM(html).window;
const bioEl = document.querySelector('.dashboard-bio-list');
const { document } = new JSDOM(html).window;
const bioEl = document.querySelector('.dashboard-bio-list');
const bioUrl = `https:${document.querySelector('.seemore a').href}`;
const bioUrl = `https:${document.querySelector('.seemore a').href}`;
const keys = Array.from(bioEl.querySelectorAll('dt'), el => el.textContent.trim());
const values = Array.from(bioEl.querySelectorAll('dd'), el => el.textContent.trim());
const keys = Array.from(bioEl.querySelectorAll('dt'), el => el.textContent.trim());
const values = Array.from(bioEl.querySelectorAll('dd'), el => el.textContent.trim());
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
const profile = {
name,
gender: 'female',
};
const profile = {
name,
gender: 'female',
};
const birthdateString = bio['Date of Birth:'];
const measurementsString = bio['Measurements:'];
const birthdateString = bio['Date of Birth:'];
const measurementsString = bio['Measurements:'];
const birthCityString = bio['Place of Birth:'];
const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString;
const birthCityString = bio['Place of Birth:'];
const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString;
const birthCountryString = bio['Country of Origin:'];
const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString;
const birthCountryString = bio['Country of Origin:'];
const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString;
const piercingsString = bio['Piercings:'];
const tattoosString = bio['Tattoos:'];
const piercingsString = bio['Piercings:'];
const tattoosString = bio['Tattoos:'];
if (birthdateString && birthdateString !== 'Unknown (add)') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate();
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
if (birthdateString && birthdateString !== 'Unknown (add)') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate();
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
if (bio['Fake Boobs:']) profile.naturalBoobs = bio['Fake Boobs:'] === 'No';
profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`;
if (bio['Fake Boobs:']) profile.naturalBoobs = bio['Fake Boobs:'] === 'No';
profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`;
profile.hair = bio['Hair Color:'].toLowerCase();
profile.eyes = bio['Eye Color:'].toLowerCase();
profile.hair = bio['Hair Color:'].toLowerCase();
profile.eyes = bio['Eye Color:'].toLowerCase();
if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString;
if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString;
if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString;
if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString;
profile.social = Array.from(bioEl.querySelectorAll('.dashboard-socialmedia a'), el => el.href);
profile.social = Array.from(bioEl.querySelectorAll('.dashboard-socialmedia a'), el => el.href);
return {
profile,
url: bioUrl,
};
return {
profile,
url: bioUrl,
};
}
async function scrapeProfileBio(html, frontpageProfile, url, name) {
const { document } = new JSDOM(html).window;
const bioEl = document.querySelector('#biographyTable');
const { document } = new JSDOM(html).window;
const bioEl = document.querySelector('#biographyTable');
const keys = Array.from(bioEl.querySelectorAll('td:nth-child(1)'), el => el.textContent.trim());
const values = Array.from(bioEl.querySelectorAll('td:nth-child(2)'), el => el.textContent.trim());
const keys = Array.from(bioEl.querySelectorAll('td:nth-child(1)'), el => el.textContent.trim());
const values = Array.from(bioEl.querySelectorAll('td:nth-child(2)'), el => el.textContent.trim());
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
const profile = {
...frontpageProfile,
name,
gender: 'female',
};
const profile = {
...frontpageProfile,
name,
gender: 'female',
};
const birthdateString = bio['Date of Birth:'];
const measurementsString = bio['Measurements:'];
const birthdateString = bio['Date of Birth:'];
const measurementsString = bio['Measurements:'];
const birthCityString = bio['Place of Birth:'];
const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString;
const birthCityString = bio['Place of Birth:'];
const birthCity = birthCityString !== undefined && birthCityString !== 'Unknown' && birthCityString !== 'Unknown (add)' && birthCityString;
const birthCountryString = bio['Country of Origin:'];
const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString;
const birthCountryString = bio['Country of Origin:'];
const birthCountry = birthCountryString !== undefined && birthCountryString !== 'Unknown' && birthCountryString !== 'Unknown (add)' && birthCountryString;
const piercingsString = bio['Piercings:'];
const tattoosString = bio['Tattoos:'];
const piercingsString = bio['Piercings:'];
const tattoosString = bio['Tattoos:'];
if (birthdateString && birthdateString !== 'Unknown') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate();
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
if (birthdateString && birthdateString !== 'Unknown') profile.birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate();
if (measurementsString) [profile.bust, profile.waist, profile.hip] = measurementsString.split('-').map(measurement => (measurement === '??' ? null : measurement));
if (bio['Fake boobs']) profile.naturalBoobs = bio['Fake boobs:'] === 'No';
profile.ethnicity = bio['Ethnicity:'];
if (bio['Fake boobs']) profile.naturalBoobs = bio['Fake boobs:'] === 'No';
profile.ethnicity = bio['Ethnicity:'];
profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`;
profile.birthPlace = `${birthCity || ''}${birthCity ? ', ' : ''}${birthCountry || ''}`;
profile.hair = bio['Hair Color:'].toLowerCase();
profile.eyes = bio['Eye Color:'].toLowerCase();
profile.height = Number(bio['Height:'].match(/\d+/)[0]);
profile.weight = Number(bio['Weight:'].match(/\d+/)[0]);
profile.hair = bio['Hair Color:'].toLowerCase();
profile.eyes = bio['Eye Color:'].toLowerCase();
profile.height = Number(bio['Height:'].match(/\d+/)[0]);
profile.weight = Number(bio['Weight:'].match(/\d+/)[0]);
if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
if (piercingsString) profile.hasPiercings = !!(piercingsString !== 'Unknown (add)' && piercingsString !== 'None');
if (tattoosString) profile.hasTattoos = !!(tattoosString !== 'Unknown (add)' && tattoosString !== 'None');
if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString;
if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString;
if (profile.hasPiercings && piercingsString !== 'various') profile.piercings = piercingsString;
if (profile.hasTattoos && tattoosString !== 'various') profile.tattoos = tattoosString;
profile.social = Array.from(bioEl.querySelectorAll('#socialmedia a'), el => el.href);
profile.social = Array.from(bioEl.querySelectorAll('#socialmedia a'), el => el.href);
return profile;
return profile;
}
async function fetchProfile(actorName) {
const slug = actorName.replace(' ', '_');
const frontpageUrl = `https://www.freeones.com/html/v_links/${slug}`;
const slug = actorName.replace(' ', '_');
const frontpageUrl = `https://www.freeones.com/html/v_links/${slug}`;
const resFrontpage = await bhttp.get(frontpageUrl);
const resFrontpage = await bhttp.get(frontpageUrl);
if (resFrontpage.statusCode === 200) {
const { url, bio } = await scrapeProfileFrontpage(resFrontpage.body.toString(), frontpageUrl, actorName);
const resBio = await bhttp.get(url);
if (resFrontpage.statusCode === 200) {
const { url, bio } = await scrapeProfileFrontpage(resFrontpage.body.toString(), frontpageUrl, actorName);
const resBio = await bhttp.get(url);
return scrapeProfileBio(resBio.body.toString(), bio, url, actorName);
}
return scrapeProfileBio(resBio.body.toString(), bio, url, actorName);
}
// apparently some actors are appended 'Babe' as their surname...
const fallbackSlug = `${slug}_Babe`;
const fallbackUrl = `https://www.freeones.com/html/s_links/${fallbackSlug}`;
const resFallback = await bhttp.get(fallbackUrl);
// apparently some actors are appended 'Babe' as their surname...
const fallbackSlug = `${slug}_Babe`;
const fallbackUrl = `https://www.freeones.com/html/s_links/${fallbackSlug}`;
const resFallback = await bhttp.get(fallbackUrl);
if (resFallback.statusCode === 200) {
const { url, profile } = await scrapeProfileFrontpage(resFallback.body.toString(), fallbackUrl, actorName);
const resBio = await bhttp.get(url);
if (resFallback.statusCode === 200) {
const { url, profile } = await scrapeProfileFrontpage(resFallback.body.toString(), fallbackUrl, actorName);
const resBio = await bhttp.get(url);
return scrapeProfileBio(resBio.body.toString(), profile, url, actorName);
}
return scrapeProfileBio(resBio.body.toString(), profile, url, actorName);
}
return null;
return null;
}
module.exports = {
fetchProfile,
fetchProfile,
};

104
src/scrapers/fullpornnetwork.js
View File

@ -4,93 +4,93 @@ const { get, geta, ctxa } = require('../utils/q');
const slugify = require('../utils/slugify');
function scrapeAll(scenes) {
return scenes.map(({ el, qu }) => {
const release = {};
return scenes.map(({ el, qu }) => {
const release = {};
release.entryId = el.dataset.setid || qu.q('.update_thumb', 'id').match(/\w+-\w+-(\d+)-\d+/)[1];
release.url = qu.url('.title');
release.entryId = el.dataset.setid || qu.q('.update_thumb', 'id').match(/\w+-\w+-(\d+)-\d+/)[1];
release.url = qu.url('.title');
release.title = qu.q('.title', true);
release.description = qu.q('.title', 'title');
release.title = qu.q('.title', true);
release.description = qu.q('.title', 'title');
release.date = qu.date('.video-data > span:last-child', 'YYYY-MM-DD');
release.duration = qu.dur('.video-data > span');
release.date = qu.date('.video-data > span:last-child', 'YYYY-MM-DD');
release.duration = qu.dur('.video-data > span');
release.actors = qu.all('.update_models a', true);
release.actors = qu.all('.update_models a', true);
const poster = qu.q('.update_thumb', 'src0_1x');
release.poster = [
poster.replace('-1x', '-2x'),
poster,
];
const poster = qu.q('.update_thumb', 'src0_1x');
release.poster = [
poster.replace('-1x', '-2x'),
poster,
];
return release;
});
return release;
});
}
function scrapeScene({ q, qa, qd, qtx }, url, _site) {
const release = { url };
const release = { url };
release.entryId = q('#image_parent img', 'id').match(/\w+-\w+-(\d+)-\d+/)[1];
release.entryId = q('#image_parent img', 'id').match(/\w+-\w+-(\d+)-\d+/)[1];
release.title = q('.trailer_title', true);
release.description = qtx('.text p');
release.date = qd('span[data-dateadded]', 'YYYY-MM-DD', null, 'data-dateadded');
release.title = q('.trailer_title', true);
release.description = qtx('.text p');
release.date = qd('span[data-dateadded]', 'YYYY-MM-DD', null, 'data-dateadded');
release.actors = qa('.update_models a', true);
release.tags = qa('.video-info a[href*="/categories"]', true);
release.actors = qa('.update_models a', true);
release.tags = qa('.video-info a[href*="/categories"]', true);
const poster = q('#image_parent img', 'src0_1x');
release.poster = [
poster.replace('-1x', '-2x'),
poster,
];
const poster = q('#image_parent img', 'src0_1x');
release.poster = [
poster.replace('-1x', '-2x'),
poster,
];
return release;
return release;
}
function scrapeProfile({ el, q, qtx }) {
const profile = {};
const profile = {};
const description = qtx('.model-bio');
if (description) profile.description = description;
const description = qtx('.model-bio');
if (description) profile.description = description;
profile.avatar = [
q('.model-image img', 'src0_2x'),
q('.model-image img', 'src0_1x'),
];
profile.avatar = [
q('.model-image img', 'src0_2x'),
q('.model-image img', 'src0_1x'),
];
profile.releases = scrapeAll(ctxa(el, '.update'));
profile.releases = scrapeAll(ctxa(el, '.update'));
return profile;
return profile;
}
async function fetchLatest(site, page = 1) {
const url = `${site.url}/categories/movies_${page}_d.html`;
const res = await geta(url, '.latest-updates .update');
const url = `${site.url}/categories/movies_${page}_d.html`;
const res = await geta(url, '.latest-updates .update');
return res.ok ? scrapeAll(res.items, site) : res.status;
return res.ok ? scrapeAll(res.items, site) : res.status;
}
async function fetchScene(url, site) {
const res = await get(url, '.content-wrapper');
const res = await get(url, '.content-wrapper');
return res.ok ? scrapeScene(res.item, url, site) : res.status;
return res.ok ? scrapeScene(res.item, url, site) : res.status;
}
async function fetchProfile(actorName, scraperSlug) {
const actorSlug = slugify(actorName, '');
const url = scraperSlug === 'povperverts'
? `https://povperverts.net/models/${actorSlug}.html`
: `https://${scraperSlug}.com/models/${actorSlug}.html`;
const actorSlug = slugify(actorName, '');
const url = scraperSlug === 'povperverts'
? `https://povperverts.net/models/${actorSlug}.html`
: `https://${scraperSlug}.com/models/${actorSlug}.html`;
const res = await get(url);
const res = await get(url);
return res.ok ? scrapeProfile(res.item, actorName) : res.status;
return res.ok ? scrapeProfile(res.item, actorName) : res.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
fetchLatest,
fetchScene,
fetchProfile,
};

Some files were not shown because too many files have changed in this diff Show More