forked from DebaucheryLibrarian/traxxx
Added periodic memory logger.
This commit is contained in:
parent
a867817dc1
commit
ccb99e278c
|
@ -2,7 +2,7 @@
|
||||||
"root": true,
|
"root": true,
|
||||||
"extends": ["airbnb-base", "plugin:vue/recommended"],
|
"extends": ["airbnb-base", "plugin:vue/recommended"],
|
||||||
"parserOptions": {
|
"parserOptions": {
|
||||||
"parser": "babel-eslint",
|
"parser": "@babel/eslint-parser",
|
||||||
"ecmaVersion": 2019,
|
"ecmaVersion": 2019,
|
||||||
"sourceType": "module"
|
"sourceType": "module"
|
||||||
},
|
},
|
||||||
|
|
|
@ -5,7 +5,7 @@ $breakpoint3: 1200px;
|
||||||
$breakpoint4: 1500px;
|
$breakpoint4: 1500px;
|
||||||
|
|
||||||
:root {
|
:root {
|
||||||
--primary: #c63971;
|
--primary: #e33379;
|
||||||
--primary-strong: #f90071;
|
--primary-strong: #f90071;
|
||||||
--primary-faded: #ffcce4;
|
--primary-faded: #ffcce4;
|
||||||
|
|
||||||
|
@ -46,7 +46,7 @@ $breakpoint4: 1500px;
|
||||||
--logo-shadow: drop-shadow(1px 0 0 $shadow-weak) drop-shadow(-1px 0 0 $shadow-weak) drop-shadow(0 1px 0 $shadow-weak) drop-shadow(0 -1px 0 $shadow-weak);
|
--logo-shadow: drop-shadow(1px 0 0 $shadow-weak) drop-shadow(-1px 0 0 $shadow-weak) drop-shadow(0 1px 0 $shadow-weak) drop-shadow(0 -1px 0 $shadow-weak);
|
||||||
--logo-highlight: drop-shadow(0 0 1px $highlight);
|
--logo-highlight: drop-shadow(0 0 1px $highlight);
|
||||||
|
|
||||||
--info: #361723;
|
--info: #321b24;
|
||||||
|
|
||||||
--male: #0af;
|
--male: #0af;
|
||||||
--female: #f0a;
|
--female: #f0a;
|
||||||
|
|
|
@ -281,7 +281,7 @@ function initActorActions(store, router) {
|
||||||
|
|
||||||
return {
|
return {
|
||||||
actor: curateActor(actor, null, curateRelease),
|
actor: curateActor(actor, null, curateRelease),
|
||||||
releases: actor.scenesConnection.releases.map(release => curateRelease(release)),
|
releases: actor.scenesConnection.releases.map((release) => curateRelease(release)),
|
||||||
totalCount: actor.scenesConnection.totalCount,
|
totalCount: actor.scenesConnection.totalCount,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -429,7 +429,7 @@ function initActorActions(store, router) {
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
actors: actors.map(actor => curateActor(actor)),
|
actors: actors.map((actor) => curateActor(actor)),
|
||||||
totalCount,
|
totalCount,
|
||||||
countries,
|
countries,
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
function favoritesStash(state) {
|
function favoritesStash(state) {
|
||||||
return state.user.stashes.find(stash => stash.slug === 'favorites');
|
return state.user.stashes.find((stash) => stash.slug === 'favorites');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -31,8 +31,8 @@ function curateActor(actor, release) {
|
||||||
|
|
||||||
if (actor.profiles) {
|
if (actor.profiles) {
|
||||||
const photos = actor.profiles
|
const photos = actor.profiles
|
||||||
.map(profile => ({ entity: profile.entity, ...profile.avatar }))
|
.map((profile) => ({ entity: profile.entity, ...profile.avatar }))
|
||||||
.filter(avatar => avatar.id && (!curatedActor.avatar || avatar.hash !== curatedActor.avatar.hash));
|
.filter((avatar) => avatar.id && (!curatedActor.avatar || avatar.hash !== curatedActor.avatar.hash));
|
||||||
|
|
||||||
const descriptions = actor.profiles.reduce((acc, profile) => ({
|
const descriptions = actor.profiles.reduce((acc, profile) => ({
|
||||||
...acc,
|
...acc,
|
||||||
|
@ -57,10 +57,10 @@ function curateActor(actor, release) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (actor.stashes) {
|
if (actor.stashes) {
|
||||||
curatedActor.stashes = actor.stashes.filter(Boolean).map(stash => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
|
curatedActor.stashes = actor.stashes.filter(Boolean).map((stash) => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
|
||||||
}
|
}
|
||||||
|
|
||||||
curatedActor.stashes = actor.stashes?.map(stash => stash.stash || stash) || [];
|
curatedActor.stashes = actor.stashes?.map((stash) => stash.stash || stash) || [];
|
||||||
|
|
||||||
return curatedActor;
|
return curatedActor;
|
||||||
}
|
}
|
||||||
|
@ -70,21 +70,21 @@ function curateRelease(release) {
|
||||||
...release,
|
...release,
|
||||||
actors: [],
|
actors: [],
|
||||||
poster: release.poster && release.poster.media,
|
poster: release.poster && release.poster.media,
|
||||||
tags: release.tags ? release.tags.map(tag => tag.tag || tag) : [],
|
tags: release.tags ? release.tags.map((tag) => tag.tag || tag) : [],
|
||||||
};
|
};
|
||||||
|
|
||||||
if (release.scenes) curatedRelease.scenes = release.scenes.filter(Boolean).map(({ scene }) => curateRelease(scene));
|
if (release.scenes) curatedRelease.scenes = release.scenes.filter(Boolean).map(({ scene }) => curateRelease(scene));
|
||||||
if (release.movies) curatedRelease.movies = release.movies.filter(Boolean).map(({ movie }) => curateRelease(movie));
|
if (release.movies) curatedRelease.movies = release.movies.filter(Boolean).map(({ movie }) => curateRelease(movie));
|
||||||
if (release.chapters) curatedRelease.chapters = release.chapters.filter(Boolean).map(chapter => curateRelease(chapter));
|
if (release.chapters) curatedRelease.chapters = release.chapters.filter(Boolean).map((chapter) => curateRelease(chapter));
|
||||||
if (release.photos) curatedRelease.photos = release.photos.filter(Boolean).map(photo => photo.media || photo);
|
if (release.photos) curatedRelease.photos = release.photos.filter(Boolean).map((photo) => photo.media || photo);
|
||||||
if (release.covers) curatedRelease.covers = release.covers.filter(Boolean).map(({ media }) => media);
|
if (release.covers) curatedRelease.covers = release.covers.filter(Boolean).map(({ media }) => media);
|
||||||
if (release.trailer) curatedRelease.trailer = release.trailer.media;
|
if (release.trailer) curatedRelease.trailer = release.trailer.media;
|
||||||
if (release.teaser) curatedRelease.teaser = release.teaser.media;
|
if (release.teaser) curatedRelease.teaser = release.teaser.media;
|
||||||
if (release.actors) curatedRelease.actors = release.actors.filter(Boolean).map(actor => curateActor(actor.actor || actor, curatedRelease));
|
if (release.actors) curatedRelease.actors = release.actors.filter(Boolean).map((actor) => curateActor(actor.actor || actor, curatedRelease));
|
||||||
if (release.directors) curatedRelease.directors = release.directors.filter(Boolean).map(director => curateActor(director.director || director, curatedRelease));
|
if (release.directors) curatedRelease.directors = release.directors.filter(Boolean).map((director) => curateActor(director.director || director, curatedRelease));
|
||||||
if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.filter(Boolean).map(({ tag }) => tag);
|
if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.filter(Boolean).map(({ tag }) => tag);
|
||||||
if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.filter(Boolean).map(({ actor }) => curateActor(actor, curatedRelease));
|
if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.filter(Boolean).map(({ actor }) => curateActor(actor, curatedRelease));
|
||||||
if (release.stashes) curatedRelease.stashes = release.stashes.filter(Boolean).map(stash => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
|
if (release.stashes) curatedRelease.stashes = release.stashes.filter(Boolean).map((stash) => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
|
||||||
|
|
||||||
if (release.productionLocation) {
|
if (release.productionLocation) {
|
||||||
curatedRelease.productionLocation = {
|
curatedRelease.productionLocation = {
|
||||||
|
@ -108,14 +108,14 @@ function curateEntity(entity, parent, releases) {
|
||||||
|
|
||||||
if (entity.children) {
|
if (entity.children) {
|
||||||
if (entity.children.nodes) {
|
if (entity.children.nodes) {
|
||||||
curatedEntity.children = entity.children.nodes.map(childEntity => curateEntity(childEntity, curatedEntity));
|
curatedEntity.children = entity.children.nodes.map((childEntity) => curateEntity(childEntity, curatedEntity));
|
||||||
}
|
}
|
||||||
|
|
||||||
curatedEntity.childrenTotal = entity.children.totalCount;
|
curatedEntity.childrenTotal = entity.children.totalCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (entity.parent || parent) curatedEntity.parent = curateEntity(entity.parent || parent);
|
if (entity.parent || parent) curatedEntity.parent = curateEntity(entity.parent || parent);
|
||||||
if (releases) curatedEntity.releases = releases.map(release => curateRelease(release));
|
if (releases) curatedEntity.releases = releases.map((release) => curateRelease(release));
|
||||||
|
|
||||||
if (entity.connection) {
|
if (entity.connection) {
|
||||||
curatedEntity.sceneTotal = entity.connection.totalCount;
|
curatedEntity.sceneTotal = entity.connection.totalCount;
|
||||||
|
@ -142,7 +142,7 @@ function curateStash(stash) {
|
||||||
|
|
||||||
if (stash.scenes || stash.scenesConnection?.scenes) {
|
if (stash.scenes || stash.scenesConnection?.scenes) {
|
||||||
curatedStash.sceneTotal = stash.scenesConnection?.totalCount || null;
|
curatedStash.sceneTotal = stash.scenesConnection?.totalCount || null;
|
||||||
curatedStash.scenes = (stash.scenesConnection?.scenes || stash.scenes).map(item => ({
|
curatedStash.scenes = (stash.scenesConnection?.scenes || stash.scenes).map((item) => ({
|
||||||
...item,
|
...item,
|
||||||
scene: curateRelease(item.scene),
|
scene: curateRelease(item.scene),
|
||||||
}));
|
}));
|
||||||
|
@ -150,7 +150,7 @@ function curateStash(stash) {
|
||||||
|
|
||||||
if (stash.actors || stash.actorsConnection?.actors) {
|
if (stash.actors || stash.actorsConnection?.actors) {
|
||||||
curatedStash.actorTotal = stash.actorsConnection?.totalCount || null;
|
curatedStash.actorTotal = stash.actorsConnection?.totalCount || null;
|
||||||
curatedStash.actors = (stash.actorsConnection?.actors || stash.actors).map(item => ({
|
curatedStash.actors = (stash.actorsConnection?.actors || stash.actors).map((item) => ({
|
||||||
...item,
|
...item,
|
||||||
actor: curateActor(item.actor),
|
actor: curateActor(item.actor),
|
||||||
}));
|
}));
|
||||||
|
@ -158,7 +158,7 @@ function curateStash(stash) {
|
||||||
|
|
||||||
if (stash.movies || stash.moviesConnection?.movies) {
|
if (stash.movies || stash.moviesConnection?.movies) {
|
||||||
curatedStash.movieTotal = stash.moviesConnection?.totalCount || null;
|
curatedStash.movieTotal = stash.moviesConnection?.totalCount || null;
|
||||||
curatedStash.movies = (stash.moviesConnection?.movies || stash.movies).map(item => ({
|
curatedStash.movies = (stash.moviesConnection?.movies || stash.movies).map((item) => ({
|
||||||
...item,
|
...item,
|
||||||
movie: curateRelease(item.movie),
|
movie: curateRelease(item.movie),
|
||||||
}));
|
}));
|
||||||
|
@ -175,11 +175,11 @@ function curateAlert(alert) {
|
||||||
const curatedAlert = alert;
|
const curatedAlert = alert;
|
||||||
|
|
||||||
if (alert.actors) {
|
if (alert.actors) {
|
||||||
curatedAlert.actors = alert.actors.map(actor => curateActor(actor.actor || actor));
|
curatedAlert.actors = alert.actors.map((actor) => curateActor(actor.actor || actor));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (alert.tags) {
|
if (alert.tags) {
|
||||||
curatedAlert.tags = alert.tags.map(tag => curateTag(tag.tag || tag));
|
curatedAlert.tags = alert.tags.map((tag) => curateTag(tag.tag || tag));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (alert.entity) {
|
if (alert.entity) {
|
||||||
|
@ -187,7 +187,7 @@ function curateAlert(alert) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (alert.stashes) {
|
if (alert.stashes) {
|
||||||
curatedAlert.stashes = alert.stashes.map(stash => curateStash(stash.stash || stash));
|
curatedAlert.stashes = alert.stashes.map((stash) => curateStash(stash.stash || stash));
|
||||||
}
|
}
|
||||||
|
|
||||||
return curatedAlert;
|
return curatedAlert;
|
||||||
|
@ -201,11 +201,11 @@ function curateUser(user) {
|
||||||
const curatedUser = user;
|
const curatedUser = user;
|
||||||
|
|
||||||
if (user.stashes) {
|
if (user.stashes) {
|
||||||
curatedUser.stashes = user.stashes.map(stash => curateStash(stash.stash || stash));
|
curatedUser.stashes = user.stashes.map((stash) => curateStash(stash.stash || stash));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (user.alerts) {
|
if (user.alerts) {
|
||||||
curatedUser.alerts = user.alerts.map(alert => curateAlert(alert.alert || alert));
|
curatedUser.alerts = user.alerts.map((alert) => curateAlert(alert.alert || alert));
|
||||||
}
|
}
|
||||||
|
|
||||||
return curatedUser;
|
return curatedUser;
|
||||||
|
|
|
@ -212,7 +212,7 @@ function initEntitiesActions(store, router) {
|
||||||
entitySlugs,
|
entitySlugs,
|
||||||
});
|
});
|
||||||
|
|
||||||
return entities.map(entity => curateEntity(entity));
|
return entities.map((entity) => curateEntity(entity));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function searchEntities({ _commit }, { query, limit = 20 }) {
|
async function searchEntities({ _commit }, { query, limit = 20 }) {
|
||||||
|
@ -246,7 +246,7 @@ function initEntitiesActions(store, router) {
|
||||||
limit,
|
limit,
|
||||||
});
|
});
|
||||||
|
|
||||||
return entities.map(entity => curateEntity(entity));
|
return entities.map((entity) => curateEntity(entity));
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|
|
@ -5,7 +5,7 @@ export function formatDuration(duration, forceHours) {
|
||||||
const minutes = Math.floor((duration % 3600) / 60);
|
const minutes = Math.floor((duration % 3600) / 60);
|
||||||
const seconds = Math.floor(duration % 60);
|
const seconds = Math.floor(duration % 60);
|
||||||
|
|
||||||
const [formattedHours, formattedMinutes, formattedSeconds] = [hours, minutes, seconds].map(segment => segment.toString().padStart(2, '0'));
|
const [formattedHours, formattedMinutes, formattedSeconds] = [hours, minutes, seconds].map((segment) => segment.toString().padStart(2, '0'));
|
||||||
|
|
||||||
if (duration >= 3600 || forceHours) {
|
if (duration >= 3600 || forceHours) {
|
||||||
return `${formattedHours}:${formattedMinutes}:${formattedSeconds}`;
|
return `${formattedHours}:${formattedMinutes}:${formattedSeconds}`;
|
||||||
|
|
|
@ -260,7 +260,7 @@ const releaseTagsFragment = `
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const releasePosterFragment = `
|
const releasePosterFragment = `
|
||||||
poster: releasesPosterByReleaseId {
|
poster: releasesPoster {
|
||||||
media {
|
media {
|
||||||
id
|
id
|
||||||
index
|
index
|
||||||
|
@ -335,7 +335,7 @@ const releasePhotosFragment = `
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const releaseTrailerFragment = `
|
const releaseTrailerFragment = `
|
||||||
trailer: releasesTrailerByReleaseId {
|
trailer: releasesTrailer {
|
||||||
media {
|
media {
|
||||||
id
|
id
|
||||||
index
|
index
|
||||||
|
@ -349,7 +349,7 @@ const releaseTrailerFragment = `
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const releaseTeaserFragment = `
|
const releaseTeaserFragment = `
|
||||||
teaser: releasesTeaserByReleaseId {
|
teaser: releasesTeaser {
|
||||||
media {
|
media {
|
||||||
id
|
id
|
||||||
index
|
index
|
||||||
|
@ -487,7 +487,7 @@ const releaseFragment = `
|
||||||
slug
|
slug
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
poster: chaptersPosterByChapterId {
|
poster: chaptersPoster {
|
||||||
media {
|
media {
|
||||||
id
|
id
|
||||||
index
|
index
|
||||||
|
|
|
@ -109,6 +109,10 @@ async function init() {
|
||||||
document.title = 'traxxx';
|
document.title = 'traxxx';
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
beforeCreate() {
|
||||||
|
this.uid = uid;
|
||||||
|
uid += 1;
|
||||||
|
},
|
||||||
methods: {
|
methods: {
|
||||||
formatDate,
|
formatDate,
|
||||||
formatDuration,
|
formatDuration,
|
||||||
|
@ -117,10 +121,6 @@ async function init() {
|
||||||
getPath,
|
getPath,
|
||||||
getBgPath: (media, type) => `url(${getPath(media, type)})`,
|
getBgPath: (media, type) => `url(${getPath(media, type)})`,
|
||||||
},
|
},
|
||||||
beforeCreate() {
|
|
||||||
this.uid = uid;
|
|
||||||
uid += 1;
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
|
||||||
app.directive('tooltip', {
|
app.directive('tooltip', {
|
||||||
|
|
|
@ -37,7 +37,7 @@ function initReleasesActions(store, router) {
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
releases: releases.map(release => curateRelease(release)),
|
releases: releases.map((release) => curateRelease(release)),
|
||||||
totalCount,
|
totalCount,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -99,7 +99,7 @@ function initReleasesActions(store, router) {
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
movies: movies.map(release => curateRelease(release)),
|
movies: movies.map((release) => curateRelease(release)),
|
||||||
totalCount,
|
totalCount,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -74,7 +74,7 @@ const routes = [
|
||||||
{
|
{
|
||||||
path: '/actor/:actorId/:actorSlug',
|
path: '/actor/:actorId/:actorSlug',
|
||||||
name: 'actor',
|
name: 'actor',
|
||||||
redirect: from => ({
|
redirect: (from) => ({
|
||||||
name: 'actorRange',
|
name: 'actorRange',
|
||||||
params: {
|
params: {
|
||||||
...from.params,
|
...from.params,
|
||||||
|
@ -91,7 +91,7 @@ const routes = [
|
||||||
{
|
{
|
||||||
path: '/director/:actorId/:actorSlug',
|
path: '/director/:actorId/:actorSlug',
|
||||||
name: 'director',
|
name: 'director',
|
||||||
redirect: from => ({
|
redirect: (from) => ({
|
||||||
name: 'directorRange',
|
name: 'directorRange',
|
||||||
params: {
|
params: {
|
||||||
...from.params,
|
...from.params,
|
||||||
|
@ -107,7 +107,7 @@ const routes = [
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: '/channel/:entitySlug',
|
path: '/channel/:entitySlug',
|
||||||
redirect: from => ({
|
redirect: (from) => ({
|
||||||
name: 'channel',
|
name: 'channel',
|
||||||
params: {
|
params: {
|
||||||
...from.params,
|
...from.params,
|
||||||
|
@ -123,7 +123,7 @@ const routes = [
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: '/network/:entitySlug',
|
path: '/network/:entitySlug',
|
||||||
redirect: from => ({
|
redirect: (from) => ({
|
||||||
name: 'network',
|
name: 'network',
|
||||||
params: {
|
params: {
|
||||||
...from.params,
|
...from.params,
|
||||||
|
@ -139,7 +139,7 @@ const routes = [
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: '/studio/:entitySlug',
|
path: '/studio/:entitySlug',
|
||||||
redirect: from => ({
|
redirect: (from) => ({
|
||||||
name: 'studio',
|
name: 'studio',
|
||||||
params: {
|
params: {
|
||||||
...from.params,
|
...from.params,
|
||||||
|
@ -155,7 +155,7 @@ const routes = [
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: '/tag/:tagSlug',
|
path: '/tag/:tagSlug',
|
||||||
redirect: from => ({
|
redirect: (from) => ({
|
||||||
name: 'tag',
|
name: 'tag',
|
||||||
params: {
|
params: {
|
||||||
...from.params,
|
...from.params,
|
||||||
|
@ -171,7 +171,7 @@ const routes = [
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: '/actors',
|
path: '/actors',
|
||||||
redirect: from => ({
|
redirect: (from) => ({
|
||||||
name: 'actors',
|
name: 'actors',
|
||||||
params: {
|
params: {
|
||||||
...from.params,
|
...from.params,
|
||||||
|
@ -229,7 +229,7 @@ const routes = [
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: '/stash/:stashId/:stashSlug',
|
path: '/stash/:stashId/:stashSlug',
|
||||||
redirect: from => ({
|
redirect: (from) => ({
|
||||||
name: 'stash',
|
name: 'stash',
|
||||||
params: {
|
params: {
|
||||||
...from.params,
|
...from.params,
|
||||||
|
|
|
@ -35,7 +35,7 @@ function initTagsActions(store, _router) {
|
||||||
name
|
name
|
||||||
slug
|
slug
|
||||||
}
|
}
|
||||||
poster: tagsPosterByTagId {
|
poster: tagsPoster {
|
||||||
media {
|
media {
|
||||||
id
|
id
|
||||||
thumbnail
|
thumbnail
|
||||||
|
@ -188,14 +188,14 @@ function initTagsActions(store, _router) {
|
||||||
before,
|
before,
|
||||||
orderBy,
|
orderBy,
|
||||||
offset: Math.max(0, (pageNumber - 1)) * limit,
|
offset: Math.max(0, (pageNumber - 1)) * limit,
|
||||||
exclude: store.state.ui.tagFilter.filter(tagFilter => tagFilter !== tagSlug),
|
exclude: store.state.ui.tagFilter.filter((tagFilter) => tagFilter !== tagSlug),
|
||||||
hasAuth: !!store.state.auth.user,
|
hasAuth: !!store.state.auth.user,
|
||||||
userId: store.state.auth.user?.id,
|
userId: store.state.auth.user?.id,
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
tag: curateTag(tagBySlug, null, curateRelease),
|
tag: curateTag(tagBySlug, null, curateRelease),
|
||||||
releases: tagBySlug.scenesConnection.releases.map(release => curateRelease(release)),
|
releases: tagBySlug.scenesConnection.releases.map((release) => curateRelease(release)),
|
||||||
totalCount: tagBySlug.scenesConnection.totalCount,
|
totalCount: tagBySlug.scenesConnection.totalCount,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -218,7 +218,7 @@ function initTagsActions(store, _router) {
|
||||||
id
|
id
|
||||||
name
|
name
|
||||||
slug
|
slug
|
||||||
poster: tagsPosterByTagId {
|
poster: tagsPoster {
|
||||||
media {
|
media {
|
||||||
thumbnail
|
thumbnail
|
||||||
comment
|
comment
|
||||||
|
@ -259,7 +259,7 @@ function initTagsActions(store, _router) {
|
||||||
limit,
|
limit,
|
||||||
});
|
});
|
||||||
|
|
||||||
return tags.map(tag => curateTag(tag, store.state.ui.sfw));
|
return tags.map((tag) => curateTag(tag, store.state.ui.sfw));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function searchTags({ _commit }, {
|
async function searchTags({ _commit }, {
|
||||||
|
@ -325,7 +325,7 @@ function initTagsActions(store, _router) {
|
||||||
minLength,
|
minLength,
|
||||||
});
|
});
|
||||||
|
|
||||||
return tags.map(tag => curateTag(tag, store.state.ui.sfw));
|
return tags.map((tag) => curateTag(tag, store.state.ui.sfw));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchTagReleases({ _commit }, tagId) {
|
async function fetchTagReleases({ _commit }, tagId) {
|
||||||
|
|
|
@ -70,7 +70,7 @@ function initUiActions(store, _router) {
|
||||||
slug
|
slug
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
entity: alertsEntityByAlertId {
|
entity: alertsEntity {
|
||||||
entity {
|
entity {
|
||||||
id
|
id
|
||||||
name
|
name
|
||||||
|
@ -103,7 +103,7 @@ function initUiActions(store, _router) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const curatedNotifications = notifications.nodes.map(notification => curateNotification(notification));
|
const curatedNotifications = notifications.nodes.map((notification) => curateNotification(notification));
|
||||||
|
|
||||||
return {
|
return {
|
||||||
notifications: curatedNotifications,
|
notifications: curatedNotifications,
|
||||||
|
@ -222,8 +222,8 @@ function initUiActions(store, _router) {
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
releases: res?.results.map(result => curateRelease(result.release)) || [],
|
releases: res?.results.map((result) => curateRelease(result.release)) || [],
|
||||||
actors: res?.actors.map(actor => curateActor(actor)) || [],
|
actors: res?.actors.map((actor) => curateActor(actor)) || [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@ async function initUiObservers(store, _router) {
|
||||||
|
|
||||||
body.classList.add(store.state.ui.theme);
|
body.classList.add(store.state.ui.theme);
|
||||||
|
|
||||||
store.watch(state => state.ui.theme, (newTheme, oldTheme) => {
|
store.watch((state) => state.ui.theme, (newTheme, oldTheme) => {
|
||||||
body.classList.add(newTheme);
|
body.classList.add(newTheme);
|
||||||
body.classList.remove(oldTheme);
|
body.classList.remove(oldTheme);
|
||||||
});
|
});
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
82
package.json
82
package.json
|
@ -7,7 +7,7 @@
|
||||||
"start": "node -r source-map-support/register src/init.js",
|
"start": "node -r source-map-support/register src/init.js",
|
||||||
"webpack": "webpack --env=production --mode=production",
|
"webpack": "webpack --env=production --mode=production",
|
||||||
"webpack-dev": "webpack --env=development --mode=development",
|
"webpack-dev": "webpack --env=development --mode=development",
|
||||||
"webpack-watch": "webpack --progress --colors --watch --env=development --mode=development",
|
"webpack-watch": "webpack --progress --color --watch --env=development --mode=development",
|
||||||
"babel": "babel src --source-maps -d dist",
|
"babel": "babel src --source-maps -d dist",
|
||||||
"babel-watch": "babel src -w --source-maps -d dist",
|
"babel-watch": "babel src -w --source-maps -d dist",
|
||||||
"build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
|
"build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
|
||||||
|
@ -40,38 +40,37 @@
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/cli": "^7.12.10",
|
"@babel/cli": "^7.12.10",
|
||||||
"@babel/core": "^7.8.4",
|
"@babel/core": "^7.8.4",
|
||||||
|
"@babel/eslint-parser": "^7.16.0",
|
||||||
"@babel/plugin-proposal-optional-chaining": "^7.8.3",
|
"@babel/plugin-proposal-optional-chaining": "^7.8.3",
|
||||||
"@babel/preset-env": "^7.8.4",
|
"@babel/preset-env": "^7.8.4",
|
||||||
"@babel/register": "^7.8.3",
|
"@babel/register": "^7.8.3",
|
||||||
"@vue/compiler-sfc": "^3.0.4",
|
"autoprefixer": "^10.4.0",
|
||||||
"autoprefixer": "^9.7.4",
|
|
||||||
"babel-eslint": "^10.1.0",
|
|
||||||
"babel-loader": "^8.0.6",
|
"babel-loader": "^8.0.6",
|
||||||
"babel-preset-airbnb": "^3.3.2",
|
"babel-preset-airbnb": "^5.0.0",
|
||||||
"css-loader": "^5.0.1",
|
"css-loader": "^6.5.0",
|
||||||
"eslint": "^7.20.0",
|
"eslint": "^8.1.0",
|
||||||
"eslint-config-airbnb": "^17.1.1",
|
"eslint-config-airbnb": "^18.2.1",
|
||||||
"eslint-config-airbnb-base": "^13.2.0",
|
"eslint-config-airbnb-base": "^14.2.1",
|
||||||
"eslint-plugin-import": "^2.20.1",
|
"eslint-plugin-import": "^2.20.1",
|
||||||
"eslint-plugin-jsx-a11y": "^6.2.3",
|
"eslint-plugin-jsx-a11y": "^6.2.3",
|
||||||
"eslint-plugin-react": "^7.18.3",
|
"eslint-plugin-react": "^7.18.3",
|
||||||
"eslint-plugin-vue": "^6.2.1",
|
"eslint-plugin-vue": "^8.0.3",
|
||||||
"eslint-watch": "^4.0.2",
|
"eslint-watch": "^7.0.0",
|
||||||
"eslint-webpack-plugin": "^2.5.2",
|
"eslint-webpack-plugin": "^3.1.0",
|
||||||
"mini-css-extract-plugin": "^1.3.3",
|
"mini-css-extract-plugin": "^2.4.3",
|
||||||
"node-sass": "^5.0.0",
|
"node-sass": "^6.0.1",
|
||||||
"postcss-loader": "^3.0.0",
|
"postcss-loader": "^6.2.0",
|
||||||
"raw-loader": "^4.0.2",
|
"raw-loader": "^4.0.2",
|
||||||
"sass-loader": "^11.0.1",
|
"sass-loader": "^12.3.0",
|
||||||
"style-loader": "^0.23.1",
|
"style-loader": "^3.3.1",
|
||||||
"vue-loader": "^16.1.2",
|
"vue-loader": "^16.8.2",
|
||||||
"webpack": "^5.11.0",
|
"webpack": "^5.11.0",
|
||||||
"webpack-cli": "^3.3.11"
|
"webpack-cli": "^4.9.1"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@casl/ability": "^5.2.2",
|
"@casl/ability": "^5.2.2",
|
||||||
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
|
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
|
||||||
"@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
|
"@graphile-contrib/pg-simplify-inflector": "^6.1.0",
|
||||||
"acorn": "^8.0.4",
|
"acorn": "^8.0.4",
|
||||||
"array-equal": "^1.0.0",
|
"array-equal": "^1.0.0",
|
||||||
"aws-sdk": "^2.847.0",
|
"aws-sdk": "^2.847.0",
|
||||||
|
@ -88,63 +87,60 @@
|
||||||
"cloudscraper": "^4.6.0",
|
"cloudscraper": "^4.6.0",
|
||||||
"config": "^3.2.5",
|
"config": "^3.2.5",
|
||||||
"connect-session-knex": "^2.0.0",
|
"connect-session-knex": "^2.0.0",
|
||||||
"convert": "^1.6.2",
|
"convert": "^4.2.4",
|
||||||
"cookie": "^0.4.0",
|
"cookie": "^0.4.0",
|
||||||
"csv-stringify": "^5.3.6",
|
"csv-stringify": "^5.3.6",
|
||||||
"dayjs": "^1.8.21",
|
"dayjs": "^1.8.21",
|
||||||
"dompurify": "^2.0.11",
|
"dompurify": "^2.0.11",
|
||||||
"ejs": "^3.0.1",
|
"ejs": "^3.0.1",
|
||||||
"express": "^4.17.1",
|
"express": "^4.17.1",
|
||||||
"express-promise-router": "^3.0.3",
|
"express-promise-router": "^4.1.0",
|
||||||
"express-react-views": "^0.11.0",
|
"express-react-views": "^0.11.0",
|
||||||
"express-session": "^1.17.1",
|
"express-session": "^1.17.1",
|
||||||
"face-api.js": "^0.22.2",
|
"face-api.js": "^0.22.2",
|
||||||
"faker": "^5.1.0",
|
"faker": "^5.1.0",
|
||||||
"file-type": "^14.1.4",
|
"file-type": "^16.5.3",
|
||||||
"fluent-ffmpeg": "^2.1.2",
|
"fluent-ffmpeg": "^2.1.2",
|
||||||
"fs-extra": "^7.0.1",
|
"fs-extra": "^10.0.0",
|
||||||
"graphile-utils": "^4.5.6",
|
"graphile-utils": "^4.12.2",
|
||||||
"graphql": "^14.6.0",
|
"graphql": "^15.4.0",
|
||||||
"html-entities": "^2.3.2",
|
"html-entities": "^2.3.2",
|
||||||
"iconv-lite": "^0.5.1",
|
"iconv-lite": "^0.6.3",
|
||||||
"inquirer": "^7.3.3",
|
"inquirer": "^8.2.0",
|
||||||
"inspector-api": "^1.4.2",
|
"inspector-api": "^1.4.2",
|
||||||
"jsdom": "^16.3.0",
|
"jsdom": "^18.0.0",
|
||||||
"knex": "^0.21.13",
|
"knex": "^0.95.12",
|
||||||
"knex-migrate": "^1.7.4",
|
"knex-migrate": "^1.7.4",
|
||||||
"longjohn": "^0.2.12",
|
"longjohn": "^0.2.12",
|
||||||
"mime": "^2.4.4",
|
"mime": "^2.4.4",
|
||||||
"mitt": "^2.1.0",
|
"mitt": "^3.0.0",
|
||||||
"moment": "^2.24.0",
|
"moment": "^2.24.0",
|
||||||
"nanoid": "^2.1.11",
|
"nanoid": "^3.1.30",
|
||||||
"object-merge-advanced": "^12.1.0",
|
"object-merge-advanced": "^12.1.0",
|
||||||
"object.omit": "^3.0.0",
|
"object.omit": "^3.0.0",
|
||||||
"opn": "^5.5.0",
|
"opn": "^6.0.0",
|
||||||
"pg": "^8.5.1",
|
"pg": "^8.5.1",
|
||||||
"postgraphile": "^4.10.0",
|
"postgraphile": "^4.10.0",
|
||||||
"postgraphile-plugin-connection-filter": "^1.1.3",
|
"postgraphile-plugin-connection-filter": "^2.2.2",
|
||||||
"promise-task-queue": "^1.2.0",
|
"promise-task-queue": "^1.2.0",
|
||||||
"prop-types": "^15.7.2",
|
"prop-types": "^15.7.2",
|
||||||
"react": "^16.13.0",
|
"sharp": "^0.29.2",
|
||||||
"react-dom": "^16.13.0",
|
|
||||||
"sharp": "^0.27.2",
|
|
||||||
"showdown": "^1.9.1",
|
"showdown": "^1.9.1",
|
||||||
"source-map-support": "^0.5.16",
|
"source-map-support": "^0.5.16",
|
||||||
"template-format": "^1.2.5",
|
"template-format": "^1.2.5",
|
||||||
"tippy.js": "^6.3.1",
|
"tippy.js": "^6.3.1",
|
||||||
"tough-cookie": "^3.0.1",
|
"tough-cookie": "^4.0.0",
|
||||||
"tty-table": "^2.8.12",
|
|
||||||
"tunnel": "0.0.6",
|
"tunnel": "0.0.6",
|
||||||
"url-pattern": "^1.0.3",
|
"url-pattern": "^1.0.3",
|
||||||
"v-tooltip": "^2.0.3",
|
"v-tooltip": "^2.0.3",
|
||||||
"video.js": "^7.11.4",
|
"video.js": "^7.11.4",
|
||||||
"videojs-vr": "^1.7.1",
|
"videojs-vr": "^1.7.1",
|
||||||
"vue": "^3.0.4",
|
"vue": "^3.2.20",
|
||||||
"vue-router": "^4.0.1",
|
"vue-router": "^4.0.12",
|
||||||
"vuex": "^4.0.0-rc.2",
|
"vuex": "^4.0.2",
|
||||||
"why-is-node-running": "^2.2.0",
|
"why-is-node-running": "^2.2.0",
|
||||||
"winston": "^3.2.1",
|
"winston": "^3.2.1",
|
||||||
"winston-daily-rotate-file": "^4.4.2",
|
"winston-daily-rotate-file": "^4.4.2",
|
||||||
"yargs": "^13.3.0"
|
"yargs": "^17.2.1"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Binary file not shown.
After Width: | Height: | Size: 680 KiB |
Binary file not shown.
After Width: | Height: | Size: 7.4 KiB |
Binary file not shown.
After Width: | Height: | Size: 35 KiB |
|
@ -900,6 +900,7 @@ const tagMedia = [
|
||||||
['flexible', 'lara_frost_legalporno', 'Lara Frost in NRX059', 'legalporno'],
|
['flexible', 'lara_frost_legalporno', 'Lara Frost in NRX059', 'legalporno'],
|
||||||
['free-use', 'jeni_angel_brazzersexxtra', 'Jeni Angel in "Gamer Girl Threesome Action"', 'brazzersexxtra'],
|
['free-use', 'jeni_angel_brazzersexxtra', 'Jeni Angel in "Gamer Girl Threesome Action"', 'brazzersexxtra'],
|
||||||
['free-use', 'veruca_james_brazzersexxtra', 'Veruca James in "The Perfect Maid"', 'brazzersexxtra'],
|
['free-use', 'veruca_james_brazzersexxtra', 'Veruca James in "The Perfect Maid"', 'brazzersexxtra'],
|
||||||
|
['free-use', 'gia_dibella_freeusefantasy', 'Gia Dibella in "Learning to Freeuse"', 'freeusefantasy'],
|
||||||
['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9"', 'julesjordan'],
|
['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9"', 'julesjordan'],
|
||||||
['gangbang', 'kristen_scott_julesjordan', 'Kristen Scott in "Interracial Gangbang!"', 'julesjordan'],
|
['gangbang', 'kristen_scott_julesjordan', 'Kristen Scott in "Interracial Gangbang!"', 'julesjordan'],
|
||||||
['gangbang', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
|
['gangbang', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
|
||||||
|
@ -1066,14 +1067,14 @@ const tagMedia = [
|
||||||
}));
|
}));
|
||||||
|
|
||||||
/* eslint-disable max-len */
|
/* eslint-disable max-len */
|
||||||
exports.seed = knex => Promise.resolve()
|
exports.seed = (knex) => Promise.resolve()
|
||||||
.then(async () => {
|
.then(async () => {
|
||||||
await upsert('media', sfw, 'id');
|
await upsert('media', sfw, 'id');
|
||||||
|
|
||||||
const tags = await knex('tags').whereIn('slug', tagMedia.map(item => item.tagSlug));
|
const tags = await knex('tags').whereIn('slug', tagMedia.map((item) => item.tagSlug));
|
||||||
|
|
||||||
const entities = await knex('entities')
|
const entities = await knex('entities')
|
||||||
.whereIn('slug', tagMedia.map(item => item.entitySlug).filter(Boolean))
|
.whereIn('slug', tagMedia.map((item) => item.entitySlug).filter(Boolean))
|
||||||
.orderBy('type', 'DESC');
|
.orderBy('type', 'DESC');
|
||||||
|
|
||||||
const entitiesBySlug = entities.reduce((acc, entity) => ({
|
const entitiesBySlug = entities.reduce((acc, entity) => ({
|
||||||
|
@ -1093,7 +1094,7 @@ exports.seed = knex => Promise.resolve()
|
||||||
concurrency: 20,
|
concurrency: 20,
|
||||||
});
|
});
|
||||||
|
|
||||||
const { inserted, updated } = await upsert('media', tagMediaWithDimensions.map(media => ({
|
const { inserted, updated } = await upsert('media', tagMediaWithDimensions.map((media) => ({
|
||||||
id: media.id,
|
id: media.id,
|
||||||
path: media.path,
|
path: media.path,
|
||||||
thumbnail: media.thumbnail,
|
thumbnail: media.thumbnail,
|
||||||
|
@ -1114,15 +1115,15 @@ exports.seed = knex => Promise.resolve()
|
||||||
[tagPhoto.tagSlug]: (acc[tagPhoto.tagSlug] || []).concat(tagPhoto),
|
[tagPhoto.tagSlug]: (acc[tagPhoto.tagSlug] || []).concat(tagPhoto),
|
||||||
}), {});
|
}), {});
|
||||||
|
|
||||||
const tagPosters = Object.values(tagMediaBySlug).map(tag => tag[0]);
|
const tagPosters = Object.values(tagMediaBySlug).map((tag) => tag[0]);
|
||||||
const tagPhotos = Object.values(tagMediaBySlug).map(tag => tag.slice(1)).flat();
|
const tagPhotos = Object.values(tagMediaBySlug).map((tag) => tag.slice(1)).flat();
|
||||||
|
|
||||||
const tagPosterEntries = tagPosters.map(poster => ({
|
const tagPosterEntries = tagPosters.map((poster) => ({
|
||||||
tag_id: tagIdsBySlug[poster.tagSlug],
|
tag_id: tagIdsBySlug[poster.tagSlug],
|
||||||
media_id: mediaIdsByPath[poster.path],
|
media_id: mediaIdsByPath[poster.path],
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const tagPhotoEntries = tagPhotos.map(photo => ({
|
const tagPhotoEntries = tagPhotos.map((photo) => ({
|
||||||
tag_id: tagIdsBySlug[photo.tagSlug],
|
tag_id: tagIdsBySlug[photo.tagSlug],
|
||||||
media_id: mediaIdsByPath[photo.path],
|
media_id: mediaIdsByPath[photo.path],
|
||||||
}));
|
}));
|
||||||
|
@ -1135,10 +1136,10 @@ exports.seed = knex => Promise.resolve()
|
||||||
// clean up (re)moved tag media
|
// clean up (re)moved tag media
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
knex('tags_posters')
|
knex('tags_posters')
|
||||||
.whereNotIn('media_id', tagPosters.map(photo => photo.id))
|
.whereNotIn('media_id', tagPosters.map((photo) => photo.id))
|
||||||
.delete(),
|
.delete(),
|
||||||
knex('tags_photos')
|
knex('tags_photos')
|
||||||
.whereNotIn('media_id', tagPhotos.map(photo => photo.id))
|
.whereNotIn('media_id', tagPhotos.map((photo) => photo.id))
|
||||||
.delete(),
|
.delete(),
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"extends": "airbnb-base",
|
"extends": "airbnb-base",
|
||||||
"parserOptions": {
|
"parserOptions": {
|
||||||
"parser": "babel-eslint",
|
"parser": "@babel/eslint-parser",
|
||||||
"sourceType": "script"
|
"sourceType": "script"
|
||||||
},
|
},
|
||||||
"rules": {
|
"rules": {
|
||||||
|
|
104
src/actors.js
104
src/actors.js
|
@ -124,9 +124,9 @@ function getMostFrequent(items) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function getMostFrequentDate(dates) {
|
function getMostFrequentDate(dates) {
|
||||||
const year = getMostFrequent(dates.map(dateX => dateX.getFullYear()));
|
const year = getMostFrequent(dates.map((dateX) => dateX.getFullYear()));
|
||||||
const month = getMostFrequent(dates.map(dateX => dateX.getMonth()));
|
const month = getMostFrequent(dates.map((dateX) => dateX.getMonth()));
|
||||||
const date = getMostFrequent(dates.map(dateX => dateX.getDate()));
|
const date = getMostFrequent(dates.map((dateX) => dateX.getDate()));
|
||||||
|
|
||||||
if (year === null || month === null || date === null) {
|
if (year === null || month === null || date === null) {
|
||||||
return null;
|
return null;
|
||||||
|
@ -153,7 +153,7 @@ function toBaseActors(actorsOrNames, release) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const baseActors = actorsOrNames
|
const baseActors = actorsOrNames
|
||||||
.filter(actorOrName => actorOrName && (typeof actorOrName === 'string' || actorOrName.name))
|
.filter((actorOrName) => actorOrName && (typeof actorOrName === 'string' || actorOrName.name))
|
||||||
.map((actorOrName) => {
|
.map((actorOrName) => {
|
||||||
const [baseName, entryId] = (actorOrName.name || actorOrName).split(':');
|
const [baseName, entryId] = (actorOrName.name || actorOrName).split(':');
|
||||||
|
|
||||||
|
@ -265,7 +265,7 @@ function curateActor(actor, withDetails = false, isProfile = false) {
|
||||||
size: actor.avatar.size,
|
size: actor.avatar.size,
|
||||||
source: actor.avatar.source,
|
source: actor.avatar.source,
|
||||||
},
|
},
|
||||||
...(actor.profiles && { profiles: actor.profiles?.map(profile => curateActor(profile, true, true)) }),
|
...(actor.profiles && { profiles: actor.profiles?.map((profile) => curateActor(profile, true, true)) }),
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -285,7 +285,7 @@ function curateActorEntry(baseActor, batchId) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function curateActorEntries(baseActors, batchId) {
|
function curateActorEntries(baseActors, batchId) {
|
||||||
return baseActors.map(baseActor => curateActorEntry(baseActor, batchId));
|
return baseActors.map((baseActor) => curateActorEntry(baseActor, batchId));
|
||||||
}
|
}
|
||||||
|
|
||||||
function curateProfileEntry(profile) {
|
function curateProfileEntry(profile) {
|
||||||
|
@ -448,7 +448,7 @@ async function curateProfile(profile, actor) {
|
||||||
curatedProfile.scenes = toBaseReleases(profile.scenes || profile.releases, profile.entity, actor)
|
curatedProfile.scenes = toBaseReleases(profile.scenes || profile.releases, profile.entity, actor)
|
||||||
// attach actor to base scene, in case it was not scraped
|
// attach actor to base scene, in case it was not scraped
|
||||||
.map((scene) => {
|
.map((scene) => {
|
||||||
if (actor && !scene.actors?.find(sceneActor => slugify(sceneActor) === actor.slug || slugify(sceneActor.name) === actor.slug)) {
|
if (actor && !scene.actors?.find((sceneActor) => slugify(sceneActor) === actor.slug || slugify(sceneActor.name) === actor.slug)) {
|
||||||
return {
|
return {
|
||||||
...scene,
|
...scene,
|
||||||
actors: [actor, ...(scene.actors || [])],
|
actors: [actor, ...(scene.actors || [])],
|
||||||
|
@ -477,10 +477,10 @@ async function fetchProfiles(actorIdsOrNames) {
|
||||||
.modify((query) => {
|
.modify((query) => {
|
||||||
if (actorIdsOrNames) {
|
if (actorIdsOrNames) {
|
||||||
query
|
query
|
||||||
.whereIn('actor_id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number'))
|
.whereIn('actor_id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
|
||||||
.orWhere((builder) => {
|
.orWhere((builder) => {
|
||||||
builder
|
builder
|
||||||
.whereIn('actors.name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string'))
|
.whereIn('actors.name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
|
||||||
.whereNull('actors.entity_id');
|
.whereNull('actors.entity_id');
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -517,12 +517,12 @@ async function interpolateProfiles(actorIdsOrNames) {
|
||||||
...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }),
|
...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }),
|
||||||
...(profile.birth_state && { state: profile.birth_state }),
|
...(profile.birth_state && { state: profile.birth_state }),
|
||||||
...(profile.birth_city && { city: profile.birth_city }),
|
...(profile.birth_city && { city: profile.birth_city }),
|
||||||
}].filter(location => Object.keys(location).length > 0),
|
}].filter((location) => Object.keys(location).length > 0),
|
||||||
residence: [...acc.residence || [], {
|
residence: [...acc.residence || [], {
|
||||||
...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }),
|
...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }),
|
||||||
...(profile.residence_state && { state: profile.residence_state }),
|
...(profile.residence_state && { state: profile.residence_state }),
|
||||||
...(profile.residence_city && { city: profile.residence_city }),
|
...(profile.residence_city && { city: profile.residence_city }),
|
||||||
}].filter(location => Object.keys(location).length > 0),
|
}].filter((location) => Object.keys(location).length > 0),
|
||||||
}), {});
|
}), {});
|
||||||
|
|
||||||
const mostFrequentValues = [
|
const mostFrequentValues = [
|
||||||
|
@ -549,7 +549,7 @@ async function interpolateProfiles(actorIdsOrNames) {
|
||||||
...mostFrequentValues,
|
...mostFrequentValues,
|
||||||
};
|
};
|
||||||
|
|
||||||
profile.height = getMostFrequent(valuesByProperty.height.filter(height => height > 50 && height < 300)); // remove unlikely values
|
profile.height = getMostFrequent(valuesByProperty.height.filter((height) => height > 50 && height < 300)); // remove unlikely values
|
||||||
|
|
||||||
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
|
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
|
||||||
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
|
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
|
||||||
|
@ -558,21 +558,21 @@ async function interpolateProfiles(actorIdsOrNames) {
|
||||||
profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs);
|
profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs);
|
||||||
|
|
||||||
// ensure most frequent country, city and state match up
|
// ensure most frequent country, city and state match up
|
||||||
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map(location => location.country));
|
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map((location) => location.country));
|
||||||
const remainingOriginCountries = valuesByProperty.origin.filter(location => location.country === profile.birth_country_alpha2);
|
const remainingOriginCountries = valuesByProperty.origin.filter((location) => location.country === profile.birth_country_alpha2);
|
||||||
|
|
||||||
profile.birth_state = getMostFrequent(remainingOriginCountries.map(location => location.state));
|
profile.birth_state = getMostFrequent(remainingOriginCountries.map((location) => location.state));
|
||||||
const remainingOriginStates = remainingOriginCountries.filter(location => !profile.birth_state || location.state === profile.birth_state);
|
const remainingOriginStates = remainingOriginCountries.filter((location) => !profile.birth_state || location.state === profile.birth_state);
|
||||||
|
|
||||||
profile.birth_city = getMostFrequent(remainingOriginStates.map(location => location.city));
|
profile.birth_city = getMostFrequent(remainingOriginStates.map((location) => location.city));
|
||||||
|
|
||||||
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map(location => location.country));
|
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map((location) => location.country));
|
||||||
const remainingResidenceCountries = valuesByProperty.residence.filter(location => location.country === profile.residence_country_alpha2);
|
const remainingResidenceCountries = valuesByProperty.residence.filter((location) => location.country === profile.residence_country_alpha2);
|
||||||
|
|
||||||
profile.residence_state = getMostFrequent(remainingResidenceCountries.map(location => location.state));
|
profile.residence_state = getMostFrequent(remainingResidenceCountries.map((location) => location.state));
|
||||||
const remainingResidenceStates = remainingResidenceCountries.filter(location => !profile.residence_state || location.state === profile.residence_state);
|
const remainingResidenceStates = remainingResidenceCountries.filter((location) => !profile.residence_state || location.state === profile.residence_state);
|
||||||
|
|
||||||
profile.residence_city = getMostFrequent(remainingResidenceStates.map(location => location.city));
|
profile.residence_city = getMostFrequent(remainingResidenceStates.map((location) => location.city));
|
||||||
|
|
||||||
profile.weight = getAverage(valuesByProperty.weight);
|
profile.weight = getAverage(valuesByProperty.weight);
|
||||||
|
|
||||||
|
@ -580,8 +580,8 @@ async function interpolateProfiles(actorIdsOrNames) {
|
||||||
profile.piercings = getLongest(valuesByProperty.piercings);
|
profile.piercings = getLongest(valuesByProperty.piercings);
|
||||||
|
|
||||||
profile.avatar_media_id = actorProfiles
|
profile.avatar_media_id = actorProfiles
|
||||||
.map(actorProfile => actorProfile.avatar)
|
.map((actorProfile) => actorProfile.avatar)
|
||||||
.filter(avatar => avatar && (avatar.entropy === null || avatar.entropy > 5.5))
|
.filter((avatar) => avatar && (avatar.entropy === null || avatar.entropy > 5.5))
|
||||||
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
|
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
|
@ -598,10 +598,10 @@ async function interpolateProfiles(actorIdsOrNames) {
|
||||||
.modify((modifyBuilder) => {
|
.modify((modifyBuilder) => {
|
||||||
if (actorIdsOrNames) {
|
if (actorIdsOrNames) {
|
||||||
modifyBuilder
|
modifyBuilder
|
||||||
.whereIn('id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number'))
|
.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
|
||||||
.orWhere((whereBuilder) => {
|
.orWhere((whereBuilder) => {
|
||||||
whereBuilder
|
whereBuilder
|
||||||
.whereIn('name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string'))
|
.whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
|
||||||
.whereNull('entity_id');
|
.whereNull('entity_id');
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -610,7 +610,7 @@ async function interpolateProfiles(actorIdsOrNames) {
|
||||||
.transacting(transaction);
|
.transacting(transaction);
|
||||||
|
|
||||||
// insert new interpolated data
|
// insert new interpolated data
|
||||||
const queries = interpolatedProfiles.map(profile => knex('actors')
|
const queries = interpolatedProfiles.map((profile) => knex('actors')
|
||||||
.where('id', profile.id)
|
.where('id', profile.id)
|
||||||
.update(profile)
|
.update(profile)
|
||||||
.transacting(transaction));
|
.transacting(transaction));
|
||||||
|
@ -621,8 +621,8 @@ async function interpolateProfiles(actorIdsOrNames) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function upsertProfiles(profiles) {
|
async function upsertProfiles(profiles) {
|
||||||
const newProfileEntries = profiles.filter(profile => !profile.update).map(profile => curateProfileEntry(profile)).filter(Boolean);
|
const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
|
||||||
const updatingProfileEntries = profiles.filter(profile => profile.update).map(profile => curateProfileEntry(profile)).filter(Boolean);
|
const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
|
||||||
|
|
||||||
if (newProfileEntries.length > 0) {
|
if (newProfileEntries.length > 0) {
|
||||||
await bulkInsert('actors_profiles', newProfileEntries);
|
await bulkInsert('actors_profiles', newProfileEntries);
|
||||||
|
@ -632,7 +632,7 @@ async function upsertProfiles(profiles) {
|
||||||
|
|
||||||
if (argv.force && updatingProfileEntries.length > 0) {
|
if (argv.force && updatingProfileEntries.length > 0) {
|
||||||
const transaction = await knex.transaction();
|
const transaction = await knex.transaction();
|
||||||
const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles')
|
const queries = updatingProfileEntries.map((profileEntry) => knex('actors_profiles')
|
||||||
.where('id', profileEntry.id)
|
.where('id', profileEntry.id)
|
||||||
.update(profileEntry)
|
.update(profileEntry)
|
||||||
.returning(['id', 'actor_id'])
|
.returning(['id', 'actor_id'])
|
||||||
|
@ -647,7 +647,7 @@ async function upsertProfiles(profiles) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) {
|
async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) {
|
||||||
const validSources = actor.entity ? sources.filter(source => source === actor.entity.slug) : sources;
|
const validSources = actor.entity ? sources.filter((source) => source === actor.entity.slug) : sources;
|
||||||
|
|
||||||
const profiles = Promise.map(validSources, async (source) => {
|
const profiles = Promise.map(validSources, async (source) => {
|
||||||
try {
|
try {
|
||||||
|
@ -748,12 +748,12 @@ async function getActorNames(actorNames) {
|
||||||
)
|
)
|
||||||
`, [argv.actorsUpdate || new Date()]);
|
`, [argv.actorsUpdate || new Date()]);
|
||||||
|
|
||||||
return actorsWithoutProfiles.rows.map(actor => actor.name);
|
return actorsWithoutProfiles.rows.map((actor) => actor.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function storeProfiles(profiles) {
|
async function storeProfiles(profiles) {
|
||||||
const profilesWithAvatarIds = await associateAvatars(profiles);
|
const profilesWithAvatarIds = await associateAvatars(profiles);
|
||||||
const actorIds = Array.from(new Set(profiles.map(profile => profile.id)));
|
const actorIds = Array.from(new Set(profiles.map((profile) => profile.id)));
|
||||||
|
|
||||||
await upsertProfiles(profilesWithAvatarIds);
|
await upsertProfiles(profilesWithAvatarIds);
|
||||||
await interpolateProfiles(actorIds);
|
await interpolateProfiles(actorIds);
|
||||||
|
@ -772,7 +772,7 @@ async function scrapeActors(argNames) {
|
||||||
fetchEntitiesBySlug(entitySlugs, 'desc'),
|
fetchEntitiesBySlug(entitySlugs, 'desc'),
|
||||||
knex('actors')
|
knex('actors')
|
||||||
.select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
|
.select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
|
||||||
.whereIn('actors.slug', baseActors.map(baseActor => baseActor.slug))
|
.whereIn('actors.slug', baseActors.map((baseActor) => baseActor.slug))
|
||||||
.whereNull('actors.alias_for')
|
.whereNull('actors.alias_for')
|
||||||
.leftJoin('entities', 'entities.id', 'actors.entity_id')
|
.leftJoin('entities', 'entities.id', 'actors.entity_id')
|
||||||
.groupBy('actors.id', 'entities.id'),
|
.groupBy('actors.id', 'entities.id'),
|
||||||
|
@ -786,7 +786,7 @@ async function scrapeActors(argNames) {
|
||||||
},
|
},
|
||||||
}), {});
|
}), {});
|
||||||
|
|
||||||
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlugAndEntryId[baseActor.slug]?.[baseActor.entryId]);
|
const newBaseActors = baseActors.filter((baseActor) => !existingActorEntriesBySlugAndEntryId[baseActor.slug]?.[baseActor.entryId]);
|
||||||
|
|
||||||
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
|
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
|
||||||
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
|
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
|
||||||
|
@ -799,7 +799,7 @@ async function scrapeActors(argNames) {
|
||||||
|
|
||||||
const existingProfiles = await knex('actors_profiles')
|
const existingProfiles = await knex('actors_profiles')
|
||||||
.select(knex.raw('actors_profiles.*, row_to_json(avatars) as avatar'))
|
.select(knex.raw('actors_profiles.*, row_to_json(avatars) as avatar'))
|
||||||
.whereIn('actor_id', actors.map(actor => actor.id))
|
.whereIn('actor_id', actors.map((actor) => actor.id))
|
||||||
.leftJoin('media as avatars', 'avatars.id', 'actors_profiles.avatar_media_id');
|
.leftJoin('media as avatars', 'avatars.id', 'actors_profiles.avatar_media_id');
|
||||||
|
|
||||||
const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({
|
const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({
|
||||||
|
@ -812,7 +812,7 @@ async function scrapeActors(argNames) {
|
||||||
|
|
||||||
const profilesPerActor = await Promise.map(
|
const profilesPerActor = await Promise.map(
|
||||||
actors,
|
actors,
|
||||||
async actor => scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId),
|
async (actor) => scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId),
|
||||||
{ concurrency: 10 },
|
{ concurrency: 10 },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -833,7 +833,7 @@ async function scrapeActors(argNames) {
|
||||||
|
|
||||||
async function getOrCreateActors(baseActors, batchId) {
|
async function getOrCreateActors(baseActors, batchId) {
|
||||||
// WHERE IN causes stack depth error and performance issues with a large amount of values, no knex VALUES helper available
|
// WHERE IN causes stack depth error and performance issues with a large amount of values, no knex VALUES helper available
|
||||||
const actorValues = baseActors.map(actor => knex.raw('(:slug, :entityId, :entryId, :collisionLikely)', {
|
const actorValues = baseActors.map((actor) => knex.raw('(:slug, :entityId, :entryId, :collisionLikely)', {
|
||||||
slug: actor.slug,
|
slug: actor.slug,
|
||||||
entityId: actor.entity.id,
|
entityId: actor.entity.id,
|
||||||
entryId: actor.entryId,
|
entryId: actor.entryId,
|
||||||
|
@ -867,7 +867,7 @@ async function getOrCreateActors(baseActors, batchId) {
|
||||||
},
|
},
|
||||||
}), {});
|
}), {});
|
||||||
|
|
||||||
const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.entity.id]?.[baseActor.entryId]?.[baseActor.slug] && !existingActorSlugs.null?.null?.[baseActor.slug]);
|
const uniqueBaseActors = baseActors.filter((baseActor) => !existingActorSlugs[baseActor.entity.id]?.[baseActor.entryId]?.[baseActor.slug] && !existingActorSlugs.null?.null?.[baseActor.slug]);
|
||||||
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
|
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
|
||||||
|
|
||||||
const newActors = await bulkInsert('actors', curatedActorEntries);
|
const newActors = await bulkInsert('actors', curatedActorEntries);
|
||||||
|
@ -884,13 +884,13 @@ async function getOrCreateActors(baseActors, batchId) {
|
||||||
}), {});
|
}), {});
|
||||||
|
|
||||||
const newActorProfiles = await Promise.all(baseActors
|
const newActorProfiles = await Promise.all(baseActors
|
||||||
.filter(actor => actor.hasProfile)
|
.filter((actor) => actor.hasProfile)
|
||||||
.map(actor => ({
|
.map((actor) => ({
|
||||||
...actor,
|
...actor,
|
||||||
id: newActorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || newActorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug],
|
id: newActorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || newActorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug],
|
||||||
}))
|
}))
|
||||||
.filter(actor => !!actor.id)
|
.filter((actor) => !!actor.id)
|
||||||
.map(actor => curateProfile(actor)));
|
.map((actor) => curateProfile(actor)));
|
||||||
|
|
||||||
await storeProfiles(newActorProfiles);
|
await storeProfiles(newActorProfiles);
|
||||||
|
|
||||||
|
@ -950,16 +950,16 @@ async function associatePeople(releases, batchId, type = 'actor') {
|
||||||
|
|
||||||
const releaseActorAssociations = Object.entries(baseActorsByReleaseId)
|
const releaseActorAssociations = Object.entries(baseActorsByReleaseId)
|
||||||
.map(([releaseId, releaseActors]) => releaseActors
|
.map(([releaseId, releaseActors]) => releaseActors
|
||||||
.map(releaseActor => ({
|
.map((releaseActor) => ({
|
||||||
release_id: releaseId,
|
release_id: releaseId,
|
||||||
...(actorIdsByEntityIdEntryIdAndSlug[releaseActor.entity?.id]?.[releaseActor.entryId]?.[releaseActor.slug] || actorIdsByEntityIdEntryIdAndSlug.null.null[releaseActor.slug]),
|
...(actorIdsByEntityIdEntryIdAndSlug[releaseActor.entity?.id]?.[releaseActor.entryId]?.[releaseActor.slug] || actorIdsByEntityIdEntryIdAndSlug.null.null[releaseActor.slug]),
|
||||||
})))
|
})))
|
||||||
.flat();
|
.flat();
|
||||||
|
|
||||||
const validReleaseActorAssociations = releaseActorAssociations.filter(association => association.release_id && association[personKey]);
|
const validReleaseActorAssociations = releaseActorAssociations.filter((association) => association.release_id && association[personKey]);
|
||||||
|
|
||||||
if (releaseActorAssociations.length > validReleaseActorAssociations.length) {
|
if (releaseActorAssociations.length > validReleaseActorAssociations.length) {
|
||||||
const invalidReleaseActorAssociations = releaseActorAssociations.filter(association => !association.release_id || !association[personKey]);
|
const invalidReleaseActorAssociations = releaseActorAssociations.filter((association) => !association.release_id || !association[personKey]);
|
||||||
|
|
||||||
logger.error(invalidReleaseActorAssociations);
|
logger.error(invalidReleaseActorAssociations);
|
||||||
}
|
}
|
||||||
|
@ -1021,15 +1021,15 @@ async function searchActors(query) {
|
||||||
.from(knex.raw('search_actors(?) as actors', [query]))
|
.from(knex.raw('search_actors(?) as actors', [query]))
|
||||||
.limit(100);
|
.limit(100);
|
||||||
|
|
||||||
return actors.map(actor => curateActor(actor));
|
return actors.map((actor) => curateActor(actor));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function flushProfiles(actorIdsOrNames) {
|
async function flushProfiles(actorIdsOrNames) {
|
||||||
const profiles = await fetchProfiles(actorIdsOrNames);
|
const profiles = await fetchProfiles(actorIdsOrNames);
|
||||||
const actorNames = Array.from(new Set(profiles.map(profile => profile.actor.name)));
|
const actorNames = Array.from(new Set(profiles.map((profile) => profile.actor.name)));
|
||||||
|
|
||||||
const deleteCount = await knex('actors_profiles')
|
const deleteCount = await knex('actors_profiles')
|
||||||
.whereIn('id', profiles.map(profile => profile.id))
|
.whereIn('id', profiles.map((profile) => profile.id))
|
||||||
.delete();
|
.delete();
|
||||||
|
|
||||||
await interpolateProfiles(actorIdsOrNames);
|
await interpolateProfiles(actorIdsOrNames);
|
||||||
|
@ -1050,14 +1050,14 @@ async function flushProfiles(actorIdsOrNames) {
|
||||||
|
|
||||||
async function deleteActors(actorIdsOrNames) {
|
async function deleteActors(actorIdsOrNames) {
|
||||||
const actors = await knex('actors')
|
const actors = await knex('actors')
|
||||||
.whereIn('id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number'))
|
.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
|
||||||
.orWhere((builder) => {
|
.orWhere((builder) => {
|
||||||
builder
|
builder
|
||||||
.whereIn('name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string'))
|
.whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
|
||||||
.whereNull('entity_id');
|
.whereNull('entity_id');
|
||||||
});
|
});
|
||||||
|
|
||||||
const actorIds = actors.map(actor => actor.id);
|
const actorIds = actors.map((actor) => actor.id);
|
||||||
|
|
||||||
const sceneIds = await knex('releases_actors')
|
const sceneIds = await knex('releases_actors')
|
||||||
.select('releases.id')
|
.select('releases.id')
|
||||||
|
|
|
@ -22,15 +22,15 @@ async function addAlert(alert, sessionUser) {
|
||||||
.returning('id');
|
.returning('id');
|
||||||
|
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
alert.actors?.length > 0 && bulkInsert('alerts_actors', alert.actors.map(actorId => ({
|
alert.actors?.length > 0 && bulkInsert('alerts_actors', alert.actors.map((actorId) => ({
|
||||||
alert_id: alertId,
|
alert_id: alertId,
|
||||||
actor_id: actorId,
|
actor_id: actorId,
|
||||||
})), false),
|
})), false),
|
||||||
alert.tags?.length > 0 && bulkInsert('alerts_tags', alert.tags.map(tagId => ({
|
alert.tags?.length > 0 && bulkInsert('alerts_tags', alert.tags.map((tagId) => ({
|
||||||
alert_id: alertId,
|
alert_id: alertId,
|
||||||
tag_id: tagId,
|
tag_id: tagId,
|
||||||
})), false),
|
})), false),
|
||||||
alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map(stashId => ({
|
alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map((stashId) => ({
|
||||||
alert_id: alertId,
|
alert_id: alertId,
|
||||||
stash_id: stashId,
|
stash_id: stashId,
|
||||||
})), false),
|
})), false),
|
||||||
|
@ -106,20 +106,20 @@ async function notify(scenes) {
|
||||||
))))
|
))))
|
||||||
GROUP BY releases.id, users.id, alerts.id;
|
GROUP BY releases.id, users.id, alerts.id;
|
||||||
`, {
|
`, {
|
||||||
sceneIds: scenes.map(scene => scene.id),
|
sceneIds: scenes.map((scene) => scene.id),
|
||||||
});
|
});
|
||||||
|
|
||||||
const notifications = releases.rows
|
const notifications = releases.rows
|
||||||
.filter(alert => alert.notify)
|
.filter((alert) => alert.notify)
|
||||||
.map(notification => ({
|
.map((notification) => ({
|
||||||
user_id: notification.user_id,
|
user_id: notification.user_id,
|
||||||
alert_id: notification.alert_id,
|
alert_id: notification.alert_id,
|
||||||
scene_id: notification.scene_id,
|
scene_id: notification.scene_id,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const stashes = releases.rows
|
const stashes = releases.rows
|
||||||
.filter(release => release.stashes.length > 0)
|
.filter((release) => release.stashes.length > 0)
|
||||||
.flatMap(release => release.stashes.map(stash => ({
|
.flatMap((release) => release.stashes.map((stash) => ({
|
||||||
scene_id: release.scene_id,
|
scene_id: release.scene_id,
|
||||||
stash_id: stash,
|
stash_id: stash,
|
||||||
})));
|
})));
|
||||||
|
|
55
src/app.js
55
src/app.js
|
@ -22,6 +22,7 @@ const { flushOrphanedMedia } = require('./media');
|
||||||
const getFileEntries = require('./utils/file-entries');
|
const getFileEntries = require('./utils/file-entries');
|
||||||
|
|
||||||
const inspector = new Inspector();
|
const inspector = new Inspector();
|
||||||
|
let done = false;
|
||||||
|
|
||||||
function logActive() {
|
function logActive() {
|
||||||
console.log('log active!');
|
console.log('log active!');
|
||||||
|
@ -32,24 +33,47 @@ function logActive() {
|
||||||
}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
|
}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
function monitorMemory() {
|
||||||
|
logger.debug(`Memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
|
||||||
|
|
||||||
|
if (!done) {
|
||||||
|
setTimeout(() => monitorMemory(), 10000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
async function stopMemorySample() {
|
async function stopMemorySample() {
|
||||||
const profile = await inspector.heap.stopSampling();
|
const profile = await inspector.heap.stopSampling();
|
||||||
const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
|
const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
|
||||||
|
|
||||||
await inspector.heap.disable();
|
await inspector.heap.disable();
|
||||||
|
await fs.writeFile(filepath, JSON.stringify(profile));
|
||||||
fs.writeFile(filepath, JSON.stringify(profile));
|
|
||||||
|
|
||||||
logger.info(`Saved heap sample to ${filepath}`);
|
logger.info(`Saved heap sample to ${filepath}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function startMemorySample() {
|
||||||
|
await inspector.heap.enable();
|
||||||
|
await inspector.heap.startSampling();
|
||||||
|
|
||||||
|
// monitorMemory();
|
||||||
|
|
||||||
|
logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
|
||||||
|
|
||||||
|
setTimeout(async () => {
|
||||||
|
await stopMemorySample();
|
||||||
|
|
||||||
|
if (!done) {
|
||||||
|
await startMemorySample();
|
||||||
|
}
|
||||||
|
}, 30000);
|
||||||
|
}
|
||||||
|
|
||||||
async function init() {
|
async function init() {
|
||||||
try {
|
try {
|
||||||
if (argv.memory) {
|
if (argv.memory) {
|
||||||
await inspector.heap.enable();
|
await startMemorySample();
|
||||||
await inspector.heap.startSampling();
|
|
||||||
|
|
||||||
logger.info('Started heap sampling');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (argv.logActive) {
|
if (argv.logActive) {
|
||||||
|
@ -122,7 +146,7 @@ async function init() {
|
||||||
const actorNames = (argv.actors || []).concat(actorsFromFile || []);
|
const actorNames = (argv.actors || []).concat(actorsFromFile || []);
|
||||||
|
|
||||||
const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
|
const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
|
||||||
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.scenes).flat().filter(Boolean);
|
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map((actor) => actor.scenes).flat().filter(Boolean);
|
||||||
|
|
||||||
const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
|
const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
|
||||||
|
|
||||||
|
@ -133,10 +157,10 @@ async function init() {
|
||||||
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
|
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
|
||||||
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
|
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
|
||||||
|
|
||||||
const sceneMovies = deepScenes ? deepScenes.filter(scene => scene.movie).map(scene => ({ ...scene.movie, entity: scene.entity })) : [];
|
const sceneMovies = deepScenes ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
|
||||||
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
|
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
|
||||||
|
|
||||||
const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
|
const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
|
||||||
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
|
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
|
||||||
|
|
||||||
if (argv.report) {
|
if (argv.report) {
|
||||||
|
@ -150,23 +174,12 @@ async function init() {
|
||||||
|
|
||||||
await associateMovieScenes(storedMovies, storedScenes);
|
await associateMovieScenes(storedMovies, storedScenes);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (argv.memory) {
|
|
||||||
await stopMemorySample();
|
|
||||||
}
|
|
||||||
|
|
||||||
knex.destroy();
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(error);
|
logger.error(error);
|
||||||
|
|
||||||
if (argv.memory) {
|
|
||||||
await stopMemorySample();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
knex.destroy();
|
knex.destroy();
|
||||||
|
done = true;
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = init;
|
module.exports = init;
|
||||||
|
|
|
@ -4,7 +4,11 @@ const config = require('config');
|
||||||
const yargs = require('yargs');
|
const yargs = require('yargs');
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
function interpretAfter(after) {
|
function interpretAfter(after, ignoreIfEmpty = false) {
|
||||||
|
if (!after && ignoreIfEmpty) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
if (!after) {
|
if (!after) {
|
||||||
return new Date(0, 0, 0);
|
return new Date(0, 0, 0);
|
||||||
}
|
}
|
||||||
|
@ -313,6 +317,6 @@ const { argv } = yargs
|
||||||
default: 60000,
|
default: 60000,
|
||||||
})
|
})
|
||||||
.coerce('after', interpretAfter)
|
.coerce('after', interpretAfter)
|
||||||
.coerce('actors-update', interpretAfter);
|
.coerce('actors-update', (after) => interpretAfter(after, true));
|
||||||
|
|
||||||
module.exports = argv;
|
module.exports = argv;
|
||||||
|
|
|
@ -185,7 +185,7 @@ async function scrapeReleases(baseReleases, entitiesBySlug, type) {
|
||||||
|
|
||||||
return Promise.map(
|
return Promise.map(
|
||||||
baseReleases,
|
baseReleases,
|
||||||
async baseRelease => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
|
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
|
||||||
{ concurrency: 10 },
|
{ concurrency: 10 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,7 +46,7 @@ function curateEntity(entity, includeParameters = false) {
|
||||||
} : {};
|
} : {};
|
||||||
|
|
||||||
if (entity.tags) {
|
if (entity.tags) {
|
||||||
curatedEntity.tags = entity.tags.map(tag => ({
|
curatedEntity.tags = entity.tags.map((tag) => ({
|
||||||
id: tag.id,
|
id: tag.id,
|
||||||
name: tag.name,
|
name: tag.name,
|
||||||
slug: tag.slug,
|
slug: tag.slug,
|
||||||
|
@ -59,14 +59,14 @@ function curateEntity(entity, includeParameters = false) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (entity.children) {
|
if (entity.children) {
|
||||||
curatedEntity.children = entity.children.map(child => curateEntity({
|
curatedEntity.children = entity.children.map((child) => curateEntity({
|
||||||
...child,
|
...child,
|
||||||
parent: curatedEntity.id ? curatedEntity : null,
|
parent: curatedEntity.id ? curatedEntity : null,
|
||||||
}, includeParameters));
|
}, includeParameters));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (entity.included_children) {
|
if (entity.included_children) {
|
||||||
curatedEntity.includedChildren = entity.included_children.map(child => curateEntity({
|
curatedEntity.includedChildren = entity.included_children.map((child) => curateEntity({
|
||||||
...child,
|
...child,
|
||||||
parent: curatedEntity.id ? curatedEntity : null,
|
parent: curatedEntity.id ? curatedEntity : null,
|
||||||
}, includeParameters));
|
}, includeParameters));
|
||||||
|
@ -79,7 +79,7 @@ function curateEntity(entity, includeParameters = false) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function curateEntities(entities, includeParameters) {
|
async function curateEntities(entities, includeParameters) {
|
||||||
return Promise.all(entities.map(async entity => curateEntity(entity, includeParameters)));
|
return Promise.all(entities.map(async (entity) => curateEntity(entity, includeParameters)));
|
||||||
}
|
}
|
||||||
|
|
||||||
function urlToSiteSlug(url) {
|
function urlToSiteSlug(url) {
|
||||||
|
@ -102,8 +102,8 @@ async function fetchIncludedEntities() {
|
||||||
includeAll: !argv.networks && !argv.channels && !config.include?.networks && !config.include?.channels,
|
includeAll: !argv.networks && !argv.channels && !config.include?.networks && !config.include?.channels,
|
||||||
includedNetworks: argv.networks || (!argv.channels && config.include?.networks) || [],
|
includedNetworks: argv.networks || (!argv.channels && config.include?.networks) || [],
|
||||||
includedChannels: argv.channels || (!argv.networks && config.include?.channels) || [],
|
includedChannels: argv.channels || (!argv.networks && config.include?.channels) || [],
|
||||||
excludedNetworks: argv.excludeNetworks || config.exclude?.networks.filter(network => !argv.networks?.includes(network)) || [], // ignore explicitly included networks
|
excludedNetworks: argv.excludeNetworks || config.exclude?.networks.filter((network) => !argv.networks?.includes(network)) || [], // ignore explicitly included networks
|
||||||
excludedChannels: argv.excludeChannels || config.exclude?.channels.filter(channel => !argv.channels?.includes(channel)) || [], // ignore explicitly included channels
|
excludedChannels: argv.excludeChannels || config.exclude?.channels.filter((channel) => !argv.channels?.includes(channel)) || [], // ignore explicitly included channels
|
||||||
};
|
};
|
||||||
|
|
||||||
const rawNetworks = await knex.raw(`
|
const rawNetworks = await knex.raw(`
|
||||||
|
@ -228,11 +228,11 @@ async function fetchEntitiesBySlug(entitySlugs, sort = 'asc') {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchReleaseEntities(baseReleases) {
|
async function fetchReleaseEntities(baseReleases) {
|
||||||
const baseReleasesWithoutEntity = baseReleases.filter(release => release.url && !release.site && !release.entity);
|
const baseReleasesWithoutEntity = baseReleases.filter((release) => release.url && !release.site && !release.entity);
|
||||||
|
|
||||||
const entitySlugs = Array.from(new Set(
|
const entitySlugs = Array.from(new Set(
|
||||||
baseReleasesWithoutEntity
|
baseReleasesWithoutEntity
|
||||||
.map(baseRelease => urlToSiteSlug(baseRelease.url))
|
.map((baseRelease) => urlToSiteSlug(baseRelease.url))
|
||||||
.filter(Boolean),
|
.filter(Boolean),
|
||||||
));
|
));
|
||||||
|
|
||||||
|
|
|
@ -16,7 +16,7 @@ function logger(filepath) {
|
||||||
return winston.createLogger({
|
return winston.createLogger({
|
||||||
format: winston.format.combine(
|
format: winston.format.combine(
|
||||||
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
|
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
|
||||||
winston.format(info => (info instanceof Error
|
winston.format((info) => (info instanceof Error
|
||||||
? { ...info, message: info.stack }
|
? { ...info, message: info.stack }
|
||||||
: { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(),
|
: { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(),
|
||||||
winston.format.colorize(),
|
winston.format.colorize(),
|
||||||
|
|
38
src/media.js
38
src/media.js
|
@ -190,7 +190,7 @@ function sortBaseTrailersByQuality(sources, role) {
|
||||||
|
|
||||||
function fallbackMediaToBaseMedia(rawMedia, role, metadata) {
|
function fallbackMediaToBaseMedia(rawMedia, role, metadata) {
|
||||||
const baseSources = rawMedia
|
const baseSources = rawMedia
|
||||||
.map(source => toBaseSource(source))
|
.map((source) => toBaseSource(source))
|
||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
|
|
||||||
const sortedBaseSources = sortBaseTrailersByQuality(baseSources, role);
|
const sortedBaseSources = sortBaseTrailersByQuality(baseSources, role);
|
||||||
|
@ -225,12 +225,12 @@ function toBaseMedias(rawMedias, role, metadata) {
|
||||||
|
|
||||||
async function findSourceDuplicates(baseMedias) {
|
async function findSourceDuplicates(baseMedias) {
|
||||||
const sourceUrls = baseMedias
|
const sourceUrls = baseMedias
|
||||||
.map(baseMedia => baseMedia.sources.map(source => source.src))
|
.map((baseMedia) => baseMedia.sources.map((source) => source.src))
|
||||||
.flat()
|
.flat()
|
||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
|
|
||||||
const extractUrls = baseMedias
|
const extractUrls = baseMedias
|
||||||
.map(baseMedia => baseMedia.sources.map(source => source.url))
|
.map((baseMedia) => baseMedia.sources.map((source) => source.url))
|
||||||
.flat()
|
.flat()
|
||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
|
|
||||||
|
@ -246,12 +246,12 @@ async function findSourceDuplicates(baseMedias) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function findHashDuplicates(medias) {
|
async function findHashDuplicates(medias) {
|
||||||
const hashes = medias.map(media => media.meta?.hash || media.entry?.hash).filter(Boolean);
|
const hashes = medias.map((media) => media.meta?.hash || media.entry?.hash).filter(Boolean);
|
||||||
|
|
||||||
const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
|
const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
|
||||||
const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
|
const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
|
||||||
|
|
||||||
const uniqueHashMedias = medias.filter(media => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
|
const uniqueHashMedias = medias.filter((media) => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
|
||||||
|
|
||||||
const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
|
const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
|
||||||
if (!media.meta?.hash) {
|
if (!media.meta?.hash) {
|
||||||
|
@ -278,8 +278,8 @@ async function findHashDuplicates(medias) {
|
||||||
const selfUniqueHashMedias = Object.values(selfUniqueMediasByHash);
|
const selfUniqueHashMedias = Object.values(selfUniqueMediasByHash);
|
||||||
|
|
||||||
const existingHashMedias = medias
|
const existingHashMedias = medias
|
||||||
.filter(media => existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash])
|
.filter((media) => existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash])
|
||||||
.map(media => ({
|
.map((media) => ({
|
||||||
...media,
|
...media,
|
||||||
entry: existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash],
|
entry: existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash],
|
||||||
}))
|
}))
|
||||||
|
@ -563,8 +563,8 @@ streamQueue.define('fetchStreamSource', async ({ source, tempFileTarget, hashStr
|
||||||
const video = ffmpeg(source.stream)
|
const video = ffmpeg(source.stream)
|
||||||
.format('mp4')
|
.format('mp4')
|
||||||
.outputOptions(['-movflags frag_keyframe+empty_moov'])
|
.outputOptions(['-movflags frag_keyframe+empty_moov'])
|
||||||
.on('start', cmd => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
|
.on('start', (cmd) => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
|
||||||
.on('error', error => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
|
.on('error', (error) => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
|
||||||
.pipe();
|
.pipe();
|
||||||
|
|
||||||
await pipeline(video, hashStream, tempFileTarget);
|
await pipeline(video, hashStream, tempFileTarget);
|
||||||
|
@ -745,7 +745,7 @@ async function storeMedias(baseMedias, options) {
|
||||||
|
|
||||||
const fetchedMedias = await Promise.map(
|
const fetchedMedias = await Promise.map(
|
||||||
baseMedias,
|
baseMedias,
|
||||||
async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
|
async (baseMedia) => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
|
||||||
{ concurrency: 100 }, // don't overload disk (or network, although this has its own throttling)
|
{ concurrency: 100 }, // don't overload disk (or network, although this has its own throttling)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -753,7 +753,7 @@ async function storeMedias(baseMedias, options) {
|
||||||
|
|
||||||
const savedMedias = await Promise.map(
|
const savedMedias = await Promise.map(
|
||||||
uniqueHashMedias,
|
uniqueHashMedias,
|
||||||
async baseMedia => storeFile(baseMedia, options),
|
async (baseMedia) => storeFile(baseMedia, options),
|
||||||
{ concurrency: 100 }, // don't overload disk
|
{ concurrency: 100 }, // don't overload disk
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -761,13 +761,13 @@ async function storeMedias(baseMedias, options) {
|
||||||
// overwrite files in case image processing was changed
|
// overwrite files in case image processing was changed
|
||||||
await Promise.map(
|
await Promise.map(
|
||||||
existingHashMedias,
|
existingHashMedias,
|
||||||
async baseMedia => storeFile(baseMedia, options),
|
async (baseMedia) => storeFile(baseMedia, options),
|
||||||
{ concurrency: 100 }, // don't overload disk
|
{ concurrency: 100 }, // don't overload disk
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const newMediaWithEntries = savedMedias.filter(Boolean).map((media, index) => curateMediaEntry(media, index));
|
const newMediaWithEntries = savedMedias.filter(Boolean).map((media, index) => curateMediaEntry(media, index));
|
||||||
const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry);
|
const newMediaEntries = newMediaWithEntries.filter((media) => media.newEntry).map((media) => media.entry);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await bulkInsert('media', newMediaEntries, false);
|
await bulkInsert('media', newMediaEntries, false);
|
||||||
|
@ -851,7 +851,7 @@ async function associateAvatars(profiles) {
|
||||||
return profiles;
|
return profiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
const profilesWithBaseMedias = profiles.map(profile => (profile.avatar
|
const profilesWithBaseMedias = profiles.map((profile) => (profile.avatar
|
||||||
? {
|
? {
|
||||||
...profile,
|
...profile,
|
||||||
avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', {
|
avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', {
|
||||||
|
@ -862,7 +862,7 @@ async function associateAvatars(profiles) {
|
||||||
: profile
|
: profile
|
||||||
));
|
));
|
||||||
|
|
||||||
const baseMedias = profilesWithBaseMedias.map(profile => profile.avatarBaseMedia).filter(Boolean);
|
const baseMedias = profilesWithBaseMedias.map((profile) => profile.avatarBaseMedia).filter(Boolean);
|
||||||
|
|
||||||
const storedMedias = await storeMedias(baseMedias, { stats: true });
|
const storedMedias = await storeMedias(baseMedias, { stats: true });
|
||||||
const storedMediasById = itemsByKey(storedMedias, 'id');
|
const storedMediasById = itemsByKey(storedMedias, 'id');
|
||||||
|
@ -885,13 +885,13 @@ async function associateAvatars(profiles) {
|
||||||
|
|
||||||
async function deleteS3Objects(media) {
|
async function deleteS3Objects(media) {
|
||||||
const objects = media
|
const objects = media
|
||||||
.map(item => [
|
.map((item) => [
|
||||||
{ Key: item.path },
|
{ Key: item.path },
|
||||||
{ Key: item.thumbnail },
|
{ Key: item.thumbnail },
|
||||||
{ Key: item.lazy },
|
{ Key: item.lazy },
|
||||||
])
|
])
|
||||||
.flat()
|
.flat()
|
||||||
.filter(item => item.Key);
|
.filter((item) => item.Key);
|
||||||
|
|
||||||
const status = await s3.deleteObjects({
|
const status = await s3.deleteObjects({
|
||||||
Bucket: config.s3.bucket,
|
Bucket: config.s3.bucket,
|
||||||
|
@ -936,7 +936,7 @@ async function flushOrphanedMedia() {
|
||||||
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
|
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
|
||||||
.delete();
|
.delete();
|
||||||
|
|
||||||
await Promise.all(orphanedMedia.filter(media => !media.is_s3).map(media => Promise.all([
|
await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([
|
||||||
media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),
|
media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),
|
||||||
media.thumbnail && fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }),
|
media.thumbnail && fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }),
|
||||||
media.lazy && fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }),
|
media.lazy && fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }),
|
||||||
|
@ -945,7 +945,7 @@ async function flushOrphanedMedia() {
|
||||||
logger.info(`Removed ${orphanedMedia.length} media files from database and storage`);
|
logger.info(`Removed ${orphanedMedia.length} media files from database and storage`);
|
||||||
|
|
||||||
if (config.s3.enabled) {
|
if (config.s3.enabled) {
|
||||||
await deleteS3Objects(orphanedMedia.filter(media => media.is_s3));
|
await deleteS3Objects(orphanedMedia.filter((media) => media.is_s3));
|
||||||
}
|
}
|
||||||
|
|
||||||
await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });
|
await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });
|
||||||
|
|
|
@ -142,7 +142,7 @@ function curateRelease(release, withMedia = false, withPoster = true) {
|
||||||
slug: release.parent.slug,
|
slug: release.parent.slug,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
actors: (release.actors || []).map(actor => ({
|
actors: (release.actors || []).map((actor) => ({
|
||||||
id: actor.id,
|
id: actor.id,
|
||||||
name: actor.name,
|
name: actor.name,
|
||||||
slug: actor.slug,
|
slug: actor.slug,
|
||||||
|
@ -150,12 +150,12 @@ function curateRelease(release, withMedia = false, withPoster = true) {
|
||||||
entityId: actor.entity_id,
|
entityId: actor.entity_id,
|
||||||
aliasFor: actor.alias_for,
|
aliasFor: actor.alias_for,
|
||||||
})),
|
})),
|
||||||
tags: (release.tags || []).map(tag => ({
|
tags: (release.tags || []).map((tag) => ({
|
||||||
id: tag.id,
|
id: tag.id,
|
||||||
name: tag.name,
|
name: tag.name,
|
||||||
slug: tag.slug,
|
slug: tag.slug,
|
||||||
})),
|
})),
|
||||||
chapters: (release.chapters || []).map(chapter => ({
|
chapters: (release.chapters || []).map((chapter) => ({
|
||||||
id: chapter.id,
|
id: chapter.id,
|
||||||
index: chapter.index,
|
index: chapter.index,
|
||||||
time: chapter.time,
|
time: chapter.time,
|
||||||
|
@ -174,7 +174,7 @@ function curateRelease(release, withMedia = false, withPoster = true) {
|
||||||
} : null,
|
} : null,
|
||||||
}),
|
}),
|
||||||
...(withMedia && {
|
...(withMedia && {
|
||||||
photos: (release.photos || []).map(photo => ({
|
photos: (release.photos || []).map((photo) => ({
|
||||||
id: photo.id,
|
id: photo.id,
|
||||||
path: photo.path,
|
path: photo.path,
|
||||||
thumbnail: release.poster.thumbnail,
|
thumbnail: release.poster.thumbnail,
|
||||||
|
@ -207,16 +207,16 @@ function curateGraphqlRelease(release) {
|
||||||
description: release.description || null,
|
description: release.description || null,
|
||||||
duration: release.duration,
|
duration: release.duration,
|
||||||
entity: release.entity,
|
entity: release.entity,
|
||||||
actors: release.actors.map(actor => actor.actor),
|
actors: release.actors.map((actor) => actor.actor),
|
||||||
tags: release.tags.map(tag => tag.tag),
|
tags: release.tags.map((tag) => tag.tag),
|
||||||
...(release.chapters && { chapters: release.chapters.map(chapter => ({
|
...(release.chapters && { chapters: release.chapters.map((chapter) => ({
|
||||||
...chapter,
|
...chapter,
|
||||||
tags: chapter.tags.map(tag => tag.tag),
|
tags: chapter.tags.map((tag) => tag.tag),
|
||||||
poster: chapter.poster?.media || null,
|
poster: chapter.poster?.media || null,
|
||||||
photos: chapter.photos.map(photo => photo.media),
|
photos: chapter.photos.map((photo) => photo.media),
|
||||||
})) }),
|
})) }),
|
||||||
poster: release.poster?.media || null,
|
poster: release.poster?.media || null,
|
||||||
...(release.photos && { photos: release.photos.map(photo => photo.media) }),
|
...(release.photos && { photos: release.photos.map((photo) => photo.media) }),
|
||||||
trailer: release.trailer?.media || null,
|
trailer: release.trailer?.media || null,
|
||||||
createdAt: release.createdAt,
|
createdAt: release.createdAt,
|
||||||
};
|
};
|
||||||
|
@ -256,7 +256,7 @@ async function fetchScenes(limit = 100) {
|
||||||
limit: Math.min(limit, 10000),
|
limit: Math.min(limit, 10000),
|
||||||
});
|
});
|
||||||
|
|
||||||
return releases.map(release => curateGraphqlRelease(release));
|
return releases.map((release) => curateGraphqlRelease(release));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function searchScenes(query, limit = 100, relevance = 0) {
|
async function searchScenes(query, limit = 100, relevance = 0) {
|
||||||
|
@ -289,7 +289,7 @@ async function searchScenes(query, limit = 100, relevance = 0) {
|
||||||
relevance,
|
relevance,
|
||||||
});
|
});
|
||||||
|
|
||||||
return releases.map(release => curateGraphqlRelease({ ...release.release, relevance: release.rank }));
|
return releases.map((release) => curateGraphqlRelease({ ...release.release, relevance: release.rank }));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function deleteScenes(sceneIds) {
|
async function deleteScenes(sceneIds) {
|
||||||
|
|
|
@ -25,7 +25,7 @@ function scrapeAllTour(scenes, channel) {
|
||||||
release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, '');
|
release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, '');
|
||||||
|
|
||||||
release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY');
|
release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY');
|
||||||
release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map(actor => actor.trim());
|
release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map((actor) => actor.trim());
|
||||||
|
|
||||||
const poster = query.img('.scene-img-wrapper img');
|
const poster = query.img('.scene-img-wrapper img');
|
||||||
release.poster = [
|
release.poster = [
|
||||||
|
@ -124,7 +124,7 @@ async function scrapeRelease({ query, html }, url, channel, baseRelease, options
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
release.photos = query.imgs('#dv_frames a > img').map(photo => [
|
release.photos = query.imgs('#dv_frames a > img').map((photo) => [
|
||||||
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`),
|
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`),
|
||||||
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`),
|
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`),
|
||||||
photo,
|
photo,
|
||||||
|
@ -301,7 +301,7 @@ async function fetchProfile(baseActor, channel, include) {
|
||||||
const searchRes = await http.get(`${channel.url}/search/SearchAutoComplete_Agg_ByMedia?rows=9&name_startsWith=${slugify(baseActor.name, '+')}`);
|
const searchRes = await http.get(`${channel.url}/search/SearchAutoComplete_Agg_ByMedia?rows=9&name_startsWith=${slugify(baseActor.name, '+')}`);
|
||||||
|
|
||||||
if (searchRes.ok) {
|
if (searchRes.ok) {
|
||||||
const actorResult = searchRes.body.Results.find(result => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description));
|
const actorResult = searchRes.body.Results.find((result) => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description));
|
||||||
|
|
||||||
if (actorResult) {
|
if (actorResult) {
|
||||||
return fetchProfilePage(`${channel.url}${actorResult.BasicResponseGroup.id}`, channel, include);
|
return fetchProfilePage(`${channel.url}${actorResult.BasicResponseGroup.id}`, channel, include);
|
||||||
|
|
|
@ -22,13 +22,13 @@ async function networkFetchScene(url, site, release) {
|
||||||
async function fetchLatest(site, page = 1) {
|
async function fetchLatest(site, page = 1) {
|
||||||
const releases = await fetchApiLatest(site, page, false);
|
const releases = await fetchApiLatest(site, page, false);
|
||||||
|
|
||||||
return releases.map(release => curateRelease(release, site));
|
return releases.map((release) => curateRelease(release, site));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchUpcoming(site, page = 1) {
|
async function fetchUpcoming(site, page = 1) {
|
||||||
const releases = await fetchApiUpcoming(site, page, false);
|
const releases = await fetchApiUpcoming(site, page, false);
|
||||||
|
|
||||||
return releases.map(release => curateRelease(release, site));
|
return releases.map((release) => curateRelease(release, site));
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -34,7 +34,7 @@ function extractActors(scene) {
|
||||||
async function fetchLatestWrap(site, page = 1, include, preData) {
|
async function fetchLatestWrap(site, page = 1, include, preData) {
|
||||||
const latest = await fetchLatest(site, page, include, preData);
|
const latest = await fetchLatest(site, page, include, preData);
|
||||||
|
|
||||||
return latest.map(scene => extractActors(scene));
|
return latest.map((scene) => extractActors(scene));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchSceneWrap(url, channel, baseRelease, include) {
|
async function fetchSceneWrap(url, channel, baseRelease, include) {
|
||||||
|
|
|
@ -13,7 +13,7 @@ function scrapeScene({ query }, channel) {
|
||||||
release.description = query.cnt('.latest_update_description');
|
release.description = query.cnt('.latest_update_description');
|
||||||
|
|
||||||
release.date = query.date('.update_date', 'MM/DD/YYYY');
|
release.date = query.date('.update_date', 'MM/DD/YYYY');
|
||||||
release.actors = query.all('.tour_update_models a').map(actorEl => ({
|
release.actors = query.all('.tour_update_models a').map((actorEl) => ({
|
||||||
name: query.cnt(actorEl),
|
name: query.cnt(actorEl),
|
||||||
url: query.url(actorEl, null),
|
url: query.url(actorEl, null),
|
||||||
}));
|
}));
|
||||||
|
@ -30,7 +30,7 @@ function scrapeScene({ query }, channel) {
|
||||||
poster,
|
poster,
|
||||||
];
|
];
|
||||||
|
|
||||||
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map(img => [
|
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
|
||||||
img.replace(/.jpg$/, '-full.jpg'),
|
img.replace(/.jpg$/, '-full.jpg'),
|
||||||
img,
|
img,
|
||||||
]);
|
]);
|
||||||
|
|
|
@ -6,8 +6,8 @@ function extractActors(actorString) {
|
||||||
return actorString
|
return actorString
|
||||||
?.replace(/.*:|\(.*\)|\d+(-|\s)year(-|\s)old|nurses?|tangled/ig, '') // remove Patient:, (date) and other nonsense
|
?.replace(/.*:|\(.*\)|\d+(-|\s)year(-|\s)old|nurses?|tangled/ig, '') // remove Patient:, (date) and other nonsense
|
||||||
.split(/\band\b|\bvs\b|\/|,|&/ig)
|
.split(/\band\b|\bvs\b|\/|,|&/ig)
|
||||||
.map(actor => actor.trim())
|
.map((actor) => actor.trim())
|
||||||
.filter(actor => !!actor && !/\banal\b|\bschool\b|\bgamer\b|\breturn\b|\bfor\b|\bare\b|\bpart\b|realdoll|bimbo|p\d+/ig.test(actor))
|
.filter((actor) => !!actor && !/\banal\b|\bschool\b|\bgamer\b|\breturn\b|\bfor\b|\bare\b|\bpart\b|realdoll|bimbo|p\d+/ig.test(actor))
|
||||||
|| [];
|
|| [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -16,7 +16,7 @@ function matchActors(actorString, models) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
return models.filter(model => new RegExp(model.name, 'i').test(actorString));
|
return models.filter((model) => new RegExp(model.name, 'i').test(actorString));
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeLatest(scenes, site, models) {
|
function scrapeLatest(scenes, site, models) {
|
||||||
|
@ -61,7 +61,7 @@ function scrapeScene({ html, qu }, url, site, include, models) {
|
||||||
|
|
||||||
release.tags = qu.all('.tags a', true);
|
release.tags = qu.all('.tags a', true);
|
||||||
|
|
||||||
release.photos = qu.imgs('.stills img').map(photoPath => `${site.url}/${photoPath}`);
|
release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
|
||||||
|
|
||||||
const posterIndex = 'splash:';
|
const posterIndex = 'splash:';
|
||||||
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
|
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
|
||||||
|
@ -101,7 +101,7 @@ async function fetchModels(site, page = 1, accModels = []) {
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const models = extractModels(res.item, site);
|
const models = extractModels(res.item, site);
|
||||||
const nextPage = res.item.qa('.pagenumbers', true)
|
const nextPage = res.item.qa('.pagenumbers', true)
|
||||||
.map(pageX => Number(pageX))
|
.map((pageX) => Number(pageX))
|
||||||
.filter(Boolean) // remove << and >>
|
.filter(Boolean) // remove << and >>
|
||||||
.includes(page + 1);
|
.includes(page + 1);
|
||||||
|
|
||||||
|
|
|
@ -46,7 +46,7 @@ function scrapeScene({ html, qu }, url) {
|
||||||
|
|
||||||
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
|
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
|
||||||
|
|
||||||
release.actors = qu.all('h5:not(.video_categories) a').map(actor => ({
|
release.actors = qu.all('h5:not(.video_categories) a').map((actor) => ({
|
||||||
name: qu.q(actor, null, true),
|
name: qu.q(actor, null, true),
|
||||||
url: qu.url(actor, null),
|
url: qu.url(actor, null),
|
||||||
}));
|
}));
|
||||||
|
@ -58,7 +58,7 @@ function scrapeScene({ html, qu }, url) {
|
||||||
const poster = qu.img('a img');
|
const poster = qu.img('a img');
|
||||||
|
|
||||||
release.poster = getFallbacks(poster);
|
release.poster = getFallbacks(poster);
|
||||||
release.photos = qu.imgs('.featured-video img', 'src0_1x').map(source => getFallbacks(source));
|
release.photos = qu.imgs('.featured-video img', 'src0_1x').map((source) => getFallbacks(source));
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ function scrapeAll(scenes, channel) {
|
||||||
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
|
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
|
||||||
release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
|
release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
|
||||||
|
|
||||||
release.actors = query.all('.video-card-details--cast a').map(el => ({
|
release.actors = query.all('.video-card-details--cast a').map((el) => ({
|
||||||
name: qu.query.cnt(el),
|
name: qu.query.cnt(el),
|
||||||
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
||||||
}));
|
}));
|
||||||
|
@ -57,7 +57,7 @@ function scrapeScene({ query }, url, channel) {
|
||||||
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||||
release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
|
release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
|
||||||
|
|
||||||
release.actors = query.all('.video-actors a').map(el => ({
|
release.actors = query.all('.video-actors a').map((el) => ({
|
||||||
name: qu.query.cnt(el),
|
name: qu.query.cnt(el),
|
||||||
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
||||||
}));
|
}));
|
||||||
|
|
|
@ -22,13 +22,13 @@ function scrapeAll(scenes, site) {
|
||||||
if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
|
if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
|
||||||
|
|
||||||
[release.poster, ...release.photos] = qu.all('.item-thumbs img')
|
[release.poster, ...release.photos] = qu.all('.item-thumbs img')
|
||||||
.map(source => [
|
.map((source) => [
|
||||||
source.getAttribute('src0_3x'),
|
source.getAttribute('src0_3x'),
|
||||||
source.getAttribute('src0_2x'),
|
source.getAttribute('src0_2x'),
|
||||||
source.getAttribute('src0_1x'),
|
source.getAttribute('src0_1x'),
|
||||||
]
|
]
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.map(fallback => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`)));
|
.map((fallback) => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`)));
|
||||||
|
|
||||||
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
|
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
|
||||||
|
|
||||||
|
@ -116,7 +116,7 @@ async function scrapeProfile({ qu }, site, withScenes) {
|
||||||
qu.q('.profile-pic img', 'src0_3x'),
|
qu.q('.profile-pic img', 'src0_3x'),
|
||||||
qu.q('.profile-pic img', 'src0_2x'),
|
qu.q('.profile-pic img', 'src0_2x'),
|
||||||
qu.q('.profile-pic img', 'src0_1x'),
|
qu.q('.profile-pic img', 'src0_1x'),
|
||||||
].filter(Boolean).map(source => (/^http/.test(source) ? source : `${site.url}${source}`));
|
].filter(Boolean).map((source) => (/^http/.test(source) ? source : `${site.url}${source}`));
|
||||||
|
|
||||||
if (withScenes) {
|
if (withScenes) {
|
||||||
const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1];
|
const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1];
|
||||||
|
|
|
@ -48,7 +48,7 @@ async function fetchPhotos(scene) {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.ok && res.body.images) {
|
if (res.ok && res.body.images) {
|
||||||
return res.body.images.map(image => qu.prefixUrl(image, 'https://photos.bang.com'));
|
return res.body.images.map((image) => qu.prefixUrl(image, 'https://photos.bang.com'));
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
@ -59,7 +59,7 @@ async function scrapeScene(scene, entity, options) {
|
||||||
entryId: scene.id,
|
entryId: scene.id,
|
||||||
title: scene.name,
|
title: scene.name,
|
||||||
description: scene.description,
|
description: scene.description,
|
||||||
tags: scene.genres.concat(scene.actions).map(genre => genre.name),
|
tags: scene.genres.concat(scene.actions).map((genre) => genre.name),
|
||||||
duration: scene.duration,
|
duration: scene.duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -69,19 +69,19 @@ async function scrapeScene(scene, entity, options) {
|
||||||
const date = new Date(scene.releaseDate);
|
const date = new Date(scene.releaseDate);
|
||||||
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
||||||
|
|
||||||
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: genderMap[actor.gender] }));
|
release.actors = scene.actors.map((actor) => ({ name: actor.name, gender: genderMap[actor.gender] }));
|
||||||
|
|
||||||
if (scene.is4k) release.tags.push('4k');
|
if (scene.is4k) release.tags.push('4k');
|
||||||
if (scene.gay) release.tags.push('gay');
|
if (scene.gay) release.tags.push('gay');
|
||||||
|
|
||||||
const defaultPoster = scene.screenshots.find(photo => photo.default === true);
|
const defaultPoster = scene.screenshots.find((photo) => photo.default === true);
|
||||||
const screens = scene.screenshots.filter(photo => photo.default === false);
|
const screens = scene.screenshots.filter((photo) => photo.default === false);
|
||||||
|
|
||||||
const remainingScreens = defaultPoster ? screens : screens.slice(1);
|
const remainingScreens = defaultPoster ? screens : screens.slice(1);
|
||||||
const poster = defaultPoster || screens[0];
|
const poster = defaultPoster || screens[0];
|
||||||
|
|
||||||
release.poster = getScreenUrl(poster, scene);
|
release.poster = getScreenUrl(poster, scene);
|
||||||
release.photos = remainingScreens.map(photo => getScreenUrl(photo, scene));
|
release.photos = remainingScreens.map((photo) => getScreenUrl(photo, scene));
|
||||||
|
|
||||||
if (options?.includePhotos) {
|
if (options?.includePhotos) {
|
||||||
const photos = await fetchPhotos(scene);
|
const photos = await fetchPhotos(scene);
|
||||||
|
@ -399,7 +399,7 @@ async function fetchProfile({ name: actorName }, context, include) {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase());
|
const actor = res.body.hits.hits.find((hit) => hit._source.name.toLowerCase() === actorName.toLowerCase());
|
||||||
|
|
||||||
if (actor) {
|
if (actor) {
|
||||||
return scrapeProfile(actor._source, context.entity, include);
|
return scrapeProfile(actor._source, context.entity, include);
|
||||||
|
|
|
@ -8,10 +8,9 @@ function scrapeProfile(html) {
|
||||||
const profile = {};
|
const profile = {};
|
||||||
|
|
||||||
const bio = qu.all('.infobox tr[valign="top"]')
|
const bio = qu.all('.infobox tr[valign="top"]')
|
||||||
.map(detail => qu.all(detail, 'td', true))
|
.map((detail) => qu.all(detail, 'td', true))
|
||||||
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
|
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
|
||||||
|
|
||||||
|
|
||||||
/* unreliable, see: Syren De Mer
|
/* unreliable, see: Syren De Mer
|
||||||
const catlinks = qa('#mw-normal-catlinks a', true);
|
const catlinks = qa('#mw-normal-catlinks a', true);
|
||||||
const isTrans = catlinks.some(link => link.match(/shemale|transgender/i));
|
const isTrans = catlinks.some(link => link.match(/shemale|transgender/i));
|
||||||
|
|
|
@ -26,7 +26,7 @@ function scrapeAll(scenes) {
|
||||||
release.entryId = new URL(release.url).pathname.match(/\/videos\/([\w-]+)/)[1];
|
release.entryId = new URL(release.url).pathname.match(/\/videos\/([\w-]+)/)[1];
|
||||||
|
|
||||||
release.title = query.cnt('.title') || query.q('img', 'title');
|
release.title = query.cnt('.title') || query.q('img', 'title');
|
||||||
release.actors = subtitle.slice(subtitle.indexOf(':') + 1).split(',').map(actor => actor.trim()).filter(Boolean);
|
release.actors = subtitle.slice(subtitle.indexOf(':') + 1).split(',').map((actor) => actor.trim()).filter(Boolean);
|
||||||
|
|
||||||
release.poster = query.img('.thumb img');
|
release.poster = query.img('.thumb img');
|
||||||
|
|
||||||
|
@ -48,13 +48,13 @@ function scrapeScene({ query, html }, url, channel) {
|
||||||
|
|
||||||
const dataString = query.html('.yoast-schema-graph');
|
const dataString = query.html('.yoast-schema-graph');
|
||||||
const data = dataString && JSON.parse(dataString)['@graph'];
|
const data = dataString && JSON.parse(dataString)['@graph'];
|
||||||
const pageData = data.find(item => item['@type'] === 'WebPage');
|
const pageData = data.find((item) => item['@type'] === 'WebPage');
|
||||||
const imageData = data.find(item => item['@type'] === 'ImageObject');
|
const imageData = data.find((item) => item['@type'] === 'ImageObject');
|
||||||
|
|
||||||
release.entryId = new URL(url).pathname.match(/\/videos\/([\w-]+)/)[1];
|
release.entryId = new URL(url).pathname.match(/\/videos\/([\w-]+)/)[1];
|
||||||
|
|
||||||
release.title = query.cnt('.video .title h1')
|
release.title = query.cnt('.video .title h1')
|
||||||
|| data.find(item => item['@type'] === 'BreadcrumbList')?.itemListElement.slice(-1)[0].item.name
|
|| data.find((item) => item['@type'] === 'BreadcrumbList')?.itemListElement.slice(-1)[0].item.name
|
||||||
|| pageData?.name.slice(0, pageData.name.lastIndexOf('-')).trim();
|
|| pageData?.name.slice(0, pageData.name.lastIndexOf('-')).trim();
|
||||||
|
|
||||||
release.description = query.cnt('.video .descript');
|
release.description = query.cnt('.video .descript');
|
||||||
|
|
|
@ -66,7 +66,7 @@ function scrapeProfile({ query }) {
|
||||||
const profile = {};
|
const profile = {};
|
||||||
|
|
||||||
const keys = query.all('.model-descr_line:not(.model-descr_rait) p.text span', true);
|
const keys = query.all('.model-descr_line:not(.model-descr_rait) p.text span', true);
|
||||||
const values = query.all('.model-descr_line:not(.model-descr_rait) p.text').map(el => query.text(el));
|
const values = query.all('.model-descr_line:not(.model-descr_rait) p.text').map((el) => query.text(el));
|
||||||
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
|
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
|
||||||
|
|
||||||
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)\s*cm\)/)?.[1]);
|
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)\s*cm\)/)?.[1]);
|
||||||
|
@ -100,7 +100,7 @@ function scrapeProfile({ query }) {
|
||||||
profile.piercings = bio.piercings;
|
profile.piercings = bio.piercings;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
|
if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim());
|
||||||
|
|
||||||
const avatar = query.q('.model-img img');
|
const avatar = query.q('.model-img img');
|
||||||
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;
|
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;
|
||||||
|
|
|
@ -43,7 +43,7 @@ function scrapeScene({ query }, channel, html) {
|
||||||
release.date = date;
|
release.date = date;
|
||||||
release.datePrecision = precision;
|
release.datePrecision = precision;
|
||||||
|
|
||||||
release.actors = query.all('.sub-video .pornstar-link').map(el => ({
|
release.actors = query.all('.sub-video .pornstar-link').map((el) => ({
|
||||||
name: query.cnt(el, null),
|
name: query.cnt(el, null),
|
||||||
url: query.url(el, null, 'href', { origin: 'https://www.cumlouder.com' }),
|
url: query.url(el, null, 'href', { origin: 'https://www.cumlouder.com' }),
|
||||||
}));
|
}));
|
||||||
|
|
|
@ -52,10 +52,10 @@ async function scrapeScene({ query }, url, channel) {
|
||||||
release.poster = query.poster() || query.poster('dl8-video') || query.img('#videoBlock img');
|
release.poster = query.poster() || query.poster('dl8-video') || query.img('#videoBlock img');
|
||||||
release.photos = query.urls('.photo-slider-guest .card a');
|
release.photos = query.urls('.photo-slider-guest .card a');
|
||||||
|
|
||||||
release.trailer = query.all('source[type="video/mp4"]').map(trailer => ({
|
release.trailer = query.all('source[type="video/mp4"]').map((trailer) => ({
|
||||||
src: trailer.src,
|
src: trailer.src,
|
||||||
quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null,
|
quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null,
|
||||||
vr: channel.tags?.some(tag => tag.slug === 'vr'),
|
vr: channel.tags?.some((tag) => tag.slug === 'vr'),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
|
@ -63,7 +63,7 @@ async function scrapeScene({ query }, url, channel) {
|
||||||
|
|
||||||
async function fetchActorReleases(urls) {
|
async function fetchActorReleases(urls) {
|
||||||
// DDF Network and DDF Network Stream list all scenes, exclude
|
// DDF Network and DDF Network Stream list all scenes, exclude
|
||||||
const sources = urls.filter(url => !/ddfnetwork/.test(url));
|
const sources = urls.filter((url) => !/ddfnetwork/.test(url));
|
||||||
|
|
||||||
const releases = await Promise.all(sources.map(async (url) => {
|
const releases = await Promise.all(sources.map(async (url) => {
|
||||||
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)');
|
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)');
|
||||||
|
@ -79,10 +79,10 @@ async function fetchActorReleases(urls) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrapeProfile({ query }, _url, actorName) {
|
async function scrapeProfile({ query }, _url, actorName) {
|
||||||
const keys = query.all('.about-title', true).map(key => slugify(key, '_'));
|
const keys = query.all('.about-title', true).map((key) => slugify(key, '_'));
|
||||||
const values = query.all('.about-info').map((el) => {
|
const values = query.all('.about-info').map((el) => {
|
||||||
if (el.children.length > 0) {
|
if (el.children.length > 0) {
|
||||||
return Array.from(el.children, child => child.textContent.trim()).join(', ');
|
return Array.from(el.children, (child) => child.textContent.trim()).join(', ');
|
||||||
}
|
}
|
||||||
|
|
||||||
return el.textContent.trim();
|
return el.textContent.trim();
|
||||||
|
|
|
@ -47,7 +47,7 @@ function scrapeLatest(html, site, filter = true) {
|
||||||
const entryId = `${site.slug}_${pathname.split('/')[4]}`;
|
const entryId = `${site.slug}_${pathname.split('/')[4]}`;
|
||||||
|
|
||||||
const title = element.querySelector('.scene-title').textContent;
|
const title = element.querySelector('.scene-title').textContent;
|
||||||
const actors = title.split(/[,&]|\band\b/).map(actor => actor.replace(/BTS/i, '').trim());
|
const actors = title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim());
|
||||||
|
|
||||||
const poster = `https:${element.querySelector('img').src}`;
|
const poster = `https:${element.querySelector('img').src}`;
|
||||||
const teaser = sceneLinkElement.dataset.preview_clip_url;
|
const teaser = sceneLinkElement.dataset.preview_clip_url;
|
||||||
|
|
|
@ -12,7 +12,7 @@ function scrapeAll(scenes, channel) {
|
||||||
|
|
||||||
release.title = query.cnt('.title');
|
release.title = query.cnt('.title');
|
||||||
|
|
||||||
release.actors = query.all('.actors a').map(actorEl => ({
|
release.actors = query.all('.actors a').map((actorEl) => ({
|
||||||
name: query.cnt(actorEl),
|
name: query.cnt(actorEl),
|
||||||
url: query.url(actorEl, null, 'href', { origin: channel.url }),
|
url: query.url(actorEl, null, 'href', { origin: channel.url }),
|
||||||
}));
|
}));
|
||||||
|
@ -40,7 +40,7 @@ function scrapeScene({ query }, url, channel) {
|
||||||
release.date = query.date('.publish_date', 'MMMM DD, YYYY');
|
release.date = query.date('.publish_date', 'MMMM DD, YYYY');
|
||||||
release.duration = query.dur('.duration');
|
release.duration = query.dur('.duration');
|
||||||
|
|
||||||
release.actors = query.all('.actress a').map(actorEl => ({
|
release.actors = query.all('.actress a').map((actorEl) => ({
|
||||||
name: query.cnt(actorEl),
|
name: query.cnt(actorEl),
|
||||||
url: query.url(actorEl, null, 'href', { origin: channel.url }),
|
url: query.url(actorEl, null, 'href', { origin: channel.url }),
|
||||||
}));
|
}));
|
||||||
|
@ -91,7 +91,7 @@ function scrapeMovie({ query, el }, url, channel) {
|
||||||
|
|
||||||
release.duration = query.dur('.duration');
|
release.duration = query.dur('.duration');
|
||||||
|
|
||||||
release.actors = query.all('.actors .actor').map(actorEl => ({
|
release.actors = query.all('.actors .actor').map((actorEl) => ({
|
||||||
name: query.cnt(actorEl, '.name'),
|
name: query.cnt(actorEl, '.name'),
|
||||||
url: query.url(actorEl, 'a', 'href', { origin: channel.url }),
|
url: query.url(actorEl, 'a', 'href', { origin: channel.url }),
|
||||||
avatar: query.sourceSet(actorEl, '.thumbnail img', 'data-srcset'),
|
avatar: query.sourceSet(actorEl, '.thumbnail img', 'data-srcset'),
|
||||||
|
@ -99,7 +99,7 @@ function scrapeMovie({ query, el }, url, channel) {
|
||||||
|
|
||||||
release.poster = query.sourceSet('.banner', 'data-src')?.[0];
|
release.poster = query.sourceSet('.banner', 'data-src')?.[0];
|
||||||
release.covers = [query.all(query.el('.cover').parentElement, 'source')
|
release.covers = [query.all(query.el('.cover').parentElement, 'source')
|
||||||
?.map(coverEl => query.sourceSet(coverEl, null, 'data-srcset'))
|
?.map((coverEl) => query.sourceSet(coverEl, null, 'data-srcset'))
|
||||||
.flat()
|
.flat()
|
||||||
.sort((coverA, coverB) => {
|
.sort((coverA, coverB) => {
|
||||||
const resA = Number(coverA.match(/_(\d{3,})_/)?.[1]);
|
const resA = Number(coverA.match(/_(\d{3,})_/)?.[1]);
|
||||||
|
|
|
@ -53,7 +53,7 @@ function getImageWithFallbacks(q, selector, site, el) {
|
||||||
q(selector, 'src0_1x'),
|
q(selector, 'src0_1x'),
|
||||||
];
|
];
|
||||||
|
|
||||||
return sources.filter(Boolean).map(src => `${site.parameters?.media || site.url}${src}`);
|
return sources.filter(Boolean).map((src) => `${site.parameters?.media || site.url}${src}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeAllClassic(scenes, channel) {
|
function scrapeAllClassic(scenes, channel) {
|
||||||
|
@ -107,7 +107,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
|
||||||
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
|
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
|
||||||
release.entryId = deriveEntryId(release);
|
release.entryId = deriveEntryId(release);
|
||||||
|
|
||||||
if (channel.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
|
if (channel.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
|
||||||
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
|
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -143,7 +143,7 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
|
||||||
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||||
release.duration = query.dur('.update-info-row:nth-child(2)');
|
release.duration = query.dur('.update-info-row:nth-child(2)');
|
||||||
|
|
||||||
release.actors = query.all('.models-list-thumbs a').map(el => ({
|
release.actors = query.all('.models-list-thumbs a').map((el) => ({
|
||||||
name: query.cnt(el, 'span'),
|
name: query.cnt(el, 'span'),
|
||||||
avatar: getImageWithFallbacks(query.q, 'img', entity, el),
|
avatar: getImageWithFallbacks(query.q, 'img', entity, el),
|
||||||
url: query.url(el, null),
|
url: query.url(el, null),
|
||||||
|
@ -164,8 +164,8 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
|
||||||
if (stars) release.stars = Number(stars);
|
if (stars) release.stars = Number(stars);
|
||||||
|
|
||||||
if (entity.type === 'network') {
|
if (entity.type === 'network') {
|
||||||
const channelRegExp = new RegExp(entity.children.map(channel => channel.parameters?.match || channel.name).join('|'), 'i');
|
const channelRegExp = new RegExp(entity.children.map((channel) => channel.parameters?.match || channel.name).join('|'), 'i');
|
||||||
const channel = release.tags.find(tag => channelRegExp.test(tag));
|
const channel = release.tags.find((tag) => channelRegExp.test(tag));
|
||||||
|
|
||||||
if (channel) {
|
if (channel) {
|
||||||
release.channel = slugify(channel, '');
|
release.channel = slugify(channel, '');
|
||||||
|
@ -199,8 +199,8 @@ async function scrapeProfile({ query }, entity, parameters) {
|
||||||
avatarEl.getAttribute('src0'),
|
avatarEl.getAttribute('src0'),
|
||||||
avatarEl.getAttribute('src'),
|
avatarEl.getAttribute('src'),
|
||||||
]
|
]
|
||||||
.filter(avatar => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
|
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
|
||||||
.map(avatar => qu.prefixUrl(avatar, entity.url));
|
.map((avatar) => qu.prefixUrl(avatar, entity.url));
|
||||||
|
|
||||||
if (avatarSources.length) profile.avatar = avatarSources;
|
if (avatarSources.length) profile.avatar = avatarSources;
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,7 @@ function extractLowArtActors(release) {
|
||||||
const actors = release.title
|
const actors = release.title
|
||||||
.replace(/solo/i, '')
|
.replace(/solo/i, '')
|
||||||
.split(/,|\band\b/ig)
|
.split(/,|\band\b/ig)
|
||||||
.map(actor => actor.trim());
|
.map((actor) => actor.trim());
|
||||||
|
|
||||||
return {
|
return {
|
||||||
...release,
|
...release,
|
||||||
|
@ -32,7 +32,7 @@ async function networkFetchLatest(site, page = 1) {
|
||||||
const releases = await fetchLatest(site, page);
|
const releases = await fetchLatest(site, page);
|
||||||
|
|
||||||
if (site.slug === 'lowartfilms') {
|
if (site.slug === 'lowartfilms') {
|
||||||
return releases.map(release => extractLowArtActors(release));
|
return releases.map((release) => extractLowArtActors(release));
|
||||||
}
|
}
|
||||||
|
|
||||||
return releases;
|
return releases;
|
||||||
|
@ -76,7 +76,7 @@ async function fetchClassicProfile(actorName, { site }) {
|
||||||
if (!pornstarsRes.ok) return null;
|
if (!pornstarsRes.ok) return null;
|
||||||
|
|
||||||
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
|
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
|
||||||
.find(el => slugify(el.textContent) === actorSlug)
|
.find((el) => slugify(el.textContent) === actorSlug)
|
||||||
?.value;
|
?.value;
|
||||||
|
|
||||||
if (actorPath) {
|
if (actorPath) {
|
||||||
|
|
|
@ -14,7 +14,7 @@ function scrapeAllA(scenes, channel) {
|
||||||
release.date = query.date('.thumb-added, .date', ['MMM D, YYYY', 'MMMM DD, YYYY'], /\w+ \d{1,2}, \d{4}/);
|
release.date = query.date('.thumb-added, .date', ['MMM D, YYYY', 'MMMM DD, YYYY'], /\w+ \d{1,2}, \d{4}/);
|
||||||
release.duration = query.dur('.thumb-duration');
|
release.duration = query.dur('.thumb-duration');
|
||||||
|
|
||||||
release.actors = query.all('.thumb-models a, .models a').map(actorEl => ({
|
release.actors = query.all('.thumb-models a, .models a').map((actorEl) => ({
|
||||||
name: query.cnt(actorEl),
|
name: query.cnt(actorEl),
|
||||||
url: query.url(actorEl, null, 'href', { origin: channel.url }),
|
url: query.url(actorEl, null, 'href', { origin: channel.url }),
|
||||||
}));
|
}));
|
||||||
|
@ -70,7 +70,7 @@ function scrapeSceneA({ query }, url, channel) {
|
||||||
|
|
||||||
release.duration = query.dur('.media-body li span, .duration');
|
release.duration = query.dur('.media-body li span, .duration');
|
||||||
|
|
||||||
release.actors = query.all('.media-body a[href*="models/"], .models a').map(actorEl => ({
|
release.actors = query.all('.media-body a[href*="models/"], .models a').map((actorEl) => ({
|
||||||
name: query.cnt(actorEl),
|
name: query.cnt(actorEl),
|
||||||
url: query.url(actorEl, null, 'href', { origin: channel.url }),
|
url: query.url(actorEl, null, 'href', { origin: channel.url }),
|
||||||
}));
|
}));
|
||||||
|
|
|
@ -9,7 +9,7 @@ function scrapeProfile(html, actorName) {
|
||||||
const { document } = new JSDOM(html).window;
|
const { document } = new JSDOM(html).window;
|
||||||
const profile = { name: actorName };
|
const profile = { name: actorName };
|
||||||
|
|
||||||
const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), el => decodeURI(el.href)).reduce((acc, item) => {
|
const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), (el) => decodeURI(el.href)).reduce((acc, item) => {
|
||||||
const keyMatch = item.match(/\[\w+\]/);
|
const keyMatch = item.match(/\[\w+\]/);
|
||||||
|
|
||||||
if (keyMatch) {
|
if (keyMatch) {
|
||||||
|
@ -52,7 +52,7 @@ function scrapeProfile(html, actorName) {
|
||||||
if (bio.height) profile.height = Number(bio.height.split(',')[0]);
|
if (bio.height) profile.height = Number(bio.height.split(',')[0]);
|
||||||
if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]);
|
if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]);
|
||||||
|
|
||||||
profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), el => el.href);
|
profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), (el) => el.href);
|
||||||
|
|
||||||
const avatar = document.querySelector('.profile-image-large img').src;
|
const avatar = document.querySelector('.profile-image-large img').src;
|
||||||
if (!avatar.match('placeholder')) profile.avatar = { src: avatar, credit: null };
|
if (!avatar.match('placeholder')) profile.avatar = { src: avatar, credit: null };
|
||||||
|
|
|
@ -33,7 +33,7 @@ async function fetchApiCredentials(referer, site) {
|
||||||
const res = await http.get(referer);
|
const res = await http.get(referer);
|
||||||
const body = res.body.toString();
|
const body = res.body.toString();
|
||||||
|
|
||||||
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
|
const apiLine = body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
|
||||||
|
|
||||||
if (!apiLine) {
|
if (!apiLine) {
|
||||||
throw new Error(`No Gamma API key found for ${referer}`);
|
throw new Error(`No Gamma API key found for ${referer}`);
|
||||||
|
@ -169,7 +169,7 @@ async function getThumbs(entryId, site, parameters) {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) {
|
if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) {
|
||||||
return res.body.results[0].hits[0].set_pictures.map(img => ([
|
return res.body.results[0].hits[0].set_pictures.map((img) => ([
|
||||||
`https://transform.gammacdn.com/photo_set${img.thumb_path}`,
|
`https://transform.gammacdn.com/photo_set${img.thumb_path}`,
|
||||||
`https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`,
|
`https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`,
|
||||||
]));
|
]));
|
||||||
|
@ -214,7 +214,7 @@ async function scrapeApiReleases(json, site) {
|
||||||
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
|
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
|
||||||
release.director = scene.directors[0]?.name || null;
|
release.director = scene.directors[0]?.name || null;
|
||||||
|
|
||||||
release.actors = scene.actors.map(actor => ({
|
release.actors = scene.actors.map((actor) => ({
|
||||||
entryId: actor.actor_id,
|
entryId: actor.actor_id,
|
||||||
name: actor.name,
|
name: actor.name,
|
||||||
gender: actor.gender,
|
gender: actor.gender,
|
||||||
|
@ -226,7 +226,7 @@ async function scrapeApiReleases(json, site) {
|
||||||
}));
|
}));
|
||||||
|
|
||||||
release.tags = scene.master_categories
|
release.tags = scene.master_categories
|
||||||
.concat(scene.categories?.map(category => category.name))
|
.concat(scene.categories?.map((category) => category.name))
|
||||||
.filter(Boolean); // some categories don't have a name
|
.filter(Boolean); // some categories don't have a name
|
||||||
|
|
||||||
const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
|
const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
|
||||||
|
@ -272,7 +272,7 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
|
||||||
|
|
||||||
[release.likes, release.dislikes] = $(element).find('.value')
|
[release.likes, release.dislikes] = $(element).find('.value')
|
||||||
.toArray()
|
.toArray()
|
||||||
.map(value => Number($(value).text()));
|
.map((value) => Number($(value).text()));
|
||||||
|
|
||||||
const posterEl = $(element).find('.imgLink img, .tlcImageItem');
|
const posterEl = $(element).find('.imgLink img, .tlcImageItem');
|
||||||
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
|
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
|
||||||
|
@ -327,13 +327,13 @@ async function scrapeScene(html, url, site, baseRelease, mobileHtml, options) {
|
||||||
const actors = data?.actor || data2?.actor;
|
const actors = data?.actor || data2?.actor;
|
||||||
|
|
||||||
if (actors) {
|
if (actors) {
|
||||||
release.actors = actors.map(actor => ({
|
release.actors = actors.map((actor) => ({
|
||||||
name: actor.name,
|
name: actor.name,
|
||||||
gender: actor.gender,
|
gender: actor.gender,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
const hasTrans = release.actors?.some(actor => actor.gender === 'shemale');
|
const hasTrans = release.actors?.some((actor) => actor.gender === 'shemale');
|
||||||
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
|
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
|
||||||
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
|
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
|
||||||
|
|
||||||
|
@ -420,7 +420,7 @@ async function scrapeSceneApi(data, site, options) {
|
||||||
release.duration = data.length;
|
release.duration = data.length;
|
||||||
release.date = new Date(data.date * 1000) || qu.parseDate(data.release_date, 'YYYY-MM-DD');
|
release.date = new Date(data.date * 1000) || qu.parseDate(data.release_date, 'YYYY-MM-DD');
|
||||||
|
|
||||||
release.actors = data.actors.map(actor => ({
|
release.actors = data.actors.map((actor) => ({
|
||||||
entryId: actor.actor_id,
|
entryId: actor.actor_id,
|
||||||
name: actor.name,
|
name: actor.name,
|
||||||
gender: actor.gender,
|
gender: actor.gender,
|
||||||
|
@ -429,7 +429,7 @@ async function scrapeSceneApi(data, site, options) {
|
||||||
: qu.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
|
: qu.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
release.tags = data.categories.map(category => category.name);
|
release.tags = data.categories.map((category) => category.name);
|
||||||
|
|
||||||
if (data.pictures) {
|
if (data.pictures) {
|
||||||
release.poster = [
|
release.poster = [
|
||||||
|
@ -501,7 +501,7 @@ async function scrapeMovie({ query, html }, window, url, entity, options) {
|
||||||
release.date = qu.extractDate(data.dvdReleaseDate);
|
release.date = qu.extractDate(data.dvdReleaseDate);
|
||||||
release.title = data.dvdName;
|
release.title = data.dvdName;
|
||||||
|
|
||||||
release.actors = data.dvdActors.map(actor => ({ name: actor.actorName, entryId: actor.actorId }));
|
release.actors = data.dvdActors.map((actor) => ({ name: actor.actorName, entryId: actor.actorId }));
|
||||||
release.tags = query.cnts('.dvdCol a');
|
release.tags = query.cnts('.dvdCol a');
|
||||||
|
|
||||||
release.scenes = scrapeAll(html, entity, entity.url);
|
release.scenes = scrapeAll(html, entity, entity.url);
|
||||||
|
@ -602,9 +602,9 @@ function scrapeApiProfile(data, releases, siteSlug) {
|
||||||
if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
|
if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
|
||||||
|
|
||||||
const avatarPaths = Object.values(data.pictures).reverse();
|
const avatarPaths = Object.values(data.pictures).reverse();
|
||||||
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map(avatarPath => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
|
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map((avatarPath) => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
|
||||||
|
|
||||||
if (releases) profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
|
if (releases) profile.releases = releases.map((release) => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
@ -723,7 +723,7 @@ function getDeepUrl(url, site, baseRelease, mobile) {
|
||||||
const filter = new Set(['en', 'video', 'scene', site.slug, site.parent.slug]);
|
const filter = new Set(['en', 'video', 'scene', site.slug, site.parent.slug]);
|
||||||
const pathname = baseRelease?.path || new URL(url).pathname
|
const pathname = baseRelease?.path || new URL(url).pathname
|
||||||
.split('/')
|
.split('/')
|
||||||
.filter(component => !filter.has(component))
|
.filter((component) => !filter.has(component))
|
||||||
.join('/'); // reduce to scene ID and title slug
|
.join('/'); // reduce to scene ID and title slug
|
||||||
|
|
||||||
const sceneId = baseRelease?.entryId || pathname.match(/\/(\d+)\//)?.[1];
|
const sceneId = baseRelease?.entryId || pathname.match(/\/(\d+)\//)?.[1];
|
||||||
|
@ -863,7 +863,7 @@ async function fetchApiProfile({ name: actorName }, context, include) {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.status === 200 && res.body.results[0].hits.length > 0) {
|
if (res.status === 200 && res.body.results[0].hits.length > 0) {
|
||||||
const actorData = res.body.results[0].hits.find(actor => slugify(actor.name) === slugify(actorName));
|
const actorData = res.body.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
|
||||||
|
|
||||||
if (actorData) {
|
if (actorData) {
|
||||||
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);
|
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);
|
||||||
|
|
|
@ -42,13 +42,13 @@ function scrapeScene({ query }, url) {
|
||||||
release.duration = query.dur('.content-metas span:nth-child(2)');
|
release.duration = query.dur('.content-metas span:nth-child(2)');
|
||||||
release.likes = query.number('.content-metas span:nth-child(6)');
|
release.likes = query.number('.content-metas span:nth-child(6)');
|
||||||
|
|
||||||
release.actors = query.all('.model-thumb img').map(el => ({
|
release.actors = query.all('.model-thumb img').map((el) => ({
|
||||||
name: query.q(el, null, 'alt'),
|
name: query.q(el, null, 'alt'),
|
||||||
avatar: query.img(el, null, 'src'),
|
avatar: query.img(el, null, 'src'),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
release.poster = query.poster('.content-video video');
|
release.poster = query.poster('.content-video video');
|
||||||
release.photos = query.urls('#photo-carousel a').map(photo => [
|
release.photos = query.urls('#photo-carousel a').map((photo) => [
|
||||||
photo.replace('/full', ''),
|
photo.replace('/full', ''),
|
||||||
photo,
|
photo,
|
||||||
photo.replace('/full', '/thumbs'),
|
photo.replace('/full', '/thumbs'),
|
||||||
|
@ -135,7 +135,7 @@ async function fetchProfile(baseActor, entity, include) {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (searchRes.ok) {
|
if (searchRes.ok) {
|
||||||
const actor = searchRes.body.find(result => result.type === 'model' && result.title === baseActor.name);
|
const actor = searchRes.body.find((result) => result.type === 'model' && result.title === baseActor.name);
|
||||||
|
|
||||||
if (actor) {
|
if (actor) {
|
||||||
const actorRes = await qu.get(actor.url);
|
const actorRes = await qu.get(actor.url);
|
||||||
|
|
|
@ -21,7 +21,7 @@ function scrapeAll(scenes) {
|
||||||
avatar: [
|
avatar: [
|
||||||
avatarEl.src.replace(/-\d+x\d+/, ''),
|
avatarEl.src.replace(/-\d+x\d+/, ''),
|
||||||
avatarEl.src,
|
avatarEl.src,
|
||||||
].map(src => ({ src, interval: 1000, concurrency: 1 })),
|
].map((src) => ({ src, interval: 1000, concurrency: 1 })),
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
}).concat({
|
}).concat({
|
||||||
|
|
|
@ -53,7 +53,7 @@ function getImageWithFallbacks(q, selector, site, el) {
|
||||||
q(selector, 'src0_1x'),
|
q(selector, 'src0_1x'),
|
||||||
];
|
];
|
||||||
|
|
||||||
return sources.filter(Boolean).map(src => `${site.parameters?.media || site.url}${src}`);
|
return sources.filter(Boolean).map((src) => `${site.parameters?.media || site.url}${src}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeAll(scenes, channel) {
|
function scrapeAll(scenes, channel) {
|
||||||
|
@ -108,7 +108,7 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
|
||||||
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
|
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
|
||||||
release.entryId = deriveEntryId(release);
|
release.entryId = deriveEntryId(release);
|
||||||
|
|
||||||
if (site.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
|
if (site.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
|
||||||
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
|
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -132,7 +132,7 @@ function scrapeScene({ html, query }, channel, url) {
|
||||||
const poster = qu.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
|
const poster = qu.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
|
||||||
|
|
||||||
[release.poster, ...release.photos] = [poster, ...query.imgs('.item-thumb img', 'src0_1x', { origin: channel.url })]
|
[release.poster, ...release.photos] = [poster, ...query.imgs('.item-thumb img', 'src0_1x', { origin: channel.url })]
|
||||||
.map(src => src && [
|
.map((src) => src && [
|
||||||
src.replace('-1x', '-3x'),
|
src.replace('-1x', '-3x'),
|
||||||
src.replace('-1x', '-2x'),
|
src.replace('-1x', '-2x'),
|
||||||
src,
|
src,
|
||||||
|
@ -161,7 +161,7 @@ function scrapeSceneT1({ html, query }, site, url, baseRelease) {
|
||||||
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||||
release.duration = query.dur('.update-info-row:nth-child(2)');
|
release.duration = query.dur('.update-info-row:nth-child(2)');
|
||||||
|
|
||||||
release.actors = query.all('.models-list-thumbs a').map(el => ({
|
release.actors = query.all('.models-list-thumbs a').map((el) => ({
|
||||||
name: query.q(el, 'span', true),
|
name: query.q(el, 'span', true),
|
||||||
avatar: getImageWithFallbacks(query.q, 'img', site, el),
|
avatar: getImageWithFallbacks(query.q, 'img', site, el),
|
||||||
}));
|
}));
|
||||||
|
@ -180,8 +180,8 @@ function scrapeSceneT1({ html, query }, site, url, baseRelease) {
|
||||||
if (stars) release.stars = Number(stars);
|
if (stars) release.stars = Number(stars);
|
||||||
|
|
||||||
if (site.type === 'network') {
|
if (site.type === 'network') {
|
||||||
const channelRegExp = new RegExp(site.children.map(channel => channel.parameters?.match || channel.name).join('|'), 'i');
|
const channelRegExp = new RegExp(site.children.map((channel) => channel.parameters?.match || channel.name).join('|'), 'i');
|
||||||
const channel = release.tags.find(tag => channelRegExp.test(tag));
|
const channel = release.tags.find((tag) => channelRegExp.test(tag));
|
||||||
|
|
||||||
if (channel) {
|
if (channel) {
|
||||||
release.channel = slugify(channel, '');
|
release.channel = slugify(channel, '');
|
||||||
|
@ -290,7 +290,7 @@ async function scrapeProfile({ query, el }, channel, options) {
|
||||||
if (bio.piercings && /yes/i.test(bio.piercings)) profile.hasPiercings = true;
|
if (bio.piercings && /yes/i.test(bio.piercings)) profile.hasPiercings = true;
|
||||||
if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false;
|
if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false;
|
||||||
|
|
||||||
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
|
if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim());
|
||||||
|
|
||||||
profile.social = [bio.onlyfans, bio.twitter, bio.instagram].filter(Boolean);
|
profile.social = [bio.onlyfans, bio.twitter, bio.instagram].filter(Boolean);
|
||||||
|
|
||||||
|
|
|
@ -26,7 +26,7 @@ function scrapeLatest(scenes, site) {
|
||||||
release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
|
release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
|
||||||
}
|
}
|
||||||
|
|
||||||
release.actors = actors.map(actor => actor.trim());
|
release.actors = actors.map((actor) => actor.trim());
|
||||||
|
|
||||||
const description = query.q('.articleCopyText', true);
|
const description = query.q('.articleCopyText', true);
|
||||||
if (description) release.description = description.slice(0, description.lastIndexOf('('));
|
if (description) release.description = description.slice(0, description.lastIndexOf('('));
|
||||||
|
@ -81,7 +81,7 @@ function scrapeScene({ query }, site) {
|
||||||
release.title = title.trim();
|
release.title = title.trim();
|
||||||
release.description = query.q('.articleCopyText', true);
|
release.description = query.q('.articleCopyText', true);
|
||||||
|
|
||||||
release.actors = actors.map(actor => actor.trim());
|
release.actors = actors.map((actor) => actor.trim());
|
||||||
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
|
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
|
||||||
release.duration = query.dur('.articlePostDateText a:nth-child(2)');
|
release.duration = query.dur('.articlePostDateText a:nth-child(2)');
|
||||||
|
|
||||||
|
|
|
@ -117,7 +117,7 @@ function scrapeProfile({ query }, actorName, actorAvatar, channel, releasesFromS
|
||||||
profile.releases = releasesFromScene?.[profile.name] || scrapeProfileScenes(qu.initAll(query.all('.Models li')), actorName, channel);
|
profile.releases = releasesFromScene?.[profile.name] || scrapeProfileScenes(qu.initAll(query.all('.Models li')), actorName, channel);
|
||||||
|
|
||||||
// avatar is the poster of a scene, find scene and use its high quality poster instead
|
// avatar is the poster of a scene, find scene and use its high quality poster instead
|
||||||
const avatarRelease = profile.releases.find(release => new URL(release.poster[1]).pathname === new URL(actorAvatar).pathname);
|
const avatarRelease = profile.releases.find((release) => new URL(release.poster[1]).pathname === new URL(actorAvatar).pathname);
|
||||||
profile.avatar = avatarRelease?.poster[0];
|
profile.avatar = avatarRelease?.poster[0];
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
|
|
|
@ -13,7 +13,7 @@ async function fetchActors(entryId, channel, { token, time }) {
|
||||||
const res = await http.get(url);
|
const res = await http.get(url);
|
||||||
|
|
||||||
if (res.statusCode === 200 && res.body.status === true) {
|
if (res.statusCode === 200 && res.body.status === true) {
|
||||||
return Object.values(res.body.response.collection).map(actor => Object.values(actor.modelId.collection)[0].stageName);
|
return Object.values(res.body.response.collection).map((actor) => Object.values(actor.modelId.collection)[0].stageName);
|
||||||
}
|
}
|
||||||
|
|
||||||
return [];
|
return [];
|
||||||
|
@ -46,7 +46,7 @@ function scrapeLatest(items, channel) {
|
||||||
|
|
||||||
release.title = query.cnt('h5 a');
|
release.title = query.cnt('h5 a');
|
||||||
|
|
||||||
[release.poster, ...release.photos] = query.imgs('.screenshot').map(src => [
|
[release.poster, ...release.photos] = query.imgs('.screenshot').map((src) => [
|
||||||
// unnecessarily large
|
// unnecessarily large
|
||||||
// src.replace(/\/\d+/, 3840),
|
// src.replace(/\/\d+/, 3840),
|
||||||
// src.replace(/\/\d+/, '/2000'),
|
// src.replace(/\/\d+/, '/2000'),
|
||||||
|
@ -99,7 +99,7 @@ function scrapeScene({ query, html }, url, channel) {
|
||||||
[release.poster, ...release.photos] = [poster]
|
[release.poster, ...release.photos] = [poster]
|
||||||
.concat(photos)
|
.concat(photos)
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.map(src => [
|
.map((src) => [
|
||||||
src.replace(/\/(\d+)\/\d+/, '/$1/1500'),
|
src.replace(/\/(\d+)\/\d+/, '/$1/1500'),
|
||||||
src.replace(/\/(\d+)\/\d+/, '/$1/1000'),
|
src.replace(/\/(\d+)\/\d+/, '/$1/1000'),
|
||||||
src,
|
src,
|
||||||
|
@ -128,8 +128,8 @@ async function scrapeSceneApi(scene, channel, tokens, deep) {
|
||||||
release.date = new Date(scene.sites.collection[scene.id].publishDate);
|
release.date = new Date(scene.sites.collection[scene.id].publishDate);
|
||||||
release.poster = scene._resources.primary[0].url;
|
release.poster = scene._resources.primary[0].url;
|
||||||
|
|
||||||
if (scene.tags) release.tags = Object.values(scene.tags.collection).map(tag => tag.alias);
|
if (scene.tags) release.tags = Object.values(scene.tags.collection).map((tag) => tag.alias);
|
||||||
if (scene._resources.base) release.photos = scene._resources.base.map(resource => resource.url);
|
if (scene._resources.base) release.photos = scene._resources.base.map((resource) => resource.url);
|
||||||
|
|
||||||
if (deep) {
|
if (deep) {
|
||||||
// don't make external requests during update scraping, as this would happen for every scene on the page
|
// don't make external requests during update scraping, as this would happen for every scene on the page
|
||||||
|
@ -149,7 +149,7 @@ async function scrapeSceneApi(scene, channel, tokens, deep) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeLatestApi(scenes, site, tokens) {
|
function scrapeLatestApi(scenes, site, tokens) {
|
||||||
return Promise.map(scenes, async scene => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
|
return Promise.map(scenes, async (scene) => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchToken(channel) {
|
async function fetchToken(channel) {
|
||||||
|
|
|
@ -33,7 +33,7 @@ function scrapeLatest(scenes, dates, site) {
|
||||||
|
|
||||||
const poster = qu.img('img[src*="photos/"][width="400"]');
|
const poster = qu.img('img[src*="photos/"][width="400"]');
|
||||||
release.poster = `${site.url}/visitors/${poster}`;
|
release.poster = `${site.url}/visitors/${poster}`;
|
||||||
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map(source => `${site.url}/visitors/${source}`);
|
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
});
|
});
|
||||||
|
|
|
@ -55,7 +55,7 @@ async function getPhotosLegacy(entryId, site, type = 'highres', page = 1) {
|
||||||
|
|
||||||
// don't add first URL to pages to prevent unnecessary duplicate request
|
// don't add first URL to pages to prevent unnecessary duplicate request
|
||||||
const photos = scrapePhotos(html, type);
|
const photos = scrapePhotos(html, type);
|
||||||
const pages = Array.from(new Set($('.page_numbers a').toArray().map(el => $(el).attr('href'))));
|
const pages = Array.from(new Set($('.page_numbers a').toArray().map((el) => $(el).attr('href'))));
|
||||||
|
|
||||||
const otherPhotos = pages
|
const otherPhotos = pages
|
||||||
? await Promise.map(pages, async (pageX) => {
|
? await Promise.map(pages, async (pageX) => {
|
||||||
|
@ -84,7 +84,7 @@ async function getPhotos(entryId, site, type = 'highres', page = 1) {
|
||||||
const res = await http.get(albumUrl);
|
const res = await http.get(albumUrl);
|
||||||
const html = res.body.toString();
|
const html = res.body.toString();
|
||||||
|
|
||||||
const sourceLines = html.split(/\n/).filter(line => line.match(/ptx\["\w+"\]/));
|
const sourceLines = html.split(/\n/).filter((line) => line.match(/ptx\["\w+"\]/));
|
||||||
const sources = sourceLines.reduce((acc, sourceLine) => {
|
const sources = sourceLines.reduce((acc, sourceLine) => {
|
||||||
const quality = sourceLine.match(/\["\w+"\]/)[0].slice(2, -2);
|
const quality = sourceLine.match(/\["\w+"\]/)[0].slice(2, -2);
|
||||||
const sourceStart = sourceLine.match(/\/trial|\/tour|\/content/);
|
const sourceStart = sourceLine.match(/\/trial|\/tour|\/content/);
|
||||||
|
@ -261,7 +261,7 @@ async function scrapeScene({ html, query }, url, site, include) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (include.trailer && site.slug !== 'manuelferrara') {
|
if (include.trailer && site.slug !== 'manuelferrara') {
|
||||||
const trailerLines = html.split('\n').filter(line => /movie\["trailer\w*"\]\[/i.test(line));
|
const trailerLines = html.split('\n').filter((line) => /movie\["trailer\w*"\]\[/i.test(line));
|
||||||
|
|
||||||
if (trailerLines.length) {
|
if (trailerLines.length) {
|
||||||
release.trailer = trailerLines.map((trailerLine) => {
|
release.trailer = trailerLines.map((trailerLine) => {
|
||||||
|
@ -307,7 +307,7 @@ function scrapeMovie({ el, query }, url, site) {
|
||||||
const scenes = scrapeAll(sceneQus, site);
|
const scenes = scrapeAll(sceneQus, site);
|
||||||
|
|
||||||
const curatedScenes = scenes
|
const curatedScenes = scenes
|
||||||
?.map(scene => ({ ...scene, movie }))
|
?.map((scene) => ({ ...scene, movie }))
|
||||||
.sort((sceneA, sceneB) => sceneA.date - sceneB.date);
|
.sort((sceneA, sceneB) => sceneA.date - sceneB.date);
|
||||||
|
|
||||||
movie.date = curatedScenes?.[0].date;
|
movie.date = curatedScenes?.[0].date;
|
||||||
|
@ -354,13 +354,13 @@ function scrapeProfile(html, url, actorName, entity) {
|
||||||
avatarEl.getAttribute('src0'),
|
avatarEl.getAttribute('src0'),
|
||||||
avatarEl.getAttribute('src'),
|
avatarEl.getAttribute('src'),
|
||||||
]
|
]
|
||||||
.filter(avatar => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
|
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
|
||||||
.map(avatar => qu.prefixUrl(avatar, entity.url));
|
.map((avatar) => qu.prefixUrl(avatar, entity.url));
|
||||||
|
|
||||||
if (avatarSources.length) profile.avatar = avatarSources;
|
if (avatarSources.length) profile.avatar = avatarSources;
|
||||||
}
|
}
|
||||||
|
|
||||||
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href);
|
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), (el) => el.href);
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,7 +35,7 @@ function scrapeScene({ query }, url) {
|
||||||
release.title = query.cnt('.title');
|
release.title = query.cnt('.title');
|
||||||
release.date = query.date('.date .content', 'MMM Do, YYYY');
|
release.date = query.date('.date .content', 'MMM Do, YYYY');
|
||||||
|
|
||||||
release.actors = query.all('.models .content a').map(modelEl => ({
|
release.actors = query.all('.models .content a').map((modelEl) => ({
|
||||||
name: query.cnt(modelEl),
|
name: query.cnt(modelEl),
|
||||||
url: query.url(modelEl, null),
|
url: query.url(modelEl, null),
|
||||||
}));
|
}));
|
||||||
|
@ -76,7 +76,7 @@ async function fetchProfile(baseActor, entity) {
|
||||||
return searchRes.status;
|
return searchRes.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
const actorUrl = searchRes.items.find(item => slugify(item.query.cnt('.title')) === baseActor.slug)?.query.url('a');
|
const actorUrl = searchRes.items.find((item) => slugify(item.query.cnt('.title')) === baseActor.slug)?.query.url('a');
|
||||||
|
|
||||||
if (!actorUrl) {
|
if (!actorUrl) {
|
||||||
return null;
|
return null;
|
||||||
|
|
|
@ -53,7 +53,7 @@ function scrapeLatest(scenes, site) {
|
||||||
}
|
}
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
}).filter(scene => scene);
|
}).filter((scene) => scene);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrapeScene({ query, html }, url, baseRelease, channel, session) {
|
async function scrapeScene({ query, html }, url, baseRelease, channel, session) {
|
||||||
|
@ -100,7 +100,7 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session)
|
||||||
const trailerInfoRes = await http.post(trailerInfoUrl, null, { session });
|
const trailerInfoRes = await http.post(trailerInfoUrl, null, { session });
|
||||||
|
|
||||||
if (trailerInfoRes.ok && trailerInfoRes.body.sources?.length > 0) {
|
if (trailerInfoRes.ok && trailerInfoRes.body.sources?.length > 0) {
|
||||||
release.trailer = trailerInfoRes.body.sources.map(trailer => ({
|
release.trailer = trailerInfoRes.body.sources.map((trailer) => ({
|
||||||
src: trailer.src,
|
src: trailer.src,
|
||||||
type: trailer.type,
|
type: trailer.type,
|
||||||
/* unreliable, sometimes actual video is 720p
|
/* unreliable, sometimes actual video is 720p
|
||||||
|
|
|
@ -4,11 +4,11 @@ const qu = require('../utils/qu');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
|
||||||
function scrapeAll({ query }) {
|
function scrapeAll({ query }) {
|
||||||
const urls = query.urls('td > a:not([href*=joinnow])').map(pathname => `http://killergram.com/${encodeURI(pathname)}`);
|
const urls = query.urls('td > a:not([href*=joinnow])').map((pathname) => `http://killergram.com/${encodeURI(pathname)}`);
|
||||||
const posters = query.imgs('td > a img');
|
const posters = query.imgs('td > a img');
|
||||||
const titles = query.all('.episodeheadertext', true);
|
const titles = query.all('.episodeheadertext', true);
|
||||||
const actors = query.all('.episodetextinfo:nth-child(3)').map(el => query.all(el, 'a', true));
|
const actors = query.all('.episodetextinfo:nth-child(3)').map((el) => query.all(el, 'a', true));
|
||||||
const channels = query.all('.episodetextinfo:nth-child(2) a', true).map(channel => slugify(channel, ''));
|
const channels = query.all('.episodetextinfo:nth-child(2) a', true).map((channel) => slugify(channel, ''));
|
||||||
|
|
||||||
if ([urls.length, posters.length, titles.length, actors.length, channels.length].every((value, index, array) => value === array[0])) { // make sure every set has the same number of items
|
if ([urls.length, posters.length, titles.length, actors.length, channels.length].every((value, index, array) => value === array[0])) { // make sure every set has the same number of items
|
||||||
const releases = urls.map((url, index) => ({
|
const releases = urls.map((url, index) => ({
|
||||||
|
@ -51,7 +51,7 @@ function scrapeScene({ query, html }, url) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchActorReleases({ query }, url, remainingPages, actorName, accReleases = []) {
|
async function fetchActorReleases({ query }, url, remainingPages, actorName, accReleases = []) {
|
||||||
const releases = scrapeAll({ query }).filter(release => release.actors.includes(actorName));
|
const releases = scrapeAll({ query }).filter((release) => release.actors.includes(actorName));
|
||||||
|
|
||||||
if (remainingPages.length > 0) {
|
if (remainingPages.length > 0) {
|
||||||
const { origin, pathname, searchParams } = new URL(url);
|
const { origin, pathname, searchParams } = new URL(url);
|
||||||
|
|
|
@ -19,7 +19,7 @@ function scrapeAll(scenes) {
|
||||||
release.stars = query.q('.average-rating', 'data-rating') / 10;
|
release.stars = query.q('.average-rating', 'data-rating') / 10;
|
||||||
|
|
||||||
release.poster = query.img('.adimage');
|
release.poster = query.img('.adimage');
|
||||||
release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map(photo => [
|
release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map((photo) => [
|
||||||
photo.replace('410/', '830/'),
|
photo.replace('410/', '830/'),
|
||||||
photo,
|
photo,
|
||||||
]);
|
]);
|
||||||
|
@ -40,13 +40,13 @@ async function scrapeScene({ query }, url) {
|
||||||
release.description = query.q('.description-text', true);
|
release.description = query.q('.description-text', true);
|
||||||
|
|
||||||
release.date = query.date('.shoot-date', 'MMMM DD, YYYY');
|
release.date = query.date('.shoot-date', 'MMMM DD, YYYY');
|
||||||
release.actors = query.all('.names a', true).map(actor => actor.replace(/,\s*/, ''));
|
release.actors = query.all('.names a', true).map((actor) => actor.replace(/,\s*/, ''));
|
||||||
release.director = query.q('.director-name', true);
|
release.director = query.q('.director-name', true);
|
||||||
|
|
||||||
release.photos = query.imgs('.gallery .thumb img, #gallerySlider .gallery-img', 'data-image-file');
|
release.photos = query.imgs('.gallery .thumb img, #gallerySlider .gallery-img', 'data-image-file');
|
||||||
release.poster = query.poster();
|
release.poster = query.poster();
|
||||||
|
|
||||||
release.tags = query.all('.tag-list a[href*="/tag"]', true).map(tag => tag.replace(/,\s*/, ''));
|
release.tags = query.all('.tag-list a[href*="/tag"]', true).map((tag) => tag.replace(/,\s*/, ''));
|
||||||
|
|
||||||
const trailer = query.q('.player span[data-type="trailer-src"]', 'data-url');
|
const trailer = query.q('.player span[data-type="trailer-src"]', 'data-url');
|
||||||
|
|
||||||
|
@ -154,7 +154,7 @@ async function fetchProfile({ name: actorName }, entity, include) {
|
||||||
const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
|
const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
|
||||||
|
|
||||||
if (searchRes.ok) {
|
if (searchRes.ok) {
|
||||||
const actorItem = searchRes.items.find(item => item.query.exists(`.model-link img[alt="${actorName}"]`));
|
const actorItem = searchRes.items.find((item) => item.query.exists(`.model-link img[alt="${actorName}"]`));
|
||||||
|
|
||||||
if (actorItem) {
|
if (actorItem) {
|
||||||
const actorPath = actorItem.query.url('.model-link');
|
const actorPath = actorItem.query.url('.model-link');
|
||||||
|
|
|
@ -126,7 +126,7 @@ async function scrapeScene(html, url, site, useGallery) {
|
||||||
'1080p': 1080,
|
'1080p': 1080,
|
||||||
};
|
};
|
||||||
|
|
||||||
release.trailer = data.clip.qualities.map(trailer => ({
|
release.trailer = data.clip.qualities.map((trailer) => ({
|
||||||
src: trailer.src,
|
src: trailer.src,
|
||||||
type: trailer.type,
|
type: trailer.type,
|
||||||
quality: qualityMap[trailer.quality] || trailer.quality,
|
quality: qualityMap[trailer.quality] || trailer.quality,
|
||||||
|
@ -147,10 +147,10 @@ async function scrapeProfile(html, _url, actorName) {
|
||||||
};
|
};
|
||||||
|
|
||||||
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
|
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
|
||||||
const entries = Array.from(document.querySelectorAll('.model--description tr'), el => el.textContent.replace(/\n/g, '').split(':'));
|
const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
|
||||||
|
|
||||||
const bio = entries
|
const bio = entries
|
||||||
.filter(entry => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
|
.filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
|
||||||
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
|
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
|
||||||
|
|
||||||
profile.birthPlace = bio.Nationality;
|
profile.birthPlace = bio.Nationality;
|
||||||
|
@ -184,7 +184,7 @@ async function fetchProfile({ name: actorName }) {
|
||||||
const res = await http.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
|
const res = await http.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
|
||||||
const data = res.body;
|
const data = res.body;
|
||||||
|
|
||||||
const result = data.terms.find(item => item.type === 'model');
|
const result = data.terms.find((item) => item.type === 'model');
|
||||||
|
|
||||||
if (result) {
|
if (result) {
|
||||||
const bioRes = await http.get(result.url);
|
const bioRes = await http.get(result.url);
|
||||||
|
|
|
@ -65,7 +65,7 @@ async function fetchPhotos(url) {
|
||||||
const res = await qu.get(url, '.et_post_gallery');
|
const res = await qu.get(url, '.et_post_gallery');
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
return res.item.query.urls('a').map(imgUrl => ({
|
return res.item.query.urls('a').map((imgUrl) => ({
|
||||||
src: imgUrl,
|
src: imgUrl,
|
||||||
referer: url,
|
referer: url,
|
||||||
}));
|
}));
|
||||||
|
@ -89,14 +89,14 @@ async function scrapeScene({ query }, url, channel, include) {
|
||||||
release.date = query.date('.vid_date', 'MMMM D, YYYY');
|
release.date = query.date('.vid_date', 'MMMM D, YYYY');
|
||||||
release.duration = query.dur('.vid_length');
|
release.duration = query.dur('.vid_length');
|
||||||
|
|
||||||
release.actors = query.all('.vid_infos a[href*="author/"]').map(actorEl => ({
|
release.actors = query.all('.vid_infos a[href*="author/"]').map((actorEl) => ({
|
||||||
name: query.cnt(actorEl),
|
name: query.cnt(actorEl),
|
||||||
url: query.url(actorEl, null),
|
url: query.url(actorEl, null),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
release.tags = query.cnts('.vid_infos a[rel="tag"]');
|
release.tags = query.cnts('.vid_infos a[rel="tag"]');
|
||||||
|
|
||||||
const posterData = data['@graph']?.find(item => item['@type'] === 'ImageObject');
|
const posterData = data['@graph']?.find((item) => item['@type'] === 'ImageObject');
|
||||||
|
|
||||||
const poster = posterData?.url
|
const poster = posterData?.url
|
||||||
|| query.q('meta[property="og:image"]', 'content')
|
|| query.q('meta[property="og:image"]', 'content')
|
||||||
|
|
|
@ -20,7 +20,7 @@ function scrapeAll(scenes) {
|
||||||
release.duration = query.dur('.total-time');
|
release.duration = query.dur('.total-time');
|
||||||
|
|
||||||
const [poster, ...primaryPhotos] = query.imgs('a img');
|
const [poster, ...primaryPhotos] = query.imgs('a img');
|
||||||
const secondaryPhotos = query.styles('.thumb-top, .thumb-bottom, .thumb-mouseover', 'background-image').map(style => style.match(/url\((.*)\)/)[1]);
|
const secondaryPhotos = query.styles('.thumb-top, .thumb-bottom, .thumb-mouseover', 'background-image').map((style) => style.match(/url\((.*)\)/)[1]);
|
||||||
|
|
||||||
release.poster = poster;
|
release.poster = poster;
|
||||||
release.photos = primaryPhotos.concat(secondaryPhotos);
|
release.photos = primaryPhotos.concat(secondaryPhotos);
|
||||||
|
|
|
@ -14,14 +14,14 @@ const { inchesToCm, lbsToKg } = require('../utils/convert');
|
||||||
function getThumbs(scene) {
|
function getThumbs(scene) {
|
||||||
if (scene.images.poster) {
|
if (scene.images.poster) {
|
||||||
return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
|
return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
|
||||||
.filter(img => typeof img === 'object') // remove alternateText property
|
.filter((img) => typeof img === 'object') // remove alternateText property
|
||||||
.map(image => image.xl.url);
|
.map((image) => image.xl.url);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (scene.images.card_main_rect) {
|
if (scene.images.card_main_rect) {
|
||||||
return scene.images.card_main_rect
|
return scene.images.card_main_rect
|
||||||
.concat(scene.images.card_secondary_rect || [])
|
.concat(scene.images.card_secondary_rect || [])
|
||||||
.map(image => image.xl.url.replace('.thumb', ''));
|
.map((image) => image.xl.url.replace('.thumb', ''));
|
||||||
}
|
}
|
||||||
|
|
||||||
return [];
|
return [];
|
||||||
|
@ -29,14 +29,14 @@ function getThumbs(scene) {
|
||||||
|
|
||||||
function getVideos(data) {
|
function getVideos(data) {
|
||||||
const teaserSources = data.videos.mediabook?.files;
|
const teaserSources = data.videos.mediabook?.files;
|
||||||
const trailerSources = data.children.find(child => child.type === 'trailer')?.videos.full?.files;
|
const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
|
||||||
|
|
||||||
const teaser = teaserSources && Object.values(teaserSources).map(source => ({
|
const teaser = teaserSources && Object.values(teaserSources).map((source) => ({
|
||||||
src: source.urls.view,
|
src: source.urls.view,
|
||||||
quality: parseInt(source.format, 10),
|
quality: parseInt(source.format, 10),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const trailer = trailerSources && Object.values(trailerSources).map(source => ({
|
const trailer = trailerSources && Object.values(trailerSources).map((source) => ({
|
||||||
src: source.urls.view,
|
src: source.urls.view,
|
||||||
quality: parseInt(source.format, 10),
|
quality: parseInt(source.format, 10),
|
||||||
}));
|
}));
|
||||||
|
@ -59,8 +59,8 @@ function scrapeLatestX(data, site, filterChannel) {
|
||||||
release.date = new Date(data.dateReleased);
|
release.date = new Date(data.dateReleased);
|
||||||
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
|
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
|
||||||
|
|
||||||
release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
|
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
|
||||||
release.tags = data.tags.map(tag => tag.name);
|
release.tags = data.tags.map((tag) => tag.name);
|
||||||
|
|
||||||
[release.poster, ...release.photos] = getThumbs(data);
|
[release.poster, ...release.photos] = getThumbs(data);
|
||||||
|
|
||||||
|
@ -69,15 +69,15 @@ function scrapeLatestX(data, site, filterChannel) {
|
||||||
if (teaser) release.teaser = teaser;
|
if (teaser) release.teaser = teaser;
|
||||||
if (trailer) release.trailer = trailer;
|
if (trailer) release.trailer = trailer;
|
||||||
|
|
||||||
release.chapters = data.timeTags?.map(chapter => ({
|
release.chapters = data.timeTags?.map((chapter) => ({
|
||||||
time: chapter.startTime,
|
time: chapter.startTime,
|
||||||
duration: chapter.endTime - chapter.startTime,
|
duration: chapter.endTime - chapter.startTime,
|
||||||
tags: [chapter.name],
|
tags: [chapter.name],
|
||||||
}));
|
}));
|
||||||
|
|
||||||
if ((site.parameters?.extract === true && data.collections.length > 0) // release should not belong to any channel
|
if ((site.parameters?.extract === true && data.collections.length > 0) // release should not belong to any channel
|
||||||
|| (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) // release should belong to specific channel
|
|| (typeof site.parameters?.extract === 'string' && !data.collections.some((collection) => collection.shortName === site.parameters.extract)) // release should belong to specific channel
|
||||||
|| (filterChannel && !data.collections?.some(collection => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes
|
|| (filterChannel && !data.collections?.some((collection) => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes
|
||||||
return {
|
return {
|
||||||
...release,
|
...release,
|
||||||
exclude: true,
|
exclude: true,
|
||||||
|
@ -88,11 +88,11 @@ function scrapeLatestX(data, site, filterChannel) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrapeLatest(items, site, filterChannel) {
|
async function scrapeLatest(items, site, filterChannel) {
|
||||||
const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel));
|
const latestReleases = items.map((data) => scrapeLatestX(data, site, filterChannel));
|
||||||
|
|
||||||
return {
|
return {
|
||||||
scenes: latestReleases.filter(scene => !scene.exclude),
|
scenes: latestReleases.filter((scene) => !scene.exclude),
|
||||||
unextracted: latestReleases.filter(scene => scene.exclude),
|
unextracted: latestReleases.filter((scene) => scene.exclude),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -108,8 +108,8 @@ function scrapeScene(data, url, _site, networkName) {
|
||||||
release.date = new Date(data.dateReleased);
|
release.date = new Date(data.dateReleased);
|
||||||
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
|
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
|
||||||
|
|
||||||
release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
|
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
|
||||||
release.tags = data.tags.map(tag => tag.name);
|
release.tags = data.tags.map((tag) => tag.name);
|
||||||
|
|
||||||
[release.poster, ...release.photos] = getThumbs(data);
|
[release.poster, ...release.photos] = getThumbs(data);
|
||||||
|
|
||||||
|
@ -118,7 +118,7 @@ function scrapeScene(data, url, _site, networkName) {
|
||||||
if (teaser) release.teaser = teaser;
|
if (teaser) release.teaser = teaser;
|
||||||
if (trailer) release.trailer = trailer;
|
if (trailer) release.trailer = trailer;
|
||||||
|
|
||||||
release.chapters = data.timeTags?.map(chapter => ({
|
release.chapters = data.timeTags?.map((chapter) => ({
|
||||||
time: chapter.startTime,
|
time: chapter.startTime,
|
||||||
duration: chapter.endTime - chapter.startTime,
|
duration: chapter.endTime - chapter.startTime,
|
||||||
tags: [chapter.name],
|
tags: [chapter.name],
|
||||||
|
@ -213,18 +213,18 @@ function scrapeProfile(data, html, releases = [], networkName) {
|
||||||
|| data.images.card_main_rect[0].xs?.url;
|
|| data.images.card_main_rect[0].xs?.url;
|
||||||
}
|
}
|
||||||
|
|
||||||
const birthdate = query.all('li').find(el => /Date of Birth/.test(el.textContent));
|
const birthdate = query.all('li').find((el) => /Date of Birth/.test(el.textContent));
|
||||||
if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');
|
if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');
|
||||||
|
|
||||||
if (data.tags.some(tag => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
|
if (data.tags.some((tag) => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
|
||||||
profile.naturalBoobs = true;
|
profile.naturalBoobs = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (data.tags.some(tag => /boob type/i.test(tag.category) && /enhanced/i.test(tag.name))) {
|
if (data.tags.some((tag) => /boob type/i.test(tag.category) && /enhanced/i.test(tag.name))) {
|
||||||
profile.naturalBoobs = false;
|
profile.naturalBoobs = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
profile.releases = releases.map(release => scrapeScene(release, null, null, networkName));
|
profile.releases = releases.map((release) => scrapeScene(release, null, null, networkName));
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
@ -325,7 +325,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.statusCode === 200) {
|
if (res.statusCode === 200) {
|
||||||
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
|
const actorData = res.body.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
|
||||||
|
|
||||||
if (actorData) {
|
if (actorData) {
|
||||||
const actorUrl = `https://www.${entity.slug}.com/${entity.parameters?.actorPath || 'model'}/${actorData.id}/${actorSlug}`;
|
const actorUrl = `https://www.${entity.slug}.com/${entity.parameters?.actorPath || 'model'}/${actorData.id}/${actorSlug}`;
|
||||||
|
|
|
@ -69,7 +69,7 @@ function scrapeScene(html, url, site) {
|
||||||
|
|
||||||
const posterPath = $('video, dl8-video').attr('poster') || $('img.start-card').attr('src');
|
const posterPath = $('video, dl8-video').attr('poster') || $('img.start-card').attr('src');
|
||||||
const poster = posterPath && `https:${posterPath}`;
|
const poster = posterPath && `https:${posterPath}`;
|
||||||
const photos = $('.contain-scene-images.desktop-only a').map((index, el) => $(el).attr('href')).toArray().filter(Boolean).map(photo => `https:${photo}`);
|
const photos = $('.contain-scene-images.desktop-only a').map((index, el) => $(el).attr('href')).toArray().filter(Boolean).map((photo) => `https:${photo}`);
|
||||||
|
|
||||||
const trailerEl = $('source');
|
const trailerEl = $('source');
|
||||||
const trailerSrc = trailerEl.attr('src');
|
const trailerSrc = trailerEl.attr('src');
|
||||||
|
@ -120,7 +120,7 @@ async function scrapeProfile(html) {
|
||||||
|
|
||||||
const releases = query.urls('.scene-item > a:first-child');
|
const releases = query.urls('.scene-item > a:first-child');
|
||||||
const otherPages = query.urls('.pagination a:not([rel=next]):not([rel=prev])');
|
const otherPages = query.urls('.pagination a:not([rel=next]):not([rel=prev])');
|
||||||
const olderReleases = await Promise.all(otherPages.map(async page => fetchActorReleases(page)));
|
const olderReleases = await Promise.all(otherPages.map(async (page) => fetchActorReleases(page)));
|
||||||
|
|
||||||
profile.releases = releases.concat(olderReleases.flat());
|
profile.releases = releases.concat(olderReleases.flat());
|
||||||
|
|
||||||
|
|
|
@ -78,7 +78,7 @@ async function scrapeScene({ query }, url, site) {
|
||||||
release.tags = query.all('.categories a', true);
|
release.tags = query.all('.categories a', true);
|
||||||
|
|
||||||
release.poster = query.poster() || query.img('.fake-video-player img');
|
release.poster = query.poster() || query.img('.fake-video-player img');
|
||||||
release.trailer = query.all('source').map(source => ({
|
release.trailer = query.all('source').map((source) => ({
|
||||||
src: source.src,
|
src: source.src,
|
||||||
quality: Number(source.getAttribute('res')),
|
quality: Number(source.getAttribute('res')),
|
||||||
}));
|
}));
|
||||||
|
@ -106,11 +106,11 @@ function scrapeProfile({ query }, _actorName, origin) {
|
||||||
profile.residencePlace = bio.location;
|
profile.residencePlace = bio.location;
|
||||||
|
|
||||||
profile.height = heightToCm(bio.height);
|
profile.height = heightToCm(bio.height);
|
||||||
[profile.bust, profile.waist, profile.hip] = bio.figure.split('-').map(v => Number(v) || v);
|
[profile.bust, profile.waist, profile.hip] = bio.figure.split('-').map((v) => Number(v) || v);
|
||||||
|
|
||||||
profile.avatar = query.img('.model-profile img');
|
profile.avatar = query.img('.model-profile img');
|
||||||
|
|
||||||
const releases = query.all('.content-grid-item').filter(el => /video\//.test(query.url(el, '.img-wrapper a'))); // filter out photos
|
const releases = query.all('.content-grid-item').filter((el) => /video\//.test(query.url(el, '.img-wrapper a'))); // filter out photos
|
||||||
profile.releases = scrapeAll(query.initAll(releases), null, origin);
|
profile.releases = scrapeAll(query.initAll(releases), null, origin);
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
|
@ -143,7 +143,7 @@ async function fetchProfile({ name: actorName }, { site }) {
|
||||||
|
|
||||||
if (!resModels.ok) return resModels.status;
|
if (!resModels.ok) return resModels.status;
|
||||||
|
|
||||||
const modelPath = resModels.item.qu.all('.content-grid-item a.title').find(el => slugify(el.textContent) === slugify(actorName));
|
const modelPath = resModels.item.qu.all('.content-grid-item a.title').find((el) => slugify(el.textContent) === slugify(actorName));
|
||||||
|
|
||||||
if (modelPath) {
|
if (modelPath) {
|
||||||
const modelUrl = `${origin}${modelPath}`;
|
const modelUrl = `${origin}${modelPath}`;
|
||||||
|
|
|
@ -26,7 +26,7 @@ function scrapeAll(months, channel, year) {
|
||||||
gender: 'female',
|
gender: 'female',
|
||||||
url: query.url('a.video-pop-up', 'data-modellink', { origin: `${channel.url}/submissive` }),
|
url: query.url('a.video-pop-up', 'data-modellink', { origin: `${channel.url}/submissive` }),
|
||||||
}]
|
}]
|
||||||
.filter(actor => !/lockdown/i.test(actor.name))
|
.filter((actor) => !/lockdown/i.test(actor.name))
|
||||||
.concat({
|
.concat({
|
||||||
name: 'Pascal White',
|
name: 'Pascal White',
|
||||||
gender: 'male',
|
gender: 'male',
|
||||||
|
|
|
@ -17,21 +17,21 @@ function extractMaleModelsFromTags(tagContainer) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
const tagEls = Array.from(tagContainer.childNodes, node => ({ type: node.nodeType, text: node.textContent.trim() })).filter(node => node.text.length > 0);
|
const tagEls = Array.from(tagContainer.childNodes, (node) => ({ type: node.nodeType, text: node.textContent.trim() })).filter((node) => node.text.length > 0);
|
||||||
const modelLabelIndex = tagEls.findIndex(node => node.text === 'Male Models');
|
const modelLabelIndex = tagEls.findIndex((node) => node.text === 'Male Models');
|
||||||
|
|
||||||
if (modelLabelIndex > -1) {
|
if (modelLabelIndex > -1) {
|
||||||
const nextLabelIndex = tagEls.findIndex((node, index) => index > modelLabelIndex && node.type === 3);
|
const nextLabelIndex = tagEls.findIndex((node, index) => index > modelLabelIndex && node.type === 3);
|
||||||
const maleModels = tagEls.slice(modelLabelIndex + 1, nextLabelIndex);
|
const maleModels = tagEls.slice(modelLabelIndex + 1, nextLabelIndex);
|
||||||
|
|
||||||
return maleModels.map(model => model.text);
|
return maleModels.map((model) => model.text);
|
||||||
}
|
}
|
||||||
|
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
async function extractChannelFromPhoto(photo, channel) {
|
async function extractChannelFromPhoto(photo, channel) {
|
||||||
const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug);
|
const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map((child) => child.slug);
|
||||||
const channelMatch = photo.match(new RegExp(siteSlugs.join('|')));
|
const channelMatch = photo.match(new RegExp(siteSlugs.join('|')));
|
||||||
|
|
||||||
if (channelMatch) {
|
if (channelMatch) {
|
||||||
|
@ -52,7 +52,7 @@ async function scrapeLatest(scenes, site) {
|
||||||
const slug = new URL(release.url).pathname.split('/')[2];
|
const slug = new URL(release.url).pathname.split('/')[2];
|
||||||
release.entryId = getHash(`${site.slug}${slug}${release.date.toISOString()}`);
|
release.entryId = getHash(`${site.slug}${slug}${release.date.toISOString()}`);
|
||||||
|
|
||||||
release.actors = release.title.split('&').map(actor => actor.trim());
|
release.actors = release.title.split('&').map((actor) => actor.trim());
|
||||||
|
|
||||||
[release.poster, ...release.photos] = query.imgs('.bloc-link img');
|
[release.poster, ...release.photos] = query.imgs('.bloc-link img');
|
||||||
|
|
||||||
|
@ -78,7 +78,7 @@ async function scrapeScene({ query }, site, url) {
|
||||||
const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
|
const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
|
||||||
if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
|
if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
|
||||||
|
|
||||||
release.photos = query.all('.bxslider_pics img').map(el => el.dataset.original || el.src);
|
release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
|
||||||
release.poster = query.poster();
|
release.poster = query.poster();
|
||||||
|
|
||||||
const trailer = query.trailer();
|
const trailer = query.trailer();
|
||||||
|
|
|
@ -67,7 +67,7 @@ function scrapeScene({ query, html }, url, entity) {
|
||||||
release.description = query.cnt('.info_container .description');
|
release.description = query.cnt('.info_container .description');
|
||||||
|
|
||||||
release.date = query.date('.info_container .info_line:nth-child(1)', 'YYYY-MM-DD') || query.date('.description', 'DD MMMM YYYY', /\d{1,2} \w+ \d{4}/);
|
release.date = query.date('.info_container .info_line:nth-child(1)', 'YYYY-MM-DD') || query.date('.description', 'DD MMMM YYYY', /\d{1,2} \w+ \d{4}/);
|
||||||
release.actors = query.all('.girl_item, .starring .item').map(actorEl => mapActor(actorEl, query, entity));
|
release.actors = query.all('.girl_item, .starring .item').map((actorEl) => mapActor(actorEl, query, entity));
|
||||||
|
|
||||||
release.duration = query.duration('.infos .description');
|
release.duration = query.duration('.infos .description');
|
||||||
|
|
||||||
|
@ -81,7 +81,7 @@ function scrapeScene({ query, html }, url, entity) {
|
||||||
release.tags = query.cnts('.tags a:not(.more_tag)');
|
release.tags = query.cnts('.tags a:not(.more_tag)');
|
||||||
release.poster = removeImageBorder(html.match(/image: "(.*?)"/)?.[1]);
|
release.poster = removeImageBorder(html.match(/image: "(.*?)"/)?.[1]);
|
||||||
|
|
||||||
release.trailer = html.match(/url: "(.*mp4.*)"/g)?.map(src => ({
|
release.trailer = html.match(/url: "(.*mp4.*)"/g)?.map((src) => ({
|
||||||
src: src.match(/"(.*)"/)?.[1],
|
src: src.match(/"(.*)"/)?.[1],
|
||||||
quality: Number(src.match(/[-/](\d+)p/)?.[1]),
|
quality: Number(src.match(/[-/](\d+)p/)?.[1]),
|
||||||
}));
|
}));
|
||||||
|
|
|
@ -40,7 +40,7 @@ function scrapeScene({ query }, url, channel) {
|
||||||
release.date = date;
|
release.date = date;
|
||||||
release.datePrecision = precision;
|
release.datePrecision = precision;
|
||||||
|
|
||||||
release.actors = query.cnts(details.actors, 'a').map(actor => capitalize(actor, { uncapitalize: true }));
|
release.actors = query.cnts(details.actors, 'a').map((actor) => capitalize(actor, { uncapitalize: true }));
|
||||||
release.duration = query.duration(details.duration);
|
release.duration = query.duration(details.duration);
|
||||||
release.tags = query.cnts(details.genres, 'a');
|
release.tags = query.cnts(details.genres, 'a');
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,7 @@ function scrapeAll(scenes) {
|
||||||
release.title = query.cnt('[class*="item-title"] a') || query.q('.bottom .link', 'title');
|
release.title = query.cnt('[class*="item-title"] a') || query.q('.bottom .link', 'title');
|
||||||
release.date = query.date('[class*="item-date"]', 'MMM DD, YYYY');
|
release.date = query.date('[class*="item-date"]', 'MMM DD, YYYY');
|
||||||
|
|
||||||
release.actors = query.all('[class*="item-actors"] a').map(el => ({
|
release.actors = query.all('[class*="item-actors"] a').map((el) => ({
|
||||||
name: query.cnt(el),
|
name: query.cnt(el),
|
||||||
url: query.url(el, null),
|
url: query.url(el, null),
|
||||||
}));
|
}));
|
||||||
|
@ -44,7 +44,7 @@ function scrapeScene({ query }, url) {
|
||||||
release.description = query.meta('name=description') || query.q('read-even-more', true);
|
release.description = query.meta('name=description') || query.q('read-even-more', true);
|
||||||
|
|
||||||
release.date = query.date('.h5-published', 'MMM DD, YYYY', /\w{3} \d{1,2}, \d{4}/);
|
release.date = query.date('.h5-published', 'MMM DD, YYYY', /\w{3} \d{1,2}, \d{4}/);
|
||||||
release.actors = query.all('.video-top-details .actors a[href*="/models"]').map(el => ({
|
release.actors = query.all('.video-top-details .actors a[href*="/models"]').map((el) => ({
|
||||||
name: query.cnt(el),
|
name: query.cnt(el),
|
||||||
url: query.url(el, null),
|
url: query.url(el, null),
|
||||||
}));
|
}));
|
||||||
|
@ -53,7 +53,7 @@ function scrapeScene({ query }, url) {
|
||||||
release.tags = query.all('.video-top-details a[href*="/categories"], .video-top-details a[href*="/tags"]', true);
|
release.tags = query.all('.video-top-details a[href*="/categories"], .video-top-details a[href*="/tags"]', true);
|
||||||
|
|
||||||
release.poster = query.img('.poster img') || query.meta('itemprop=thumbnailUrl');
|
release.poster = query.img('.poster img') || query.meta('itemprop=thumbnailUrl');
|
||||||
release.photos = query.imgs('#gallery-thumbs [class*="thumb"]', 'data-bg').slice(1).map(photo => [ // first image is poster
|
release.photos = query.imgs('#gallery-thumbs [class*="thumb"]', 'data-bg').slice(1).map((photo) => [ // first image is poster
|
||||||
photo.replace('512x288', '1472x828'),
|
photo.replace('512x288', '1472x828'),
|
||||||
photo,
|
photo,
|
||||||
]);
|
]);
|
||||||
|
|
|
@ -16,7 +16,7 @@ const hairMap = {
|
||||||
async function scrapeProfile(html, _url, actorName) {
|
async function scrapeProfile(html, _url, actorName) {
|
||||||
const { document } = new JSDOM(html).window;
|
const { document } = new JSDOM(html).window;
|
||||||
|
|
||||||
const entries = Array.from(document.querySelectorAll('.infoPiece'), el => el.textContent.replace(/\n|\t/g, '').split(':'));
|
const entries = Array.from(document.querySelectorAll('.infoPiece'), (el) => el.textContent.replace(/\n|\t/g, '').split(':'));
|
||||||
const bio = entries.reduce((acc, [key, value]) => (key ? { ...acc, [key.trim()]: value.trim() } : acc), {});
|
const bio = entries.reduce((acc, [key, value]) => (key ? { ...acc, [key.trim()]: value.trim() } : acc), {});
|
||||||
|
|
||||||
const profile = {
|
const profile = {
|
||||||
|
@ -47,7 +47,7 @@ async function scrapeProfile(html, _url, actorName) {
|
||||||
if (bio.Tattoos) profile.hasTattoos = bio.Tattoos === 'Yes';
|
if (bio.Tattoos) profile.hasTattoos = bio.Tattoos === 'Yes';
|
||||||
|
|
||||||
if (avatarEl && !/default\//.test(avatarEl.src)) profile.avatar = avatarEl.src;
|
if (avatarEl && !/default\//.test(avatarEl.src)) profile.avatar = avatarEl.src;
|
||||||
profile.social = Array.from(document.querySelectorAll('.socialList a'), el => el.href).filter(link => link !== 'https://www.twitter.com/'); // PH links to Twitter itself for some reason
|
profile.social = Array.from(document.querySelectorAll('.socialList a'), (el) => el.href).filter((link) => link !== 'https://www.twitter.com/'); // PH links to Twitter itself for some reason
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,7 +9,7 @@ function scrapePhotos(html) {
|
||||||
const { qis } = ex(html, '#photos-page');
|
const { qis } = ex(html, '#photos-page');
|
||||||
const photos = qis('img');
|
const photos = qis('img');
|
||||||
|
|
||||||
return photos.map(photo => [
|
return photos.map((photo) => [
|
||||||
photo
|
photo
|
||||||
.replace('x_800', 'x_xl')
|
.replace('x_800', 'x_xl')
|
||||||
.replace('_tn', ''),
|
.replace('_tn', ''),
|
||||||
|
@ -76,22 +76,22 @@ async function scrapeScene(html, url, site) {
|
||||||
release.actors = qu.all('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true);
|
release.actors = qu.all('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true);
|
||||||
|
|
||||||
if (release.actors.length === 0) {
|
if (release.actors.length === 0) {
|
||||||
const actorEl = qu.all('.stat').find(stat => /Featuring/.test(stat.textContent));
|
const actorEl = qu.all('.stat').find((stat) => /Featuring/.test(stat.textContent));
|
||||||
const actorString = qu.text(actorEl);
|
const actorString = qu.text(actorEl);
|
||||||
|
|
||||||
release.actors = actorString?.split(/,\band\b|,/g).map(actor => actor.trim()) || [];
|
release.actors = actorString?.split(/,\band\b|,/g).map((actor) => actor.trim()) || [];
|
||||||
}
|
}
|
||||||
|
|
||||||
if (release.actors.length === 0 && site.parameters?.actors) release.actors = site.parameters.actors;
|
if (release.actors.length === 0 && site.parameters?.actors) release.actors = site.parameters.actors;
|
||||||
|
|
||||||
release.tags = qu.all('a[href*=tag]', true);
|
release.tags = qu.all('a[href*=tag]', true);
|
||||||
|
|
||||||
const dateEl = qu.all('.value').find(el => /\w+ \d+\w+, \d{4}/.test(el.textContent));
|
const dateEl = qu.all('.value').find((el) => /\w+ \d+\w+, \d{4}/.test(el.textContent));
|
||||||
release.date = qu.date(dateEl, null, 'MMMM Do, YYYY')
|
release.date = qu.date(dateEl, null, 'MMMM Do, YYYY')
|
||||||
|| qu.date('.date', 'MMMM Do, YYYY', /\w+ \d{1,2}\w+, \d{4}/)
|
|| qu.date('.date', 'MMMM Do, YYYY', /\w+ \d{1,2}\w+, \d{4}/)
|
||||||
|| qu.date('.info .holder', 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/);
|
|| qu.date('.info .holder', 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/);
|
||||||
|
|
||||||
const durationEl = qu.all('value').find(el => /\d{1,3}:\d{2}/.test(el.textContent));
|
const durationEl = qu.all('value').find((el) => /\d{1,3}:\d{2}/.test(el.textContent));
|
||||||
release.duration = qu.dur(durationEl);
|
release.duration = qu.dur(durationEl);
|
||||||
|
|
||||||
release.poster = qu.poster('video') || qu.img('.flowplayer img') || html.match(/posterImage: '(.*\.jpg)'/)?.[1] || null; // _800.jpg is larger than _xl.jpg in landscape
|
release.poster = qu.poster('video') || qu.img('.flowplayer img') || html.match(/posterImage: '(.*\.jpg)'/)?.[1] || null; // _800.jpg is larger than _xl.jpg in landscape
|
||||||
|
@ -100,7 +100,7 @@ async function scrapeScene(html, url, site) {
|
||||||
if (photosUrl) {
|
if (photosUrl) {
|
||||||
release.photos = await fetchPhotos(photosUrl);
|
release.photos = await fetchPhotos(photosUrl);
|
||||||
} else {
|
} else {
|
||||||
release.photos = qu.imgs('img[src*=ThumbNails], .p-photos .tn img').map(photo => [
|
release.photos = qu.imgs('img[src*=ThumbNails], .p-photos .tn img').map((photo) => [
|
||||||
photo.replace('_tn', ''),
|
photo.replace('_tn', ''),
|
||||||
photo,
|
photo,
|
||||||
]);
|
]);
|
||||||
|
@ -126,7 +126,7 @@ async function scrapeScene(html, url, site) {
|
||||||
|
|
||||||
function scrapeModels(html, actorName) {
|
function scrapeModels(html, actorName) {
|
||||||
const { qa } = ex(html);
|
const { qa } = ex(html);
|
||||||
const model = qa('.model a').find(link => link.title === actorName);
|
const model = qa('.model a').find((link) => link.title === actorName);
|
||||||
|
|
||||||
return model?.href || null;
|
return model?.href || null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,7 +15,7 @@ function scrapeAll(scenes) {
|
||||||
release.entryId = getEntryId(release.url);
|
release.entryId = getEntryId(release.url);
|
||||||
|
|
||||||
release.title = query.cnt('.title-label a');
|
release.title = query.cnt('.title-label a');
|
||||||
release.actors = query.all('.update_models a').map(el => ({
|
release.actors = query.all('.update_models a').map((el) => ({
|
||||||
name: query.cnt(el),
|
name: query.cnt(el),
|
||||||
url: query.url(el, null),
|
url: query.url(el, null),
|
||||||
}));
|
}));
|
||||||
|
@ -37,7 +37,7 @@ function scrapeScene({ query }, url) {
|
||||||
|
|
||||||
release.description = query.cnt('#sceneInfo .description');
|
release.description = query.cnt('#sceneInfo .description');
|
||||||
|
|
||||||
release.actors = query.all('#sceneInfo .data-others a[href*="/models"]').map(el => ({
|
release.actors = query.all('#sceneInfo .data-others a[href*="/models"]').map((el) => ({
|
||||||
name: query.el(el, null, 'title'),
|
name: query.el(el, null, 'title'),
|
||||||
url: query.url(el, null),
|
url: query.url(el, null),
|
||||||
}));
|
}));
|
||||||
|
@ -50,8 +50,8 @@ function scrapeScene({ query }, url) {
|
||||||
release.poster = [poster, poster?.replace(/imgw=\w+/, 'imgw=680')];
|
release.poster = [poster, poster?.replace(/imgw=\w+/, 'imgw=680')];
|
||||||
|
|
||||||
release.photos = query.imgs('.photos-holder img')
|
release.photos = query.imgs('.photos-holder img')
|
||||||
.filter(src => new URL(src).pathname !== posterPathname)
|
.filter((src) => new URL(src).pathname !== posterPathname)
|
||||||
.map(src => [
|
.map((src) => [
|
||||||
src.replace(/imgw=\d+/, 'imgw=1284'),
|
src.replace(/imgw=\d+/, 'imgw=1284'),
|
||||||
src,
|
src,
|
||||||
]);
|
]);
|
||||||
|
@ -74,7 +74,7 @@ function scrapeProfileScenes(scenes) {
|
||||||
|
|
||||||
release.description = query.cnt('.model-update-description');
|
release.description = query.cnt('.model-update-description');
|
||||||
|
|
||||||
release.actors = query.all('.model-labels a').map(el => ({
|
release.actors = query.all('.model-labels a').map((el) => ({
|
||||||
name: query.cnt(el),
|
name: query.cnt(el),
|
||||||
url: query.url(el, null),
|
url: query.url(el, null),
|
||||||
}));
|
}));
|
||||||
|
|
|
@ -13,7 +13,7 @@ function getChannelSlug(channelName, entity) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const channelSlug = slugify(channelName, '', { removePunctuation: true });
|
const channelSlug = slugify(channelName, '', { removePunctuation: true });
|
||||||
const channel = entity.children.find(child => new RegExp(channelSlug).test(child.slug));
|
const channel = entity.children.find((child) => new RegExp(channelSlug).test(child.slug));
|
||||||
|
|
||||||
return channel?.slug || null;
|
return channel?.slug || null;
|
||||||
}
|
}
|
||||||
|
@ -27,8 +27,8 @@ function scrapeScene(scene, channel) {
|
||||||
release.title = scene.title;
|
release.title = scene.title;
|
||||||
release.date = qu.extractDate(scene.publishedDate);
|
release.date = qu.extractDate(scene.publishedDate);
|
||||||
|
|
||||||
release.actors = scene.models?.map(model => model.modelName) || [];
|
release.actors = scene.models?.map((model) => model.modelName) || [];
|
||||||
release.actors = scene.models?.map(model => ({
|
release.actors = scene.models?.map((model) => ({
|
||||||
name: model.modelName,
|
name: model.modelName,
|
||||||
avatar: `https://images.mylfcdn.net/tsv4/model/profiles/${slugify(model.modelName, '_')}.jpg`,
|
avatar: `https://images.mylfcdn.net/tsv4/model/profiles/${slugify(model.modelName, '_')}.jpg`,
|
||||||
url: `${channel.url}/models/www.mylf.com/models/${model.modelId}`,
|
url: `${channel.url}/models/www.mylf.com/models/${model.modelId}`,
|
||||||
|
@ -113,7 +113,7 @@ function scrapeProfile(actor, entity) {
|
||||||
}
|
}
|
||||||
|
|
||||||
profile.avatar = actor.img;
|
profile.avatar = actor.img;
|
||||||
profile.scenes = actor.movies?.map(scene => scrapeScene(scene, entity));
|
profile.scenes = actor.movies?.map((scene) => scrapeScene(scene, entity));
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,15 +25,15 @@ function scrapeAll(scenes, entity) {
|
||||||
release.date = moment.utc(scene.year, 'YYYY').toDate();
|
release.date = moment.utc(scene.year, 'YYYY').toDate();
|
||||||
release.datePrecision = 'year';
|
release.datePrecision = 'year';
|
||||||
|
|
||||||
release.actors = scene.actors.map(actor => ({
|
release.actors = scene.actors.map((actor) => ({
|
||||||
name: actor.name.trim(),
|
name: actor.name.trim(),
|
||||||
avatar: actor.image || null,
|
avatar: actor.image || null,
|
||||||
})).filter(actor => actor.name && slugify(actor.name) !== 'amateur-girl');
|
})).filter((actor) => actor.name && slugify(actor.name) !== 'amateur-girl');
|
||||||
|
|
||||||
release.duration = scene.duration;
|
release.duration = scene.duration;
|
||||||
release.stars = scene.video_rating_score;
|
release.stars = scene.video_rating_score;
|
||||||
|
|
||||||
[release.poster, ...release.photos] = scene.screenshots.map(url => prefixUrl(url));
|
[release.poster, ...release.photos] = scene.screenshots.map((url) => prefixUrl(url));
|
||||||
|
|
||||||
if (scene.is_gay) {
|
if (scene.is_gay) {
|
||||||
release.tags = ['gay'];
|
release.tags = ['gay'];
|
||||||
|
@ -64,7 +64,7 @@ async function scrapeScene({ query }, url) {
|
||||||
release.description = query.q('.detail-description', true);
|
release.description = query.q('.detail-description', true);
|
||||||
release.duration = query.dur('.detail-meta li:first-child');
|
release.duration = query.dur('.detail-meta li:first-child');
|
||||||
|
|
||||||
const actors = [query.q('.detail-hero-title h1', true)?.trim()].filter(name => name && slugify(name) !== 'amateur-girl');
|
const actors = [query.q('.detail-hero-title h1', true)?.trim()].filter((name) => name && slugify(name) !== 'amateur-girl');
|
||||||
|
|
||||||
if (actors.length > 0) {
|
if (actors.length > 0) {
|
||||||
release.actors = actors;
|
release.actors = actors;
|
||||||
|
@ -143,7 +143,7 @@ async function fetchProfile({ name: actorName }, { entity }, include) {
|
||||||
const res = await http.get(`https://teencoreclub.com/api/actors?query=${actorName}`);
|
const res = await http.get(`https://teencoreclub.com/api/actors?query=${actorName}`);
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const actor = res.body.data.find(item => slugify(item.name) === slugify(actorName));
|
const actor = res.body.data.find((item) => slugify(item.name) === slugify(actorName));
|
||||||
|
|
||||||
if (actor) {
|
if (actor) {
|
||||||
return scrapeProfile(actor, entity, include);
|
return scrapeProfile(actor, entity, include);
|
||||||
|
|
|
@ -14,7 +14,7 @@ function scrapeAll(scenes, channel) {
|
||||||
release.title = query.cnt('.title');
|
release.title = query.cnt('.title');
|
||||||
|
|
||||||
release.date = query.date('time', 'MMMM D, YYYY');
|
release.date = query.date('time', 'MMMM D, YYYY');
|
||||||
release.actors = query.all('.actors a').map(el => ({
|
release.actors = query.all('.actors a').map((el) => ({
|
||||||
name: query.cnt(el),
|
name: query.cnt(el),
|
||||||
url: query.url(el, null),
|
url: query.url(el, null),
|
||||||
}));
|
}));
|
||||||
|
@ -29,7 +29,7 @@ function scrapeAll(scenes, channel) {
|
||||||
const siteId = query.url('.site a', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
|
const siteId = query.url('.site a', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
|
||||||
|
|
||||||
if (siteId) {
|
if (siteId) {
|
||||||
release.channel = network.children.find(child => child.parameters.siteId.toString() === siteId)?.slug;
|
release.channel = network.children.find((child) => child.parameters.siteId.toString() === siteId)?.slug;
|
||||||
}
|
}
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
|
@ -48,7 +48,7 @@ function scrapeScene({ query }, url, channel) {
|
||||||
release.date = query.date('.title-line .date', 'MMMM D, YYYY');
|
release.date = query.date('.title-line .date', 'MMMM D, YYYY');
|
||||||
release.duration = query.number('.dur') * 60;
|
release.duration = query.number('.dur') * 60;
|
||||||
|
|
||||||
release.actors = query.all('.site a[href*="/models"]').map(el => ({
|
release.actors = query.all('.site a[href*="/models"]').map((el) => ({
|
||||||
name: query.cnt(el),
|
name: query.cnt(el),
|
||||||
url: query.url(el, null),
|
url: query.url(el, null),
|
||||||
}));
|
}));
|
||||||
|
@ -63,7 +63,7 @@ function scrapeScene({ query }, url, channel) {
|
||||||
const siteId = query.url('.site a[href*="site[]"]', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
|
const siteId = query.url('.site a[href*="site[]"]', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
|
||||||
|
|
||||||
if (siteId) {
|
if (siteId) {
|
||||||
release.channel = network.children.find(child => child.parameters.siteId.toString() === siteId)?.slug;
|
release.channel = network.children.find((child) => child.parameters.siteId.toString() === siteId)?.slug;
|
||||||
}
|
}
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
|
|
|
@ -20,7 +20,7 @@ function scrapeSceneX(scene) {
|
||||||
release.date = new Date(scene.release_date);
|
release.date = new Date(scene.release_date);
|
||||||
|
|
||||||
release.actors = scene.models
|
release.actors = scene.models
|
||||||
.map(actor => (/&/.test(actor.name)
|
.map((actor) => (/&/.test(actor.name)
|
||||||
? actor.name.split(/\s*&\s*/)
|
? actor.name.split(/\s*&\s*/)
|
||||||
: {
|
: {
|
||||||
name: actor.name,
|
name: actor.name,
|
||||||
|
@ -31,7 +31,7 @@ function scrapeSceneX(scene) {
|
||||||
.flat();
|
.flat();
|
||||||
|
|
||||||
release.stars = scene.rating;
|
release.stars = scene.rating;
|
||||||
release.tags = scene.tags.map(tag => tag.name);
|
release.tags = scene.tags.map((tag) => tag.name);
|
||||||
|
|
||||||
if (mime.getType(scene.thumb) === 'image/gif') {
|
if (mime.getType(scene.thumb) === 'image/gif') {
|
||||||
release.teaser = scene.thumb;
|
release.teaser = scene.thumb;
|
||||||
|
@ -128,7 +128,7 @@ async function fetchProfile(baseActor, entity, options) {
|
||||||
return searchRes.status;
|
return searchRes.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
const actor = searchRes.body.models.items.find(model => slugify(model.name) === slugify(baseActor.name));
|
const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
|
||||||
|
|
||||||
if (actor) {
|
if (actor) {
|
||||||
return scrapeProfile(actor, options);
|
return scrapeProfile(actor, options);
|
||||||
|
|
|
@ -217,7 +217,7 @@ function gender() {
|
||||||
}
|
}
|
||||||
|
|
||||||
function actors(release) {
|
function actors(release) {
|
||||||
const length = release.tags.some(tag => ['dp', 'dap', 'gangbang'].includes(tag))
|
const length = release.tags.some((tag) => ['dp', 'dap', 'gangbang'].includes(tag))
|
||||||
? Math.floor(Math.random() * 6) + 3
|
? Math.floor(Math.random() * 6) + 3
|
||||||
: Math.floor(Math.random() * 3) + 2;
|
: Math.floor(Math.random() * 3) + 2;
|
||||||
|
|
||||||
|
@ -254,7 +254,7 @@ async function fetchLatest(entity, page, options) {
|
||||||
// const poster = 'sfw/kittens/thumbs/iNEXVlX-RLs.jpeg';
|
// const poster = 'sfw/kittens/thumbs/iNEXVlX-RLs.jpeg';
|
||||||
|
|
||||||
release.poster = `http://${config.web.host}:${config.web.port}/img/${poster}?id=${nanoid()}`; // ensure source is unique
|
release.poster = `http://${config.web.host}:${config.web.port}/img/${poster}?id=${nanoid()}`; // ensure source is unique
|
||||||
release.photos = photos.map(photo => `http://${config.web.host}:${config.web.port}/img/${photo}?id=${nanoid()}`);
|
release.photos = photos.map((photo) => `http://${config.web.host}:${config.web.port}/img/${photo}?id=${nanoid()}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
release.tags = await knex('tags')
|
release.tags = await knex('tags')
|
||||||
|
|
|
@ -17,7 +17,7 @@ function scrapeLatestNative(scenes, site) {
|
||||||
release.date = ed(scene.release_date, 'YYYY-MM-DD');
|
release.date = ed(scene.release_date, 'YYYY-MM-DD');
|
||||||
release.duration = parseInt(scene.runtime, 10) * 60;
|
release.duration = parseInt(scene.runtime, 10) * 60;
|
||||||
|
|
||||||
release.actors = scene.cast?.map(actor => ({
|
release.actors = scene.cast?.map((actor) => ({
|
||||||
name: actor.stagename,
|
name: actor.stagename,
|
||||||
gender: actor.gender.toLowerCase(),
|
gender: actor.gender.toLowerCase(),
|
||||||
avatar: actor.placard,
|
avatar: actor.placard,
|
||||||
|
@ -38,10 +38,10 @@ function scrapeSceneNative({ html, q, qa }, url, _site) {
|
||||||
release.title = q('.scene-h2-heading', true);
|
release.title = q('.scene-h2-heading', true);
|
||||||
release.description = q('.indie-model-p', true);
|
release.description = q('.indie-model-p', true);
|
||||||
|
|
||||||
const dateString = qa('h5').find(el => /Released/.test(el.textContent)).textContent;
|
const dateString = qa('h5').find((el) => /Released/.test(el.textContent)).textContent;
|
||||||
release.date = ed(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
release.date = ed(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||||
|
|
||||||
const duration = qa('h5').find(el => /Runtime/.test(el.textContent)).textContent;
|
const duration = qa('h5').find((el) => /Runtime/.test(el.textContent)).textContent;
|
||||||
const [hours, minutes] = duration.match(/\d+/g);
|
const [hours, minutes] = duration.match(/\d+/g);
|
||||||
|
|
||||||
if (minutes) release.duration = (hours * 3600) + (minutes * 60);
|
if (minutes) release.duration = (hours * 3600) + (minutes * 60);
|
||||||
|
@ -111,7 +111,7 @@ async function fetchSceneWrapper(url, site, release) {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (searchRes.statusCode === 200 && searchRes.body.code === 200) {
|
if (searchRes.statusCode === 200 && searchRes.body.code === 200) {
|
||||||
const sceneMatch = searchRes.body.responseData.find(item => slugify(item.name) === slugify(scene.title));
|
const sceneMatch = searchRes.body.responseData.find((item) => slugify(item.name) === slugify(scene.title));
|
||||||
|
|
||||||
if (sceneMatch) {
|
if (sceneMatch) {
|
||||||
return {
|
return {
|
||||||
|
|
|
@ -15,7 +15,7 @@ const genderMap = {
|
||||||
|
|
||||||
function getPosterFallbacks(poster) {
|
function getPosterFallbacks(poster) {
|
||||||
return poster
|
return poster
|
||||||
.filter(image => /landscape/i.test(image.name))
|
.filter((image) => /landscape/i.test(image.name))
|
||||||
.sort((imageA, imageB) => imageB.height - imageA.height)
|
.sort((imageA, imageB) => imageB.height - imageA.height)
|
||||||
.map((image) => {
|
.map((image) => {
|
||||||
const sources = [image.src, image.highdpi?.['2x'], image.highdpi?.['3x']];
|
const sources = [image.src, image.highdpi?.['2x'], image.highdpi?.['3x']];
|
||||||
|
@ -23,7 +23,7 @@ function getPosterFallbacks(poster) {
|
||||||
return image.height === 1080 ? sources : sources.reverse();
|
return image.height === 1080 ? sources : sources.reverse();
|
||||||
})
|
})
|
||||||
.flat()
|
.flat()
|
||||||
.map(src => ({
|
.map((src) => ({
|
||||||
src,
|
src,
|
||||||
expectType: {
|
expectType: {
|
||||||
'binary/octet-stream': 'image/jpeg',
|
'binary/octet-stream': 'image/jpeg',
|
||||||
|
@ -33,8 +33,8 @@ function getPosterFallbacks(poster) {
|
||||||
|
|
||||||
function getTeaserFallbacks(teaser) {
|
function getTeaserFallbacks(teaser) {
|
||||||
return teaser
|
return teaser
|
||||||
.filter(video => /landscape/i.test(video.name))
|
.filter((video) => /landscape/i.test(video.name))
|
||||||
.map(video => ({
|
.map((video) => ({
|
||||||
src: video.src,
|
src: video.src,
|
||||||
type: video.type,
|
type: video.type,
|
||||||
quality: Number(String(video.height).replace('353', '360')),
|
quality: Number(String(video.height).replace('353', '360')),
|
||||||
|
@ -44,7 +44,7 @@ function getTeaserFallbacks(teaser) {
|
||||||
function getAvatarFallbacks(avatar) {
|
function getAvatarFallbacks(avatar) {
|
||||||
return avatar
|
return avatar
|
||||||
.sort((imageA, imageB) => imageB.height - imageA.height)
|
.sort((imageA, imageB) => imageB.height - imageA.height)
|
||||||
.map(image => [image.highdpi?.['3x'], image.highdpi?.['2x'], image.src])
|
.map((image) => [image.highdpi?.['3x'], image.highdpi?.['2x'], image.src])
|
||||||
.flat();
|
.flat();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -149,7 +149,7 @@ async function getPhotos(url) {
|
||||||
});
|
});
|
||||||
|
|
||||||
const state = htmlRes?.window.__APOLLO_STATE__;
|
const state = htmlRes?.window.__APOLLO_STATE__;
|
||||||
const key = Object.values(state.ROOT_QUERY).find(query => query?.__ref)?.__ref;
|
const key = Object.values(state.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
|
||||||
const data = state[key];
|
const data = state[key];
|
||||||
|
|
||||||
console.log(data);
|
console.log(data);
|
||||||
|
@ -158,7 +158,7 @@ async function getPhotos(url) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
return data.carousel.slice(1).map(photo => photo.main?.[0].src).filter(Boolean);
|
return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeAll(scenes, site, origin) {
|
function scrapeAll(scenes, site, origin) {
|
||||||
|
@ -191,7 +191,7 @@ function scrapeUpcoming(scene, site) {
|
||||||
release.title = scene.targetUrl
|
release.title = scene.targetUrl
|
||||||
.slice(1)
|
.slice(1)
|
||||||
.split('-')
|
.split('-')
|
||||||
.map(component => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
|
.map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
|
||||||
.join(' ');
|
.join(' ');
|
||||||
|
|
||||||
release.url = `${site.url}/videos${scene.targetUrl}`;
|
release.url = `${site.url}/videos${scene.targetUrl}`;
|
||||||
|
@ -243,7 +243,7 @@ async function scrapeScene(data, url, site, baseRelease, options) {
|
||||||
const trailer = await getTrailer(scene, site, url);
|
const trailer = await getTrailer(scene, site, url);
|
||||||
if (trailer) release.trailer = trailer;
|
if (trailer) release.trailer = trailer;
|
||||||
|
|
||||||
release.chapters = data.video.chapters?.video.map(chapter => ({
|
release.chapters = data.video.chapters?.video.map((chapter) => ({
|
||||||
tags: [chapter.title],
|
tags: [chapter.title],
|
||||||
time: chapter.seconds,
|
time: chapter.seconds,
|
||||||
}));
|
}));
|
||||||
|
|
|
@ -131,7 +131,7 @@ function scrapeScene(html, url) {
|
||||||
release.actors = qu.all('.info-video-models a', true);
|
release.actors = qu.all('.info-video-models a', true);
|
||||||
release.tags = qu.all('.info-video-category a', true);
|
release.tags = qu.all('.info-video-category a', true);
|
||||||
|
|
||||||
release.photos = qu.urls('.swiper-wrapper .swiper-slide a').map(source => source.replace('.jpg/', '.jpg'));
|
release.photos = qu.urls('.swiper-wrapper .swiper-slide a').map((source) => source.replace('.jpg/', '.jpg'));
|
||||||
release.poster = qu.meta('meta[property="og:image"]');
|
release.poster = qu.meta('meta[property="og:image"]');
|
||||||
|
|
||||||
if (!release.poster) {
|
if (!release.poster) {
|
||||||
|
|
|
@ -22,7 +22,7 @@ async function getTrailerUrl(release, channel, request) {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const trailers = res.body.streams.map(trailer => ({
|
const trailers = res.body.streams.map((trailer) => ({
|
||||||
src: trailer.url,
|
src: trailer.url,
|
||||||
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
|
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
|
||||||
vr: true,
|
vr: true,
|
||||||
|
@ -47,7 +47,7 @@ function scrapeAll(scenes, channel) {
|
||||||
release.title = query.cnt('.card__h');
|
release.title = query.cnt('.card__h');
|
||||||
release.date = query.date('.card__date', 'D MMMM, YYYY');
|
release.date = query.date('.card__date', 'D MMMM, YYYY');
|
||||||
|
|
||||||
release.actors = query.all('.card__links a').map(el => ({
|
release.actors = query.all('.card__links a').map((el) => ({
|
||||||
name: qu.query.cnt(el),
|
name: qu.query.cnt(el),
|
||||||
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
||||||
}));
|
}));
|
||||||
|
@ -82,14 +82,14 @@ async function scrapeScene({ query }, url, channel, baseRelease, options, reques
|
||||||
release.date = query.date('.detail__date', 'D MMMM, YYYY');
|
release.date = query.date('.detail__date', 'D MMMM, YYYY');
|
||||||
release.duration = query.number('.time') * 60;
|
release.duration = query.number('.time') * 60;
|
||||||
|
|
||||||
release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map(el => ({
|
release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map((el) => ({
|
||||||
name: qu.query.cnt(el),
|
name: qu.query.cnt(el),
|
||||||
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
release.tags = query.cnts('.tag-list .tag').concat(query.cnts('.detail__specs-list .detail__specs-item'));
|
release.tags = query.cnts('.tag-list .tag').concat(query.cnts('.detail__specs-list .detail__specs-item'));
|
||||||
|
|
||||||
release.photos = query.all('.photo-strip__slide').map(el => ([
|
release.photos = query.all('.photo-strip__slide').map((el) => ([
|
||||||
qu.query.img(el, null, 'data-src'),
|
qu.query.img(el, null, 'data-src'),
|
||||||
qu.query.img(el, 'img', 'src'),
|
qu.query.img(el, 'img', 'src'),
|
||||||
]));
|
]));
|
||||||
|
|
|
@ -18,7 +18,7 @@ function scrapeLatest(html, site) {
|
||||||
release.entryId = scene.dataset.videoId;
|
release.entryId = scene.dataset.videoId;
|
||||||
release.title = scene.querySelector('.card-title').textContent;
|
release.title = scene.querySelector('.card-title').textContent;
|
||||||
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
|
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
|
||||||
release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent);
|
release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent);
|
||||||
|
|
||||||
// slow CDN?
|
// slow CDN?
|
||||||
const poster = scene.querySelector('.single-image').dataset.src;
|
const poster = scene.querySelector('.single-image').dataset.src;
|
||||||
|
@ -32,7 +32,7 @@ function scrapeLatest(html, site) {
|
||||||
concurrency: 1,
|
concurrency: 1,
|
||||||
};
|
};
|
||||||
|
|
||||||
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({
|
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), (el) => ({
|
||||||
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
|
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
|
||||||
referer: site.url,
|
referer: site.url,
|
||||||
attempts: 5,
|
attempts: 5,
|
||||||
|
@ -63,7 +63,7 @@ function scrapeScene(html, site, url) {
|
||||||
release.url = url;
|
release.url = url;
|
||||||
release.title = scene.querySelector('.t2019-stitle').textContent.trim();
|
release.title = scene.querySelector('.t2019-stitle').textContent.trim();
|
||||||
release.description = scene.querySelector('#t2019-description').textContent.trim();
|
release.description = scene.querySelector('#t2019-description').textContent.trim();
|
||||||
release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), el => el.textContent);
|
release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), (el) => el.textContent);
|
||||||
|
|
||||||
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
|
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
|
||||||
|
|
||||||
|
@ -75,7 +75,7 @@ function scrapeScene(html, site, url) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// unreliable CDN
|
// unreliable CDN
|
||||||
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => ({
|
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), (el) => ({
|
||||||
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
|
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
|
||||||
referer: site.url,
|
referer: site.url,
|
||||||
attempts: 5,
|
attempts: 5,
|
||||||
|
|
|
@ -39,7 +39,7 @@ function curateSite(site, includeParameters = false) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function curateSites(sites, includeParameters) {
|
async function curateSites(sites, includeParameters) {
|
||||||
return Promise.all(sites.map(async site => curateSite(site, includeParameters)));
|
return Promise.all(sites.map(async (site) => curateSite(site, includeParameters)));
|
||||||
}
|
}
|
||||||
|
|
||||||
function destructConfigNetworks(networks = []) {
|
function destructConfigNetworks(networks = []) {
|
||||||
|
@ -165,7 +165,7 @@ async function fetchIncludedSites() {
|
||||||
|
|
||||||
async function fetchSites(queryObject) {
|
async function fetchSites(queryObject) {
|
||||||
const sites = await knex('sites')
|
const sites = await knex('sites')
|
||||||
.where(builder => whereOr(queryObject, 'sites', builder))
|
.where((builder) => whereOr(queryObject, 'sites', builder))
|
||||||
.select(
|
.select(
|
||||||
'sites.*',
|
'sites.*',
|
||||||
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
|
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
|
||||||
|
|
|
@ -58,7 +58,7 @@ async function fetchStashes(domain, itemId, sessionUser) {
|
||||||
})
|
})
|
||||||
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
|
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
|
||||||
|
|
||||||
return stashes.map(stash => curateStash(stash));
|
return stashes.map((stash) => curateStash(stash));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createStash(newStash, sessionUser) {
|
async function createStash(newStash, sessionUser) {
|
||||||
|
|
|
@ -17,7 +17,7 @@ const { notify } = require('./alerts');
|
||||||
|
|
||||||
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
|
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
|
||||||
const slugBase = release.title
|
const slugBase = release.title
|
||||||
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map(actor => actor.name).join(' ')}`)
|
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|
||||||
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
|
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
|
||||||
|| null;
|
|| null;
|
||||||
|
|
||||||
|
@ -74,11 +74,11 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce
|
||||||
}
|
}
|
||||||
|
|
||||||
async function attachChannelEntities(releases) {
|
async function attachChannelEntities(releases) {
|
||||||
const releasesWithoutEntity = releases.filter(release => release.channel && (!release.entity || release.entity.type === 'network'));
|
const releasesWithoutEntity = releases.filter((release) => release.channel && (!release.entity || release.entity.type === 'network'));
|
||||||
|
|
||||||
const channelEntities = await knex('entities')
|
const channelEntities = await knex('entities')
|
||||||
.select(knex.raw('entities.*, row_to_json(parents) as parent'))
|
.select(knex.raw('entities.*, row_to_json(parents) as parent'))
|
||||||
.whereIn('entities.slug', releasesWithoutEntity.map(release => release.channel))
|
.whereIn('entities.slug', releasesWithoutEntity.map((release) => release.channel))
|
||||||
.where('entities.type', 'channel')
|
.where('entities.type', 'channel')
|
||||||
.leftJoin('entities AS parents', 'parents.id', 'entities.parent_id');
|
.leftJoin('entities AS parents', 'parents.id', 'entities.parent_id');
|
||||||
|
|
||||||
|
@ -108,7 +108,7 @@ async function attachChannelEntities(releases) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function attachStudios(releases) {
|
async function attachStudios(releases) {
|
||||||
const studioSlugs = releases.map(release => release.studio).filter(Boolean);
|
const studioSlugs = releases.map((release) => release.studio).filter(Boolean);
|
||||||
|
|
||||||
const studios = await knex('entities')
|
const studios = await knex('entities')
|
||||||
.whereIn('slug', studioSlugs)
|
.whereIn('slug', studioSlugs)
|
||||||
|
@ -186,7 +186,7 @@ function filterInternalDuplicateReleases(releases) {
|
||||||
}, {});
|
}, {});
|
||||||
|
|
||||||
return Object.values(releasesByEntityIdAndEntryId)
|
return Object.values(releasesByEntityIdAndEntryId)
|
||||||
.map(entityReleases => Object.values(entityReleases))
|
.map((entityReleases) => Object.values(entityReleases))
|
||||||
.flat();
|
.flat();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -194,11 +194,11 @@ async function filterDuplicateReleases(releases) {
|
||||||
const internalUniqueReleases = filterInternalDuplicateReleases(releases);
|
const internalUniqueReleases = filterInternalDuplicateReleases(releases);
|
||||||
|
|
||||||
const duplicateReleaseEntries = await knex('releases')
|
const duplicateReleaseEntries = await knex('releases')
|
||||||
.whereIn(['entry_id', 'entity_id'], internalUniqueReleases.map(release => [release.entryId, release.entity.id]))
|
.whereIn(['entry_id', 'entity_id'], internalUniqueReleases.map((release) => [release.entryId, release.entity.id]))
|
||||||
.orWhereIn(['entry_id', 'entity_id'], internalUniqueReleases
|
.orWhereIn(['entry_id', 'entity_id'], internalUniqueReleases
|
||||||
// scene IDs shared across network, mark as duplicate so scene can be updated with channel if only available on release day (i.e. Perv City)
|
// scene IDs shared across network, mark as duplicate so scene can be updated with channel if only available on release day (i.e. Perv City)
|
||||||
.filter(release => release.entity.parent?.parameters?.networkEntryIds)
|
.filter((release) => release.entity.parent?.parameters?.networkEntryIds)
|
||||||
.map(release => [release.entryId, release.entity.parent.id]));
|
.map((release) => [release.entryId, release.entity.parent.id]));
|
||||||
|
|
||||||
const duplicateReleasesByEntityIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
|
const duplicateReleasesByEntityIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
|
||||||
if (!acc[release.entity_id]) acc[release.entity_id] = {};
|
if (!acc[release.entity_id]) acc[release.entity_id] = {};
|
||||||
|
@ -207,10 +207,10 @@ async function filterDuplicateReleases(releases) {
|
||||||
return acc;
|
return acc;
|
||||||
}, {});
|
}, {});
|
||||||
|
|
||||||
const duplicateReleases = internalUniqueReleases.filter(release => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|
const duplicateReleases = internalUniqueReleases.filter((release) => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|
||||||
|| duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
|
|| duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
|
||||||
|
|
||||||
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|
const uniqueReleases = internalUniqueReleases.filter((release) => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|
||||||
&& !duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
|
&& !duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
@ -263,7 +263,7 @@ async function updateSceneSearch(releaseIds) {
|
||||||
|
|
||||||
async function storeChapters(releases) {
|
async function storeChapters(releases) {
|
||||||
const chapters = releases
|
const chapters = releases
|
||||||
.map(release => release.chapters?.map((chapter, index) => ({
|
.map((release) => release.chapters?.map((chapter, index) => ({
|
||||||
releaseId: release.id,
|
releaseId: release.id,
|
||||||
index: index + 1,
|
index: index + 1,
|
||||||
time: chapter.time,
|
time: chapter.time,
|
||||||
|
@ -278,7 +278,7 @@ async function storeChapters(releases) {
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.sort((chapterA, chapterB) => chapterA.time - chapterB.time);
|
.sort((chapterA, chapterB) => chapterA.time - chapterB.time);
|
||||||
|
|
||||||
const curatedChapterEntries = chapters.map(chapter => ({
|
const curatedChapterEntries = chapters.map((chapter) => ({
|
||||||
index: chapter.index,
|
index: chapter.index,
|
||||||
time: chapter.time,
|
time: chapter.time,
|
||||||
duration: chapter.duration,
|
duration: chapter.duration,
|
||||||
|
@ -297,7 +297,7 @@ async function storeChapters(releases) {
|
||||||
},
|
},
|
||||||
}), {});
|
}), {});
|
||||||
|
|
||||||
const chaptersWithId = chapters.map(chapter => ({
|
const chaptersWithId = chapters.map((chapter) => ({
|
||||||
...chapter,
|
...chapter,
|
||||||
id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.index],
|
id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.index],
|
||||||
}));
|
}));
|
||||||
|
@ -316,13 +316,13 @@ async function storeScenes(releases) {
|
||||||
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
|
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
|
||||||
|
|
||||||
const releasesWithChannels = await attachChannelEntities(releases);
|
const releasesWithChannels = await attachChannelEntities(releases);
|
||||||
const releasesWithBaseActors = releasesWithChannels.map(release => ({ ...release, actors: toBaseActors(release.actors) }));
|
const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) }));
|
||||||
const releasesWithStudios = await attachStudios(releasesWithBaseActors);
|
const releasesWithStudios = await attachStudios(releasesWithBaseActors);
|
||||||
|
|
||||||
// uniqueness is entity ID + entry ID, filter uniques after adding entities
|
// uniqueness is entity ID + entry ID, filter uniques after adding entities
|
||||||
const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
|
const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
|
||||||
|
|
||||||
const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId)));
|
const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId)));
|
||||||
const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries);
|
const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries);
|
||||||
|
|
||||||
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
|
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
|
||||||
|
@ -355,13 +355,13 @@ async function storeScenes(releases) {
|
||||||
]);
|
]);
|
||||||
|
|
||||||
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
|
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
|
||||||
await updateSceneSearch(releasesWithId.map(release => release.id));
|
await updateSceneSearch(releasesWithId.map((release) => release.id));
|
||||||
|
|
||||||
// media is more error-prone, associate separately
|
// media is more error-prone, associate separately
|
||||||
await associateReleaseMedia(releasesWithId);
|
await associateReleaseMedia(releasesWithId);
|
||||||
|
|
||||||
if (argv.sceneActors && actors) {
|
if (argv.sceneActors && actors) {
|
||||||
await scrapeActors(actors.map(actor => actor.name));
|
await scrapeActors(actors.map((actor) => actor.name));
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(`Stored ${storedReleaseEntries.length}, updated ${updated.rowCount} releases`);
|
logger.info(`Stored ${storedReleaseEntries.length}, updated ${updated.rowCount} releases`);
|
||||||
|
@ -446,12 +446,12 @@ async function storeMovies(movies) {
|
||||||
const { uniqueReleases } = await filterDuplicateReleases(movies);
|
const { uniqueReleases } = await filterDuplicateReleases(movies);
|
||||||
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
|
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
|
||||||
|
|
||||||
const curatedMovieEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie')));
|
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
|
||||||
|
|
||||||
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
|
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
|
||||||
const moviesWithId = attachReleaseIds(movies, storedMovies);
|
const moviesWithId = attachReleaseIds(movies, storedMovies);
|
||||||
|
|
||||||
await updateMovieSearch(moviesWithId.map(movie => movie.id));
|
await updateMovieSearch(moviesWithId.map((movie) => movie.id));
|
||||||
await associateReleaseMedia(moviesWithId, 'movie');
|
await associateReleaseMedia(moviesWithId, 'movie');
|
||||||
|
|
||||||
return moviesWithId;
|
return moviesWithId;
|
||||||
|
|
26
src/tags.js
26
src/tags.js
|
@ -32,7 +32,7 @@ function curateTag(tag) {
|
||||||
priority: tag.priority,
|
priority: tag.priority,
|
||||||
group: curateTag(tag.group),
|
group: curateTag(tag.group),
|
||||||
aliasFor: curateTag(tag.alias),
|
aliasFor: curateTag(tag.alias),
|
||||||
aliases: (tag.aliases || []).map(aliasTag => curateTag(aliasTag)),
|
aliases: (tag.aliases || []).map((aliasTag) => curateTag(aliasTag)),
|
||||||
};
|
};
|
||||||
|
|
||||||
if (tag.poster) {
|
if (tag.poster) {
|
||||||
|
@ -40,7 +40,7 @@ function curateTag(tag) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (tag.photos) {
|
if (tag.photos) {
|
||||||
curatedTag.photos = tag.photos.map(photo => curateTagMedia(photo));
|
curatedTag.photos = tag.photos.map((photo) => curateTagMedia(photo));
|
||||||
}
|
}
|
||||||
|
|
||||||
return curatedTag;
|
return curatedTag;
|
||||||
|
@ -75,13 +75,13 @@ function withRelations(queryBuilder, withMedia) {
|
||||||
|
|
||||||
async function matchReleaseTags(releases) {
|
async function matchReleaseTags(releases) {
|
||||||
const rawTags = releases
|
const rawTags = releases
|
||||||
.map(release => release.tags).flat()
|
.map((release) => release.tags).flat()
|
||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
|
|
||||||
const casedTags = [...new Set(
|
const casedTags = [...new Set(
|
||||||
rawTags
|
rawTags
|
||||||
.concat(rawTags.map(tag => tag.toLowerCase()))
|
.concat(rawTags.map((tag) => tag.toLowerCase()))
|
||||||
.concat(rawTags.map(tag => tag.toUpperCase())),
|
.concat(rawTags.map((tag) => tag.toUpperCase())),
|
||||||
)];
|
)];
|
||||||
|
|
||||||
const tagEntries = await knex('tags')
|
const tagEntries = await knex('tags')
|
||||||
|
@ -98,7 +98,7 @@ async function matchReleaseTags(releases) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getEntityTags(releases) {
|
async function getEntityTags(releases) {
|
||||||
const entityIds = releases.map(release => release.entity?.id).filter(Boolean);
|
const entityIds = releases.map((release) => release.entity?.id).filter(Boolean);
|
||||||
const entityTags = await knex('entities_tags').whereIn('entity_id', entityIds);
|
const entityTags = await knex('entities_tags').whereIn('entity_id', entityIds);
|
||||||
|
|
||||||
const entityTagIdsByEntityId = entityTags.reduce((acc, entityTag) => {
|
const entityTagIdsByEntityId = entityTags.reduce((acc, entityTag) => {
|
||||||
|
@ -117,12 +117,12 @@ async function getEntityTags(releases) {
|
||||||
function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type) {
|
function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type) {
|
||||||
const tagAssociations = releases
|
const tagAssociations = releases
|
||||||
.map((release) => {
|
.map((release) => {
|
||||||
const entityTagIds = entityTagIdsByEntityId[release.entity?.id]?.map(tag => ({ id: tag.id, origin: tag.name })) || [];
|
const entityTagIds = entityTagIdsByEntityId[release.entity?.id]?.map((tag) => ({ id: tag.id, origin: tag.name })) || [];
|
||||||
const releaseTags = release.tags?.filter(Boolean) || [];
|
const releaseTags = release.tags?.filter(Boolean) || [];
|
||||||
|
|
||||||
const releaseTagsWithIds = releaseTags.every(tag => typeof tag === 'number')
|
const releaseTagsWithIds = releaseTags.every((tag) => typeof tag === 'number')
|
||||||
? releaseTags // obsolete scraper returned pre-matched tags
|
? releaseTags // obsolete scraper returned pre-matched tags
|
||||||
: releaseTags.map(tag => ({
|
: releaseTags.map((tag) => ({
|
||||||
id: tagIdsBySlug[slugify(tag)],
|
id: tagIdsBySlug[slugify(tag)],
|
||||||
original: tag,
|
original: tag,
|
||||||
}));
|
}));
|
||||||
|
@ -133,7 +133,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit
|
||||||
.concat(entityTagIds)
|
.concat(entityTagIds)
|
||||||
.filter(Boolean),
|
.filter(Boolean),
|
||||||
)]
|
)]
|
||||||
.map(tag => ({
|
.map((tag) => ({
|
||||||
[`${type}_id`]: release.id,
|
[`${type}_id`]: release.id,
|
||||||
tag_id: tag.id,
|
tag_id: tag.id,
|
||||||
original_tag: tag.original,
|
original_tag: tag.original,
|
||||||
|
@ -161,7 +161,7 @@ async function associateReleaseTags(releases, type = 'release') {
|
||||||
|
|
||||||
async function fetchTag(tagId) {
|
async function fetchTag(tagId) {
|
||||||
const tag = await knex('tags')
|
const tag = await knex('tags')
|
||||||
.modify(queryBuilder => withRelations(queryBuilder, true))
|
.modify((queryBuilder) => withRelations(queryBuilder, true))
|
||||||
.where((builder) => {
|
.where((builder) => {
|
||||||
if (Number(tagId)) {
|
if (Number(tagId)) {
|
||||||
builder.where('tags.id', tagId);
|
builder.where('tags.id', tagId);
|
||||||
|
@ -179,10 +179,10 @@ async function fetchTag(tagId) {
|
||||||
|
|
||||||
async function fetchTags(limit = 100) {
|
async function fetchTags(limit = 100) {
|
||||||
const tags = await knex('tags')
|
const tags = await knex('tags')
|
||||||
.modify(queryBuilder => withRelations(queryBuilder, false))
|
.modify((queryBuilder) => withRelations(queryBuilder, false))
|
||||||
.limit(limit);
|
.limit(limit);
|
||||||
|
|
||||||
return tags.map(tag => curateTag(tag));
|
return tags.map((tag) => curateTag(tag));
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -28,8 +28,8 @@ function mapReleasesToEntityIdAndEntryId(acc, release) {
|
||||||
function filterLocalUniqueReleases(releases, accReleases) {
|
function filterLocalUniqueReleases(releases, accReleases) {
|
||||||
const localDuplicateReleasesBySiteIdAndEntryId = accReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
|
const localDuplicateReleasesBySiteIdAndEntryId = accReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
|
||||||
|
|
||||||
const localUniqueReleases = releases.filter(release => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
|
const localUniqueReleases = releases.filter((release) => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
|
||||||
const localDuplicateReleases = releases.filter(release => localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
|
const localDuplicateReleases = releases.filter((release) => localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
localUniqueReleases,
|
localUniqueReleases,
|
||||||
|
@ -39,7 +39,7 @@ function filterLocalUniqueReleases(releases, accReleases) {
|
||||||
|
|
||||||
async function filterUniqueReleases(releases) {
|
async function filterUniqueReleases(releases) {
|
||||||
const releaseIdentifiers = releases
|
const releaseIdentifiers = releases
|
||||||
.map(release => [release.entity.id, release.entryId]);
|
.map((release) => [release.entity.id, release.entryId]);
|
||||||
|
|
||||||
const duplicateReleaseEntries = await knex('releases')
|
const duplicateReleaseEntries = await knex('releases')
|
||||||
.select(knex.raw('releases.*, row_to_json(entities) as entity'))
|
.select(knex.raw('releases.*, row_to_json(entities) as entity'))
|
||||||
|
@ -55,13 +55,13 @@ async function filterUniqueReleases(releases) {
|
||||||
.orWhere(knex.raw('updated_at - date > INTERVAL \'1 day\'')); // scene was updated after the release date, no updates expected
|
.orWhere(knex.raw('updated_at - date > INTERVAL \'1 day\'')); // scene was updated after the release date, no updates expected
|
||||||
});
|
});
|
||||||
|
|
||||||
const duplicateReleases = duplicateReleaseEntries.map(release => curateRelease(release));
|
const duplicateReleases = duplicateReleaseEntries.map((release) => curateRelease(release));
|
||||||
const duplicateReleasesByEntityIdAndEntryId = duplicateReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
|
const duplicateReleasesByEntityIdAndEntryId = duplicateReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
|
||||||
|
|
||||||
const internalUniqueReleasesByEntityIdAndEntryId = releases.reduce((acc, release) => mapReleasesToEntityIdAndEntryId(acc, release), {});
|
const internalUniqueReleasesByEntityIdAndEntryId = releases.reduce((acc, release) => mapReleasesToEntityIdAndEntryId(acc, release), {});
|
||||||
const internalUniqueReleases = Object.values(internalUniqueReleasesByEntityIdAndEntryId).map(releasesByEntryId => Object.values(releasesByEntryId)).flat();
|
const internalUniqueReleases = Object.values(internalUniqueReleasesByEntityIdAndEntryId).map((releasesByEntryId) => Object.values(releasesByEntryId)).flat();
|
||||||
|
|
||||||
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
|
const uniqueReleases = internalUniqueReleases.filter((release) => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
|
||||||
|
|
||||||
return { uniqueReleases, duplicateReleases };
|
return { uniqueReleases, duplicateReleases };
|
||||||
}
|
}
|
||||||
|
@ -83,7 +83,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
|
||||||
return accReleases.length + pageReleases.length < argv.last;
|
return accReleases.length + pageReleases.length < argv.last;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!pageReleases.concat(unextracted).every(release => !!release.date)) { // some scenes don't have dates
|
if (!pageReleases.concat(unextracted).every((release) => !!release.date)) { // some scenes don't have dates
|
||||||
return accReleases.length + pageReleases.length < argv.missingDateLimit;
|
return accReleases.length + pageReleases.length < argv.missingDateLimit;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -124,8 +124,8 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
|
||||||
return accReleases;
|
return accReleases;
|
||||||
}
|
}
|
||||||
|
|
||||||
const validPageReleases = pageReleases.filter(release => release?.entryId); // filter out empty and unidentified releases
|
const validPageReleases = pageReleases.filter((release) => release?.entryId); // filter out empty and unidentified releases
|
||||||
const pageReleasesWithEntity = validPageReleases.map(release => ({ ...release, entity: release.entity || entity }));
|
const pageReleasesWithEntity = validPageReleases.map((release) => ({ ...release, entity: release.entity || entity }));
|
||||||
|
|
||||||
if (pageReleases.length > validPageReleases.length) {
|
if (pageReleases.length > validPageReleases.length) {
|
||||||
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
|
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
|
||||||
|
@ -140,10 +140,10 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
|
||||||
|
|
||||||
const releases = await scrapeReleasesPage(argv.page || 1, []);
|
const releases = await scrapeReleasesPage(argv.page || 1, []);
|
||||||
|
|
||||||
const hasDates = releases.every(release => !!release.date);
|
const hasDates = releases.every((release) => !!release.date);
|
||||||
|
|
||||||
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|
||||||
|| (hasDates && releases.filter(release => moment(release.date).isAfter(argv.after)))
|
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|
||||||
|| releases.slice(0, Math.max(argv.missingDateLimit, 0));
|
|| releases.slice(0, Math.max(argv.missingDateLimit, 0));
|
||||||
|
|
||||||
const { uniqueReleases, duplicateReleases } = argv.force
|
const { uniqueReleases, duplicateReleases } = argv.force
|
||||||
|
@ -280,7 +280,7 @@ async function fetchUpdates() {
|
||||||
|
|
||||||
const scrapedNetworks = await Promise.map(
|
const scrapedNetworks = await Promise.map(
|
||||||
includedNetworks,
|
includedNetworks,
|
||||||
async networkEntity => (networkEntity.parameters?.sequential
|
async (networkEntity) => (networkEntity.parameters?.sequential
|
||||||
? scrapeNetworkSequential(networkEntity)
|
? scrapeNetworkSequential(networkEntity)
|
||||||
: scrapeNetworkParallel(networkEntity)),
|
: scrapeNetworkParallel(networkEntity)),
|
||||||
{ concurrency: 5 },
|
{ concurrency: 5 },
|
||||||
|
|
|
@ -18,7 +18,7 @@ function curateUser(user) {
|
||||||
identityVerified: user.identity_verified,
|
identityVerified: user.identity_verified,
|
||||||
ability,
|
ability,
|
||||||
createdAt: user.created_at,
|
createdAt: user.created_at,
|
||||||
stashes: user.stashes?.filter(Boolean).map(stash => curateStash(stash)) || [],
|
stashes: user.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [],
|
||||||
};
|
};
|
||||||
|
|
||||||
return curatedUser;
|
return curatedUser;
|
||||||
|
|
|
@ -22,13 +22,13 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
|
||||||
const chunked = chunk(items, chunkSize);
|
const chunked = chunk(items, chunkSize);
|
||||||
|
|
||||||
const queries = chunked
|
const queries = chunked
|
||||||
.map(chunkItems => knex.raw(updated || ':query RETURNING *;', {
|
.map((chunkItems) => knex.raw(updated || ':query RETURNING *;', {
|
||||||
query: knex(table).insert(chunkItems),
|
query: knex(table).insert(chunkItems),
|
||||||
}).transacting(transaction));
|
}).transacting(transaction));
|
||||||
|
|
||||||
const responses = await Promise.all(queries);
|
const responses = await Promise.all(queries);
|
||||||
|
|
||||||
return responses.flat().map(response => response.rows).flat();
|
return responses.flat().map((response) => response.rows).flat();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,7 @@ function capitalize(string, { trim = true, uncapitalize = false } = {}) {
|
||||||
|
|
||||||
const capitalized = string
|
const capitalized = string
|
||||||
.split(/\s+/)
|
.split(/\s+/)
|
||||||
.map(component => `${component.charAt(0).toUpperCase()}${uncapitalize ? component.slice(1).toLowerCase() : component.slice(1)}`)
|
.map((component) => `${component.charAt(0).toUpperCase()}${uncapitalize ? component.slice(1).toLowerCase() : component.slice(1)}`)
|
||||||
.join(' ');
|
.join(' ');
|
||||||
|
|
||||||
return trim ? capitalized.trim() : capitalized;
|
return trim ? capitalized.trim() : capitalized;
|
||||||
|
|
|
@ -62,7 +62,7 @@ function convertManyApi(input, to) {
|
||||||
const curatedInput = input
|
const curatedInput = input
|
||||||
.replace('\'', 'ft')
|
.replace('\'', 'ft')
|
||||||
.replace(/"|''/, 'in')
|
.replace(/"|''/, 'in')
|
||||||
.replace(/\d+ft\s*\d+\s*$/, match => `${match}in`); // height without any inch symbol
|
.replace(/\d+ft\s*\d+\s*$/, (match) => `${match}in`); // height without any inch symbol
|
||||||
|
|
||||||
return Math.round(convertMany(curatedInput).to(to)) || null;
|
return Math.round(convertMany(curatedInput).to(to)) || null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,7 +8,7 @@ async function getFileEntries(location) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const file = await fs.promises.readFile(location, 'utf-8');
|
const file = await fs.promises.readFile(location, 'utf-8');
|
||||||
const entries = file.split(/\n/).map(entry => entry.trim()).filter(Boolean);
|
const entries = file.split(/\n/).map((entry) => entry.trim()).filter(Boolean);
|
||||||
|
|
||||||
return entries;
|
return entries;
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,7 +27,7 @@ async function fetchSource(link) {
|
||||||
const tempFileStream = fs.createWriteStream(tempFilePath);
|
const tempFileStream = fs.createWriteStream(tempFilePath);
|
||||||
const hashStream = new PassThrough();
|
const hashStream = new PassThrough();
|
||||||
|
|
||||||
hashStream.on('data', chunk => hasher.write(chunk));
|
hashStream.on('data', (chunk) => hasher.write(chunk));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await http.get(link, null, {
|
const res = await http.get(link, null, {
|
||||||
|
|
|
@ -102,7 +102,7 @@ function all(context, selector, attrArg, applyTrim = true) {
|
||||||
const attr = attrArg === true ? 'textContent' : attrArg;
|
const attr = attrArg === true ? 'textContent' : attrArg;
|
||||||
|
|
||||||
if (attr) {
|
if (attr) {
|
||||||
return Array.from(context.querySelectorAll(selector), el => q(el, null, attr, applyTrim));
|
return Array.from(context.querySelectorAll(selector), (el) => q(el, null, attr, applyTrim));
|
||||||
}
|
}
|
||||||
|
|
||||||
return Array.from(context.querySelectorAll(selector));
|
return Array.from(context.querySelectorAll(selector));
|
||||||
|
@ -155,7 +155,7 @@ function jsons(context, selector) {
|
||||||
function htmls(context, selector) {
|
function htmls(context, selector) {
|
||||||
const els = all(context, selector, null, true);
|
const els = all(context, selector, null, true);
|
||||||
|
|
||||||
return els.map(el => el.innerHTML);
|
return els.map((el) => el.innerHTML);
|
||||||
}
|
}
|
||||||
|
|
||||||
function texts(context, selector, applyTrim = true, filter = true) {
|
function texts(context, selector, applyTrim = true, filter = true) {
|
||||||
|
@ -163,8 +163,8 @@ function texts(context, selector, applyTrim = true, filter = true) {
|
||||||
if (!el) return null;
|
if (!el) return null;
|
||||||
|
|
||||||
const nodes = Array.from(el.childNodes)
|
const nodes = Array.from(el.childNodes)
|
||||||
.filter(node => node.nodeName === '#text')
|
.filter((node) => node.nodeName === '#text')
|
||||||
.map(node => (applyTrim ? trim(node.textContent) : node.textContent));
|
.map((node) => (applyTrim ? trim(node.textContent) : node.textContent));
|
||||||
|
|
||||||
return filter ? nodes.filter(Boolean) : nodes;
|
return filter ? nodes.filter(Boolean) : nodes;
|
||||||
}
|
}
|
||||||
|
@ -272,7 +272,7 @@ function images(context, selector = 'img', attr, { origin, protocol = 'https' }
|
||||||
|
|
||||||
const imageEls = all(context, selector, attribute);
|
const imageEls = all(context, selector, attribute);
|
||||||
|
|
||||||
return imageEls.map(imageEl => prefixUrl(imageEl, origin, protocol));
|
return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol));
|
||||||
}
|
}
|
||||||
|
|
||||||
function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
|
function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
|
||||||
|
@ -289,7 +289,7 @@ function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https
|
||||||
function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
|
function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
|
||||||
const urlEls = all(context, selector, attr);
|
const urlEls = all(context, selector, attr);
|
||||||
|
|
||||||
return attr ? urlEls.map(urlEl => prefixUrl(urlEl, origin, protocol)) : urlEls;
|
return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls;
|
||||||
}
|
}
|
||||||
|
|
||||||
function sourceSet(context, selector, attr = 'srcset', options = {}) {
|
function sourceSet(context, selector, attr = 'srcset', options = {}) {
|
||||||
|
@ -330,7 +330,7 @@ function sourceSet(context, selector, attr = 'srcset', options = {}) {
|
||||||
return sources;
|
return sources;
|
||||||
}
|
}
|
||||||
|
|
||||||
return sources.map(source => source.url);
|
return sources.map((source) => source.url);
|
||||||
}
|
}
|
||||||
|
|
||||||
function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
|
function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
|
||||||
|
@ -348,7 +348,7 @@ function video(context, selector = 'source', attr = 'src', { origin, protocol =
|
||||||
function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
|
function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
|
||||||
const trailerEls = all(context, selector, attr);
|
const trailerEls = all(context, selector, attr);
|
||||||
|
|
||||||
return attr ? trailerEls.map(trailerEl => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
|
return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
|
||||||
}
|
}
|
||||||
|
|
||||||
function duration(context, selector, match, attr = 'textContent') {
|
function duration(context, selector, match, attr = 'textContent') {
|
||||||
|
@ -499,11 +499,11 @@ function init(context, selector, window) {
|
||||||
|
|
||||||
function initAll(context, selector, window) {
|
function initAll(context, selector, window) {
|
||||||
if (Array.isArray(context)) {
|
if (Array.isArray(context)) {
|
||||||
return context.map(element => init(element, null, window));
|
return context.map((element) => init(element, null, window));
|
||||||
}
|
}
|
||||||
|
|
||||||
return Array.from(context.querySelectorAll(selector))
|
return Array.from(context.querySelectorAll(selector))
|
||||||
.map(element => init(element, null, window));
|
.map((element) => init(element, null, window));
|
||||||
}
|
}
|
||||||
|
|
||||||
function extract(htmlValue, selector, options) {
|
function extract(htmlValue, selector, options) {
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue