Added periodic memory logger.

This commit is contained in:
DebaucheryLibrarian 2021-11-20 23:59:15 +01:00
parent a867817dc1
commit ccb99e278c
109 changed files with 10238 additions and 10833 deletions

View File

@ -2,7 +2,7 @@
"root": true, "root": true,
"extends": ["airbnb-base", "plugin:vue/recommended"], "extends": ["airbnb-base", "plugin:vue/recommended"],
"parserOptions": { "parserOptions": {
"parser": "babel-eslint", "parser": "@babel/eslint-parser",
"ecmaVersion": 2019, "ecmaVersion": 2019,
"sourceType": "module" "sourceType": "module"
}, },

2
.nvmrc
View File

@ -1 +1 @@
14.15.4 16.13.0

View File

@ -5,7 +5,7 @@ $breakpoint3: 1200px;
$breakpoint4: 1500px; $breakpoint4: 1500px;
:root { :root {
--primary: #c63971; --primary: #e33379;
--primary-strong: #f90071; --primary-strong: #f90071;
--primary-faded: #ffcce4; --primary-faded: #ffcce4;
@ -46,7 +46,7 @@ $breakpoint4: 1500px;
--logo-shadow: drop-shadow(1px 0 0 $shadow-weak) drop-shadow(-1px 0 0 $shadow-weak) drop-shadow(0 1px 0 $shadow-weak) drop-shadow(0 -1px 0 $shadow-weak); --logo-shadow: drop-shadow(1px 0 0 $shadow-weak) drop-shadow(-1px 0 0 $shadow-weak) drop-shadow(0 1px 0 $shadow-weak) drop-shadow(0 -1px 0 $shadow-weak);
--logo-highlight: drop-shadow(0 0 1px $highlight); --logo-highlight: drop-shadow(0 0 1px $highlight);
--info: #361723; --info: #321b24;
--male: #0af; --male: #0af;
--female: #f0a; --female: #f0a;

View File

@ -281,7 +281,7 @@ function initActorActions(store, router) {
return { return {
actor: curateActor(actor, null, curateRelease), actor: curateActor(actor, null, curateRelease),
releases: actor.scenesConnection.releases.map(release => curateRelease(release)), releases: actor.scenesConnection.releases.map((release) => curateRelease(release)),
totalCount: actor.scenesConnection.totalCount, totalCount: actor.scenesConnection.totalCount,
}; };
} }
@ -429,7 +429,7 @@ function initActorActions(store, router) {
}); });
return { return {
actors: actors.map(actor => curateActor(actor)), actors: actors.map((actor) => curateActor(actor)),
totalCount, totalCount,
countries, countries,
}; };

View File

@ -1,5 +1,5 @@
function favoritesStash(state) { function favoritesStash(state) {
return state.user.stashes.find(stash => stash.slug === 'favorites'); return state.user.stashes.find((stash) => stash.slug === 'favorites');
} }
module.exports = { module.exports = {

View File

@ -31,8 +31,8 @@ function curateActor(actor, release) {
if (actor.profiles) { if (actor.profiles) {
const photos = actor.profiles const photos = actor.profiles
.map(profile => ({ entity: profile.entity, ...profile.avatar })) .map((profile) => ({ entity: profile.entity, ...profile.avatar }))
.filter(avatar => avatar.id && (!curatedActor.avatar || avatar.hash !== curatedActor.avatar.hash)); .filter((avatar) => avatar.id && (!curatedActor.avatar || avatar.hash !== curatedActor.avatar.hash));
const descriptions = actor.profiles.reduce((acc, profile) => ({ const descriptions = actor.profiles.reduce((acc, profile) => ({
...acc, ...acc,
@ -57,10 +57,10 @@ function curateActor(actor, release) {
} }
if (actor.stashes) { if (actor.stashes) {
curatedActor.stashes = actor.stashes.filter(Boolean).map(stash => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define curatedActor.stashes = actor.stashes.filter(Boolean).map((stash) => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
} }
curatedActor.stashes = actor.stashes?.map(stash => stash.stash || stash) || []; curatedActor.stashes = actor.stashes?.map((stash) => stash.stash || stash) || [];
return curatedActor; return curatedActor;
} }
@ -70,21 +70,21 @@ function curateRelease(release) {
...release, ...release,
actors: [], actors: [],
poster: release.poster && release.poster.media, poster: release.poster && release.poster.media,
tags: release.tags ? release.tags.map(tag => tag.tag || tag) : [], tags: release.tags ? release.tags.map((tag) => tag.tag || tag) : [],
}; };
if (release.scenes) curatedRelease.scenes = release.scenes.filter(Boolean).map(({ scene }) => curateRelease(scene)); if (release.scenes) curatedRelease.scenes = release.scenes.filter(Boolean).map(({ scene }) => curateRelease(scene));
if (release.movies) curatedRelease.movies = release.movies.filter(Boolean).map(({ movie }) => curateRelease(movie)); if (release.movies) curatedRelease.movies = release.movies.filter(Boolean).map(({ movie }) => curateRelease(movie));
if (release.chapters) curatedRelease.chapters = release.chapters.filter(Boolean).map(chapter => curateRelease(chapter)); if (release.chapters) curatedRelease.chapters = release.chapters.filter(Boolean).map((chapter) => curateRelease(chapter));
if (release.photos) curatedRelease.photos = release.photos.filter(Boolean).map(photo => photo.media || photo); if (release.photos) curatedRelease.photos = release.photos.filter(Boolean).map((photo) => photo.media || photo);
if (release.covers) curatedRelease.covers = release.covers.filter(Boolean).map(({ media }) => media); if (release.covers) curatedRelease.covers = release.covers.filter(Boolean).map(({ media }) => media);
if (release.trailer) curatedRelease.trailer = release.trailer.media; if (release.trailer) curatedRelease.trailer = release.trailer.media;
if (release.teaser) curatedRelease.teaser = release.teaser.media; if (release.teaser) curatedRelease.teaser = release.teaser.media;
if (release.actors) curatedRelease.actors = release.actors.filter(Boolean).map(actor => curateActor(actor.actor || actor, curatedRelease)); if (release.actors) curatedRelease.actors = release.actors.filter(Boolean).map((actor) => curateActor(actor.actor || actor, curatedRelease));
if (release.directors) curatedRelease.directors = release.directors.filter(Boolean).map(director => curateActor(director.director || director, curatedRelease)); if (release.directors) curatedRelease.directors = release.directors.filter(Boolean).map((director) => curateActor(director.director || director, curatedRelease));
if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.filter(Boolean).map(({ tag }) => tag); if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.filter(Boolean).map(({ tag }) => tag);
if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.filter(Boolean).map(({ actor }) => curateActor(actor, curatedRelease)); if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.filter(Boolean).map(({ actor }) => curateActor(actor, curatedRelease));
if (release.stashes) curatedRelease.stashes = release.stashes.filter(Boolean).map(stash => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define if (release.stashes) curatedRelease.stashes = release.stashes.filter(Boolean).map((stash) => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
if (release.productionLocation) { if (release.productionLocation) {
curatedRelease.productionLocation = { curatedRelease.productionLocation = {
@ -108,14 +108,14 @@ function curateEntity(entity, parent, releases) {
if (entity.children) { if (entity.children) {
if (entity.children.nodes) { if (entity.children.nodes) {
curatedEntity.children = entity.children.nodes.map(childEntity => curateEntity(childEntity, curatedEntity)); curatedEntity.children = entity.children.nodes.map((childEntity) => curateEntity(childEntity, curatedEntity));
} }
curatedEntity.childrenTotal = entity.children.totalCount; curatedEntity.childrenTotal = entity.children.totalCount;
} }
if (entity.parent || parent) curatedEntity.parent = curateEntity(entity.parent || parent); if (entity.parent || parent) curatedEntity.parent = curateEntity(entity.parent || parent);
if (releases) curatedEntity.releases = releases.map(release => curateRelease(release)); if (releases) curatedEntity.releases = releases.map((release) => curateRelease(release));
if (entity.connection) { if (entity.connection) {
curatedEntity.sceneTotal = entity.connection.totalCount; curatedEntity.sceneTotal = entity.connection.totalCount;
@ -142,7 +142,7 @@ function curateStash(stash) {
if (stash.scenes || stash.scenesConnection?.scenes) { if (stash.scenes || stash.scenesConnection?.scenes) {
curatedStash.sceneTotal = stash.scenesConnection?.totalCount || null; curatedStash.sceneTotal = stash.scenesConnection?.totalCount || null;
curatedStash.scenes = (stash.scenesConnection?.scenes || stash.scenes).map(item => ({ curatedStash.scenes = (stash.scenesConnection?.scenes || stash.scenes).map((item) => ({
...item, ...item,
scene: curateRelease(item.scene), scene: curateRelease(item.scene),
})); }));
@ -150,7 +150,7 @@ function curateStash(stash) {
if (stash.actors || stash.actorsConnection?.actors) { if (stash.actors || stash.actorsConnection?.actors) {
curatedStash.actorTotal = stash.actorsConnection?.totalCount || null; curatedStash.actorTotal = stash.actorsConnection?.totalCount || null;
curatedStash.actors = (stash.actorsConnection?.actors || stash.actors).map(item => ({ curatedStash.actors = (stash.actorsConnection?.actors || stash.actors).map((item) => ({
...item, ...item,
actor: curateActor(item.actor), actor: curateActor(item.actor),
})); }));
@ -158,7 +158,7 @@ function curateStash(stash) {
if (stash.movies || stash.moviesConnection?.movies) { if (stash.movies || stash.moviesConnection?.movies) {
curatedStash.movieTotal = stash.moviesConnection?.totalCount || null; curatedStash.movieTotal = stash.moviesConnection?.totalCount || null;
curatedStash.movies = (stash.moviesConnection?.movies || stash.movies).map(item => ({ curatedStash.movies = (stash.moviesConnection?.movies || stash.movies).map((item) => ({
...item, ...item,
movie: curateRelease(item.movie), movie: curateRelease(item.movie),
})); }));
@ -175,11 +175,11 @@ function curateAlert(alert) {
const curatedAlert = alert; const curatedAlert = alert;
if (alert.actors) { if (alert.actors) {
curatedAlert.actors = alert.actors.map(actor => curateActor(actor.actor || actor)); curatedAlert.actors = alert.actors.map((actor) => curateActor(actor.actor || actor));
} }
if (alert.tags) { if (alert.tags) {
curatedAlert.tags = alert.tags.map(tag => curateTag(tag.tag || tag)); curatedAlert.tags = alert.tags.map((tag) => curateTag(tag.tag || tag));
} }
if (alert.entity) { if (alert.entity) {
@ -187,7 +187,7 @@ function curateAlert(alert) {
} }
if (alert.stashes) { if (alert.stashes) {
curatedAlert.stashes = alert.stashes.map(stash => curateStash(stash.stash || stash)); curatedAlert.stashes = alert.stashes.map((stash) => curateStash(stash.stash || stash));
} }
return curatedAlert; return curatedAlert;
@ -201,11 +201,11 @@ function curateUser(user) {
const curatedUser = user; const curatedUser = user;
if (user.stashes) { if (user.stashes) {
curatedUser.stashes = user.stashes.map(stash => curateStash(stash.stash || stash)); curatedUser.stashes = user.stashes.map((stash) => curateStash(stash.stash || stash));
} }
if (user.alerts) { if (user.alerts) {
curatedUser.alerts = user.alerts.map(alert => curateAlert(alert.alert || alert)); curatedUser.alerts = user.alerts.map((alert) => curateAlert(alert.alert || alert));
} }
return curatedUser; return curatedUser;

View File

@ -212,7 +212,7 @@ function initEntitiesActions(store, router) {
entitySlugs, entitySlugs,
}); });
return entities.map(entity => curateEntity(entity)); return entities.map((entity) => curateEntity(entity));
} }
async function searchEntities({ _commit }, { query, limit = 20 }) { async function searchEntities({ _commit }, { query, limit = 20 }) {
@ -246,7 +246,7 @@ function initEntitiesActions(store, router) {
limit, limit,
}); });
return entities.map(entity => curateEntity(entity)); return entities.map((entity) => curateEntity(entity));
} }
return { return {

View File

@ -5,7 +5,7 @@ export function formatDuration(duration, forceHours) {
const minutes = Math.floor((duration % 3600) / 60); const minutes = Math.floor((duration % 3600) / 60);
const seconds = Math.floor(duration % 60); const seconds = Math.floor(duration % 60);
const [formattedHours, formattedMinutes, formattedSeconds] = [hours, minutes, seconds].map(segment => segment.toString().padStart(2, '0')); const [formattedHours, formattedMinutes, formattedSeconds] = [hours, minutes, seconds].map((segment) => segment.toString().padStart(2, '0'));
if (duration >= 3600 || forceHours) { if (duration >= 3600 || forceHours) {
return `${formattedHours}:${formattedMinutes}:${formattedSeconds}`; return `${formattedHours}:${formattedMinutes}:${formattedSeconds}`;

View File

@ -260,7 +260,7 @@ const releaseTagsFragment = `
`; `;
const releasePosterFragment = ` const releasePosterFragment = `
poster: releasesPosterByReleaseId { poster: releasesPoster {
media { media {
id id
index index
@ -335,7 +335,7 @@ const releasePhotosFragment = `
`; `;
const releaseTrailerFragment = ` const releaseTrailerFragment = `
trailer: releasesTrailerByReleaseId { trailer: releasesTrailer {
media { media {
id id
index index
@ -349,7 +349,7 @@ const releaseTrailerFragment = `
`; `;
const releaseTeaserFragment = ` const releaseTeaserFragment = `
teaser: releasesTeaserByReleaseId { teaser: releasesTeaser {
media { media {
id id
index index
@ -487,7 +487,7 @@ const releaseFragment = `
slug slug
} }
} }
poster: chaptersPosterByChapterId { poster: chaptersPoster {
media { media {
id id
index index

View File

@ -109,6 +109,10 @@ async function init() {
document.title = 'traxxx'; document.title = 'traxxx';
}, },
}, },
beforeCreate() {
this.uid = uid;
uid += 1;
},
methods: { methods: {
formatDate, formatDate,
formatDuration, formatDuration,
@ -117,10 +121,6 @@ async function init() {
getPath, getPath,
getBgPath: (media, type) => `url(${getPath(media, type)})`, getBgPath: (media, type) => `url(${getPath(media, type)})`,
}, },
beforeCreate() {
this.uid = uid;
uid += 1;
},
}); });
app.directive('tooltip', { app.directive('tooltip', {

View File

@ -37,7 +37,7 @@ function initReleasesActions(store, router) {
}); });
return { return {
releases: releases.map(release => curateRelease(release)), releases: releases.map((release) => curateRelease(release)),
totalCount, totalCount,
}; };
} }
@ -99,7 +99,7 @@ function initReleasesActions(store, router) {
}); });
return { return {
movies: movies.map(release => curateRelease(release)), movies: movies.map((release) => curateRelease(release)),
totalCount, totalCount,
}; };
} }

View File

@ -74,7 +74,7 @@ const routes = [
{ {
path: '/actor/:actorId/:actorSlug', path: '/actor/:actorId/:actorSlug',
name: 'actor', name: 'actor',
redirect: from => ({ redirect: (from) => ({
name: 'actorRange', name: 'actorRange',
params: { params: {
...from.params, ...from.params,
@ -91,7 +91,7 @@ const routes = [
{ {
path: '/director/:actorId/:actorSlug', path: '/director/:actorId/:actorSlug',
name: 'director', name: 'director',
redirect: from => ({ redirect: (from) => ({
name: 'directorRange', name: 'directorRange',
params: { params: {
...from.params, ...from.params,
@ -107,7 +107,7 @@ const routes = [
}, },
{ {
path: '/channel/:entitySlug', path: '/channel/:entitySlug',
redirect: from => ({ redirect: (from) => ({
name: 'channel', name: 'channel',
params: { params: {
...from.params, ...from.params,
@ -123,7 +123,7 @@ const routes = [
}, },
{ {
path: '/network/:entitySlug', path: '/network/:entitySlug',
redirect: from => ({ redirect: (from) => ({
name: 'network', name: 'network',
params: { params: {
...from.params, ...from.params,
@ -139,7 +139,7 @@ const routes = [
}, },
{ {
path: '/studio/:entitySlug', path: '/studio/:entitySlug',
redirect: from => ({ redirect: (from) => ({
name: 'studio', name: 'studio',
params: { params: {
...from.params, ...from.params,
@ -155,7 +155,7 @@ const routes = [
}, },
{ {
path: '/tag/:tagSlug', path: '/tag/:tagSlug',
redirect: from => ({ redirect: (from) => ({
name: 'tag', name: 'tag',
params: { params: {
...from.params, ...from.params,
@ -171,7 +171,7 @@ const routes = [
}, },
{ {
path: '/actors', path: '/actors',
redirect: from => ({ redirect: (from) => ({
name: 'actors', name: 'actors',
params: { params: {
...from.params, ...from.params,
@ -229,7 +229,7 @@ const routes = [
}, },
{ {
path: '/stash/:stashId/:stashSlug', path: '/stash/:stashId/:stashSlug',
redirect: from => ({ redirect: (from) => ({
name: 'stash', name: 'stash',
params: { params: {
...from.params, ...from.params,

View File

@ -35,7 +35,7 @@ function initTagsActions(store, _router) {
name name
slug slug
} }
poster: tagsPosterByTagId { poster: tagsPoster {
media { media {
id id
thumbnail thumbnail
@ -188,14 +188,14 @@ function initTagsActions(store, _router) {
before, before,
orderBy, orderBy,
offset: Math.max(0, (pageNumber - 1)) * limit, offset: Math.max(0, (pageNumber - 1)) * limit,
exclude: store.state.ui.tagFilter.filter(tagFilter => tagFilter !== tagSlug), exclude: store.state.ui.tagFilter.filter((tagFilter) => tagFilter !== tagSlug),
hasAuth: !!store.state.auth.user, hasAuth: !!store.state.auth.user,
userId: store.state.auth.user?.id, userId: store.state.auth.user?.id,
}); });
return { return {
tag: curateTag(tagBySlug, null, curateRelease), tag: curateTag(tagBySlug, null, curateRelease),
releases: tagBySlug.scenesConnection.releases.map(release => curateRelease(release)), releases: tagBySlug.scenesConnection.releases.map((release) => curateRelease(release)),
totalCount: tagBySlug.scenesConnection.totalCount, totalCount: tagBySlug.scenesConnection.totalCount,
}; };
} }
@ -218,7 +218,7 @@ function initTagsActions(store, _router) {
id id
name name
slug slug
poster: tagsPosterByTagId { poster: tagsPoster {
media { media {
thumbnail thumbnail
comment comment
@ -259,7 +259,7 @@ function initTagsActions(store, _router) {
limit, limit,
}); });
return tags.map(tag => curateTag(tag, store.state.ui.sfw)); return tags.map((tag) => curateTag(tag, store.state.ui.sfw));
} }
async function searchTags({ _commit }, { async function searchTags({ _commit }, {
@ -325,7 +325,7 @@ function initTagsActions(store, _router) {
minLength, minLength,
}); });
return tags.map(tag => curateTag(tag, store.state.ui.sfw)); return tags.map((tag) => curateTag(tag, store.state.ui.sfw));
} }
async function fetchTagReleases({ _commit }, tagId) { async function fetchTagReleases({ _commit }, tagId) {

View File

@ -70,7 +70,7 @@ function initUiActions(store, _router) {
slug slug
} }
} }
entity: alertsEntityByAlertId { entity: alertsEntity {
entity { entity {
id id
name name
@ -103,7 +103,7 @@ function initUiActions(store, _router) {
}; };
} }
const curatedNotifications = notifications.nodes.map(notification => curateNotification(notification)); const curatedNotifications = notifications.nodes.map((notification) => curateNotification(notification));
return { return {
notifications: curatedNotifications, notifications: curatedNotifications,
@ -222,8 +222,8 @@ function initUiActions(store, _router) {
}); });
return { return {
releases: res?.results.map(result => curateRelease(result.release)) || [], releases: res?.results.map((result) => curateRelease(result.release)) || [],
actors: res?.actors.map(actor => curateActor(actor)) || [], actors: res?.actors.map((actor) => curateActor(actor)) || [],
}; };
} }

View File

@ -3,7 +3,7 @@ async function initUiObservers(store, _router) {
body.classList.add(store.state.ui.theme); body.classList.add(store.state.ui.theme);
store.watch(state => state.ui.theme, (newTheme, oldTheme) => { store.watch((state) => state.ui.theme, (newTheme, oldTheme) => {
body.classList.add(newTheme); body.classList.add(newTheme);
body.classList.remove(oldTheme); body.classList.remove(oldTheme);
}); });

19986
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -7,7 +7,7 @@
"start": "node -r source-map-support/register src/init.js", "start": "node -r source-map-support/register src/init.js",
"webpack": "webpack --env=production --mode=production", "webpack": "webpack --env=production --mode=production",
"webpack-dev": "webpack --env=development --mode=development", "webpack-dev": "webpack --env=development --mode=development",
"webpack-watch": "webpack --progress --colors --watch --env=development --mode=development", "webpack-watch": "webpack --progress --color --watch --env=development --mode=development",
"babel": "babel src --source-maps -d dist", "babel": "babel src --source-maps -d dist",
"babel-watch": "babel src -w --source-maps -d dist", "babel-watch": "babel src -w --source-maps -d dist",
"build": "babel src --source-maps -d dist && webpack --env=production --mode=production", "build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
@ -40,38 +40,37 @@
"devDependencies": { "devDependencies": {
"@babel/cli": "^7.12.10", "@babel/cli": "^7.12.10",
"@babel/core": "^7.8.4", "@babel/core": "^7.8.4",
"@babel/eslint-parser": "^7.16.0",
"@babel/plugin-proposal-optional-chaining": "^7.8.3", "@babel/plugin-proposal-optional-chaining": "^7.8.3",
"@babel/preset-env": "^7.8.4", "@babel/preset-env": "^7.8.4",
"@babel/register": "^7.8.3", "@babel/register": "^7.8.3",
"@vue/compiler-sfc": "^3.0.4", "autoprefixer": "^10.4.0",
"autoprefixer": "^9.7.4",
"babel-eslint": "^10.1.0",
"babel-loader": "^8.0.6", "babel-loader": "^8.0.6",
"babel-preset-airbnb": "^3.3.2", "babel-preset-airbnb": "^5.0.0",
"css-loader": "^5.0.1", "css-loader": "^6.5.0",
"eslint": "^7.20.0", "eslint": "^8.1.0",
"eslint-config-airbnb": "^17.1.1", "eslint-config-airbnb": "^18.2.1",
"eslint-config-airbnb-base": "^13.2.0", "eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.20.1", "eslint-plugin-import": "^2.20.1",
"eslint-plugin-jsx-a11y": "^6.2.3", "eslint-plugin-jsx-a11y": "^6.2.3",
"eslint-plugin-react": "^7.18.3", "eslint-plugin-react": "^7.18.3",
"eslint-plugin-vue": "^6.2.1", "eslint-plugin-vue": "^8.0.3",
"eslint-watch": "^4.0.2", "eslint-watch": "^7.0.0",
"eslint-webpack-plugin": "^2.5.2", "eslint-webpack-plugin": "^3.1.0",
"mini-css-extract-plugin": "^1.3.3", "mini-css-extract-plugin": "^2.4.3",
"node-sass": "^5.0.0", "node-sass": "^6.0.1",
"postcss-loader": "^3.0.0", "postcss-loader": "^6.2.0",
"raw-loader": "^4.0.2", "raw-loader": "^4.0.2",
"sass-loader": "^11.0.1", "sass-loader": "^12.3.0",
"style-loader": "^0.23.1", "style-loader": "^3.3.1",
"vue-loader": "^16.1.2", "vue-loader": "^16.8.2",
"webpack": "^5.11.0", "webpack": "^5.11.0",
"webpack-cli": "^3.3.11" "webpack-cli": "^4.9.1"
}, },
"dependencies": { "dependencies": {
"@casl/ability": "^5.2.2", "@casl/ability": "^5.2.2",
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6", "@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
"@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1", "@graphile-contrib/pg-simplify-inflector": "^6.1.0",
"acorn": "^8.0.4", "acorn": "^8.0.4",
"array-equal": "^1.0.0", "array-equal": "^1.0.0",
"aws-sdk": "^2.847.0", "aws-sdk": "^2.847.0",
@ -88,63 +87,60 @@
"cloudscraper": "^4.6.0", "cloudscraper": "^4.6.0",
"config": "^3.2.5", "config": "^3.2.5",
"connect-session-knex": "^2.0.0", "connect-session-knex": "^2.0.0",
"convert": "^1.6.2", "convert": "^4.2.4",
"cookie": "^0.4.0", "cookie": "^0.4.0",
"csv-stringify": "^5.3.6", "csv-stringify": "^5.3.6",
"dayjs": "^1.8.21", "dayjs": "^1.8.21",
"dompurify": "^2.0.11", "dompurify": "^2.0.11",
"ejs": "^3.0.1", "ejs": "^3.0.1",
"express": "^4.17.1", "express": "^4.17.1",
"express-promise-router": "^3.0.3", "express-promise-router": "^4.1.0",
"express-react-views": "^0.11.0", "express-react-views": "^0.11.0",
"express-session": "^1.17.1", "express-session": "^1.17.1",
"face-api.js": "^0.22.2", "face-api.js": "^0.22.2",
"faker": "^5.1.0", "faker": "^5.1.0",
"file-type": "^14.1.4", "file-type": "^16.5.3",
"fluent-ffmpeg": "^2.1.2", "fluent-ffmpeg": "^2.1.2",
"fs-extra": "^7.0.1", "fs-extra": "^10.0.0",
"graphile-utils": "^4.5.6", "graphile-utils": "^4.12.2",
"graphql": "^14.6.0", "graphql": "^15.4.0",
"html-entities": "^2.3.2", "html-entities": "^2.3.2",
"iconv-lite": "^0.5.1", "iconv-lite": "^0.6.3",
"inquirer": "^7.3.3", "inquirer": "^8.2.0",
"inspector-api": "^1.4.2", "inspector-api": "^1.4.2",
"jsdom": "^16.3.0", "jsdom": "^18.0.0",
"knex": "^0.21.13", "knex": "^0.95.12",
"knex-migrate": "^1.7.4", "knex-migrate": "^1.7.4",
"longjohn": "^0.2.12", "longjohn": "^0.2.12",
"mime": "^2.4.4", "mime": "^2.4.4",
"mitt": "^2.1.0", "mitt": "^3.0.0",
"moment": "^2.24.0", "moment": "^2.24.0",
"nanoid": "^2.1.11", "nanoid": "^3.1.30",
"object-merge-advanced": "^12.1.0", "object-merge-advanced": "^12.1.0",
"object.omit": "^3.0.0", "object.omit": "^3.0.0",
"opn": "^5.5.0", "opn": "^6.0.0",
"pg": "^8.5.1", "pg": "^8.5.1",
"postgraphile": "^4.10.0", "postgraphile": "^4.10.0",
"postgraphile-plugin-connection-filter": "^1.1.3", "postgraphile-plugin-connection-filter": "^2.2.2",
"promise-task-queue": "^1.2.0", "promise-task-queue": "^1.2.0",
"prop-types": "^15.7.2", "prop-types": "^15.7.2",
"react": "^16.13.0", "sharp": "^0.29.2",
"react-dom": "^16.13.0",
"sharp": "^0.27.2",
"showdown": "^1.9.1", "showdown": "^1.9.1",
"source-map-support": "^0.5.16", "source-map-support": "^0.5.16",
"template-format": "^1.2.5", "template-format": "^1.2.5",
"tippy.js": "^6.3.1", "tippy.js": "^6.3.1",
"tough-cookie": "^3.0.1", "tough-cookie": "^4.0.0",
"tty-table": "^2.8.12",
"tunnel": "0.0.6", "tunnel": "0.0.6",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.0.3", "v-tooltip": "^2.0.3",
"video.js": "^7.11.4", "video.js": "^7.11.4",
"videojs-vr": "^1.7.1", "videojs-vr": "^1.7.1",
"vue": "^3.0.4", "vue": "^3.2.20",
"vue-router": "^4.0.1", "vue-router": "^4.0.12",
"vuex": "^4.0.0-rc.2", "vuex": "^4.0.2",
"why-is-node-running": "^2.2.0", "why-is-node-running": "^2.2.0",
"winston": "^3.2.1", "winston": "^3.2.1",
"winston-daily-rotate-file": "^4.4.2", "winston-daily-rotate-file": "^4.4.2",
"yargs": "^13.3.0" "yargs": "^17.2.1"
} }
} }

Binary file not shown.

After

Width:  |  Height:  |  Size: 680 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 35 KiB

View File

@ -900,6 +900,7 @@ const tagMedia = [
['flexible', 'lara_frost_legalporno', 'Lara Frost in NRX059', 'legalporno'], ['flexible', 'lara_frost_legalporno', 'Lara Frost in NRX059', 'legalporno'],
['free-use', 'jeni_angel_brazzersexxtra', 'Jeni Angel in "Gamer Girl Threesome Action"', 'brazzersexxtra'], ['free-use', 'jeni_angel_brazzersexxtra', 'Jeni Angel in "Gamer Girl Threesome Action"', 'brazzersexxtra'],
['free-use', 'veruca_james_brazzersexxtra', 'Veruca James in "The Perfect Maid"', 'brazzersexxtra'], ['free-use', 'veruca_james_brazzersexxtra', 'Veruca James in "The Perfect Maid"', 'brazzersexxtra'],
['free-use', 'gia_dibella_freeusefantasy', 'Gia Dibella in "Learning to Freeuse"', 'freeusefantasy'],
['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9"', 'julesjordan'], ['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9"', 'julesjordan'],
['gangbang', 'kristen_scott_julesjordan', 'Kristen Scott in "Interracial Gangbang!"', 'julesjordan'], ['gangbang', 'kristen_scott_julesjordan', 'Kristen Scott in "Interracial Gangbang!"', 'julesjordan'],
['gangbang', 'emily_willis_blacked', 'Emily Willis', 'blacked'], ['gangbang', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
@ -1066,14 +1067,14 @@ const tagMedia = [
})); }));
/* eslint-disable max-len */ /* eslint-disable max-len */
exports.seed = knex => Promise.resolve() exports.seed = (knex) => Promise.resolve()
.then(async () => { .then(async () => {
await upsert('media', sfw, 'id'); await upsert('media', sfw, 'id');
const tags = await knex('tags').whereIn('slug', tagMedia.map(item => item.tagSlug)); const tags = await knex('tags').whereIn('slug', tagMedia.map((item) => item.tagSlug));
const entities = await knex('entities') const entities = await knex('entities')
.whereIn('slug', tagMedia.map(item => item.entitySlug).filter(Boolean)) .whereIn('slug', tagMedia.map((item) => item.entitySlug).filter(Boolean))
.orderBy('type', 'DESC'); .orderBy('type', 'DESC');
const entitiesBySlug = entities.reduce((acc, entity) => ({ const entitiesBySlug = entities.reduce((acc, entity) => ({
@ -1093,7 +1094,7 @@ exports.seed = knex => Promise.resolve()
concurrency: 20, concurrency: 20,
}); });
const { inserted, updated } = await upsert('media', tagMediaWithDimensions.map(media => ({ const { inserted, updated } = await upsert('media', tagMediaWithDimensions.map((media) => ({
id: media.id, id: media.id,
path: media.path, path: media.path,
thumbnail: media.thumbnail, thumbnail: media.thumbnail,
@ -1114,15 +1115,15 @@ exports.seed = knex => Promise.resolve()
[tagPhoto.tagSlug]: (acc[tagPhoto.tagSlug] || []).concat(tagPhoto), [tagPhoto.tagSlug]: (acc[tagPhoto.tagSlug] || []).concat(tagPhoto),
}), {}); }), {});
const tagPosters = Object.values(tagMediaBySlug).map(tag => tag[0]); const tagPosters = Object.values(tagMediaBySlug).map((tag) => tag[0]);
const tagPhotos = Object.values(tagMediaBySlug).map(tag => tag.slice(1)).flat(); const tagPhotos = Object.values(tagMediaBySlug).map((tag) => tag.slice(1)).flat();
const tagPosterEntries = tagPosters.map(poster => ({ const tagPosterEntries = tagPosters.map((poster) => ({
tag_id: tagIdsBySlug[poster.tagSlug], tag_id: tagIdsBySlug[poster.tagSlug],
media_id: mediaIdsByPath[poster.path], media_id: mediaIdsByPath[poster.path],
})); }));
const tagPhotoEntries = tagPhotos.map(photo => ({ const tagPhotoEntries = tagPhotos.map((photo) => ({
tag_id: tagIdsBySlug[photo.tagSlug], tag_id: tagIdsBySlug[photo.tagSlug],
media_id: mediaIdsByPath[photo.path], media_id: mediaIdsByPath[photo.path],
})); }));
@ -1135,10 +1136,10 @@ exports.seed = knex => Promise.resolve()
// clean up (re)moved tag media // clean up (re)moved tag media
await Promise.all([ await Promise.all([
knex('tags_posters') knex('tags_posters')
.whereNotIn('media_id', tagPosters.map(photo => photo.id)) .whereNotIn('media_id', tagPosters.map((photo) => photo.id))
.delete(), .delete(),
knex('tags_photos') knex('tags_photos')
.whereNotIn('media_id', tagPhotos.map(photo => photo.id)) .whereNotIn('media_id', tagPhotos.map((photo) => photo.id))
.delete(), .delete(),
]); ]);
}); });

View File

@ -1,7 +1,7 @@
{ {
"extends": "airbnb-base", "extends": "airbnb-base",
"parserOptions": { "parserOptions": {
"parser": "babel-eslint", "parser": "@babel/eslint-parser",
"sourceType": "script" "sourceType": "script"
}, },
"rules": { "rules": {

View File

@ -124,9 +124,9 @@ function getMostFrequent(items) {
} }
function getMostFrequentDate(dates) { function getMostFrequentDate(dates) {
const year = getMostFrequent(dates.map(dateX => dateX.getFullYear())); const year = getMostFrequent(dates.map((dateX) => dateX.getFullYear()));
const month = getMostFrequent(dates.map(dateX => dateX.getMonth())); const month = getMostFrequent(dates.map((dateX) => dateX.getMonth()));
const date = getMostFrequent(dates.map(dateX => dateX.getDate())); const date = getMostFrequent(dates.map((dateX) => dateX.getDate()));
if (year === null || month === null || date === null) { if (year === null || month === null || date === null) {
return null; return null;
@ -153,7 +153,7 @@ function toBaseActors(actorsOrNames, release) {
} }
const baseActors = actorsOrNames const baseActors = actorsOrNames
.filter(actorOrName => actorOrName && (typeof actorOrName === 'string' || actorOrName.name)) .filter((actorOrName) => actorOrName && (typeof actorOrName === 'string' || actorOrName.name))
.map((actorOrName) => { .map((actorOrName) => {
const [baseName, entryId] = (actorOrName.name || actorOrName).split(':'); const [baseName, entryId] = (actorOrName.name || actorOrName).split(':');
@ -265,7 +265,7 @@ function curateActor(actor, withDetails = false, isProfile = false) {
size: actor.avatar.size, size: actor.avatar.size,
source: actor.avatar.source, source: actor.avatar.source,
}, },
...(actor.profiles && { profiles: actor.profiles?.map(profile => curateActor(profile, true, true)) }), ...(actor.profiles && { profiles: actor.profiles?.map((profile) => curateActor(profile, true, true)) }),
}), }),
}; };
@ -285,7 +285,7 @@ function curateActorEntry(baseActor, batchId) {
} }
function curateActorEntries(baseActors, batchId) { function curateActorEntries(baseActors, batchId) {
return baseActors.map(baseActor => curateActorEntry(baseActor, batchId)); return baseActors.map((baseActor) => curateActorEntry(baseActor, batchId));
} }
function curateProfileEntry(profile) { function curateProfileEntry(profile) {
@ -448,7 +448,7 @@ async function curateProfile(profile, actor) {
curatedProfile.scenes = toBaseReleases(profile.scenes || profile.releases, profile.entity, actor) curatedProfile.scenes = toBaseReleases(profile.scenes || profile.releases, profile.entity, actor)
// attach actor to base scene, in case it was not scraped // attach actor to base scene, in case it was not scraped
.map((scene) => { .map((scene) => {
if (actor && !scene.actors?.find(sceneActor => slugify(sceneActor) === actor.slug || slugify(sceneActor.name) === actor.slug)) { if (actor && !scene.actors?.find((sceneActor) => slugify(sceneActor) === actor.slug || slugify(sceneActor.name) === actor.slug)) {
return { return {
...scene, ...scene,
actors: [actor, ...(scene.actors || [])], actors: [actor, ...(scene.actors || [])],
@ -477,10 +477,10 @@ async function fetchProfiles(actorIdsOrNames) {
.modify((query) => { .modify((query) => {
if (actorIdsOrNames) { if (actorIdsOrNames) {
query query
.whereIn('actor_id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number')) .whereIn('actor_id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((builder) => { .orWhere((builder) => {
builder builder
.whereIn('actors.name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string')) .whereIn('actors.name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('actors.entity_id'); .whereNull('actors.entity_id');
}); });
} }
@ -517,12 +517,12 @@ async function interpolateProfiles(actorIdsOrNames) {
...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }), ...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }),
...(profile.birth_state && { state: profile.birth_state }), ...(profile.birth_state && { state: profile.birth_state }),
...(profile.birth_city && { city: profile.birth_city }), ...(profile.birth_city && { city: profile.birth_city }),
}].filter(location => Object.keys(location).length > 0), }].filter((location) => Object.keys(location).length > 0),
residence: [...acc.residence || [], { residence: [...acc.residence || [], {
...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }), ...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }),
...(profile.residence_state && { state: profile.residence_state }), ...(profile.residence_state && { state: profile.residence_state }),
...(profile.residence_city && { city: profile.residence_city }), ...(profile.residence_city && { city: profile.residence_city }),
}].filter(location => Object.keys(location).length > 0), }].filter((location) => Object.keys(location).length > 0),
}), {}); }), {});
const mostFrequentValues = [ const mostFrequentValues = [
@ -549,7 +549,7 @@ async function interpolateProfiles(actorIdsOrNames) {
...mostFrequentValues, ...mostFrequentValues,
}; };
profile.height = getMostFrequent(valuesByProperty.height.filter(height => height > 50 && height < 300)); // remove unlikely values profile.height = getMostFrequent(valuesByProperty.height.filter((height) => height > 50 && height < 300)); // remove unlikely values
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth); profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death); profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
@ -558,21 +558,21 @@ async function interpolateProfiles(actorIdsOrNames) {
profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs); profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs);
// ensure most frequent country, city and state match up // ensure most frequent country, city and state match up
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map(location => location.country)); profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map((location) => location.country));
const remainingOriginCountries = valuesByProperty.origin.filter(location => location.country === profile.birth_country_alpha2); const remainingOriginCountries = valuesByProperty.origin.filter((location) => location.country === profile.birth_country_alpha2);
profile.birth_state = getMostFrequent(remainingOriginCountries.map(location => location.state)); profile.birth_state = getMostFrequent(remainingOriginCountries.map((location) => location.state));
const remainingOriginStates = remainingOriginCountries.filter(location => !profile.birth_state || location.state === profile.birth_state); const remainingOriginStates = remainingOriginCountries.filter((location) => !profile.birth_state || location.state === profile.birth_state);
profile.birth_city = getMostFrequent(remainingOriginStates.map(location => location.city)); profile.birth_city = getMostFrequent(remainingOriginStates.map((location) => location.city));
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map(location => location.country)); profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map((location) => location.country));
const remainingResidenceCountries = valuesByProperty.residence.filter(location => location.country === profile.residence_country_alpha2); const remainingResidenceCountries = valuesByProperty.residence.filter((location) => location.country === profile.residence_country_alpha2);
profile.residence_state = getMostFrequent(remainingResidenceCountries.map(location => location.state)); profile.residence_state = getMostFrequent(remainingResidenceCountries.map((location) => location.state));
const remainingResidenceStates = remainingResidenceCountries.filter(location => !profile.residence_state || location.state === profile.residence_state); const remainingResidenceStates = remainingResidenceCountries.filter((location) => !profile.residence_state || location.state === profile.residence_state);
profile.residence_city = getMostFrequent(remainingResidenceStates.map(location => location.city)); profile.residence_city = getMostFrequent(remainingResidenceStates.map((location) => location.city));
profile.weight = getAverage(valuesByProperty.weight); profile.weight = getAverage(valuesByProperty.weight);
@ -580,8 +580,8 @@ async function interpolateProfiles(actorIdsOrNames) {
profile.piercings = getLongest(valuesByProperty.piercings); profile.piercings = getLongest(valuesByProperty.piercings);
profile.avatar_media_id = actorProfiles profile.avatar_media_id = actorProfiles
.map(actorProfile => actorProfile.avatar) .map((actorProfile) => actorProfile.avatar)
.filter(avatar => avatar && (avatar.entropy === null || avatar.entropy > 5.5)) .filter((avatar) => avatar && (avatar.entropy === null || avatar.entropy > 5.5))
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; .sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
return profile; return profile;
@ -598,10 +598,10 @@ async function interpolateProfiles(actorIdsOrNames) {
.modify((modifyBuilder) => { .modify((modifyBuilder) => {
if (actorIdsOrNames) { if (actorIdsOrNames) {
modifyBuilder modifyBuilder
.whereIn('id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number')) .whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((whereBuilder) => { .orWhere((whereBuilder) => {
whereBuilder whereBuilder
.whereIn('name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string')) .whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('entity_id'); .whereNull('entity_id');
}); });
} }
@ -610,7 +610,7 @@ async function interpolateProfiles(actorIdsOrNames) {
.transacting(transaction); .transacting(transaction);
// insert new interpolated data // insert new interpolated data
const queries = interpolatedProfiles.map(profile => knex('actors') const queries = interpolatedProfiles.map((profile) => knex('actors')
.where('id', profile.id) .where('id', profile.id)
.update(profile) .update(profile)
.transacting(transaction)); .transacting(transaction));
@ -621,8 +621,8 @@ async function interpolateProfiles(actorIdsOrNames) {
} }
async function upsertProfiles(profiles) { async function upsertProfiles(profiles) {
const newProfileEntries = profiles.filter(profile => !profile.update).map(profile => curateProfileEntry(profile)).filter(Boolean); const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
const updatingProfileEntries = profiles.filter(profile => profile.update).map(profile => curateProfileEntry(profile)).filter(Boolean); const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
if (newProfileEntries.length > 0) { if (newProfileEntries.length > 0) {
await bulkInsert('actors_profiles', newProfileEntries); await bulkInsert('actors_profiles', newProfileEntries);
@ -632,7 +632,7 @@ async function upsertProfiles(profiles) {
if (argv.force && updatingProfileEntries.length > 0) { if (argv.force && updatingProfileEntries.length > 0) {
const transaction = await knex.transaction(); const transaction = await knex.transaction();
const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles') const queries = updatingProfileEntries.map((profileEntry) => knex('actors_profiles')
.where('id', profileEntry.id) .where('id', profileEntry.id)
.update(profileEntry) .update(profileEntry)
.returning(['id', 'actor_id']) .returning(['id', 'actor_id'])
@ -647,7 +647,7 @@ async function upsertProfiles(profiles) {
} }
async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) { async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) {
const validSources = actor.entity ? sources.filter(source => source === actor.entity.slug) : sources; const validSources = actor.entity ? sources.filter((source) => source === actor.entity.slug) : sources;
const profiles = Promise.map(validSources, async (source) => { const profiles = Promise.map(validSources, async (source) => {
try { try {
@ -748,12 +748,12 @@ async function getActorNames(actorNames) {
) )
`, [argv.actorsUpdate || new Date()]); `, [argv.actorsUpdate || new Date()]);
return actorsWithoutProfiles.rows.map(actor => actor.name); return actorsWithoutProfiles.rows.map((actor) => actor.name);
} }
async function storeProfiles(profiles) { async function storeProfiles(profiles) {
const profilesWithAvatarIds = await associateAvatars(profiles); const profilesWithAvatarIds = await associateAvatars(profiles);
const actorIds = Array.from(new Set(profiles.map(profile => profile.id))); const actorIds = Array.from(new Set(profiles.map((profile) => profile.id)));
await upsertProfiles(profilesWithAvatarIds); await upsertProfiles(profilesWithAvatarIds);
await interpolateProfiles(actorIds); await interpolateProfiles(actorIds);
@ -772,7 +772,7 @@ async function scrapeActors(argNames) {
fetchEntitiesBySlug(entitySlugs, 'desc'), fetchEntitiesBySlug(entitySlugs, 'desc'),
knex('actors') knex('actors')
.select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity')) .select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
.whereIn('actors.slug', baseActors.map(baseActor => baseActor.slug)) .whereIn('actors.slug', baseActors.map((baseActor) => baseActor.slug))
.whereNull('actors.alias_for') .whereNull('actors.alias_for')
.leftJoin('entities', 'entities.id', 'actors.entity_id') .leftJoin('entities', 'entities.id', 'actors.entity_id')
.groupBy('actors.id', 'entities.id'), .groupBy('actors.id', 'entities.id'),
@ -786,7 +786,7 @@ async function scrapeActors(argNames) {
}, },
}), {}); }), {});
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlugAndEntryId[baseActor.slug]?.[baseActor.entryId]); const newBaseActors = baseActors.filter((baseActor) => !existingActorEntriesBySlugAndEntryId[baseActor.slug]?.[baseActor.entryId]);
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null]; const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId); const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
@ -799,7 +799,7 @@ async function scrapeActors(argNames) {
const existingProfiles = await knex('actors_profiles') const existingProfiles = await knex('actors_profiles')
.select(knex.raw('actors_profiles.*, row_to_json(avatars) as avatar')) .select(knex.raw('actors_profiles.*, row_to_json(avatars) as avatar'))
.whereIn('actor_id', actors.map(actor => actor.id)) .whereIn('actor_id', actors.map((actor) => actor.id))
.leftJoin('media as avatars', 'avatars.id', 'actors_profiles.avatar_media_id'); .leftJoin('media as avatars', 'avatars.id', 'actors_profiles.avatar_media_id');
const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({ const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({
@ -812,7 +812,7 @@ async function scrapeActors(argNames) {
const profilesPerActor = await Promise.map( const profilesPerActor = await Promise.map(
actors, actors,
async actor => scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId), async (actor) => scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId),
{ concurrency: 10 }, { concurrency: 10 },
); );
@ -833,7 +833,7 @@ async function scrapeActors(argNames) {
async function getOrCreateActors(baseActors, batchId) { async function getOrCreateActors(baseActors, batchId) {
// WHERE IN causes stack depth error and performance issues with a large amount of values, no knex VALUES helper available // WHERE IN causes stack depth error and performance issues with a large amount of values, no knex VALUES helper available
const actorValues = baseActors.map(actor => knex.raw('(:slug, :entityId, :entryId, :collisionLikely)', { const actorValues = baseActors.map((actor) => knex.raw('(:slug, :entityId, :entryId, :collisionLikely)', {
slug: actor.slug, slug: actor.slug,
entityId: actor.entity.id, entityId: actor.entity.id,
entryId: actor.entryId, entryId: actor.entryId,
@ -867,7 +867,7 @@ async function getOrCreateActors(baseActors, batchId) {
}, },
}), {}); }), {});
const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.entity.id]?.[baseActor.entryId]?.[baseActor.slug] && !existingActorSlugs.null?.null?.[baseActor.slug]); const uniqueBaseActors = baseActors.filter((baseActor) => !existingActorSlugs[baseActor.entity.id]?.[baseActor.entryId]?.[baseActor.slug] && !existingActorSlugs.null?.null?.[baseActor.slug]);
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId); const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
const newActors = await bulkInsert('actors', curatedActorEntries); const newActors = await bulkInsert('actors', curatedActorEntries);
@ -884,13 +884,13 @@ async function getOrCreateActors(baseActors, batchId) {
}), {}); }), {});
const newActorProfiles = await Promise.all(baseActors const newActorProfiles = await Promise.all(baseActors
.filter(actor => actor.hasProfile) .filter((actor) => actor.hasProfile)
.map(actor => ({ .map((actor) => ({
...actor, ...actor,
id: newActorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || newActorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug], id: newActorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || newActorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug],
})) }))
.filter(actor => !!actor.id) .filter((actor) => !!actor.id)
.map(actor => curateProfile(actor))); .map((actor) => curateProfile(actor)));
await storeProfiles(newActorProfiles); await storeProfiles(newActorProfiles);
@ -950,16 +950,16 @@ async function associatePeople(releases, batchId, type = 'actor') {
const releaseActorAssociations = Object.entries(baseActorsByReleaseId) const releaseActorAssociations = Object.entries(baseActorsByReleaseId)
.map(([releaseId, releaseActors]) => releaseActors .map(([releaseId, releaseActors]) => releaseActors
.map(releaseActor => ({ .map((releaseActor) => ({
release_id: releaseId, release_id: releaseId,
...(actorIdsByEntityIdEntryIdAndSlug[releaseActor.entity?.id]?.[releaseActor.entryId]?.[releaseActor.slug] || actorIdsByEntityIdEntryIdAndSlug.null.null[releaseActor.slug]), ...(actorIdsByEntityIdEntryIdAndSlug[releaseActor.entity?.id]?.[releaseActor.entryId]?.[releaseActor.slug] || actorIdsByEntityIdEntryIdAndSlug.null.null[releaseActor.slug]),
}))) })))
.flat(); .flat();
const validReleaseActorAssociations = releaseActorAssociations.filter(association => association.release_id && association[personKey]); const validReleaseActorAssociations = releaseActorAssociations.filter((association) => association.release_id && association[personKey]);
if (releaseActorAssociations.length > validReleaseActorAssociations.length) { if (releaseActorAssociations.length > validReleaseActorAssociations.length) {
const invalidReleaseActorAssociations = releaseActorAssociations.filter(association => !association.release_id || !association[personKey]); const invalidReleaseActorAssociations = releaseActorAssociations.filter((association) => !association.release_id || !association[personKey]);
logger.error(invalidReleaseActorAssociations); logger.error(invalidReleaseActorAssociations);
} }
@ -1021,15 +1021,15 @@ async function searchActors(query) {
.from(knex.raw('search_actors(?) as actors', [query])) .from(knex.raw('search_actors(?) as actors', [query]))
.limit(100); .limit(100);
return actors.map(actor => curateActor(actor)); return actors.map((actor) => curateActor(actor));
} }
async function flushProfiles(actorIdsOrNames) { async function flushProfiles(actorIdsOrNames) {
const profiles = await fetchProfiles(actorIdsOrNames); const profiles = await fetchProfiles(actorIdsOrNames);
const actorNames = Array.from(new Set(profiles.map(profile => profile.actor.name))); const actorNames = Array.from(new Set(profiles.map((profile) => profile.actor.name)));
const deleteCount = await knex('actors_profiles') const deleteCount = await knex('actors_profiles')
.whereIn('id', profiles.map(profile => profile.id)) .whereIn('id', profiles.map((profile) => profile.id))
.delete(); .delete();
await interpolateProfiles(actorIdsOrNames); await interpolateProfiles(actorIdsOrNames);
@ -1050,14 +1050,14 @@ async function flushProfiles(actorIdsOrNames) {
async function deleteActors(actorIdsOrNames) { async function deleteActors(actorIdsOrNames) {
const actors = await knex('actors') const actors = await knex('actors')
.whereIn('id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number')) .whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((builder) => { .orWhere((builder) => {
builder builder
.whereIn('name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string')) .whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('entity_id'); .whereNull('entity_id');
}); });
const actorIds = actors.map(actor => actor.id); const actorIds = actors.map((actor) => actor.id);
const sceneIds = await knex('releases_actors') const sceneIds = await knex('releases_actors')
.select('releases.id') .select('releases.id')

View File

@ -22,15 +22,15 @@ async function addAlert(alert, sessionUser) {
.returning('id'); .returning('id');
await Promise.all([ await Promise.all([
alert.actors?.length > 0 && bulkInsert('alerts_actors', alert.actors.map(actorId => ({ alert.actors?.length > 0 && bulkInsert('alerts_actors', alert.actors.map((actorId) => ({
alert_id: alertId, alert_id: alertId,
actor_id: actorId, actor_id: actorId,
})), false), })), false),
alert.tags?.length > 0 && bulkInsert('alerts_tags', alert.tags.map(tagId => ({ alert.tags?.length > 0 && bulkInsert('alerts_tags', alert.tags.map((tagId) => ({
alert_id: alertId, alert_id: alertId,
tag_id: tagId, tag_id: tagId,
})), false), })), false),
alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map(stashId => ({ alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map((stashId) => ({
alert_id: alertId, alert_id: alertId,
stash_id: stashId, stash_id: stashId,
})), false), })), false),
@ -106,20 +106,20 @@ async function notify(scenes) {
)))) ))))
GROUP BY releases.id, users.id, alerts.id; GROUP BY releases.id, users.id, alerts.id;
`, { `, {
sceneIds: scenes.map(scene => scene.id), sceneIds: scenes.map((scene) => scene.id),
}); });
const notifications = releases.rows const notifications = releases.rows
.filter(alert => alert.notify) .filter((alert) => alert.notify)
.map(notification => ({ .map((notification) => ({
user_id: notification.user_id, user_id: notification.user_id,
alert_id: notification.alert_id, alert_id: notification.alert_id,
scene_id: notification.scene_id, scene_id: notification.scene_id,
})); }));
const stashes = releases.rows const stashes = releases.rows
.filter(release => release.stashes.length > 0) .filter((release) => release.stashes.length > 0)
.flatMap(release => release.stashes.map(stash => ({ .flatMap((release) => release.stashes.map((stash) => ({
scene_id: release.scene_id, scene_id: release.scene_id,
stash_id: stash, stash_id: stash,
}))); })));

View File

@ -22,6 +22,7 @@ const { flushOrphanedMedia } = require('./media');
const getFileEntries = require('./utils/file-entries'); const getFileEntries = require('./utils/file-entries');
const inspector = new Inspector(); const inspector = new Inspector();
let done = false;
function logActive() { function logActive() {
console.log('log active!'); console.log('log active!');
@ -32,24 +33,47 @@ function logActive() {
}, typeof argv.logActive === 'number' ? argv.logActive : 60000); }, typeof argv.logActive === 'number' ? argv.logActive : 60000);
} }
/*
function monitorMemory() {
logger.debug(`Memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
if (!done) {
setTimeout(() => monitorMemory(), 10000);
}
}
*/
async function stopMemorySample() { async function stopMemorySample() {
const profile = await inspector.heap.stopSampling(); const profile = await inspector.heap.stopSampling();
const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`; const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
await inspector.heap.disable(); await inspector.heap.disable();
await fs.writeFile(filepath, JSON.stringify(profile));
fs.writeFile(filepath, JSON.stringify(profile));
logger.info(`Saved heap sample to ${filepath}`); logger.info(`Saved heap sample to ${filepath}`);
} }
async function startMemorySample() {
await inspector.heap.enable();
await inspector.heap.startSampling();
// monitorMemory();
logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
setTimeout(async () => {
await stopMemorySample();
if (!done) {
await startMemorySample();
}
}, 30000);
}
async function init() { async function init() {
try { try {
if (argv.memory) { if (argv.memory) {
await inspector.heap.enable(); await startMemorySample();
await inspector.heap.startSampling();
logger.info('Started heap sampling');
} }
if (argv.logActive) { if (argv.logActive) {
@ -122,7 +146,7 @@ async function init() {
const actorNames = (argv.actors || []).concat(actorsFromFile || []); const actorNames = (argv.actors || []).concat(actorsFromFile || []);
const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames); const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.scenes).flat().filter(Boolean); const actorBaseScenes = argv.actors && argv.actorScenes && actors.map((actor) => actor.scenes).flat().filter(Boolean);
const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates(); const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
@ -133,10 +157,10 @@ async function init() {
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])]) ? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])]; : [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const sceneMovies = deepScenes ? deepScenes.filter(scene => scene.movie).map(scene => ({ ...scene.movie, entity: scene.entity })) : []; const sceneMovies = deepScenes ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies; const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : []; const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes; const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
if (argv.report) { if (argv.report) {
@ -150,23 +174,12 @@ async function init() {
await associateMovieScenes(storedMovies, storedScenes); await associateMovieScenes(storedMovies, storedScenes);
} }
if (argv.memory) {
await stopMemorySample();
}
knex.destroy();
} catch (error) { } catch (error) {
logger.error(error); logger.error(error);
if (argv.memory) {
await stopMemorySample();
} }
knex.destroy(); knex.destroy();
done = true;
throw error;
}
} }
module.exports = init; module.exports = init;

View File

@ -4,7 +4,11 @@ const config = require('config');
const yargs = require('yargs'); const yargs = require('yargs');
const moment = require('moment'); const moment = require('moment');
function interpretAfter(after) { function interpretAfter(after, ignoreIfEmpty = false) {
if (!after && ignoreIfEmpty) {
return null;
}
if (!after) { if (!after) {
return new Date(0, 0, 0); return new Date(0, 0, 0);
} }
@ -313,6 +317,6 @@ const { argv } = yargs
default: 60000, default: 60000,
}) })
.coerce('after', interpretAfter) .coerce('after', interpretAfter)
.coerce('actors-update', interpretAfter); .coerce('actors-update', (after) => interpretAfter(after, true));
module.exports = argv; module.exports = argv;

View File

@ -185,7 +185,7 @@ async function scrapeReleases(baseReleases, entitiesBySlug, type) {
return Promise.map( return Promise.map(
baseReleases, baseReleases,
async baseRelease => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type), async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
{ concurrency: 10 }, { concurrency: 10 },
); );
} }

View File

@ -46,7 +46,7 @@ function curateEntity(entity, includeParameters = false) {
} : {}; } : {};
if (entity.tags) { if (entity.tags) {
curatedEntity.tags = entity.tags.map(tag => ({ curatedEntity.tags = entity.tags.map((tag) => ({
id: tag.id, id: tag.id,
name: tag.name, name: tag.name,
slug: tag.slug, slug: tag.slug,
@ -59,14 +59,14 @@ function curateEntity(entity, includeParameters = false) {
} }
if (entity.children) { if (entity.children) {
curatedEntity.children = entity.children.map(child => curateEntity({ curatedEntity.children = entity.children.map((child) => curateEntity({
...child, ...child,
parent: curatedEntity.id ? curatedEntity : null, parent: curatedEntity.id ? curatedEntity : null,
}, includeParameters)); }, includeParameters));
} }
if (entity.included_children) { if (entity.included_children) {
curatedEntity.includedChildren = entity.included_children.map(child => curateEntity({ curatedEntity.includedChildren = entity.included_children.map((child) => curateEntity({
...child, ...child,
parent: curatedEntity.id ? curatedEntity : null, parent: curatedEntity.id ? curatedEntity : null,
}, includeParameters)); }, includeParameters));
@ -79,7 +79,7 @@ function curateEntity(entity, includeParameters = false) {
} }
async function curateEntities(entities, includeParameters) { async function curateEntities(entities, includeParameters) {
return Promise.all(entities.map(async entity => curateEntity(entity, includeParameters))); return Promise.all(entities.map(async (entity) => curateEntity(entity, includeParameters)));
} }
function urlToSiteSlug(url) { function urlToSiteSlug(url) {
@ -102,8 +102,8 @@ async function fetchIncludedEntities() {
includeAll: !argv.networks && !argv.channels && !config.include?.networks && !config.include?.channels, includeAll: !argv.networks && !argv.channels && !config.include?.networks && !config.include?.channels,
includedNetworks: argv.networks || (!argv.channels && config.include?.networks) || [], includedNetworks: argv.networks || (!argv.channels && config.include?.networks) || [],
includedChannels: argv.channels || (!argv.networks && config.include?.channels) || [], includedChannels: argv.channels || (!argv.networks && config.include?.channels) || [],
excludedNetworks: argv.excludeNetworks || config.exclude?.networks.filter(network => !argv.networks?.includes(network)) || [], // ignore explicitly included networks excludedNetworks: argv.excludeNetworks || config.exclude?.networks.filter((network) => !argv.networks?.includes(network)) || [], // ignore explicitly included networks
excludedChannels: argv.excludeChannels || config.exclude?.channels.filter(channel => !argv.channels?.includes(channel)) || [], // ignore explicitly included channels excludedChannels: argv.excludeChannels || config.exclude?.channels.filter((channel) => !argv.channels?.includes(channel)) || [], // ignore explicitly included channels
}; };
const rawNetworks = await knex.raw(` const rawNetworks = await knex.raw(`
@ -228,11 +228,11 @@ async function fetchEntitiesBySlug(entitySlugs, sort = 'asc') {
} }
async function fetchReleaseEntities(baseReleases) { async function fetchReleaseEntities(baseReleases) {
const baseReleasesWithoutEntity = baseReleases.filter(release => release.url && !release.site && !release.entity); const baseReleasesWithoutEntity = baseReleases.filter((release) => release.url && !release.site && !release.entity);
const entitySlugs = Array.from(new Set( const entitySlugs = Array.from(new Set(
baseReleasesWithoutEntity baseReleasesWithoutEntity
.map(baseRelease => urlToSiteSlug(baseRelease.url)) .map((baseRelease) => urlToSiteSlug(baseRelease.url))
.filter(Boolean), .filter(Boolean),
)); ));

View File

@ -16,7 +16,7 @@ function logger(filepath) {
return winston.createLogger({ return winston.createLogger({
format: winston.format.combine( format: winston.format.combine(
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format(info => (info instanceof Error winston.format((info) => (info instanceof Error
? { ...info, message: info.stack } ? { ...info, message: info.stack }
: { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(), : { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(),
winston.format.colorize(), winston.format.colorize(),

View File

@ -190,7 +190,7 @@ function sortBaseTrailersByQuality(sources, role) {
function fallbackMediaToBaseMedia(rawMedia, role, metadata) { function fallbackMediaToBaseMedia(rawMedia, role, metadata) {
const baseSources = rawMedia const baseSources = rawMedia
.map(source => toBaseSource(source)) .map((source) => toBaseSource(source))
.filter(Boolean); .filter(Boolean);
const sortedBaseSources = sortBaseTrailersByQuality(baseSources, role); const sortedBaseSources = sortBaseTrailersByQuality(baseSources, role);
@ -225,12 +225,12 @@ function toBaseMedias(rawMedias, role, metadata) {
async function findSourceDuplicates(baseMedias) { async function findSourceDuplicates(baseMedias) {
const sourceUrls = baseMedias const sourceUrls = baseMedias
.map(baseMedia => baseMedia.sources.map(source => source.src)) .map((baseMedia) => baseMedia.sources.map((source) => source.src))
.flat() .flat()
.filter(Boolean); .filter(Boolean);
const extractUrls = baseMedias const extractUrls = baseMedias
.map(baseMedia => baseMedia.sources.map(source => source.url)) .map((baseMedia) => baseMedia.sources.map((source) => source.url))
.flat() .flat()
.filter(Boolean); .filter(Boolean);
@ -246,12 +246,12 @@ async function findSourceDuplicates(baseMedias) {
} }
async function findHashDuplicates(medias) { async function findHashDuplicates(medias) {
const hashes = medias.map(media => media.meta?.hash || media.entry?.hash).filter(Boolean); const hashes = medias.map((media) => media.meta?.hash || media.entry?.hash).filter(Boolean);
const existingHashMediaEntries = await knex('media').whereIn('hash', hashes); const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash'); const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
const uniqueHashMedias = medias.filter(media => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]); const uniqueHashMedias = medias.filter((media) => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => { const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
if (!media.meta?.hash) { if (!media.meta?.hash) {
@ -278,8 +278,8 @@ async function findHashDuplicates(medias) {
const selfUniqueHashMedias = Object.values(selfUniqueMediasByHash); const selfUniqueHashMedias = Object.values(selfUniqueMediasByHash);
const existingHashMedias = medias const existingHashMedias = medias
.filter(media => existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash]) .filter((media) => existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash])
.map(media => ({ .map((media) => ({
...media, ...media,
entry: existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash], entry: existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash],
})) }))
@ -563,8 +563,8 @@ streamQueue.define('fetchStreamSource', async ({ source, tempFileTarget, hashStr
const video = ffmpeg(source.stream) const video = ffmpeg(source.stream)
.format('mp4') .format('mp4')
.outputOptions(['-movflags frag_keyframe+empty_moov']) .outputOptions(['-movflags frag_keyframe+empty_moov'])
.on('start', cmd => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`)) .on('start', (cmd) => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
.on('error', error => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`)) .on('error', (error) => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
.pipe(); .pipe();
await pipeline(video, hashStream, tempFileTarget); await pipeline(video, hashStream, tempFileTarget);
@ -745,7 +745,7 @@ async function storeMedias(baseMedias, options) {
const fetchedMedias = await Promise.map( const fetchedMedias = await Promise.map(
baseMedias, baseMedias,
async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }), async (baseMedia) => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
{ concurrency: 100 }, // don't overload disk (or network, although this has its own throttling) { concurrency: 100 }, // don't overload disk (or network, although this has its own throttling)
); );
@ -753,7 +753,7 @@ async function storeMedias(baseMedias, options) {
const savedMedias = await Promise.map( const savedMedias = await Promise.map(
uniqueHashMedias, uniqueHashMedias,
async baseMedia => storeFile(baseMedia, options), async (baseMedia) => storeFile(baseMedia, options),
{ concurrency: 100 }, // don't overload disk { concurrency: 100 }, // don't overload disk
); );
@ -761,13 +761,13 @@ async function storeMedias(baseMedias, options) {
// overwrite files in case image processing was changed // overwrite files in case image processing was changed
await Promise.map( await Promise.map(
existingHashMedias, existingHashMedias,
async baseMedia => storeFile(baseMedia, options), async (baseMedia) => storeFile(baseMedia, options),
{ concurrency: 100 }, // don't overload disk { concurrency: 100 }, // don't overload disk
); );
} }
const newMediaWithEntries = savedMedias.filter(Boolean).map((media, index) => curateMediaEntry(media, index)); const newMediaWithEntries = savedMedias.filter(Boolean).map((media, index) => curateMediaEntry(media, index));
const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry); const newMediaEntries = newMediaWithEntries.filter((media) => media.newEntry).map((media) => media.entry);
try { try {
await bulkInsert('media', newMediaEntries, false); await bulkInsert('media', newMediaEntries, false);
@ -851,7 +851,7 @@ async function associateAvatars(profiles) {
return profiles; return profiles;
} }
const profilesWithBaseMedias = profiles.map(profile => (profile.avatar const profilesWithBaseMedias = profiles.map((profile) => (profile.avatar
? { ? {
...profile, ...profile,
avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', { avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', {
@ -862,7 +862,7 @@ async function associateAvatars(profiles) {
: profile : profile
)); ));
const baseMedias = profilesWithBaseMedias.map(profile => profile.avatarBaseMedia).filter(Boolean); const baseMedias = profilesWithBaseMedias.map((profile) => profile.avatarBaseMedia).filter(Boolean);
const storedMedias = await storeMedias(baseMedias, { stats: true }); const storedMedias = await storeMedias(baseMedias, { stats: true });
const storedMediasById = itemsByKey(storedMedias, 'id'); const storedMediasById = itemsByKey(storedMedias, 'id');
@ -885,13 +885,13 @@ async function associateAvatars(profiles) {
async function deleteS3Objects(media) { async function deleteS3Objects(media) {
const objects = media const objects = media
.map(item => [ .map((item) => [
{ Key: item.path }, { Key: item.path },
{ Key: item.thumbnail }, { Key: item.thumbnail },
{ Key: item.lazy }, { Key: item.lazy },
]) ])
.flat() .flat()
.filter(item => item.Key); .filter((item) => item.Key);
const status = await s3.deleteObjects({ const status = await s3.deleteObjects({
Bucket: config.s3.bucket, Bucket: config.s3.bucket,
@ -936,7 +936,7 @@ async function flushOrphanedMedia() {
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy']) .returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
.delete(); .delete();
await Promise.all(orphanedMedia.filter(media => !media.is_s3).map(media => Promise.all([ await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([
media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }), media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),
media.thumbnail && fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }), media.thumbnail && fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }),
media.lazy && fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }), media.lazy && fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }),
@ -945,7 +945,7 @@ async function flushOrphanedMedia() {
logger.info(`Removed ${orphanedMedia.length} media files from database and storage`); logger.info(`Removed ${orphanedMedia.length} media files from database and storage`);
if (config.s3.enabled) { if (config.s3.enabled) {
await deleteS3Objects(orphanedMedia.filter(media => media.is_s3)); await deleteS3Objects(orphanedMedia.filter((media) => media.is_s3));
} }
await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true }); await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });

View File

@ -142,7 +142,7 @@ function curateRelease(release, withMedia = false, withPoster = true) {
slug: release.parent.slug, slug: release.parent.slug,
}, },
}, },
actors: (release.actors || []).map(actor => ({ actors: (release.actors || []).map((actor) => ({
id: actor.id, id: actor.id,
name: actor.name, name: actor.name,
slug: actor.slug, slug: actor.slug,
@ -150,12 +150,12 @@ function curateRelease(release, withMedia = false, withPoster = true) {
entityId: actor.entity_id, entityId: actor.entity_id,
aliasFor: actor.alias_for, aliasFor: actor.alias_for,
})), })),
tags: (release.tags || []).map(tag => ({ tags: (release.tags || []).map((tag) => ({
id: tag.id, id: tag.id,
name: tag.name, name: tag.name,
slug: tag.slug, slug: tag.slug,
})), })),
chapters: (release.chapters || []).map(chapter => ({ chapters: (release.chapters || []).map((chapter) => ({
id: chapter.id, id: chapter.id,
index: chapter.index, index: chapter.index,
time: chapter.time, time: chapter.time,
@ -174,7 +174,7 @@ function curateRelease(release, withMedia = false, withPoster = true) {
} : null, } : null,
}), }),
...(withMedia && { ...(withMedia && {
photos: (release.photos || []).map(photo => ({ photos: (release.photos || []).map((photo) => ({
id: photo.id, id: photo.id,
path: photo.path, path: photo.path,
thumbnail: release.poster.thumbnail, thumbnail: release.poster.thumbnail,
@ -207,16 +207,16 @@ function curateGraphqlRelease(release) {
description: release.description || null, description: release.description || null,
duration: release.duration, duration: release.duration,
entity: release.entity, entity: release.entity,
actors: release.actors.map(actor => actor.actor), actors: release.actors.map((actor) => actor.actor),
tags: release.tags.map(tag => tag.tag), tags: release.tags.map((tag) => tag.tag),
...(release.chapters && { chapters: release.chapters.map(chapter => ({ ...(release.chapters && { chapters: release.chapters.map((chapter) => ({
...chapter, ...chapter,
tags: chapter.tags.map(tag => tag.tag), tags: chapter.tags.map((tag) => tag.tag),
poster: chapter.poster?.media || null, poster: chapter.poster?.media || null,
photos: chapter.photos.map(photo => photo.media), photos: chapter.photos.map((photo) => photo.media),
})) }), })) }),
poster: release.poster?.media || null, poster: release.poster?.media || null,
...(release.photos && { photos: release.photos.map(photo => photo.media) }), ...(release.photos && { photos: release.photos.map((photo) => photo.media) }),
trailer: release.trailer?.media || null, trailer: release.trailer?.media || null,
createdAt: release.createdAt, createdAt: release.createdAt,
}; };
@ -256,7 +256,7 @@ async function fetchScenes(limit = 100) {
limit: Math.min(limit, 10000), limit: Math.min(limit, 10000),
}); });
return releases.map(release => curateGraphqlRelease(release)); return releases.map((release) => curateGraphqlRelease(release));
} }
async function searchScenes(query, limit = 100, relevance = 0) { async function searchScenes(query, limit = 100, relevance = 0) {
@ -289,7 +289,7 @@ async function searchScenes(query, limit = 100, relevance = 0) {
relevance, relevance,
}); });
return releases.map(release => curateGraphqlRelease({ ...release.release, relevance: release.rank })); return releases.map((release) => curateGraphqlRelease({ ...release.release, relevance: release.rank }));
} }
async function deleteScenes(sceneIds) { async function deleteScenes(sceneIds) {

View File

@ -25,7 +25,7 @@ function scrapeAllTour(scenes, channel) {
release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, ''); release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, '');
release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY'); release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY');
release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map(actor => actor.trim()); release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map((actor) => actor.trim());
const poster = query.img('.scene-img-wrapper img'); const poster = query.img('.scene-img-wrapper img');
release.poster = [ release.poster = [
@ -124,7 +124,7 @@ async function scrapeRelease({ query, html }, url, channel, baseRelease, options
}; };
} }
release.photos = query.imgs('#dv_frames a > img').map(photo => [ release.photos = query.imgs('#dv_frames a > img').map((photo) => [
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`), photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`),
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`), photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`),
photo, photo,
@ -301,7 +301,7 @@ async function fetchProfile(baseActor, channel, include) {
const searchRes = await http.get(`${channel.url}/search/SearchAutoComplete_Agg_ByMedia?rows=9&name_startsWith=${slugify(baseActor.name, '+')}`); const searchRes = await http.get(`${channel.url}/search/SearchAutoComplete_Agg_ByMedia?rows=9&name_startsWith=${slugify(baseActor.name, '+')}`);
if (searchRes.ok) { if (searchRes.ok) {
const actorResult = searchRes.body.Results.find(result => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description)); const actorResult = searchRes.body.Results.find((result) => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description));
if (actorResult) { if (actorResult) {
return fetchProfilePage(`${channel.url}${actorResult.BasicResponseGroup.id}`, channel, include); return fetchProfilePage(`${channel.url}${actorResult.BasicResponseGroup.id}`, channel, include);

View File

@ -22,13 +22,13 @@ async function networkFetchScene(url, site, release) {
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const releases = await fetchApiLatest(site, page, false); const releases = await fetchApiLatest(site, page, false);
return releases.map(release => curateRelease(release, site)); return releases.map((release) => curateRelease(release, site));
} }
async function fetchUpcoming(site, page = 1) { async function fetchUpcoming(site, page = 1) {
const releases = await fetchApiUpcoming(site, page, false); const releases = await fetchApiUpcoming(site, page, false);
return releases.map(release => curateRelease(release, site)); return releases.map((release) => curateRelease(release, site));
} }
module.exports = { module.exports = {

View File

@ -34,7 +34,7 @@ function extractActors(scene) {
async function fetchLatestWrap(site, page = 1, include, preData) { async function fetchLatestWrap(site, page = 1, include, preData) {
const latest = await fetchLatest(site, page, include, preData); const latest = await fetchLatest(site, page, include, preData);
return latest.map(scene => extractActors(scene)); return latest.map((scene) => extractActors(scene));
} }
async function fetchSceneWrap(url, channel, baseRelease, include) { async function fetchSceneWrap(url, channel, baseRelease, include) {

View File

@ -13,7 +13,7 @@ function scrapeScene({ query }, channel) {
release.description = query.cnt('.latest_update_description'); release.description = query.cnt('.latest_update_description');
release.date = query.date('.update_date', 'MM/DD/YYYY'); release.date = query.date('.update_date', 'MM/DD/YYYY');
release.actors = query.all('.tour_update_models a').map(actorEl => ({ release.actors = query.all('.tour_update_models a').map((actorEl) => ({
name: query.cnt(actorEl), name: query.cnt(actorEl),
url: query.url(actorEl, null), url: query.url(actorEl, null),
})); }));
@ -30,7 +30,7 @@ function scrapeScene({ query }, channel) {
poster, poster,
]; ];
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map(img => [ release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
img.replace(/.jpg$/, '-full.jpg'), img.replace(/.jpg$/, '-full.jpg'),
img, img,
]); ]);

View File

@ -6,8 +6,8 @@ function extractActors(actorString) {
return actorString return actorString
?.replace(/.*:|\(.*\)|\d+(-|\s)year(-|\s)old|nurses?|tangled/ig, '') // remove Patient:, (date) and other nonsense ?.replace(/.*:|\(.*\)|\d+(-|\s)year(-|\s)old|nurses?|tangled/ig, '') // remove Patient:, (date) and other nonsense
.split(/\band\b|\bvs\b|\/|,|&/ig) .split(/\band\b|\bvs\b|\/|,|&/ig)
.map(actor => actor.trim()) .map((actor) => actor.trim())
.filter(actor => !!actor && !/\banal\b|\bschool\b|\bgamer\b|\breturn\b|\bfor\b|\bare\b|\bpart\b|realdoll|bimbo|p\d+/ig.test(actor)) .filter((actor) => !!actor && !/\banal\b|\bschool\b|\bgamer\b|\breturn\b|\bfor\b|\bare\b|\bpart\b|realdoll|bimbo|p\d+/ig.test(actor))
|| []; || [];
} }
@ -16,7 +16,7 @@ function matchActors(actorString, models) {
return []; return [];
} }
return models.filter(model => new RegExp(model.name, 'i').test(actorString)); return models.filter((model) => new RegExp(model.name, 'i').test(actorString));
} }
function scrapeLatest(scenes, site, models) { function scrapeLatest(scenes, site, models) {
@ -61,7 +61,7 @@ function scrapeScene({ html, qu }, url, site, include, models) {
release.tags = qu.all('.tags a', true); release.tags = qu.all('.tags a', true);
release.photos = qu.imgs('.stills img').map(photoPath => `${site.url}/${photoPath}`); release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
const posterIndex = 'splash:'; const posterIndex = 'splash:';
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4); const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
@ -101,7 +101,7 @@ async function fetchModels(site, page = 1, accModels = []) {
if (res.ok) { if (res.ok) {
const models = extractModels(res.item, site); const models = extractModels(res.item, site);
const nextPage = res.item.qa('.pagenumbers', true) const nextPage = res.item.qa('.pagenumbers', true)
.map(pageX => Number(pageX)) .map((pageX) => Number(pageX))
.filter(Boolean) // remove << and >> .filter(Boolean) // remove << and >>
.includes(page + 1); .includes(page + 1);

View File

@ -46,7 +46,7 @@ function scrapeScene({ html, qu }, url) {
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/); release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
release.actors = qu.all('h5:not(.video_categories) a').map(actor => ({ release.actors = qu.all('h5:not(.video_categories) a').map((actor) => ({
name: qu.q(actor, null, true), name: qu.q(actor, null, true),
url: qu.url(actor, null), url: qu.url(actor, null),
})); }));
@ -58,7 +58,7 @@ function scrapeScene({ html, qu }, url) {
const poster = qu.img('a img'); const poster = qu.img('a img');
release.poster = getFallbacks(poster); release.poster = getFallbacks(poster);
release.photos = qu.imgs('.featured-video img', 'src0_1x').map(source => getFallbacks(source)); release.photos = qu.imgs('.featured-video img', 'src0_1x').map((source) => getFallbacks(source));
return release; return release;
} }

View File

@ -30,7 +30,7 @@ function scrapeAll(scenes, channel) {
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY'); release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60; release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
release.actors = query.all('.video-card-details--cast a').map(el => ({ release.actors = query.all('.video-card-details--cast a').map((el) => ({
name: qu.query.cnt(el), name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }), url: qu.query.url(el, null, 'href', { origin: channel.url }),
})); }));
@ -57,7 +57,7 @@ function scrapeScene({ query }, url, channel) {
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/); release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60; release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
release.actors = query.all('.video-actors a').map(el => ({ release.actors = query.all('.video-actors a').map((el) => ({
name: qu.query.cnt(el), name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }), url: qu.query.url(el, null, 'href', { origin: channel.url }),
})); }));

View File

@ -22,13 +22,13 @@ function scrapeAll(scenes, site) {
if (/bts/i.test(release.title)) release.tags = ['behind the scenes']; if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
[release.poster, ...release.photos] = qu.all('.item-thumbs img') [release.poster, ...release.photos] = qu.all('.item-thumbs img')
.map(source => [ .map((source) => [
source.getAttribute('src0_3x'), source.getAttribute('src0_3x'),
source.getAttribute('src0_2x'), source.getAttribute('src0_2x'),
source.getAttribute('src0_1x'), source.getAttribute('src0_1x'),
] ]
.filter(Boolean) .filter(Boolean)
.map(fallback => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`))); .map((fallback) => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`)));
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`; release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
@ -116,7 +116,7 @@ async function scrapeProfile({ qu }, site, withScenes) {
qu.q('.profile-pic img', 'src0_3x'), qu.q('.profile-pic img', 'src0_3x'),
qu.q('.profile-pic img', 'src0_2x'), qu.q('.profile-pic img', 'src0_2x'),
qu.q('.profile-pic img', 'src0_1x'), qu.q('.profile-pic img', 'src0_1x'),
].filter(Boolean).map(source => (/^http/.test(source) ? source : `${site.url}${source}`)); ].filter(Boolean).map((source) => (/^http/.test(source) ? source : `${site.url}${source}`));
if (withScenes) { if (withScenes) {
const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1]; const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1];

View File

@ -48,7 +48,7 @@ async function fetchPhotos(scene) {
}); });
if (res.ok && res.body.images) { if (res.ok && res.body.images) {
return res.body.images.map(image => qu.prefixUrl(image, 'https://photos.bang.com')); return res.body.images.map((image) => qu.prefixUrl(image, 'https://photos.bang.com'));
} }
return null; return null;
@ -59,7 +59,7 @@ async function scrapeScene(scene, entity, options) {
entryId: scene.id, entryId: scene.id,
title: scene.name, title: scene.name,
description: scene.description, description: scene.description,
tags: scene.genres.concat(scene.actions).map(genre => genre.name), tags: scene.genres.concat(scene.actions).map((genre) => genre.name),
duration: scene.duration, duration: scene.duration,
}; };
@ -69,19 +69,19 @@ async function scrapeScene(scene, entity, options) {
const date = new Date(scene.releaseDate); const date = new Date(scene.releaseDate);
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: genderMap[actor.gender] })); release.actors = scene.actors.map((actor) => ({ name: actor.name, gender: genderMap[actor.gender] }));
if (scene.is4k) release.tags.push('4k'); if (scene.is4k) release.tags.push('4k');
if (scene.gay) release.tags.push('gay'); if (scene.gay) release.tags.push('gay');
const defaultPoster = scene.screenshots.find(photo => photo.default === true); const defaultPoster = scene.screenshots.find((photo) => photo.default === true);
const screens = scene.screenshots.filter(photo => photo.default === false); const screens = scene.screenshots.filter((photo) => photo.default === false);
const remainingScreens = defaultPoster ? screens : screens.slice(1); const remainingScreens = defaultPoster ? screens : screens.slice(1);
const poster = defaultPoster || screens[0]; const poster = defaultPoster || screens[0];
release.poster = getScreenUrl(poster, scene); release.poster = getScreenUrl(poster, scene);
release.photos = remainingScreens.map(photo => getScreenUrl(photo, scene)); release.photos = remainingScreens.map((photo) => getScreenUrl(photo, scene));
if (options?.includePhotos) { if (options?.includePhotos) {
const photos = await fetchPhotos(scene); const photos = await fetchPhotos(scene);
@ -399,7 +399,7 @@ async function fetchProfile({ name: actorName }, context, include) {
}); });
if (res.ok) { if (res.ok) {
const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase()); const actor = res.body.hits.hits.find((hit) => hit._source.name.toLowerCase() === actorName.toLowerCase());
if (actor) { if (actor) {
return scrapeProfile(actor._source, context.entity, include); return scrapeProfile(actor._source, context.entity, include);

View File

@ -8,10 +8,9 @@ function scrapeProfile(html) {
const profile = {}; const profile = {};
const bio = qu.all('.infobox tr[valign="top"]') const bio = qu.all('.infobox tr[valign="top"]')
.map(detail => qu.all(detail, 'td', true)) .map((detail) => qu.all(detail, 'td', true))
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {}); .reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
/* unreliable, see: Syren De Mer /* unreliable, see: Syren De Mer
const catlinks = qa('#mw-normal-catlinks a', true); const catlinks = qa('#mw-normal-catlinks a', true);
const isTrans = catlinks.some(link => link.match(/shemale|transgender/i)); const isTrans = catlinks.some(link => link.match(/shemale|transgender/i));

View File

@ -26,7 +26,7 @@ function scrapeAll(scenes) {
release.entryId = new URL(release.url).pathname.match(/\/videos\/([\w-]+)/)[1]; release.entryId = new URL(release.url).pathname.match(/\/videos\/([\w-]+)/)[1];
release.title = query.cnt('.title') || query.q('img', 'title'); release.title = query.cnt('.title') || query.q('img', 'title');
release.actors = subtitle.slice(subtitle.indexOf(':') + 1).split(',').map(actor => actor.trim()).filter(Boolean); release.actors = subtitle.slice(subtitle.indexOf(':') + 1).split(',').map((actor) => actor.trim()).filter(Boolean);
release.poster = query.img('.thumb img'); release.poster = query.img('.thumb img');
@ -48,13 +48,13 @@ function scrapeScene({ query, html }, url, channel) {
const dataString = query.html('.yoast-schema-graph'); const dataString = query.html('.yoast-schema-graph');
const data = dataString && JSON.parse(dataString)['@graph']; const data = dataString && JSON.parse(dataString)['@graph'];
const pageData = data.find(item => item['@type'] === 'WebPage'); const pageData = data.find((item) => item['@type'] === 'WebPage');
const imageData = data.find(item => item['@type'] === 'ImageObject'); const imageData = data.find((item) => item['@type'] === 'ImageObject');
release.entryId = new URL(url).pathname.match(/\/videos\/([\w-]+)/)[1]; release.entryId = new URL(url).pathname.match(/\/videos\/([\w-]+)/)[1];
release.title = query.cnt('.video .title h1') release.title = query.cnt('.video .title h1')
|| data.find(item => item['@type'] === 'BreadcrumbList')?.itemListElement.slice(-1)[0].item.name || data.find((item) => item['@type'] === 'BreadcrumbList')?.itemListElement.slice(-1)[0].item.name
|| pageData?.name.slice(0, pageData.name.lastIndexOf('-')).trim(); || pageData?.name.slice(0, pageData.name.lastIndexOf('-')).trim();
release.description = query.cnt('.video .descript'); release.description = query.cnt('.video .descript');

View File

@ -66,7 +66,7 @@ function scrapeProfile({ query }) {
const profile = {}; const profile = {};
const keys = query.all('.model-descr_line:not(.model-descr_rait) p.text span', true); const keys = query.all('.model-descr_line:not(.model-descr_rait) p.text span', true);
const values = query.all('.model-descr_line:not(.model-descr_rait) p.text').map(el => query.text(el)); const values = query.all('.model-descr_line:not(.model-descr_rait) p.text').map((el) => query.text(el));
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {}); const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)\s*cm\)/)?.[1]); if (bio.height) profile.height = Number(bio.height.match(/\((\d+)\s*cm\)/)?.[1]);
@ -100,7 +100,7 @@ function scrapeProfile({ query }) {
profile.piercings = bio.piercings; profile.piercings = bio.piercings;
} }
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim()); if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim());
const avatar = query.q('.model-img img'); const avatar = query.q('.model-img img');
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src; profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;

View File

@ -43,7 +43,7 @@ function scrapeScene({ query }, channel, html) {
release.date = date; release.date = date;
release.datePrecision = precision; release.datePrecision = precision;
release.actors = query.all('.sub-video .pornstar-link').map(el => ({ release.actors = query.all('.sub-video .pornstar-link').map((el) => ({
name: query.cnt(el, null), name: query.cnt(el, null),
url: query.url(el, null, 'href', { origin: 'https://www.cumlouder.com' }), url: query.url(el, null, 'href', { origin: 'https://www.cumlouder.com' }),
})); }));

View File

@ -52,10 +52,10 @@ async function scrapeScene({ query }, url, channel) {
release.poster = query.poster() || query.poster('dl8-video') || query.img('#videoBlock img'); release.poster = query.poster() || query.poster('dl8-video') || query.img('#videoBlock img');
release.photos = query.urls('.photo-slider-guest .card a'); release.photos = query.urls('.photo-slider-guest .card a');
release.trailer = query.all('source[type="video/mp4"]').map(trailer => ({ release.trailer = query.all('source[type="video/mp4"]').map((trailer) => ({
src: trailer.src, src: trailer.src,
quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null, quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null,
vr: channel.tags?.some(tag => tag.slug === 'vr'), vr: channel.tags?.some((tag) => tag.slug === 'vr'),
})); }));
return release; return release;
@ -63,7 +63,7 @@ async function scrapeScene({ query }, url, channel) {
async function fetchActorReleases(urls) { async function fetchActorReleases(urls) {
// DDF Network and DDF Network Stream list all scenes, exclude // DDF Network and DDF Network Stream list all scenes, exclude
const sources = urls.filter(url => !/ddfnetwork/.test(url)); const sources = urls.filter((url) => !/ddfnetwork/.test(url));
const releases = await Promise.all(sources.map(async (url) => { const releases = await Promise.all(sources.map(async (url) => {
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)'); const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)');
@ -79,10 +79,10 @@ async function fetchActorReleases(urls) {
} }
async function scrapeProfile({ query }, _url, actorName) { async function scrapeProfile({ query }, _url, actorName) {
const keys = query.all('.about-title', true).map(key => slugify(key, '_')); const keys = query.all('.about-title', true).map((key) => slugify(key, '_'));
const values = query.all('.about-info').map((el) => { const values = query.all('.about-info').map((el) => {
if (el.children.length > 0) { if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', '); return Array.from(el.children, (child) => child.textContent.trim()).join(', ');
} }
return el.textContent.trim(); return el.textContent.trim();

View File

@ -47,7 +47,7 @@ function scrapeLatest(html, site, filter = true) {
const entryId = `${site.slug}_${pathname.split('/')[4]}`; const entryId = `${site.slug}_${pathname.split('/')[4]}`;
const title = element.querySelector('.scene-title').textContent; const title = element.querySelector('.scene-title').textContent;
const actors = title.split(/[,&]|\band\b/).map(actor => actor.replace(/BTS/i, '').trim()); const actors = title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim());
const poster = `https:${element.querySelector('img').src}`; const poster = `https:${element.querySelector('img').src}`;
const teaser = sceneLinkElement.dataset.preview_clip_url; const teaser = sceneLinkElement.dataset.preview_clip_url;

View File

@ -12,7 +12,7 @@ function scrapeAll(scenes, channel) {
release.title = query.cnt('.title'); release.title = query.cnt('.title');
release.actors = query.all('.actors a').map(actorEl => ({ release.actors = query.all('.actors a').map((actorEl) => ({
name: query.cnt(actorEl), name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }), url: query.url(actorEl, null, 'href', { origin: channel.url }),
})); }));
@ -40,7 +40,7 @@ function scrapeScene({ query }, url, channel) {
release.date = query.date('.publish_date', 'MMMM DD, YYYY'); release.date = query.date('.publish_date', 'MMMM DD, YYYY');
release.duration = query.dur('.duration'); release.duration = query.dur('.duration');
release.actors = query.all('.actress a').map(actorEl => ({ release.actors = query.all('.actress a').map((actorEl) => ({
name: query.cnt(actorEl), name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }), url: query.url(actorEl, null, 'href', { origin: channel.url }),
})); }));
@ -91,7 +91,7 @@ function scrapeMovie({ query, el }, url, channel) {
release.duration = query.dur('.duration'); release.duration = query.dur('.duration');
release.actors = query.all('.actors .actor').map(actorEl => ({ release.actors = query.all('.actors .actor').map((actorEl) => ({
name: query.cnt(actorEl, '.name'), name: query.cnt(actorEl, '.name'),
url: query.url(actorEl, 'a', 'href', { origin: channel.url }), url: query.url(actorEl, 'a', 'href', { origin: channel.url }),
avatar: query.sourceSet(actorEl, '.thumbnail img', 'data-srcset'), avatar: query.sourceSet(actorEl, '.thumbnail img', 'data-srcset'),
@ -99,7 +99,7 @@ function scrapeMovie({ query, el }, url, channel) {
release.poster = query.sourceSet('.banner', 'data-src')?.[0]; release.poster = query.sourceSet('.banner', 'data-src')?.[0];
release.covers = [query.all(query.el('.cover').parentElement, 'source') release.covers = [query.all(query.el('.cover').parentElement, 'source')
?.map(coverEl => query.sourceSet(coverEl, null, 'data-srcset')) ?.map((coverEl) => query.sourceSet(coverEl, null, 'data-srcset'))
.flat() .flat()
.sort((coverA, coverB) => { .sort((coverA, coverB) => {
const resA = Number(coverA.match(/_(\d{3,})_/)?.[1]); const resA = Number(coverA.match(/_(\d{3,})_/)?.[1]);

View File

@ -53,7 +53,7 @@ function getImageWithFallbacks(q, selector, site, el) {
q(selector, 'src0_1x'), q(selector, 'src0_1x'),
]; ];
return sources.filter(Boolean).map(src => `${site.parameters?.media || site.url}${src}`); return sources.filter(Boolean).map((src) => `${site.parameters?.media || site.url}${src}`);
} }
function scrapeAllClassic(scenes, channel) { function scrapeAllClassic(scenes, channel) {
@ -107,7 +107,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1]; // release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release); release.entryId = deriveEntryId(release);
if (channel.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) { if (channel.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping // filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
return null; return null;
} }
@ -143,7 +143,7 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/); release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)'); release.duration = query.dur('.update-info-row:nth-child(2)');
release.actors = query.all('.models-list-thumbs a').map(el => ({ release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.cnt(el, 'span'), name: query.cnt(el, 'span'),
avatar: getImageWithFallbacks(query.q, 'img', entity, el), avatar: getImageWithFallbacks(query.q, 'img', entity, el),
url: query.url(el, null), url: query.url(el, null),
@ -164,8 +164,8 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
if (stars) release.stars = Number(stars); if (stars) release.stars = Number(stars);
if (entity.type === 'network') { if (entity.type === 'network') {
const channelRegExp = new RegExp(entity.children.map(channel => channel.parameters?.match || channel.name).join('|'), 'i'); const channelRegExp = new RegExp(entity.children.map((channel) => channel.parameters?.match || channel.name).join('|'), 'i');
const channel = release.tags.find(tag => channelRegExp.test(tag)); const channel = release.tags.find((tag) => channelRegExp.test(tag));
if (channel) { if (channel) {
release.channel = slugify(channel, ''); release.channel = slugify(channel, '');
@ -199,8 +199,8 @@ async function scrapeProfile({ query }, entity, parameters) {
avatarEl.getAttribute('src0'), avatarEl.getAttribute('src0'),
avatarEl.getAttribute('src'), avatarEl.getAttribute('src'),
] ]
.filter(avatar => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images .filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map(avatar => qu.prefixUrl(avatar, entity.url)); .map((avatar) => qu.prefixUrl(avatar, entity.url));
if (avatarSources.length) profile.avatar = avatarSources; if (avatarSources.length) profile.avatar = avatarSources;
} }

View File

@ -18,7 +18,7 @@ function extractLowArtActors(release) {
const actors = release.title const actors = release.title
.replace(/solo/i, '') .replace(/solo/i, '')
.split(/,|\band\b/ig) .split(/,|\band\b/ig)
.map(actor => actor.trim()); .map((actor) => actor.trim());
return { return {
...release, ...release,
@ -32,7 +32,7 @@ async function networkFetchLatest(site, page = 1) {
const releases = await fetchLatest(site, page); const releases = await fetchLatest(site, page);
if (site.slug === 'lowartfilms') { if (site.slug === 'lowartfilms') {
return releases.map(release => extractLowArtActors(release)); return releases.map((release) => extractLowArtActors(release));
} }
return releases; return releases;
@ -76,7 +76,7 @@ async function fetchClassicProfile(actorName, { site }) {
if (!pornstarsRes.ok) return null; if (!pornstarsRes.ok) return null;
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]') const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find(el => slugify(el.textContent) === actorSlug) .find((el) => slugify(el.textContent) === actorSlug)
?.value; ?.value;
if (actorPath) { if (actorPath) {

View File

@ -14,7 +14,7 @@ function scrapeAllA(scenes, channel) {
release.date = query.date('.thumb-added, .date', ['MMM D, YYYY', 'MMMM DD, YYYY'], /\w+ \d{1,2}, \d{4}/); release.date = query.date('.thumb-added, .date', ['MMM D, YYYY', 'MMMM DD, YYYY'], /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.thumb-duration'); release.duration = query.dur('.thumb-duration');
release.actors = query.all('.thumb-models a, .models a').map(actorEl => ({ release.actors = query.all('.thumb-models a, .models a').map((actorEl) => ({
name: query.cnt(actorEl), name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }), url: query.url(actorEl, null, 'href', { origin: channel.url }),
})); }));
@ -70,7 +70,7 @@ function scrapeSceneA({ query }, url, channel) {
release.duration = query.dur('.media-body li span, .duration'); release.duration = query.dur('.media-body li span, .duration');
release.actors = query.all('.media-body a[href*="models/"], .models a').map(actorEl => ({ release.actors = query.all('.media-body a[href*="models/"], .models a').map((actorEl) => ({
name: query.cnt(actorEl), name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }), url: query.url(actorEl, null, 'href', { origin: channel.url }),
})); }));

View File

@ -9,7 +9,7 @@ function scrapeProfile(html, actorName) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const profile = { name: actorName }; const profile = { name: actorName };
const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), el => decodeURI(el.href)).reduce((acc, item) => { const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), (el) => decodeURI(el.href)).reduce((acc, item) => {
const keyMatch = item.match(/\[\w+\]/); const keyMatch = item.match(/\[\w+\]/);
if (keyMatch) { if (keyMatch) {
@ -52,7 +52,7 @@ function scrapeProfile(html, actorName) {
if (bio.height) profile.height = Number(bio.height.split(',')[0]); if (bio.height) profile.height = Number(bio.height.split(',')[0]);
if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]); if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]);
profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), el => el.href); profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), (el) => el.href);
const avatar = document.querySelector('.profile-image-large img').src; const avatar = document.querySelector('.profile-image-large img').src;
if (!avatar.match('placeholder')) profile.avatar = { src: avatar, credit: null }; if (!avatar.match('placeholder')) profile.avatar = { src: avatar, credit: null };

View File

@ -33,7 +33,7 @@ async function fetchApiCredentials(referer, site) {
const res = await http.get(referer); const res = await http.get(referer);
const body = res.body.toString(); const body = res.body.toString();
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey')); const apiLine = body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
if (!apiLine) { if (!apiLine) {
throw new Error(`No Gamma API key found for ${referer}`); throw new Error(`No Gamma API key found for ${referer}`);
@ -169,7 +169,7 @@ async function getThumbs(entryId, site, parameters) {
}); });
if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) { if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) {
return res.body.results[0].hits[0].set_pictures.map(img => ([ return res.body.results[0].hits[0].set_pictures.map((img) => ([
`https://transform.gammacdn.com/photo_set${img.thumb_path}`, `https://transform.gammacdn.com/photo_set${img.thumb_path}`,
`https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`, `https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`,
])); ]));
@ -214,7 +214,7 @@ async function scrapeApiReleases(json, site) {
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate(); release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.director = scene.directors[0]?.name || null; release.director = scene.directors[0]?.name || null;
release.actors = scene.actors.map(actor => ({ release.actors = scene.actors.map((actor) => ({
entryId: actor.actor_id, entryId: actor.actor_id,
name: actor.name, name: actor.name,
gender: actor.gender, gender: actor.gender,
@ -226,7 +226,7 @@ async function scrapeApiReleases(json, site) {
})); }));
release.tags = scene.master_categories release.tags = scene.master_categories
.concat(scene.categories?.map(category => category.name)) .concat(scene.categories?.map((category) => category.name))
.filter(Boolean); // some categories don't have a name .filter(Boolean); // some categories don't have a name
const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]); const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
@ -272,7 +272,7 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
[release.likes, release.dislikes] = $(element).find('.value') [release.likes, release.dislikes] = $(element).find('.value')
.toArray() .toArray()
.map(value => Number($(value).text())); .map((value) => Number($(value).text()));
const posterEl = $(element).find('.imgLink img, .tlcImageItem'); const posterEl = $(element).find('.imgLink img, .tlcImageItem');
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src'); if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
@ -327,13 +327,13 @@ async function scrapeScene(html, url, site, baseRelease, mobileHtml, options) {
const actors = data?.actor || data2?.actor; const actors = data?.actor || data2?.actor;
if (actors) { if (actors) {
release.actors = actors.map(actor => ({ release.actors = actors.map((actor) => ({
name: actor.name, name: actor.name,
gender: actor.gender, gender: actor.gender,
})); }));
} }
const hasTrans = release.actors?.some(actor => actor.gender === 'shemale'); const hasTrans = release.actors?.some((actor) => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || []; const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags; release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
@ -420,7 +420,7 @@ async function scrapeSceneApi(data, site, options) {
release.duration = data.length; release.duration = data.length;
release.date = new Date(data.date * 1000) || qu.parseDate(data.release_date, 'YYYY-MM-DD'); release.date = new Date(data.date * 1000) || qu.parseDate(data.release_date, 'YYYY-MM-DD');
release.actors = data.actors.map(actor => ({ release.actors = data.actors.map((actor) => ({
entryId: actor.actor_id, entryId: actor.actor_id,
name: actor.name, name: actor.name,
gender: actor.gender, gender: actor.gender,
@ -429,7 +429,7 @@ async function scrapeSceneApi(data, site, options) {
: qu.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url), : qu.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
})); }));
release.tags = data.categories.map(category => category.name); release.tags = data.categories.map((category) => category.name);
if (data.pictures) { if (data.pictures) {
release.poster = [ release.poster = [
@ -501,7 +501,7 @@ async function scrapeMovie({ query, html }, window, url, entity, options) {
release.date = qu.extractDate(data.dvdReleaseDate); release.date = qu.extractDate(data.dvdReleaseDate);
release.title = data.dvdName; release.title = data.dvdName;
release.actors = data.dvdActors.map(actor => ({ name: actor.actorName, entryId: actor.actorId })); release.actors = data.dvdActors.map((actor) => ({ name: actor.actorName, entryId: actor.actorId }));
release.tags = query.cnts('.dvdCol a'); release.tags = query.cnts('.dvdCol a');
release.scenes = scrapeAll(html, entity, entity.url); release.scenes = scrapeAll(html, entity, entity.url);
@ -602,9 +602,9 @@ function scrapeApiProfile(data, releases, siteSlug) {
if (data.attributes.hair_color) profile.hair = data.attributes.hair_color; if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
const avatarPaths = Object.values(data.pictures).reverse(); const avatarPaths = Object.values(data.pictures).reverse();
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map(avatarPath => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`); if (avatarPaths.length > 0) profile.avatar = avatarPaths.map((avatarPath) => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
if (releases) profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`); if (releases) profile.releases = releases.map((release) => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
return profile; return profile;
} }
@ -723,7 +723,7 @@ function getDeepUrl(url, site, baseRelease, mobile) {
const filter = new Set(['en', 'video', 'scene', site.slug, site.parent.slug]); const filter = new Set(['en', 'video', 'scene', site.slug, site.parent.slug]);
const pathname = baseRelease?.path || new URL(url).pathname const pathname = baseRelease?.path || new URL(url).pathname
.split('/') .split('/')
.filter(component => !filter.has(component)) .filter((component) => !filter.has(component))
.join('/'); // reduce to scene ID and title slug .join('/'); // reduce to scene ID and title slug
const sceneId = baseRelease?.entryId || pathname.match(/\/(\d+)\//)?.[1]; const sceneId = baseRelease?.entryId || pathname.match(/\/(\d+)\//)?.[1];
@ -863,7 +863,7 @@ async function fetchApiProfile({ name: actorName }, context, include) {
}); });
if (res.status === 200 && res.body.results[0].hits.length > 0) { if (res.status === 200 && res.body.results[0].hits.length > 0) {
const actorData = res.body.results[0].hits.find(actor => slugify(actor.name) === slugify(actorName)); const actorData = res.body.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
if (actorData) { if (actorData) {
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug); const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);

View File

@ -42,13 +42,13 @@ function scrapeScene({ query }, url) {
release.duration = query.dur('.content-metas span:nth-child(2)'); release.duration = query.dur('.content-metas span:nth-child(2)');
release.likes = query.number('.content-metas span:nth-child(6)'); release.likes = query.number('.content-metas span:nth-child(6)');
release.actors = query.all('.model-thumb img').map(el => ({ release.actors = query.all('.model-thumb img').map((el) => ({
name: query.q(el, null, 'alt'), name: query.q(el, null, 'alt'),
avatar: query.img(el, null, 'src'), avatar: query.img(el, null, 'src'),
})); }));
release.poster = query.poster('.content-video video'); release.poster = query.poster('.content-video video');
release.photos = query.urls('#photo-carousel a').map(photo => [ release.photos = query.urls('#photo-carousel a').map((photo) => [
photo.replace('/full', ''), photo.replace('/full', ''),
photo, photo,
photo.replace('/full', '/thumbs'), photo.replace('/full', '/thumbs'),
@ -135,7 +135,7 @@ async function fetchProfile(baseActor, entity, include) {
}); });
if (searchRes.ok) { if (searchRes.ok) {
const actor = searchRes.body.find(result => result.type === 'model' && result.title === baseActor.name); const actor = searchRes.body.find((result) => result.type === 'model' && result.title === baseActor.name);
if (actor) { if (actor) {
const actorRes = await qu.get(actor.url); const actorRes = await qu.get(actor.url);

View File

@ -21,7 +21,7 @@ function scrapeAll(scenes) {
avatar: [ avatar: [
avatarEl.src.replace(/-\d+x\d+/, ''), avatarEl.src.replace(/-\d+x\d+/, ''),
avatarEl.src, avatarEl.src,
].map(src => ({ src, interval: 1000, concurrency: 1 })), ].map((src) => ({ src, interval: 1000, concurrency: 1 })),
}), }),
}; };
}).concat({ }).concat({

View File

@ -53,7 +53,7 @@ function getImageWithFallbacks(q, selector, site, el) {
q(selector, 'src0_1x'), q(selector, 'src0_1x'),
]; ];
return sources.filter(Boolean).map(src => `${site.parameters?.media || site.url}${src}`); return sources.filter(Boolean).map((src) => `${site.parameters?.media || site.url}${src}`);
} }
function scrapeAll(scenes, channel) { function scrapeAll(scenes, channel) {
@ -108,7 +108,7 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1]; // release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release); release.entryId = deriveEntryId(release);
if (site.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) { if (site.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping // filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
return null; return null;
} }
@ -132,7 +132,7 @@ function scrapeScene({ html, query }, channel, url) {
const poster = qu.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup const poster = qu.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
[release.poster, ...release.photos] = [poster, ...query.imgs('.item-thumb img', 'src0_1x', { origin: channel.url })] [release.poster, ...release.photos] = [poster, ...query.imgs('.item-thumb img', 'src0_1x', { origin: channel.url })]
.map(src => src && [ .map((src) => src && [
src.replace('-1x', '-3x'), src.replace('-1x', '-3x'),
src.replace('-1x', '-2x'), src.replace('-1x', '-2x'),
src, src,
@ -161,7 +161,7 @@ function scrapeSceneT1({ html, query }, site, url, baseRelease) {
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/); release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)'); release.duration = query.dur('.update-info-row:nth-child(2)');
release.actors = query.all('.models-list-thumbs a').map(el => ({ release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.q(el, 'span', true), name: query.q(el, 'span', true),
avatar: getImageWithFallbacks(query.q, 'img', site, el), avatar: getImageWithFallbacks(query.q, 'img', site, el),
})); }));
@ -180,8 +180,8 @@ function scrapeSceneT1({ html, query }, site, url, baseRelease) {
if (stars) release.stars = Number(stars); if (stars) release.stars = Number(stars);
if (site.type === 'network') { if (site.type === 'network') {
const channelRegExp = new RegExp(site.children.map(channel => channel.parameters?.match || channel.name).join('|'), 'i'); const channelRegExp = new RegExp(site.children.map((channel) => channel.parameters?.match || channel.name).join('|'), 'i');
const channel = release.tags.find(tag => channelRegExp.test(tag)); const channel = release.tags.find((tag) => channelRegExp.test(tag));
if (channel) { if (channel) {
release.channel = slugify(channel, ''); release.channel = slugify(channel, '');
@ -290,7 +290,7 @@ async function scrapeProfile({ query, el }, channel, options) {
if (bio.piercings && /yes/i.test(bio.piercings)) profile.hasPiercings = true; if (bio.piercings && /yes/i.test(bio.piercings)) profile.hasPiercings = true;
if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false; if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false;
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim()); if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim());
profile.social = [bio.onlyfans, bio.twitter, bio.instagram].filter(Boolean); profile.social = [bio.onlyfans, bio.twitter, bio.instagram].filter(Boolean);

View File

@ -26,7 +26,7 @@ function scrapeLatest(scenes, site) {
release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY'); release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
} }
release.actors = actors.map(actor => actor.trim()); release.actors = actors.map((actor) => actor.trim());
const description = query.q('.articleCopyText', true); const description = query.q('.articleCopyText', true);
if (description) release.description = description.slice(0, description.lastIndexOf('(')); if (description) release.description = description.slice(0, description.lastIndexOf('('));
@ -81,7 +81,7 @@ function scrapeScene({ query }, site) {
release.title = title.trim(); release.title = title.trim();
release.description = query.q('.articleCopyText', true); release.description = query.q('.articleCopyText', true);
release.actors = actors.map(actor => actor.trim()); release.actors = actors.map((actor) => actor.trim());
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY'); release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
release.duration = query.dur('.articlePostDateText a:nth-child(2)'); release.duration = query.dur('.articlePostDateText a:nth-child(2)');

View File

@ -117,7 +117,7 @@ function scrapeProfile({ query }, actorName, actorAvatar, channel, releasesFromS
profile.releases = releasesFromScene?.[profile.name] || scrapeProfileScenes(qu.initAll(query.all('.Models li')), actorName, channel); profile.releases = releasesFromScene?.[profile.name] || scrapeProfileScenes(qu.initAll(query.all('.Models li')), actorName, channel);
// avatar is the poster of a scene, find scene and use its high quality poster instead // avatar is the poster of a scene, find scene and use its high quality poster instead
const avatarRelease = profile.releases.find(release => new URL(release.poster[1]).pathname === new URL(actorAvatar).pathname); const avatarRelease = profile.releases.find((release) => new URL(release.poster[1]).pathname === new URL(actorAvatar).pathname);
profile.avatar = avatarRelease?.poster[0]; profile.avatar = avatarRelease?.poster[0];
return profile; return profile;

View File

@ -13,7 +13,7 @@ async function fetchActors(entryId, channel, { token, time }) {
const res = await http.get(url); const res = await http.get(url);
if (res.statusCode === 200 && res.body.status === true) { if (res.statusCode === 200 && res.body.status === true) {
return Object.values(res.body.response.collection).map(actor => Object.values(actor.modelId.collection)[0].stageName); return Object.values(res.body.response.collection).map((actor) => Object.values(actor.modelId.collection)[0].stageName);
} }
return []; return [];
@ -46,7 +46,7 @@ function scrapeLatest(items, channel) {
release.title = query.cnt('h5 a'); release.title = query.cnt('h5 a');
[release.poster, ...release.photos] = query.imgs('.screenshot').map(src => [ [release.poster, ...release.photos] = query.imgs('.screenshot').map((src) => [
// unnecessarily large // unnecessarily large
// src.replace(/\/\d+/, 3840), // src.replace(/\/\d+/, 3840),
// src.replace(/\/\d+/, '/2000'), // src.replace(/\/\d+/, '/2000'),
@ -99,7 +99,7 @@ function scrapeScene({ query, html }, url, channel) {
[release.poster, ...release.photos] = [poster] [release.poster, ...release.photos] = [poster]
.concat(photos) .concat(photos)
.filter(Boolean) .filter(Boolean)
.map(src => [ .map((src) => [
src.replace(/\/(\d+)\/\d+/, '/$1/1500'), src.replace(/\/(\d+)\/\d+/, '/$1/1500'),
src.replace(/\/(\d+)\/\d+/, '/$1/1000'), src.replace(/\/(\d+)\/\d+/, '/$1/1000'),
src, src,
@ -128,8 +128,8 @@ async function scrapeSceneApi(scene, channel, tokens, deep) {
release.date = new Date(scene.sites.collection[scene.id].publishDate); release.date = new Date(scene.sites.collection[scene.id].publishDate);
release.poster = scene._resources.primary[0].url; release.poster = scene._resources.primary[0].url;
if (scene.tags) release.tags = Object.values(scene.tags.collection).map(tag => tag.alias); if (scene.tags) release.tags = Object.values(scene.tags.collection).map((tag) => tag.alias);
if (scene._resources.base) release.photos = scene._resources.base.map(resource => resource.url); if (scene._resources.base) release.photos = scene._resources.base.map((resource) => resource.url);
if (deep) { if (deep) {
// don't make external requests during update scraping, as this would happen for every scene on the page // don't make external requests during update scraping, as this would happen for every scene on the page
@ -149,7 +149,7 @@ async function scrapeSceneApi(scene, channel, tokens, deep) {
} }
function scrapeLatestApi(scenes, site, tokens) { function scrapeLatestApi(scenes, site, tokens) {
return Promise.map(scenes, async scene => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 }); return Promise.map(scenes, async (scene) => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
} }
async function fetchToken(channel) { async function fetchToken(channel) {

View File

@ -33,7 +33,7 @@ function scrapeLatest(scenes, dates, site) {
const poster = qu.img('img[src*="photos/"][width="400"]'); const poster = qu.img('img[src*="photos/"][width="400"]');
release.poster = `${site.url}/visitors/${poster}`; release.poster = `${site.url}/visitors/${poster}`;
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map(source => `${site.url}/visitors/${source}`); release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
return release; return release;
}); });

View File

@ -55,7 +55,7 @@ async function getPhotosLegacy(entryId, site, type = 'highres', page = 1) {
// don't add first URL to pages to prevent unnecessary duplicate request // don't add first URL to pages to prevent unnecessary duplicate request
const photos = scrapePhotos(html, type); const photos = scrapePhotos(html, type);
const pages = Array.from(new Set($('.page_numbers a').toArray().map(el => $(el).attr('href')))); const pages = Array.from(new Set($('.page_numbers a').toArray().map((el) => $(el).attr('href'))));
const otherPhotos = pages const otherPhotos = pages
? await Promise.map(pages, async (pageX) => { ? await Promise.map(pages, async (pageX) => {
@ -84,7 +84,7 @@ async function getPhotos(entryId, site, type = 'highres', page = 1) {
const res = await http.get(albumUrl); const res = await http.get(albumUrl);
const html = res.body.toString(); const html = res.body.toString();
const sourceLines = html.split(/\n/).filter(line => line.match(/ptx\["\w+"\]/)); const sourceLines = html.split(/\n/).filter((line) => line.match(/ptx\["\w+"\]/));
const sources = sourceLines.reduce((acc, sourceLine) => { const sources = sourceLines.reduce((acc, sourceLine) => {
const quality = sourceLine.match(/\["\w+"\]/)[0].slice(2, -2); const quality = sourceLine.match(/\["\w+"\]/)[0].slice(2, -2);
const sourceStart = sourceLine.match(/\/trial|\/tour|\/content/); const sourceStart = sourceLine.match(/\/trial|\/tour|\/content/);
@ -261,7 +261,7 @@ async function scrapeScene({ html, query }, url, site, include) {
} }
if (include.trailer && site.slug !== 'manuelferrara') { if (include.trailer && site.slug !== 'manuelferrara') {
const trailerLines = html.split('\n').filter(line => /movie\["trailer\w*"\]\[/i.test(line)); const trailerLines = html.split('\n').filter((line) => /movie\["trailer\w*"\]\[/i.test(line));
if (trailerLines.length) { if (trailerLines.length) {
release.trailer = trailerLines.map((trailerLine) => { release.trailer = trailerLines.map((trailerLine) => {
@ -307,7 +307,7 @@ function scrapeMovie({ el, query }, url, site) {
const scenes = scrapeAll(sceneQus, site); const scenes = scrapeAll(sceneQus, site);
const curatedScenes = scenes const curatedScenes = scenes
?.map(scene => ({ ...scene, movie })) ?.map((scene) => ({ ...scene, movie }))
.sort((sceneA, sceneB) => sceneA.date - sceneB.date); .sort((sceneA, sceneB) => sceneA.date - sceneB.date);
movie.date = curatedScenes?.[0].date; movie.date = curatedScenes?.[0].date;
@ -354,13 +354,13 @@ function scrapeProfile(html, url, actorName, entity) {
avatarEl.getAttribute('src0'), avatarEl.getAttribute('src0'),
avatarEl.getAttribute('src'), avatarEl.getAttribute('src'),
] ]
.filter(avatar => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images .filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map(avatar => qu.prefixUrl(avatar, entity.url)); .map((avatar) => qu.prefixUrl(avatar, entity.url));
if (avatarSources.length) profile.avatar = avatarSources; if (avatarSources.length) profile.avatar = avatarSources;
} }
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href); profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), (el) => el.href);
return profile; return profile;
} }

View File

@ -35,7 +35,7 @@ function scrapeScene({ query }, url) {
release.title = query.cnt('.title'); release.title = query.cnt('.title');
release.date = query.date('.date .content', 'MMM Do, YYYY'); release.date = query.date('.date .content', 'MMM Do, YYYY');
release.actors = query.all('.models .content a').map(modelEl => ({ release.actors = query.all('.models .content a').map((modelEl) => ({
name: query.cnt(modelEl), name: query.cnt(modelEl),
url: query.url(modelEl, null), url: query.url(modelEl, null),
})); }));
@ -76,7 +76,7 @@ async function fetchProfile(baseActor, entity) {
return searchRes.status; return searchRes.status;
} }
const actorUrl = searchRes.items.find(item => slugify(item.query.cnt('.title')) === baseActor.slug)?.query.url('a'); const actorUrl = searchRes.items.find((item) => slugify(item.query.cnt('.title')) === baseActor.slug)?.query.url('a');
if (!actorUrl) { if (!actorUrl) {
return null; return null;

View File

@ -53,7 +53,7 @@ function scrapeLatest(scenes, site) {
} }
return release; return release;
}).filter(scene => scene); }).filter((scene) => scene);
} }
async function scrapeScene({ query, html }, url, baseRelease, channel, session) { async function scrapeScene({ query, html }, url, baseRelease, channel, session) {
@ -100,7 +100,7 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session)
const trailerInfoRes = await http.post(trailerInfoUrl, null, { session }); const trailerInfoRes = await http.post(trailerInfoUrl, null, { session });
if (trailerInfoRes.ok && trailerInfoRes.body.sources?.length > 0) { if (trailerInfoRes.ok && trailerInfoRes.body.sources?.length > 0) {
release.trailer = trailerInfoRes.body.sources.map(trailer => ({ release.trailer = trailerInfoRes.body.sources.map((trailer) => ({
src: trailer.src, src: trailer.src,
type: trailer.type, type: trailer.type,
/* unreliable, sometimes actual video is 720p /* unreliable, sometimes actual video is 720p

View File

@ -4,11 +4,11 @@ const qu = require('../utils/qu');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
function scrapeAll({ query }) { function scrapeAll({ query }) {
const urls = query.urls('td > a:not([href*=joinnow])').map(pathname => `http://killergram.com/${encodeURI(pathname)}`); const urls = query.urls('td > a:not([href*=joinnow])').map((pathname) => `http://killergram.com/${encodeURI(pathname)}`);
const posters = query.imgs('td > a img'); const posters = query.imgs('td > a img');
const titles = query.all('.episodeheadertext', true); const titles = query.all('.episodeheadertext', true);
const actors = query.all('.episodetextinfo:nth-child(3)').map(el => query.all(el, 'a', true)); const actors = query.all('.episodetextinfo:nth-child(3)').map((el) => query.all(el, 'a', true));
const channels = query.all('.episodetextinfo:nth-child(2) a', true).map(channel => slugify(channel, '')); const channels = query.all('.episodetextinfo:nth-child(2) a', true).map((channel) => slugify(channel, ''));
if ([urls.length, posters.length, titles.length, actors.length, channels.length].every((value, index, array) => value === array[0])) { // make sure every set has the same number of items if ([urls.length, posters.length, titles.length, actors.length, channels.length].every((value, index, array) => value === array[0])) { // make sure every set has the same number of items
const releases = urls.map((url, index) => ({ const releases = urls.map((url, index) => ({
@ -51,7 +51,7 @@ function scrapeScene({ query, html }, url) {
} }
async function fetchActorReleases({ query }, url, remainingPages, actorName, accReleases = []) { async function fetchActorReleases({ query }, url, remainingPages, actorName, accReleases = []) {
const releases = scrapeAll({ query }).filter(release => release.actors.includes(actorName)); const releases = scrapeAll({ query }).filter((release) => release.actors.includes(actorName));
if (remainingPages.length > 0) { if (remainingPages.length > 0) {
const { origin, pathname, searchParams } = new URL(url); const { origin, pathname, searchParams } = new URL(url);

View File

@ -19,7 +19,7 @@ function scrapeAll(scenes) {
release.stars = query.q('.average-rating', 'data-rating') / 10; release.stars = query.q('.average-rating', 'data-rating') / 10;
release.poster = query.img('.adimage'); release.poster = query.img('.adimage');
release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map(photo => [ release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map((photo) => [
photo.replace('410/', '830/'), photo.replace('410/', '830/'),
photo, photo,
]); ]);
@ -40,13 +40,13 @@ async function scrapeScene({ query }, url) {
release.description = query.q('.description-text', true); release.description = query.q('.description-text', true);
release.date = query.date('.shoot-date', 'MMMM DD, YYYY'); release.date = query.date('.shoot-date', 'MMMM DD, YYYY');
release.actors = query.all('.names a', true).map(actor => actor.replace(/,\s*/, '')); release.actors = query.all('.names a', true).map((actor) => actor.replace(/,\s*/, ''));
release.director = query.q('.director-name', true); release.director = query.q('.director-name', true);
release.photos = query.imgs('.gallery .thumb img, #gallerySlider .gallery-img', 'data-image-file'); release.photos = query.imgs('.gallery .thumb img, #gallerySlider .gallery-img', 'data-image-file');
release.poster = query.poster(); release.poster = query.poster();
release.tags = query.all('.tag-list a[href*="/tag"]', true).map(tag => tag.replace(/,\s*/, '')); release.tags = query.all('.tag-list a[href*="/tag"]', true).map((tag) => tag.replace(/,\s*/, ''));
const trailer = query.q('.player span[data-type="trailer-src"]', 'data-url'); const trailer = query.q('.player span[data-type="trailer-src"]', 'data-url');
@ -154,7 +154,7 @@ async function fetchProfile({ name: actorName }, entity, include) {
const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model'); const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
if (searchRes.ok) { if (searchRes.ok) {
const actorItem = searchRes.items.find(item => item.query.exists(`.model-link img[alt="${actorName}"]`)); const actorItem = searchRes.items.find((item) => item.query.exists(`.model-link img[alt="${actorName}"]`));
if (actorItem) { if (actorItem) {
const actorPath = actorItem.query.url('.model-link'); const actorPath = actorItem.query.url('.model-link');

View File

@ -126,7 +126,7 @@ async function scrapeScene(html, url, site, useGallery) {
'1080p': 1080, '1080p': 1080,
}; };
release.trailer = data.clip.qualities.map(trailer => ({ release.trailer = data.clip.qualities.map((trailer) => ({
src: trailer.src, src: trailer.src,
type: trailer.type, type: trailer.type,
quality: qualityMap[trailer.quality] || trailer.quality, quality: qualityMap[trailer.quality] || trailer.quality,
@ -147,10 +147,10 @@ async function scrapeProfile(html, _url, actorName) {
}; };
const avatarEl = document.querySelector('.model--avatar img[src^="http"]'); const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
const entries = Array.from(document.querySelectorAll('.model--description tr'), el => el.textContent.replace(/\n/g, '').split(':')); const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
const bio = entries const bio = entries
.filter(entry => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry) .filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {}); .reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
profile.birthPlace = bio.Nationality; profile.birthPlace = bio.Nationality;
@ -184,7 +184,7 @@ async function fetchProfile({ name: actorName }) {
const res = await http.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`); const res = await http.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
const data = res.body; const data = res.body;
const result = data.terms.find(item => item.type === 'model'); const result = data.terms.find((item) => item.type === 'model');
if (result) { if (result) {
const bioRes = await http.get(result.url); const bioRes = await http.get(result.url);

View File

@ -65,7 +65,7 @@ async function fetchPhotos(url) {
const res = await qu.get(url, '.et_post_gallery'); const res = await qu.get(url, '.et_post_gallery');
if (res.ok) { if (res.ok) {
return res.item.query.urls('a').map(imgUrl => ({ return res.item.query.urls('a').map((imgUrl) => ({
src: imgUrl, src: imgUrl,
referer: url, referer: url,
})); }));
@ -89,14 +89,14 @@ async function scrapeScene({ query }, url, channel, include) {
release.date = query.date('.vid_date', 'MMMM D, YYYY'); release.date = query.date('.vid_date', 'MMMM D, YYYY');
release.duration = query.dur('.vid_length'); release.duration = query.dur('.vid_length');
release.actors = query.all('.vid_infos a[href*="author/"]').map(actorEl => ({ release.actors = query.all('.vid_infos a[href*="author/"]').map((actorEl) => ({
name: query.cnt(actorEl), name: query.cnt(actorEl),
url: query.url(actorEl, null), url: query.url(actorEl, null),
})); }));
release.tags = query.cnts('.vid_infos a[rel="tag"]'); release.tags = query.cnts('.vid_infos a[rel="tag"]');
const posterData = data['@graph']?.find(item => item['@type'] === 'ImageObject'); const posterData = data['@graph']?.find((item) => item['@type'] === 'ImageObject');
const poster = posterData?.url const poster = posterData?.url
|| query.q('meta[property="og:image"]', 'content') || query.q('meta[property="og:image"]', 'content')

View File

@ -20,7 +20,7 @@ function scrapeAll(scenes) {
release.duration = query.dur('.total-time'); release.duration = query.dur('.total-time');
const [poster, ...primaryPhotos] = query.imgs('a img'); const [poster, ...primaryPhotos] = query.imgs('a img');
const secondaryPhotos = query.styles('.thumb-top, .thumb-bottom, .thumb-mouseover', 'background-image').map(style => style.match(/url\((.*)\)/)[1]); const secondaryPhotos = query.styles('.thumb-top, .thumb-bottom, .thumb-mouseover', 'background-image').map((style) => style.match(/url\((.*)\)/)[1]);
release.poster = poster; release.poster = poster;
release.photos = primaryPhotos.concat(secondaryPhotos); release.photos = primaryPhotos.concat(secondaryPhotos);

View File

@ -14,14 +14,14 @@ const { inchesToCm, lbsToKg } = require('../utils/convert');
function getThumbs(scene) { function getThumbs(scene) {
if (scene.images.poster) { if (scene.images.poster) {
return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
.filter(img => typeof img === 'object') // remove alternateText property .filter((img) => typeof img === 'object') // remove alternateText property
.map(image => image.xl.url); .map((image) => image.xl.url);
} }
if (scene.images.card_main_rect) { if (scene.images.card_main_rect) {
return scene.images.card_main_rect return scene.images.card_main_rect
.concat(scene.images.card_secondary_rect || []) .concat(scene.images.card_secondary_rect || [])
.map(image => image.xl.url.replace('.thumb', '')); .map((image) => image.xl.url.replace('.thumb', ''));
} }
return []; return [];
@ -29,14 +29,14 @@ function getThumbs(scene) {
function getVideos(data) { function getVideos(data) {
const teaserSources = data.videos.mediabook?.files; const teaserSources = data.videos.mediabook?.files;
const trailerSources = data.children.find(child => child.type === 'trailer')?.videos.full?.files; const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
const teaser = teaserSources && Object.values(teaserSources).map(source => ({ const teaser = teaserSources && Object.values(teaserSources).map((source) => ({
src: source.urls.view, src: source.urls.view,
quality: parseInt(source.format, 10), quality: parseInt(source.format, 10),
})); }));
const trailer = trailerSources && Object.values(trailerSources).map(source => ({ const trailer = trailerSources && Object.values(trailerSources).map((source) => ({
src: source.urls.view, src: source.urls.view,
quality: parseInt(source.format, 10), quality: parseInt(source.format, 10),
})); }));
@ -59,8 +59,8 @@ function scrapeLatestX(data, site, filterChannel) {
release.date = new Date(data.dateReleased); release.date = new Date(data.dateReleased);
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null; release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender })); release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map(tag => tag.name); release.tags = data.tags.map((tag) => tag.name);
[release.poster, ...release.photos] = getThumbs(data); [release.poster, ...release.photos] = getThumbs(data);
@ -69,15 +69,15 @@ function scrapeLatestX(data, site, filterChannel) {
if (teaser) release.teaser = teaser; if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer; if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({ release.chapters = data.timeTags?.map((chapter) => ({
time: chapter.startTime, time: chapter.startTime,
duration: chapter.endTime - chapter.startTime, duration: chapter.endTime - chapter.startTime,
tags: [chapter.name], tags: [chapter.name],
})); }));
if ((site.parameters?.extract === true && data.collections.length > 0) // release should not belong to any channel if ((site.parameters?.extract === true && data.collections.length > 0) // release should not belong to any channel
|| (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) // release should belong to specific channel || (typeof site.parameters?.extract === 'string' && !data.collections.some((collection) => collection.shortName === site.parameters.extract)) // release should belong to specific channel
|| (filterChannel && !data.collections?.some(collection => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes || (filterChannel && !data.collections?.some((collection) => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes
return { return {
...release, ...release,
exclude: true, exclude: true,
@ -88,11 +88,11 @@ function scrapeLatestX(data, site, filterChannel) {
} }
async function scrapeLatest(items, site, filterChannel) { async function scrapeLatest(items, site, filterChannel) {
const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel)); const latestReleases = items.map((data) => scrapeLatestX(data, site, filterChannel));
return { return {
scenes: latestReleases.filter(scene => !scene.exclude), scenes: latestReleases.filter((scene) => !scene.exclude),
unextracted: latestReleases.filter(scene => scene.exclude), unextracted: latestReleases.filter((scene) => scene.exclude),
}; };
} }
@ -108,8 +108,8 @@ function scrapeScene(data, url, _site, networkName) {
release.date = new Date(data.dateReleased); release.date = new Date(data.dateReleased);
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null; release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender })); release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map(tag => tag.name); release.tags = data.tags.map((tag) => tag.name);
[release.poster, ...release.photos] = getThumbs(data); [release.poster, ...release.photos] = getThumbs(data);
@ -118,7 +118,7 @@ function scrapeScene(data, url, _site, networkName) {
if (teaser) release.teaser = teaser; if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer; if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({ release.chapters = data.timeTags?.map((chapter) => ({
time: chapter.startTime, time: chapter.startTime,
duration: chapter.endTime - chapter.startTime, duration: chapter.endTime - chapter.startTime,
tags: [chapter.name], tags: [chapter.name],
@ -213,18 +213,18 @@ function scrapeProfile(data, html, releases = [], networkName) {
|| data.images.card_main_rect[0].xs?.url; || data.images.card_main_rect[0].xs?.url;
} }
const birthdate = query.all('li').find(el => /Date of Birth/.test(el.textContent)); const birthdate = query.all('li').find((el) => /Date of Birth/.test(el.textContent));
if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY'); if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');
if (data.tags.some(tag => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) { if (data.tags.some((tag) => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
profile.naturalBoobs = true; profile.naturalBoobs = true;
} }
if (data.tags.some(tag => /boob type/i.test(tag.category) && /enhanced/i.test(tag.name))) { if (data.tags.some((tag) => /boob type/i.test(tag.category) && /enhanced/i.test(tag.name))) {
profile.naturalBoobs = false; profile.naturalBoobs = false;
} }
profile.releases = releases.map(release => scrapeScene(release, null, null, networkName)); profile.releases = releases.map((release) => scrapeScene(release, null, null, networkName));
return profile; return profile;
} }
@ -325,7 +325,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
}); });
if (res.statusCode === 200) { if (res.statusCode === 200) {
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase()); const actorData = res.body.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
if (actorData) { if (actorData) {
const actorUrl = `https://www.${entity.slug}.com/${entity.parameters?.actorPath || 'model'}/${actorData.id}/${actorSlug}`; const actorUrl = `https://www.${entity.slug}.com/${entity.parameters?.actorPath || 'model'}/${actorData.id}/${actorSlug}`;

View File

@ -69,7 +69,7 @@ function scrapeScene(html, url, site) {
const posterPath = $('video, dl8-video').attr('poster') || $('img.start-card').attr('src'); const posterPath = $('video, dl8-video').attr('poster') || $('img.start-card').attr('src');
const poster = posterPath && `https:${posterPath}`; const poster = posterPath && `https:${posterPath}`;
const photos = $('.contain-scene-images.desktop-only a').map((index, el) => $(el).attr('href')).toArray().filter(Boolean).map(photo => `https:${photo}`); const photos = $('.contain-scene-images.desktop-only a').map((index, el) => $(el).attr('href')).toArray().filter(Boolean).map((photo) => `https:${photo}`);
const trailerEl = $('source'); const trailerEl = $('source');
const trailerSrc = trailerEl.attr('src'); const trailerSrc = trailerEl.attr('src');
@ -120,7 +120,7 @@ async function scrapeProfile(html) {
const releases = query.urls('.scene-item > a:first-child'); const releases = query.urls('.scene-item > a:first-child');
const otherPages = query.urls('.pagination a:not([rel=next]):not([rel=prev])'); const otherPages = query.urls('.pagination a:not([rel=next]):not([rel=prev])');
const olderReleases = await Promise.all(otherPages.map(async page => fetchActorReleases(page))); const olderReleases = await Promise.all(otherPages.map(async (page) => fetchActorReleases(page)));
profile.releases = releases.concat(olderReleases.flat()); profile.releases = releases.concat(olderReleases.flat());

View File

@ -78,7 +78,7 @@ async function scrapeScene({ query }, url, site) {
release.tags = query.all('.categories a', true); release.tags = query.all('.categories a', true);
release.poster = query.poster() || query.img('.fake-video-player img'); release.poster = query.poster() || query.img('.fake-video-player img');
release.trailer = query.all('source').map(source => ({ release.trailer = query.all('source').map((source) => ({
src: source.src, src: source.src,
quality: Number(source.getAttribute('res')), quality: Number(source.getAttribute('res')),
})); }));
@ -106,11 +106,11 @@ function scrapeProfile({ query }, _actorName, origin) {
profile.residencePlace = bio.location; profile.residencePlace = bio.location;
profile.height = heightToCm(bio.height); profile.height = heightToCm(bio.height);
[profile.bust, profile.waist, profile.hip] = bio.figure.split('-').map(v => Number(v) || v); [profile.bust, profile.waist, profile.hip] = bio.figure.split('-').map((v) => Number(v) || v);
profile.avatar = query.img('.model-profile img'); profile.avatar = query.img('.model-profile img');
const releases = query.all('.content-grid-item').filter(el => /video\//.test(query.url(el, '.img-wrapper a'))); // filter out photos const releases = query.all('.content-grid-item').filter((el) => /video\//.test(query.url(el, '.img-wrapper a'))); // filter out photos
profile.releases = scrapeAll(query.initAll(releases), null, origin); profile.releases = scrapeAll(query.initAll(releases), null, origin);
return profile; return profile;
@ -143,7 +143,7 @@ async function fetchProfile({ name: actorName }, { site }) {
if (!resModels.ok) return resModels.status; if (!resModels.ok) return resModels.status;
const modelPath = resModels.item.qu.all('.content-grid-item a.title').find(el => slugify(el.textContent) === slugify(actorName)); const modelPath = resModels.item.qu.all('.content-grid-item a.title').find((el) => slugify(el.textContent) === slugify(actorName));
if (modelPath) { if (modelPath) {
const modelUrl = `${origin}${modelPath}`; const modelUrl = `${origin}${modelPath}`;

View File

@ -26,7 +26,7 @@ function scrapeAll(months, channel, year) {
gender: 'female', gender: 'female',
url: query.url('a.video-pop-up', 'data-modellink', { origin: `${channel.url}/submissive` }), url: query.url('a.video-pop-up', 'data-modellink', { origin: `${channel.url}/submissive` }),
}] }]
.filter(actor => !/lockdown/i.test(actor.name)) .filter((actor) => !/lockdown/i.test(actor.name))
.concat({ .concat({
name: 'Pascal White', name: 'Pascal White',
gender: 'male', gender: 'male',

View File

@ -17,21 +17,21 @@ function extractMaleModelsFromTags(tagContainer) {
return []; return [];
} }
const tagEls = Array.from(tagContainer.childNodes, node => ({ type: node.nodeType, text: node.textContent.trim() })).filter(node => node.text.length > 0); const tagEls = Array.from(tagContainer.childNodes, (node) => ({ type: node.nodeType, text: node.textContent.trim() })).filter((node) => node.text.length > 0);
const modelLabelIndex = tagEls.findIndex(node => node.text === 'Male Models'); const modelLabelIndex = tagEls.findIndex((node) => node.text === 'Male Models');
if (modelLabelIndex > -1) { if (modelLabelIndex > -1) {
const nextLabelIndex = tagEls.findIndex((node, index) => index > modelLabelIndex && node.type === 3); const nextLabelIndex = tagEls.findIndex((node, index) => index > modelLabelIndex && node.type === 3);
const maleModels = tagEls.slice(modelLabelIndex + 1, nextLabelIndex); const maleModels = tagEls.slice(modelLabelIndex + 1, nextLabelIndex);
return maleModels.map(model => model.text); return maleModels.map((model) => model.text);
} }
return []; return [];
} }
async function extractChannelFromPhoto(photo, channel) { async function extractChannelFromPhoto(photo, channel) {
const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug); const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map((child) => child.slug);
const channelMatch = photo.match(new RegExp(siteSlugs.join('|'))); const channelMatch = photo.match(new RegExp(siteSlugs.join('|')));
if (channelMatch) { if (channelMatch) {
@ -52,7 +52,7 @@ async function scrapeLatest(scenes, site) {
const slug = new URL(release.url).pathname.split('/')[2]; const slug = new URL(release.url).pathname.split('/')[2];
release.entryId = getHash(`${site.slug}${slug}${release.date.toISOString()}`); release.entryId = getHash(`${site.slug}${slug}${release.date.toISOString()}`);
release.actors = release.title.split('&').map(actor => actor.trim()); release.actors = release.title.split('&').map((actor) => actor.trim());
[release.poster, ...release.photos] = query.imgs('.bloc-link img'); [release.poster, ...release.photos] = query.imgs('.bloc-link img');
@ -78,7 +78,7 @@ async function scrapeScene({ query }, site, url) {
const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)'); const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
if (/4K/.test(uhd)) release.tags = release.tags.concat('4k'); if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
release.photos = query.all('.bxslider_pics img').map(el => el.dataset.original || el.src); release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
release.poster = query.poster(); release.poster = query.poster();
const trailer = query.trailer(); const trailer = query.trailer();

View File

@ -67,7 +67,7 @@ function scrapeScene({ query, html }, url, entity) {
release.description = query.cnt('.info_container .description'); release.description = query.cnt('.info_container .description');
release.date = query.date('.info_container .info_line:nth-child(1)', 'YYYY-MM-DD') || query.date('.description', 'DD MMMM YYYY', /\d{1,2} \w+ \d{4}/); release.date = query.date('.info_container .info_line:nth-child(1)', 'YYYY-MM-DD') || query.date('.description', 'DD MMMM YYYY', /\d{1,2} \w+ \d{4}/);
release.actors = query.all('.girl_item, .starring .item').map(actorEl => mapActor(actorEl, query, entity)); release.actors = query.all('.girl_item, .starring .item').map((actorEl) => mapActor(actorEl, query, entity));
release.duration = query.duration('.infos .description'); release.duration = query.duration('.infos .description');
@ -81,7 +81,7 @@ function scrapeScene({ query, html }, url, entity) {
release.tags = query.cnts('.tags a:not(.more_tag)'); release.tags = query.cnts('.tags a:not(.more_tag)');
release.poster = removeImageBorder(html.match(/image: "(.*?)"/)?.[1]); release.poster = removeImageBorder(html.match(/image: "(.*?)"/)?.[1]);
release.trailer = html.match(/url: "(.*mp4.*)"/g)?.map(src => ({ release.trailer = html.match(/url: "(.*mp4.*)"/g)?.map((src) => ({
src: src.match(/"(.*)"/)?.[1], src: src.match(/"(.*)"/)?.[1],
quality: Number(src.match(/[-/](\d+)p/)?.[1]), quality: Number(src.match(/[-/](\d+)p/)?.[1]),
})); }));

View File

@ -40,7 +40,7 @@ function scrapeScene({ query }, url, channel) {
release.date = date; release.date = date;
release.datePrecision = precision; release.datePrecision = precision;
release.actors = query.cnts(details.actors, 'a').map(actor => capitalize(actor, { uncapitalize: true })); release.actors = query.cnts(details.actors, 'a').map((actor) => capitalize(actor, { uncapitalize: true }));
release.duration = query.duration(details.duration); release.duration = query.duration(details.duration);
release.tags = query.cnts(details.genres, 'a'); release.tags = query.cnts(details.genres, 'a');

View File

@ -13,7 +13,7 @@ function scrapeAll(scenes) {
release.title = query.cnt('[class*="item-title"] a') || query.q('.bottom .link', 'title'); release.title = query.cnt('[class*="item-title"] a') || query.q('.bottom .link', 'title');
release.date = query.date('[class*="item-date"]', 'MMM DD, YYYY'); release.date = query.date('[class*="item-date"]', 'MMM DD, YYYY');
release.actors = query.all('[class*="item-actors"] a').map(el => ({ release.actors = query.all('[class*="item-actors"] a').map((el) => ({
name: query.cnt(el), name: query.cnt(el),
url: query.url(el, null), url: query.url(el, null),
})); }));
@ -44,7 +44,7 @@ function scrapeScene({ query }, url) {
release.description = query.meta('name=description') || query.q('read-even-more', true); release.description = query.meta('name=description') || query.q('read-even-more', true);
release.date = query.date('.h5-published', 'MMM DD, YYYY', /\w{3} \d{1,2}, \d{4}/); release.date = query.date('.h5-published', 'MMM DD, YYYY', /\w{3} \d{1,2}, \d{4}/);
release.actors = query.all('.video-top-details .actors a[href*="/models"]').map(el => ({ release.actors = query.all('.video-top-details .actors a[href*="/models"]').map((el) => ({
name: query.cnt(el), name: query.cnt(el),
url: query.url(el, null), url: query.url(el, null),
})); }));
@ -53,7 +53,7 @@ function scrapeScene({ query }, url) {
release.tags = query.all('.video-top-details a[href*="/categories"], .video-top-details a[href*="/tags"]', true); release.tags = query.all('.video-top-details a[href*="/categories"], .video-top-details a[href*="/tags"]', true);
release.poster = query.img('.poster img') || query.meta('itemprop=thumbnailUrl'); release.poster = query.img('.poster img') || query.meta('itemprop=thumbnailUrl');
release.photos = query.imgs('#gallery-thumbs [class*="thumb"]', 'data-bg').slice(1).map(photo => [ // first image is poster release.photos = query.imgs('#gallery-thumbs [class*="thumb"]', 'data-bg').slice(1).map((photo) => [ // first image is poster
photo.replace('512x288', '1472x828'), photo.replace('512x288', '1472x828'),
photo, photo,
]); ]);

View File

@ -16,7 +16,7 @@ const hairMap = {
async function scrapeProfile(html, _url, actorName) { async function scrapeProfile(html, _url, actorName) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const entries = Array.from(document.querySelectorAll('.infoPiece'), el => el.textContent.replace(/\n|\t/g, '').split(':')); const entries = Array.from(document.querySelectorAll('.infoPiece'), (el) => el.textContent.replace(/\n|\t/g, '').split(':'));
const bio = entries.reduce((acc, [key, value]) => (key ? { ...acc, [key.trim()]: value.trim() } : acc), {}); const bio = entries.reduce((acc, [key, value]) => (key ? { ...acc, [key.trim()]: value.trim() } : acc), {});
const profile = { const profile = {
@ -47,7 +47,7 @@ async function scrapeProfile(html, _url, actorName) {
if (bio.Tattoos) profile.hasTattoos = bio.Tattoos === 'Yes'; if (bio.Tattoos) profile.hasTattoos = bio.Tattoos === 'Yes';
if (avatarEl && !/default\//.test(avatarEl.src)) profile.avatar = avatarEl.src; if (avatarEl && !/default\//.test(avatarEl.src)) profile.avatar = avatarEl.src;
profile.social = Array.from(document.querySelectorAll('.socialList a'), el => el.href).filter(link => link !== 'https://www.twitter.com/'); // PH links to Twitter itself for some reason profile.social = Array.from(document.querySelectorAll('.socialList a'), (el) => el.href).filter((link) => link !== 'https://www.twitter.com/'); // PH links to Twitter itself for some reason
return profile; return profile;
} }

View File

@ -9,7 +9,7 @@ function scrapePhotos(html) {
const { qis } = ex(html, '#photos-page'); const { qis } = ex(html, '#photos-page');
const photos = qis('img'); const photos = qis('img');
return photos.map(photo => [ return photos.map((photo) => [
photo photo
.replace('x_800', 'x_xl') .replace('x_800', 'x_xl')
.replace('_tn', ''), .replace('_tn', ''),
@ -76,22 +76,22 @@ async function scrapeScene(html, url, site) {
release.actors = qu.all('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true); release.actors = qu.all('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true);
if (release.actors.length === 0) { if (release.actors.length === 0) {
const actorEl = qu.all('.stat').find(stat => /Featuring/.test(stat.textContent)); const actorEl = qu.all('.stat').find((stat) => /Featuring/.test(stat.textContent));
const actorString = qu.text(actorEl); const actorString = qu.text(actorEl);
release.actors = actorString?.split(/,\band\b|,/g).map(actor => actor.trim()) || []; release.actors = actorString?.split(/,\band\b|,/g).map((actor) => actor.trim()) || [];
} }
if (release.actors.length === 0 && site.parameters?.actors) release.actors = site.parameters.actors; if (release.actors.length === 0 && site.parameters?.actors) release.actors = site.parameters.actors;
release.tags = qu.all('a[href*=tag]', true); release.tags = qu.all('a[href*=tag]', true);
const dateEl = qu.all('.value').find(el => /\w+ \d+\w+, \d{4}/.test(el.textContent)); const dateEl = qu.all('.value').find((el) => /\w+ \d+\w+, \d{4}/.test(el.textContent));
release.date = qu.date(dateEl, null, 'MMMM Do, YYYY') release.date = qu.date(dateEl, null, 'MMMM Do, YYYY')
|| qu.date('.date', 'MMMM Do, YYYY', /\w+ \d{1,2}\w+, \d{4}/) || qu.date('.date', 'MMMM Do, YYYY', /\w+ \d{1,2}\w+, \d{4}/)
|| qu.date('.info .holder', 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/); || qu.date('.info .holder', 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/);
const durationEl = qu.all('value').find(el => /\d{1,3}:\d{2}/.test(el.textContent)); const durationEl = qu.all('value').find((el) => /\d{1,3}:\d{2}/.test(el.textContent));
release.duration = qu.dur(durationEl); release.duration = qu.dur(durationEl);
release.poster = qu.poster('video') || qu.img('.flowplayer img') || html.match(/posterImage: '(.*\.jpg)'/)?.[1] || null; // _800.jpg is larger than _xl.jpg in landscape release.poster = qu.poster('video') || qu.img('.flowplayer img') || html.match(/posterImage: '(.*\.jpg)'/)?.[1] || null; // _800.jpg is larger than _xl.jpg in landscape
@ -100,7 +100,7 @@ async function scrapeScene(html, url, site) {
if (photosUrl) { if (photosUrl) {
release.photos = await fetchPhotos(photosUrl); release.photos = await fetchPhotos(photosUrl);
} else { } else {
release.photos = qu.imgs('img[src*=ThumbNails], .p-photos .tn img').map(photo => [ release.photos = qu.imgs('img[src*=ThumbNails], .p-photos .tn img').map((photo) => [
photo.replace('_tn', ''), photo.replace('_tn', ''),
photo, photo,
]); ]);
@ -126,7 +126,7 @@ async function scrapeScene(html, url, site) {
function scrapeModels(html, actorName) { function scrapeModels(html, actorName) {
const { qa } = ex(html); const { qa } = ex(html);
const model = qa('.model a').find(link => link.title === actorName); const model = qa('.model a').find((link) => link.title === actorName);
return model?.href || null; return model?.href || null;
} }

View File

@ -15,7 +15,7 @@ function scrapeAll(scenes) {
release.entryId = getEntryId(release.url); release.entryId = getEntryId(release.url);
release.title = query.cnt('.title-label a'); release.title = query.cnt('.title-label a');
release.actors = query.all('.update_models a').map(el => ({ release.actors = query.all('.update_models a').map((el) => ({
name: query.cnt(el), name: query.cnt(el),
url: query.url(el, null), url: query.url(el, null),
})); }));
@ -37,7 +37,7 @@ function scrapeScene({ query }, url) {
release.description = query.cnt('#sceneInfo .description'); release.description = query.cnt('#sceneInfo .description');
release.actors = query.all('#sceneInfo .data-others a[href*="/models"]').map(el => ({ release.actors = query.all('#sceneInfo .data-others a[href*="/models"]').map((el) => ({
name: query.el(el, null, 'title'), name: query.el(el, null, 'title'),
url: query.url(el, null), url: query.url(el, null),
})); }));
@ -50,8 +50,8 @@ function scrapeScene({ query }, url) {
release.poster = [poster, poster?.replace(/imgw=\w+/, 'imgw=680')]; release.poster = [poster, poster?.replace(/imgw=\w+/, 'imgw=680')];
release.photos = query.imgs('.photos-holder img') release.photos = query.imgs('.photos-holder img')
.filter(src => new URL(src).pathname !== posterPathname) .filter((src) => new URL(src).pathname !== posterPathname)
.map(src => [ .map((src) => [
src.replace(/imgw=\d+/, 'imgw=1284'), src.replace(/imgw=\d+/, 'imgw=1284'),
src, src,
]); ]);
@ -74,7 +74,7 @@ function scrapeProfileScenes(scenes) {
release.description = query.cnt('.model-update-description'); release.description = query.cnt('.model-update-description');
release.actors = query.all('.model-labels a').map(el => ({ release.actors = query.all('.model-labels a').map((el) => ({
name: query.cnt(el), name: query.cnt(el),
url: query.url(el, null), url: query.url(el, null),
})); }));

View File

@ -13,7 +13,7 @@ function getChannelSlug(channelName, entity) {
} }
const channelSlug = slugify(channelName, '', { removePunctuation: true }); const channelSlug = slugify(channelName, '', { removePunctuation: true });
const channel = entity.children.find(child => new RegExp(channelSlug).test(child.slug)); const channel = entity.children.find((child) => new RegExp(channelSlug).test(child.slug));
return channel?.slug || null; return channel?.slug || null;
} }
@ -27,8 +27,8 @@ function scrapeScene(scene, channel) {
release.title = scene.title; release.title = scene.title;
release.date = qu.extractDate(scene.publishedDate); release.date = qu.extractDate(scene.publishedDate);
release.actors = scene.models?.map(model => model.modelName) || []; release.actors = scene.models?.map((model) => model.modelName) || [];
release.actors = scene.models?.map(model => ({ release.actors = scene.models?.map((model) => ({
name: model.modelName, name: model.modelName,
avatar: `https://images.mylfcdn.net/tsv4/model/profiles/${slugify(model.modelName, '_')}.jpg`, avatar: `https://images.mylfcdn.net/tsv4/model/profiles/${slugify(model.modelName, '_')}.jpg`,
url: `${channel.url}/models/www.mylf.com/models/${model.modelId}`, url: `${channel.url}/models/www.mylf.com/models/${model.modelId}`,
@ -113,7 +113,7 @@ function scrapeProfile(actor, entity) {
} }
profile.avatar = actor.img; profile.avatar = actor.img;
profile.scenes = actor.movies?.map(scene => scrapeScene(scene, entity)); profile.scenes = actor.movies?.map((scene) => scrapeScene(scene, entity));
return profile; return profile;
} }

View File

@ -25,15 +25,15 @@ function scrapeAll(scenes, entity) {
release.date = moment.utc(scene.year, 'YYYY').toDate(); release.date = moment.utc(scene.year, 'YYYY').toDate();
release.datePrecision = 'year'; release.datePrecision = 'year';
release.actors = scene.actors.map(actor => ({ release.actors = scene.actors.map((actor) => ({
name: actor.name.trim(), name: actor.name.trim(),
avatar: actor.image || null, avatar: actor.image || null,
})).filter(actor => actor.name && slugify(actor.name) !== 'amateur-girl'); })).filter((actor) => actor.name && slugify(actor.name) !== 'amateur-girl');
release.duration = scene.duration; release.duration = scene.duration;
release.stars = scene.video_rating_score; release.stars = scene.video_rating_score;
[release.poster, ...release.photos] = scene.screenshots.map(url => prefixUrl(url)); [release.poster, ...release.photos] = scene.screenshots.map((url) => prefixUrl(url));
if (scene.is_gay) { if (scene.is_gay) {
release.tags = ['gay']; release.tags = ['gay'];
@ -64,7 +64,7 @@ async function scrapeScene({ query }, url) {
release.description = query.q('.detail-description', true); release.description = query.q('.detail-description', true);
release.duration = query.dur('.detail-meta li:first-child'); release.duration = query.dur('.detail-meta li:first-child');
const actors = [query.q('.detail-hero-title h1', true)?.trim()].filter(name => name && slugify(name) !== 'amateur-girl'); const actors = [query.q('.detail-hero-title h1', true)?.trim()].filter((name) => name && slugify(name) !== 'amateur-girl');
if (actors.length > 0) { if (actors.length > 0) {
release.actors = actors; release.actors = actors;
@ -143,7 +143,7 @@ async function fetchProfile({ name: actorName }, { entity }, include) {
const res = await http.get(`https://teencoreclub.com/api/actors?query=${actorName}`); const res = await http.get(`https://teencoreclub.com/api/actors?query=${actorName}`);
if (res.ok) { if (res.ok) {
const actor = res.body.data.find(item => slugify(item.name) === slugify(actorName)); const actor = res.body.data.find((item) => slugify(item.name) === slugify(actorName));
if (actor) { if (actor) {
return scrapeProfile(actor, entity, include); return scrapeProfile(actor, entity, include);

View File

@ -14,7 +14,7 @@ function scrapeAll(scenes, channel) {
release.title = query.cnt('.title'); release.title = query.cnt('.title');
release.date = query.date('time', 'MMMM D, YYYY'); release.date = query.date('time', 'MMMM D, YYYY');
release.actors = query.all('.actors a').map(el => ({ release.actors = query.all('.actors a').map((el) => ({
name: query.cnt(el), name: query.cnt(el),
url: query.url(el, null), url: query.url(el, null),
})); }));
@ -29,7 +29,7 @@ function scrapeAll(scenes, channel) {
const siteId = query.url('.site a', 'href', { origin: network.url, object: true })?.searchParams.get('site[]'); const siteId = query.url('.site a', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
if (siteId) { if (siteId) {
release.channel = network.children.find(child => child.parameters.siteId.toString() === siteId)?.slug; release.channel = network.children.find((child) => child.parameters.siteId.toString() === siteId)?.slug;
} }
return release; return release;
@ -48,7 +48,7 @@ function scrapeScene({ query }, url, channel) {
release.date = query.date('.title-line .date', 'MMMM D, YYYY'); release.date = query.date('.title-line .date', 'MMMM D, YYYY');
release.duration = query.number('.dur') * 60; release.duration = query.number('.dur') * 60;
release.actors = query.all('.site a[href*="/models"]').map(el => ({ release.actors = query.all('.site a[href*="/models"]').map((el) => ({
name: query.cnt(el), name: query.cnt(el),
url: query.url(el, null), url: query.url(el, null),
})); }));
@ -63,7 +63,7 @@ function scrapeScene({ query }, url, channel) {
const siteId = query.url('.site a[href*="site[]"]', 'href', { origin: network.url, object: true })?.searchParams.get('site[]'); const siteId = query.url('.site a[href*="site[]"]', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
if (siteId) { if (siteId) {
release.channel = network.children.find(child => child.parameters.siteId.toString() === siteId)?.slug; release.channel = network.children.find((child) => child.parameters.siteId.toString() === siteId)?.slug;
} }
return release; return release;

View File

@ -20,7 +20,7 @@ function scrapeSceneX(scene) {
release.date = new Date(scene.release_date); release.date = new Date(scene.release_date);
release.actors = scene.models release.actors = scene.models
.map(actor => (/&/.test(actor.name) .map((actor) => (/&/.test(actor.name)
? actor.name.split(/\s*&\s*/) ? actor.name.split(/\s*&\s*/)
: { : {
name: actor.name, name: actor.name,
@ -31,7 +31,7 @@ function scrapeSceneX(scene) {
.flat(); .flat();
release.stars = scene.rating; release.stars = scene.rating;
release.tags = scene.tags.map(tag => tag.name); release.tags = scene.tags.map((tag) => tag.name);
if (mime.getType(scene.thumb) === 'image/gif') { if (mime.getType(scene.thumb) === 'image/gif') {
release.teaser = scene.thumb; release.teaser = scene.thumb;
@ -128,7 +128,7 @@ async function fetchProfile(baseActor, entity, options) {
return searchRes.status; return searchRes.status;
} }
const actor = searchRes.body.models.items.find(model => slugify(model.name) === slugify(baseActor.name)); const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
if (actor) { if (actor) {
return scrapeProfile(actor, options); return scrapeProfile(actor, options);

View File

@ -217,7 +217,7 @@ function gender() {
} }
function actors(release) { function actors(release) {
const length = release.tags.some(tag => ['dp', 'dap', 'gangbang'].includes(tag)) const length = release.tags.some((tag) => ['dp', 'dap', 'gangbang'].includes(tag))
? Math.floor(Math.random() * 6) + 3 ? Math.floor(Math.random() * 6) + 3
: Math.floor(Math.random() * 3) + 2; : Math.floor(Math.random() * 3) + 2;
@ -254,7 +254,7 @@ async function fetchLatest(entity, page, options) {
// const poster = 'sfw/kittens/thumbs/iNEXVlX-RLs.jpeg'; // const poster = 'sfw/kittens/thumbs/iNEXVlX-RLs.jpeg';
release.poster = `http://${config.web.host}:${config.web.port}/img/${poster}?id=${nanoid()}`; // ensure source is unique release.poster = `http://${config.web.host}:${config.web.port}/img/${poster}?id=${nanoid()}`; // ensure source is unique
release.photos = photos.map(photo => `http://${config.web.host}:${config.web.port}/img/${photo}?id=${nanoid()}`); release.photos = photos.map((photo) => `http://${config.web.host}:${config.web.port}/img/${photo}?id=${nanoid()}`);
} }
release.tags = await knex('tags') release.tags = await knex('tags')

View File

@ -17,7 +17,7 @@ function scrapeLatestNative(scenes, site) {
release.date = ed(scene.release_date, 'YYYY-MM-DD'); release.date = ed(scene.release_date, 'YYYY-MM-DD');
release.duration = parseInt(scene.runtime, 10) * 60; release.duration = parseInt(scene.runtime, 10) * 60;
release.actors = scene.cast?.map(actor => ({ release.actors = scene.cast?.map((actor) => ({
name: actor.stagename, name: actor.stagename,
gender: actor.gender.toLowerCase(), gender: actor.gender.toLowerCase(),
avatar: actor.placard, avatar: actor.placard,
@ -38,10 +38,10 @@ function scrapeSceneNative({ html, q, qa }, url, _site) {
release.title = q('.scene-h2-heading', true); release.title = q('.scene-h2-heading', true);
release.description = q('.indie-model-p', true); release.description = q('.indie-model-p', true);
const dateString = qa('h5').find(el => /Released/.test(el.textContent)).textContent; const dateString = qa('h5').find((el) => /Released/.test(el.textContent)).textContent;
release.date = ed(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/); release.date = ed(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
const duration = qa('h5').find(el => /Runtime/.test(el.textContent)).textContent; const duration = qa('h5').find((el) => /Runtime/.test(el.textContent)).textContent;
const [hours, minutes] = duration.match(/\d+/g); const [hours, minutes] = duration.match(/\d+/g);
if (minutes) release.duration = (hours * 3600) + (minutes * 60); if (minutes) release.duration = (hours * 3600) + (minutes * 60);
@ -111,7 +111,7 @@ async function fetchSceneWrapper(url, site, release) {
}); });
if (searchRes.statusCode === 200 && searchRes.body.code === 200) { if (searchRes.statusCode === 200 && searchRes.body.code === 200) {
const sceneMatch = searchRes.body.responseData.find(item => slugify(item.name) === slugify(scene.title)); const sceneMatch = searchRes.body.responseData.find((item) => slugify(item.name) === slugify(scene.title));
if (sceneMatch) { if (sceneMatch) {
return { return {

View File

@ -15,7 +15,7 @@ const genderMap = {
function getPosterFallbacks(poster) { function getPosterFallbacks(poster) {
return poster return poster
.filter(image => /landscape/i.test(image.name)) .filter((image) => /landscape/i.test(image.name))
.sort((imageA, imageB) => imageB.height - imageA.height) .sort((imageA, imageB) => imageB.height - imageA.height)
.map((image) => { .map((image) => {
const sources = [image.src, image.highdpi?.['2x'], image.highdpi?.['3x']]; const sources = [image.src, image.highdpi?.['2x'], image.highdpi?.['3x']];
@ -23,7 +23,7 @@ function getPosterFallbacks(poster) {
return image.height === 1080 ? sources : sources.reverse(); return image.height === 1080 ? sources : sources.reverse();
}) })
.flat() .flat()
.map(src => ({ .map((src) => ({
src, src,
expectType: { expectType: {
'binary/octet-stream': 'image/jpeg', 'binary/octet-stream': 'image/jpeg',
@ -33,8 +33,8 @@ function getPosterFallbacks(poster) {
function getTeaserFallbacks(teaser) { function getTeaserFallbacks(teaser) {
return teaser return teaser
.filter(video => /landscape/i.test(video.name)) .filter((video) => /landscape/i.test(video.name))
.map(video => ({ .map((video) => ({
src: video.src, src: video.src,
type: video.type, type: video.type,
quality: Number(String(video.height).replace('353', '360')), quality: Number(String(video.height).replace('353', '360')),
@ -44,7 +44,7 @@ function getTeaserFallbacks(teaser) {
function getAvatarFallbacks(avatar) { function getAvatarFallbacks(avatar) {
return avatar return avatar
.sort((imageA, imageB) => imageB.height - imageA.height) .sort((imageA, imageB) => imageB.height - imageA.height)
.map(image => [image.highdpi?.['3x'], image.highdpi?.['2x'], image.src]) .map((image) => [image.highdpi?.['3x'], image.highdpi?.['2x'], image.src])
.flat(); .flat();
} }
@ -149,7 +149,7 @@ async function getPhotos(url) {
}); });
const state = htmlRes?.window.__APOLLO_STATE__; const state = htmlRes?.window.__APOLLO_STATE__;
const key = Object.values(state.ROOT_QUERY).find(query => query?.__ref)?.__ref; const key = Object.values(state.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
const data = state[key]; const data = state[key];
console.log(data); console.log(data);
@ -158,7 +158,7 @@ async function getPhotos(url) {
return []; return [];
} }
return data.carousel.slice(1).map(photo => photo.main?.[0].src).filter(Boolean); return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
} }
function scrapeAll(scenes, site, origin) { function scrapeAll(scenes, site, origin) {
@ -191,7 +191,7 @@ function scrapeUpcoming(scene, site) {
release.title = scene.targetUrl release.title = scene.targetUrl
.slice(1) .slice(1)
.split('-') .split('-')
.map(component => `${component.charAt(0).toUpperCase()}${component.slice(1)}`) .map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
.join(' '); .join(' ');
release.url = `${site.url}/videos${scene.targetUrl}`; release.url = `${site.url}/videos${scene.targetUrl}`;
@ -243,7 +243,7 @@ async function scrapeScene(data, url, site, baseRelease, options) {
const trailer = await getTrailer(scene, site, url); const trailer = await getTrailer(scene, site, url);
if (trailer) release.trailer = trailer; if (trailer) release.trailer = trailer;
release.chapters = data.video.chapters?.video.map(chapter => ({ release.chapters = data.video.chapters?.video.map((chapter) => ({
tags: [chapter.title], tags: [chapter.title],
time: chapter.seconds, time: chapter.seconds,
})); }));

View File

@ -131,7 +131,7 @@ function scrapeScene(html, url) {
release.actors = qu.all('.info-video-models a', true); release.actors = qu.all('.info-video-models a', true);
release.tags = qu.all('.info-video-category a', true); release.tags = qu.all('.info-video-category a', true);
release.photos = qu.urls('.swiper-wrapper .swiper-slide a').map(source => source.replace('.jpg/', '.jpg')); release.photos = qu.urls('.swiper-wrapper .swiper-slide a').map((source) => source.replace('.jpg/', '.jpg'));
release.poster = qu.meta('meta[property="og:image"]'); release.poster = qu.meta('meta[property="og:image"]');
if (!release.poster) { if (!release.poster) {

View File

@ -22,7 +22,7 @@ async function getTrailerUrl(release, channel, request) {
}); });
if (res.ok) { if (res.ok) {
const trailers = res.body.streams.map(trailer => ({ const trailers = res.body.streams.map((trailer) => ({
src: trailer.url, src: trailer.url,
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]), quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
vr: true, vr: true,
@ -47,7 +47,7 @@ function scrapeAll(scenes, channel) {
release.title = query.cnt('.card__h'); release.title = query.cnt('.card__h');
release.date = query.date('.card__date', 'D MMMM, YYYY'); release.date = query.date('.card__date', 'D MMMM, YYYY');
release.actors = query.all('.card__links a').map(el => ({ release.actors = query.all('.card__links a').map((el) => ({
name: qu.query.cnt(el), name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }), url: qu.query.url(el, null, 'href', { origin: channel.url }),
})); }));
@ -82,14 +82,14 @@ async function scrapeScene({ query }, url, channel, baseRelease, options, reques
release.date = query.date('.detail__date', 'D MMMM, YYYY'); release.date = query.date('.detail__date', 'D MMMM, YYYY');
release.duration = query.number('.time') * 60; release.duration = query.number('.time') * 60;
release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map(el => ({ release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map((el) => ({
name: qu.query.cnt(el), name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }), url: qu.query.url(el, null, 'href', { origin: channel.url }),
})); }));
release.tags = query.cnts('.tag-list .tag').concat(query.cnts('.detail__specs-list .detail__specs-item')); release.tags = query.cnts('.tag-list .tag').concat(query.cnts('.detail__specs-list .detail__specs-item'));
release.photos = query.all('.photo-strip__slide').map(el => ([ release.photos = query.all('.photo-strip__slide').map((el) => ([
qu.query.img(el, null, 'data-src'), qu.query.img(el, null, 'data-src'),
qu.query.img(el, 'img', 'src'), qu.query.img(el, 'img', 'src'),
])); ]));

View File

@ -18,7 +18,7 @@ function scrapeLatest(html, site) {
release.entryId = scene.dataset.videoId; release.entryId = scene.dataset.videoId;
release.title = scene.querySelector('.card-title').textContent; release.title = scene.querySelector('.card-title').textContent;
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate(); release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent); release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent);
// slow CDN? // slow CDN?
const poster = scene.querySelector('.single-image').dataset.src; const poster = scene.querySelector('.single-image').dataset.src;
@ -32,7 +32,7 @@ function scrapeLatest(html, site) {
concurrency: 1, concurrency: 1,
}; };
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({ release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), (el) => ({
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`), src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,
@ -63,7 +63,7 @@ function scrapeScene(html, site, url) {
release.url = url; release.url = url;
release.title = scene.querySelector('.t2019-stitle').textContent.trim(); release.title = scene.querySelector('.t2019-stitle').textContent.trim();
release.description = scene.querySelector('#t2019-description').textContent.trim(); release.description = scene.querySelector('#t2019-description').textContent.trim();
release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), el => el.textContent); release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), (el) => el.textContent);
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span')); const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
@ -75,7 +75,7 @@ function scrapeScene(html, site, url) {
} }
// unreliable CDN // unreliable CDN
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => ({ release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), (el) => ({
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`), src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,

View File

@ -39,7 +39,7 @@ function curateSite(site, includeParameters = false) {
} }
async function curateSites(sites, includeParameters) { async function curateSites(sites, includeParameters) {
return Promise.all(sites.map(async site => curateSite(site, includeParameters))); return Promise.all(sites.map(async (site) => curateSite(site, includeParameters)));
} }
function destructConfigNetworks(networks = []) { function destructConfigNetworks(networks = []) {
@ -165,7 +165,7 @@ async function fetchIncludedSites() {
async function fetchSites(queryObject) { async function fetchSites(queryObject) {
const sites = await knex('sites') const sites = await knex('sites')
.where(builder => whereOr(queryObject, 'sites', builder)) .where((builder) => whereOr(queryObject, 'sites', builder))
.select( .select(
'sites.*', 'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',

View File

@ -58,7 +58,7 @@ async function fetchStashes(domain, itemId, sessionUser) {
}) })
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`); .leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
return stashes.map(stash => curateStash(stash)); return stashes.map((stash) => curateStash(stash));
} }
async function createStash(newStash, sessionUser) { async function createStash(newStash, sessionUser) {

View File

@ -17,7 +17,7 @@ const { notify } = require('./alerts');
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') { async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
const slugBase = release.title const slugBase = release.title
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map(actor => actor.name).join(' ')}`) || (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`) || (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
|| null; || null;
@ -74,11 +74,11 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce
} }
async function attachChannelEntities(releases) { async function attachChannelEntities(releases) {
const releasesWithoutEntity = releases.filter(release => release.channel && (!release.entity || release.entity.type === 'network')); const releasesWithoutEntity = releases.filter((release) => release.channel && (!release.entity || release.entity.type === 'network'));
const channelEntities = await knex('entities') const channelEntities = await knex('entities')
.select(knex.raw('entities.*, row_to_json(parents) as parent')) .select(knex.raw('entities.*, row_to_json(parents) as parent'))
.whereIn('entities.slug', releasesWithoutEntity.map(release => release.channel)) .whereIn('entities.slug', releasesWithoutEntity.map((release) => release.channel))
.where('entities.type', 'channel') .where('entities.type', 'channel')
.leftJoin('entities AS parents', 'parents.id', 'entities.parent_id'); .leftJoin('entities AS parents', 'parents.id', 'entities.parent_id');
@ -108,7 +108,7 @@ async function attachChannelEntities(releases) {
} }
async function attachStudios(releases) { async function attachStudios(releases) {
const studioSlugs = releases.map(release => release.studio).filter(Boolean); const studioSlugs = releases.map((release) => release.studio).filter(Boolean);
const studios = await knex('entities') const studios = await knex('entities')
.whereIn('slug', studioSlugs) .whereIn('slug', studioSlugs)
@ -186,7 +186,7 @@ function filterInternalDuplicateReleases(releases) {
}, {}); }, {});
return Object.values(releasesByEntityIdAndEntryId) return Object.values(releasesByEntityIdAndEntryId)
.map(entityReleases => Object.values(entityReleases)) .map((entityReleases) => Object.values(entityReleases))
.flat(); .flat();
} }
@ -194,11 +194,11 @@ async function filterDuplicateReleases(releases) {
const internalUniqueReleases = filterInternalDuplicateReleases(releases); const internalUniqueReleases = filterInternalDuplicateReleases(releases);
const duplicateReleaseEntries = await knex('releases') const duplicateReleaseEntries = await knex('releases')
.whereIn(['entry_id', 'entity_id'], internalUniqueReleases.map(release => [release.entryId, release.entity.id])) .whereIn(['entry_id', 'entity_id'], internalUniqueReleases.map((release) => [release.entryId, release.entity.id]))
.orWhereIn(['entry_id', 'entity_id'], internalUniqueReleases .orWhereIn(['entry_id', 'entity_id'], internalUniqueReleases
// scene IDs shared across network, mark as duplicate so scene can be updated with channel if only available on release day (i.e. Perv City) // scene IDs shared across network, mark as duplicate so scene can be updated with channel if only available on release day (i.e. Perv City)
.filter(release => release.entity.parent?.parameters?.networkEntryIds) .filter((release) => release.entity.parent?.parameters?.networkEntryIds)
.map(release => [release.entryId, release.entity.parent.id])); .map((release) => [release.entryId, release.entity.parent.id]));
const duplicateReleasesByEntityIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => { const duplicateReleasesByEntityIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
if (!acc[release.entity_id]) acc[release.entity_id] = {}; if (!acc[release.entity_id]) acc[release.entity_id] = {};
@ -207,10 +207,10 @@ async function filterDuplicateReleases(releases) {
return acc; return acc;
}, {}); }, {});
const duplicateReleases = internalUniqueReleases.filter(release => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId] const duplicateReleases = internalUniqueReleases.filter((release) => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|| duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]); || duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId] const uniqueReleases = internalUniqueReleases.filter((release) => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
&& !duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]); && !duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
return { return {
@ -263,7 +263,7 @@ async function updateSceneSearch(releaseIds) {
async function storeChapters(releases) { async function storeChapters(releases) {
const chapters = releases const chapters = releases
.map(release => release.chapters?.map((chapter, index) => ({ .map((release) => release.chapters?.map((chapter, index) => ({
releaseId: release.id, releaseId: release.id,
index: index + 1, index: index + 1,
time: chapter.time, time: chapter.time,
@ -278,7 +278,7 @@ async function storeChapters(releases) {
.filter(Boolean) .filter(Boolean)
.sort((chapterA, chapterB) => chapterA.time - chapterB.time); .sort((chapterA, chapterB) => chapterA.time - chapterB.time);
const curatedChapterEntries = chapters.map(chapter => ({ const curatedChapterEntries = chapters.map((chapter) => ({
index: chapter.index, index: chapter.index,
time: chapter.time, time: chapter.time,
duration: chapter.duration, duration: chapter.duration,
@ -297,7 +297,7 @@ async function storeChapters(releases) {
}, },
}), {}); }), {});
const chaptersWithId = chapters.map(chapter => ({ const chaptersWithId = chapters.map((chapter) => ({
...chapter, ...chapter,
id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.index], id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.index],
})); }));
@ -316,13 +316,13 @@ async function storeScenes(releases) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id'); const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const releasesWithChannels = await attachChannelEntities(releases); const releasesWithChannels = await attachChannelEntities(releases);
const releasesWithBaseActors = releasesWithChannels.map(release => ({ ...release, actors: toBaseActors(release.actors) })); const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) }));
const releasesWithStudios = await attachStudios(releasesWithBaseActors); const releasesWithStudios = await attachStudios(releasesWithBaseActors);
// uniqueness is entity ID + entry ID, filter uniques after adding entities // uniqueness is entity ID + entry ID, filter uniques after adding entities
const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios); const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId))); const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId)));
const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries); const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries);
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : []; const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
@ -355,13 +355,13 @@ async function storeScenes(releases) {
]); ]);
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
await updateSceneSearch(releasesWithId.map(release => release.id)); await updateSceneSearch(releasesWithId.map((release) => release.id));
// media is more error-prone, associate separately // media is more error-prone, associate separately
await associateReleaseMedia(releasesWithId); await associateReleaseMedia(releasesWithId);
if (argv.sceneActors && actors) { if (argv.sceneActors && actors) {
await scrapeActors(actors.map(actor => actor.name)); await scrapeActors(actors.map((actor) => actor.name));
} }
logger.info(`Stored ${storedReleaseEntries.length}, updated ${updated.rowCount} releases`); logger.info(`Stored ${storedReleaseEntries.length}, updated ${updated.rowCount} releases`);
@ -446,12 +446,12 @@ async function storeMovies(movies) {
const { uniqueReleases } = await filterDuplicateReleases(movies); const { uniqueReleases } = await filterDuplicateReleases(movies);
const [batchId] = await knex('batches').insert({ comment: null }).returning('id'); const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const curatedMovieEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie'))); const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true); const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
const moviesWithId = attachReleaseIds(movies, storedMovies); const moviesWithId = attachReleaseIds(movies, storedMovies);
await updateMovieSearch(moviesWithId.map(movie => movie.id)); await updateMovieSearch(moviesWithId.map((movie) => movie.id));
await associateReleaseMedia(moviesWithId, 'movie'); await associateReleaseMedia(moviesWithId, 'movie');
return moviesWithId; return moviesWithId;

View File

@ -32,7 +32,7 @@ function curateTag(tag) {
priority: tag.priority, priority: tag.priority,
group: curateTag(tag.group), group: curateTag(tag.group),
aliasFor: curateTag(tag.alias), aliasFor: curateTag(tag.alias),
aliases: (tag.aliases || []).map(aliasTag => curateTag(aliasTag)), aliases: (tag.aliases || []).map((aliasTag) => curateTag(aliasTag)),
}; };
if (tag.poster) { if (tag.poster) {
@ -40,7 +40,7 @@ function curateTag(tag) {
} }
if (tag.photos) { if (tag.photos) {
curatedTag.photos = tag.photos.map(photo => curateTagMedia(photo)); curatedTag.photos = tag.photos.map((photo) => curateTagMedia(photo));
} }
return curatedTag; return curatedTag;
@ -75,13 +75,13 @@ function withRelations(queryBuilder, withMedia) {
async function matchReleaseTags(releases) { async function matchReleaseTags(releases) {
const rawTags = releases const rawTags = releases
.map(release => release.tags).flat() .map((release) => release.tags).flat()
.filter(Boolean); .filter(Boolean);
const casedTags = [...new Set( const casedTags = [...new Set(
rawTags rawTags
.concat(rawTags.map(tag => tag.toLowerCase())) .concat(rawTags.map((tag) => tag.toLowerCase()))
.concat(rawTags.map(tag => tag.toUpperCase())), .concat(rawTags.map((tag) => tag.toUpperCase())),
)]; )];
const tagEntries = await knex('tags') const tagEntries = await knex('tags')
@ -98,7 +98,7 @@ async function matchReleaseTags(releases) {
} }
async function getEntityTags(releases) { async function getEntityTags(releases) {
const entityIds = releases.map(release => release.entity?.id).filter(Boolean); const entityIds = releases.map((release) => release.entity?.id).filter(Boolean);
const entityTags = await knex('entities_tags').whereIn('entity_id', entityIds); const entityTags = await knex('entities_tags').whereIn('entity_id', entityIds);
const entityTagIdsByEntityId = entityTags.reduce((acc, entityTag) => { const entityTagIdsByEntityId = entityTags.reduce((acc, entityTag) => {
@ -117,12 +117,12 @@ async function getEntityTags(releases) {
function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type) { function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type) {
const tagAssociations = releases const tagAssociations = releases
.map((release) => { .map((release) => {
const entityTagIds = entityTagIdsByEntityId[release.entity?.id]?.map(tag => ({ id: tag.id, origin: tag.name })) || []; const entityTagIds = entityTagIdsByEntityId[release.entity?.id]?.map((tag) => ({ id: tag.id, origin: tag.name })) || [];
const releaseTags = release.tags?.filter(Boolean) || []; const releaseTags = release.tags?.filter(Boolean) || [];
const releaseTagsWithIds = releaseTags.every(tag => typeof tag === 'number') const releaseTagsWithIds = releaseTags.every((tag) => typeof tag === 'number')
? releaseTags // obsolete scraper returned pre-matched tags ? releaseTags // obsolete scraper returned pre-matched tags
: releaseTags.map(tag => ({ : releaseTags.map((tag) => ({
id: tagIdsBySlug[slugify(tag)], id: tagIdsBySlug[slugify(tag)],
original: tag, original: tag,
})); }));
@ -133,7 +133,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit
.concat(entityTagIds) .concat(entityTagIds)
.filter(Boolean), .filter(Boolean),
)] )]
.map(tag => ({ .map((tag) => ({
[`${type}_id`]: release.id, [`${type}_id`]: release.id,
tag_id: tag.id, tag_id: tag.id,
original_tag: tag.original, original_tag: tag.original,
@ -161,7 +161,7 @@ async function associateReleaseTags(releases, type = 'release') {
async function fetchTag(tagId) { async function fetchTag(tagId) {
const tag = await knex('tags') const tag = await knex('tags')
.modify(queryBuilder => withRelations(queryBuilder, true)) .modify((queryBuilder) => withRelations(queryBuilder, true))
.where((builder) => { .where((builder) => {
if (Number(tagId)) { if (Number(tagId)) {
builder.where('tags.id', tagId); builder.where('tags.id', tagId);
@ -179,10 +179,10 @@ async function fetchTag(tagId) {
async function fetchTags(limit = 100) { async function fetchTags(limit = 100) {
const tags = await knex('tags') const tags = await knex('tags')
.modify(queryBuilder => withRelations(queryBuilder, false)) .modify((queryBuilder) => withRelations(queryBuilder, false))
.limit(limit); .limit(limit);
return tags.map(tag => curateTag(tag)); return tags.map((tag) => curateTag(tag));
} }
module.exports = { module.exports = {

View File

@ -28,8 +28,8 @@ function mapReleasesToEntityIdAndEntryId(acc, release) {
function filterLocalUniqueReleases(releases, accReleases) { function filterLocalUniqueReleases(releases, accReleases) {
const localDuplicateReleasesBySiteIdAndEntryId = accReleases.reduce(mapReleasesToEntityIdAndEntryId, {}); const localDuplicateReleasesBySiteIdAndEntryId = accReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
const localUniqueReleases = releases.filter(release => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]); const localUniqueReleases = releases.filter((release) => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
const localDuplicateReleases = releases.filter(release => localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]); const localDuplicateReleases = releases.filter((release) => localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
return { return {
localUniqueReleases, localUniqueReleases,
@ -39,7 +39,7 @@ function filterLocalUniqueReleases(releases, accReleases) {
async function filterUniqueReleases(releases) { async function filterUniqueReleases(releases) {
const releaseIdentifiers = releases const releaseIdentifiers = releases
.map(release => [release.entity.id, release.entryId]); .map((release) => [release.entity.id, release.entryId]);
const duplicateReleaseEntries = await knex('releases') const duplicateReleaseEntries = await knex('releases')
.select(knex.raw('releases.*, row_to_json(entities) as entity')) .select(knex.raw('releases.*, row_to_json(entities) as entity'))
@ -55,13 +55,13 @@ async function filterUniqueReleases(releases) {
.orWhere(knex.raw('updated_at - date > INTERVAL \'1 day\'')); // scene was updated after the release date, no updates expected .orWhere(knex.raw('updated_at - date > INTERVAL \'1 day\'')); // scene was updated after the release date, no updates expected
}); });
const duplicateReleases = duplicateReleaseEntries.map(release => curateRelease(release)); const duplicateReleases = duplicateReleaseEntries.map((release) => curateRelease(release));
const duplicateReleasesByEntityIdAndEntryId = duplicateReleases.reduce(mapReleasesToEntityIdAndEntryId, {}); const duplicateReleasesByEntityIdAndEntryId = duplicateReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
const internalUniqueReleasesByEntityIdAndEntryId = releases.reduce((acc, release) => mapReleasesToEntityIdAndEntryId(acc, release), {}); const internalUniqueReleasesByEntityIdAndEntryId = releases.reduce((acc, release) => mapReleasesToEntityIdAndEntryId(acc, release), {});
const internalUniqueReleases = Object.values(internalUniqueReleasesByEntityIdAndEntryId).map(releasesByEntryId => Object.values(releasesByEntryId)).flat(); const internalUniqueReleases = Object.values(internalUniqueReleasesByEntityIdAndEntryId).map((releasesByEntryId) => Object.values(releasesByEntryId)).flat();
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]); const uniqueReleases = internalUniqueReleases.filter((release) => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
return { uniqueReleases, duplicateReleases }; return { uniqueReleases, duplicateReleases };
} }
@ -83,7 +83,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
return accReleases.length + pageReleases.length < argv.last; return accReleases.length + pageReleases.length < argv.last;
} }
if (!pageReleases.concat(unextracted).every(release => !!release.date)) { // some scenes don't have dates if (!pageReleases.concat(unextracted).every((release) => !!release.date)) { // some scenes don't have dates
return accReleases.length + pageReleases.length < argv.missingDateLimit; return accReleases.length + pageReleases.length < argv.missingDateLimit;
} }
@ -124,8 +124,8 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
return accReleases; return accReleases;
} }
const validPageReleases = pageReleases.filter(release => release?.entryId); // filter out empty and unidentified releases const validPageReleases = pageReleases.filter((release) => release?.entryId); // filter out empty and unidentified releases
const pageReleasesWithEntity = validPageReleases.map(release => ({ ...release, entity: release.entity || entity })); const pageReleasesWithEntity = validPageReleases.map((release) => ({ ...release, entity: release.entity || entity }));
if (pageReleases.length > validPageReleases.length) { if (pageReleases.length > validPageReleases.length) {
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`); logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
@ -140,10 +140,10 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const releases = await scrapeReleasesPage(argv.page || 1, []); const releases = await scrapeReleasesPage(argv.page || 1, []);
const hasDates = releases.every(release => !!release.date); const hasDates = releases.every((release) => !!release.date);
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0))) const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|| (hasDates && releases.filter(release => moment(release.date).isAfter(argv.after))) || (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|| releases.slice(0, Math.max(argv.missingDateLimit, 0)); || releases.slice(0, Math.max(argv.missingDateLimit, 0));
const { uniqueReleases, duplicateReleases } = argv.force const { uniqueReleases, duplicateReleases } = argv.force
@ -280,7 +280,7 @@ async function fetchUpdates() {
const scrapedNetworks = await Promise.map( const scrapedNetworks = await Promise.map(
includedNetworks, includedNetworks,
async networkEntity => (networkEntity.parameters?.sequential async (networkEntity) => (networkEntity.parameters?.sequential
? scrapeNetworkSequential(networkEntity) ? scrapeNetworkSequential(networkEntity)
: scrapeNetworkParallel(networkEntity)), : scrapeNetworkParallel(networkEntity)),
{ concurrency: 5 }, { concurrency: 5 },

View File

@ -18,7 +18,7 @@ function curateUser(user) {
identityVerified: user.identity_verified, identityVerified: user.identity_verified,
ability, ability,
createdAt: user.created_at, createdAt: user.created_at,
stashes: user.stashes?.filter(Boolean).map(stash => curateStash(stash)) || [], stashes: user.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [],
}; };
return curatedUser; return curatedUser;

View File

@ -22,13 +22,13 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
const chunked = chunk(items, chunkSize); const chunked = chunk(items, chunkSize);
const queries = chunked const queries = chunked
.map(chunkItems => knex.raw(updated || ':query RETURNING *;', { .map((chunkItems) => knex.raw(updated || ':query RETURNING *;', {
query: knex(table).insert(chunkItems), query: knex(table).insert(chunkItems),
}).transacting(transaction)); }).transacting(transaction));
const responses = await Promise.all(queries); const responses = await Promise.all(queries);
return responses.flat().map(response => response.rows).flat(); return responses.flat().map((response) => response.rows).flat();
}); });
} }

View File

@ -7,7 +7,7 @@ function capitalize(string, { trim = true, uncapitalize = false } = {}) {
const capitalized = string const capitalized = string
.split(/\s+/) .split(/\s+/)
.map(component => `${component.charAt(0).toUpperCase()}${uncapitalize ? component.slice(1).toLowerCase() : component.slice(1)}`) .map((component) => `${component.charAt(0).toUpperCase()}${uncapitalize ? component.slice(1).toLowerCase() : component.slice(1)}`)
.join(' '); .join(' ');
return trim ? capitalized.trim() : capitalized; return trim ? capitalized.trim() : capitalized;

View File

@ -62,7 +62,7 @@ function convertManyApi(input, to) {
const curatedInput = input const curatedInput = input
.replace('\'', 'ft') .replace('\'', 'ft')
.replace(/"|''/, 'in') .replace(/"|''/, 'in')
.replace(/\d+ft\s*\d+\s*$/, match => `${match}in`); // height without any inch symbol .replace(/\d+ft\s*\d+\s*$/, (match) => `${match}in`); // height without any inch symbol
return Math.round(convertMany(curatedInput).to(to)) || null; return Math.round(convertMany(curatedInput).to(to)) || null;
} }

View File

@ -8,7 +8,7 @@ async function getFileEntries(location) {
} }
const file = await fs.promises.readFile(location, 'utf-8'); const file = await fs.promises.readFile(location, 'utf-8');
const entries = file.split(/\n/).map(entry => entry.trim()).filter(Boolean); const entries = file.split(/\n/).map((entry) => entry.trim()).filter(Boolean);
return entries; return entries;
} }

View File

@ -27,7 +27,7 @@ async function fetchSource(link) {
const tempFileStream = fs.createWriteStream(tempFilePath); const tempFileStream = fs.createWriteStream(tempFilePath);
const hashStream = new PassThrough(); const hashStream = new PassThrough();
hashStream.on('data', chunk => hasher.write(chunk)); hashStream.on('data', (chunk) => hasher.write(chunk));
try { try {
const res = await http.get(link, null, { const res = await http.get(link, null, {

View File

@ -102,7 +102,7 @@ function all(context, selector, attrArg, applyTrim = true) {
const attr = attrArg === true ? 'textContent' : attrArg; const attr = attrArg === true ? 'textContent' : attrArg;
if (attr) { if (attr) {
return Array.from(context.querySelectorAll(selector), el => q(el, null, attr, applyTrim)); return Array.from(context.querySelectorAll(selector), (el) => q(el, null, attr, applyTrim));
} }
return Array.from(context.querySelectorAll(selector)); return Array.from(context.querySelectorAll(selector));
@ -155,7 +155,7 @@ function jsons(context, selector) {
function htmls(context, selector) { function htmls(context, selector) {
const els = all(context, selector, null, true); const els = all(context, selector, null, true);
return els.map(el => el.innerHTML); return els.map((el) => el.innerHTML);
} }
function texts(context, selector, applyTrim = true, filter = true) { function texts(context, selector, applyTrim = true, filter = true) {
@ -163,8 +163,8 @@ function texts(context, selector, applyTrim = true, filter = true) {
if (!el) return null; if (!el) return null;
const nodes = Array.from(el.childNodes) const nodes = Array.from(el.childNodes)
.filter(node => node.nodeName === '#text') .filter((node) => node.nodeName === '#text')
.map(node => (applyTrim ? trim(node.textContent) : node.textContent)); .map((node) => (applyTrim ? trim(node.textContent) : node.textContent));
return filter ? nodes.filter(Boolean) : nodes; return filter ? nodes.filter(Boolean) : nodes;
} }
@ -272,7 +272,7 @@ function images(context, selector = 'img', attr, { origin, protocol = 'https' }
const imageEls = all(context, selector, attribute); const imageEls = all(context, selector, attribute);
return imageEls.map(imageEl => prefixUrl(imageEl, origin, protocol)); return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol));
} }
function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) { function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
@ -289,7 +289,7 @@ function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https
function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) { function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
const urlEls = all(context, selector, attr); const urlEls = all(context, selector, attr);
return attr ? urlEls.map(urlEl => prefixUrl(urlEl, origin, protocol)) : urlEls; return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls;
} }
function sourceSet(context, selector, attr = 'srcset', options = {}) { function sourceSet(context, selector, attr = 'srcset', options = {}) {
@ -330,7 +330,7 @@ function sourceSet(context, selector, attr = 'srcset', options = {}) {
return sources; return sources;
} }
return sources.map(source => source.url); return sources.map((source) => source.url);
} }
function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) { function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
@ -348,7 +348,7 @@ function video(context, selector = 'source', attr = 'src', { origin, protocol =
function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) { function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
const trailerEls = all(context, selector, attr); const trailerEls = all(context, selector, attr);
return attr ? trailerEls.map(trailerEl => prefixUrl(trailerEl, origin, protocol)) : trailerEls; return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
} }
function duration(context, selector, match, attr = 'textContent') { function duration(context, selector, match, attr = 'textContent') {
@ -499,11 +499,11 @@ function init(context, selector, window) {
function initAll(context, selector, window) { function initAll(context, selector, window) {
if (Array.isArray(context)) { if (Array.isArray(context)) {
return context.map(element => init(element, null, window)); return context.map((element) => init(element, null, window));
} }
return Array.from(context.querySelectorAll(selector)) return Array.from(context.querySelectorAll(selector))
.map(element => init(element, null, window)); .map((element) => init(element, null, window));
} }
function extract(htmlValue, selector, options) { function extract(htmlValue, selector, options) {

Some files were not shown because too many files have changed in this diff Show More