boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated dependencies. Added periodic memory logger.

This commit is contained in:
DebaucheryLibrarian 2021-11-20 23:59:15 +01:00
parent a867817dc1
commit 26539b74a5
109 changed files with 10238 additions and 10833 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.eslintrc
View File

@ -2,7 +2,7 @@
"root": true,
"extends": ["airbnb-base", "plugin:vue/recommended"],
"parserOptions": {
"parser": "babel-eslint",
"parser": "@babel/eslint-parser",
"ecmaVersion": 2019,
"sourceType": "module"
},

2
.nvmrc
View File

@ -1 +1 @@
14.15.4
16.13.0

4
assets/css/_theme.scss
View File

@ -5,7 +5,7 @@ $breakpoint3: 1200px;
$breakpoint4: 1500px;
:root {
--primary: #c63971;
--primary: #e33379;
--primary-strong: #f90071;
--primary-faded: #ffcce4;
@ -46,7 +46,7 @@ $breakpoint4: 1500px;
--logo-shadow: drop-shadow(1px 0 0 $shadow-weak) drop-shadow(-1px 0 0 $shadow-weak) drop-shadow(0 1px 0 $shadow-weak) drop-shadow(0 -1px 0 $shadow-weak);
--logo-highlight: drop-shadow(0 0 1px $highlight);
--info: #361723;
--info: #321b24;
--male: #0af;
--female: #f0a;

4
assets/js/actors/actions.js
View File

@ -281,7 +281,7 @@ function initActorActions(store, router) {
return {
actor: curateActor(actor, null, curateRelease),
releases: actor.scenesConnection.releases.map(release => curateRelease(release)),
releases: actor.scenesConnection.releases.map((release) => curateRelease(release)),
totalCount: actor.scenesConnection.totalCount,
};
}
@ -429,7 +429,7 @@ function initActorActions(store, router) {
});
return {
actors: actors.map(actor => curateActor(actor)),
actors: actors.map((actor) => curateActor(actor)),
totalCount,
countries,
};

2
assets/js/auth/getters.js
View File

@ -1,5 +1,5 @@
function favoritesStash(state) {
return state.user.stashes.find(stash => stash.slug === 'favorites');
return state.user.stashes.find((stash) => stash.slug === 'favorites');
}
module.exports = {

40
assets/js/curate.js
View File

@ -31,8 +31,8 @@ function curateActor(actor, release) {
if (actor.profiles) {
const photos = actor.profiles
.map(profile => ({ entity: profile.entity, ...profile.avatar }))
.filter(avatar => avatar.id && (!curatedActor.avatar || avatar.hash !== curatedActor.avatar.hash));
.map((profile) => ({ entity: profile.entity, ...profile.avatar }))
.filter((avatar) => avatar.id && (!curatedActor.avatar || avatar.hash !== curatedActor.avatar.hash));
const descriptions = actor.profiles.reduce((acc, profile) => ({
...acc,
@ -57,10 +57,10 @@ function curateActor(actor, release) {
}
if (actor.stashes) {
curatedActor.stashes = actor.stashes.filter(Boolean).map(stash => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
curatedActor.stashes = actor.stashes.filter(Boolean).map((stash) => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
}
curatedActor.stashes = actor.stashes?.map(stash => stash.stash || stash) || [];
curatedActor.stashes = actor.stashes?.map((stash) => stash.stash || stash) || [];
return curatedActor;
}
@ -70,21 +70,21 @@ function curateRelease(release) {
...release,
actors: [],
poster: release.poster && release.poster.media,
tags: release.tags ? release.tags.map(tag => tag.tag || tag) : [],
tags: release.tags ? release.tags.map((tag) => tag.tag || tag) : [],
};
if (release.scenes) curatedRelease.scenes = release.scenes.filter(Boolean).map(({ scene }) => curateRelease(scene));
if (release.movies) curatedRelease.movies = release.movies.filter(Boolean).map(({ movie }) => curateRelease(movie));
if (release.chapters) curatedRelease.chapters = release.chapters.filter(Boolean).map(chapter => curateRelease(chapter));
if (release.photos) curatedRelease.photos = release.photos.filter(Boolean).map(photo => photo.media || photo);
if (release.chapters) curatedRelease.chapters = release.chapters.filter(Boolean).map((chapter) => curateRelease(chapter));
if (release.photos) curatedRelease.photos = release.photos.filter(Boolean).map((photo) => photo.media || photo);
if (release.covers) curatedRelease.covers = release.covers.filter(Boolean).map(({ media }) => media);
if (release.trailer) curatedRelease.trailer = release.trailer.media;
if (release.teaser) curatedRelease.teaser = release.teaser.media;
if (release.actors) curatedRelease.actors = release.actors.filter(Boolean).map(actor => curateActor(actor.actor || actor, curatedRelease));
if (release.directors) curatedRelease.directors = release.directors.filter(Boolean).map(director => curateActor(director.director || director, curatedRelease));
if (release.actors) curatedRelease.actors = release.actors.filter(Boolean).map((actor) => curateActor(actor.actor || actor, curatedRelease));
if (release.directors) curatedRelease.directors = release.directors.filter(Boolean).map((director) => curateActor(director.director || director, curatedRelease));
if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.filter(Boolean).map(({ tag }) => tag);
if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.filter(Boolean).map(({ actor }) => curateActor(actor, curatedRelease));
if (release.stashes) curatedRelease.stashes = release.stashes.filter(Boolean).map(stash => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
if (release.stashes) curatedRelease.stashes = release.stashes.filter(Boolean).map((stash) => curateStash(stash.stash || stash)); // eslint-disable-line no-use-before-define
if (release.productionLocation) {
curatedRelease.productionLocation = {
@ -108,14 +108,14 @@ function curateEntity(entity, parent, releases) {
if (entity.children) {
if (entity.children.nodes) {
curatedEntity.children = entity.children.nodes.map(childEntity => curateEntity(childEntity, curatedEntity));
curatedEntity.children = entity.children.nodes.map((childEntity) => curateEntity(childEntity, curatedEntity));
}
curatedEntity.childrenTotal = entity.children.totalCount;
}
if (entity.parent || parent) curatedEntity.parent = curateEntity(entity.parent || parent);
if (releases) curatedEntity.releases = releases.map(release => curateRelease(release));
if (releases) curatedEntity.releases = releases.map((release) => curateRelease(release));
if (entity.connection) {
curatedEntity.sceneTotal = entity.connection.totalCount;
@ -142,7 +142,7 @@ function curateStash(stash) {
if (stash.scenes || stash.scenesConnection?.scenes) {
curatedStash.sceneTotal = stash.scenesConnection?.totalCount || null;
curatedStash.scenes = (stash.scenesConnection?.scenes || stash.scenes).map(item => ({
curatedStash.scenes = (stash.scenesConnection?.scenes || stash.scenes).map((item) => ({
...item,
scene: curateRelease(item.scene),
}));
@ -150,7 +150,7 @@ function curateStash(stash) {
if (stash.actors || stash.actorsConnection?.actors) {
curatedStash.actorTotal = stash.actorsConnection?.totalCount || null;
curatedStash.actors = (stash.actorsConnection?.actors || stash.actors).map(item => ({
curatedStash.actors = (stash.actorsConnection?.actors || stash.actors).map((item) => ({
...item,
actor: curateActor(item.actor),
}));
@ -158,7 +158,7 @@ function curateStash(stash) {
if (stash.movies || stash.moviesConnection?.movies) {
curatedStash.movieTotal = stash.moviesConnection?.totalCount || null;
curatedStash.movies = (stash.moviesConnection?.movies || stash.movies).map(item => ({
curatedStash.movies = (stash.moviesConnection?.movies || stash.movies).map((item) => ({
...item,
movie: curateRelease(item.movie),
}));
@ -175,11 +175,11 @@ function curateAlert(alert) {
const curatedAlert = alert;
if (alert.actors) {
curatedAlert.actors = alert.actors.map(actor => curateActor(actor.actor || actor));
curatedAlert.actors = alert.actors.map((actor) => curateActor(actor.actor || actor));
}
if (alert.tags) {
curatedAlert.tags = alert.tags.map(tag => curateTag(tag.tag || tag));
curatedAlert.tags = alert.tags.map((tag) => curateTag(tag.tag || tag));
}
if (alert.entity) {
@ -187,7 +187,7 @@ function curateAlert(alert) {
}
if (alert.stashes) {
curatedAlert.stashes = alert.stashes.map(stash => curateStash(stash.stash || stash));
curatedAlert.stashes = alert.stashes.map((stash) => curateStash(stash.stash || stash));
}
return curatedAlert;
@ -201,11 +201,11 @@ function curateUser(user) {
const curatedUser = user;
if (user.stashes) {
curatedUser.stashes = user.stashes.map(stash => curateStash(stash.stash || stash));
curatedUser.stashes = user.stashes.map((stash) => curateStash(stash.stash || stash));
}
if (user.alerts) {
curatedUser.alerts = user.alerts.map(alert => curateAlert(alert.alert || alert));
curatedUser.alerts = user.alerts.map((alert) => curateAlert(alert.alert || alert));
}
return curatedUser;

4
assets/js/entities/actions.js
View File

@ -212,7 +212,7 @@ function initEntitiesActions(store, router) {
entitySlugs,
});
return entities.map(entity => curateEntity(entity));
return entities.map((entity) => curateEntity(entity));
}
async function searchEntities({ _commit }, { query, limit = 20 }) {
@ -246,7 +246,7 @@ function initEntitiesActions(store, router) {
limit,
});
return entities.map(entity => curateEntity(entity));
return entities.map((entity) => curateEntity(entity));
}
return {

2
assets/js/format.js
View File

@ -5,7 +5,7 @@ export function formatDuration(duration, forceHours) {
const minutes = Math.floor((duration % 3600) / 60);
const seconds = Math.floor(duration % 60);
const [formattedHours, formattedMinutes, formattedSeconds] = [hours, minutes, seconds].map(segment => segment.toString().padStart(2, '0'));
const [formattedHours, formattedMinutes, formattedSeconds] = [hours, minutes, seconds].map((segment) => segment.toString().padStart(2, '0'));
if (duration >= 3600 || forceHours) {
return `${formattedHours}:${formattedMinutes}:${formattedSeconds}`;

8
assets/js/fragments.js
View File

@ -260,7 +260,7 @@ const releaseTagsFragment = `
`;
const releasePosterFragment = `
poster: releasesPosterByReleaseId {
poster: releasesPoster {
media {
id
index
@ -335,7 +335,7 @@ const releasePhotosFragment = `
`;
const releaseTrailerFragment = `
trailer: releasesTrailerByReleaseId {
trailer: releasesTrailer {
media {
id
index
@ -349,7 +349,7 @@ const releaseTrailerFragment = `
`;
const releaseTeaserFragment = `
teaser: releasesTeaserByReleaseId {
teaser: releasesTeaser {
media {
id
index
@ -487,7 +487,7 @@ const releaseFragment = `
slug
}
}
poster: chaptersPosterByChapterId {
poster: chaptersPoster {
media {
id
index

8
assets/js/main.js
View File

@ -109,6 +109,10 @@ async function init() {
document.title = 'traxxx';
},
},
beforeCreate() {
this.uid = uid;
uid += 1;
},
methods: {
formatDate,
formatDuration,
@ -117,10 +121,6 @@ async function init() {
getPath,
getBgPath: (media, type) => `url(${getPath(media, type)})`,
},
beforeCreate() {
this.uid = uid;
uid += 1;
},
});
app.directive('tooltip', {

4
assets/js/releases/actions.js
View File

@ -37,7 +37,7 @@ function initReleasesActions(store, router) {
});
return {
releases: releases.map(release => curateRelease(release)),
releases: releases.map((release) => curateRelease(release)),
totalCount,
};
}
@ -99,7 +99,7 @@ function initReleasesActions(store, router) {
});
return {
movies: movies.map(release => curateRelease(release)),
movies: movies.map((release) => curateRelease(release)),
totalCount,
};
}

16
assets/js/router.js
View File

@ -74,7 +74,7 @@ const routes = [
{
path: '/actor/:actorId/:actorSlug',
name: 'actor',
redirect: from => ({
redirect: (from) => ({
name: 'actorRange',
params: {
...from.params,
@ -91,7 +91,7 @@ const routes = [
{
path: '/director/:actorId/:actorSlug',
name: 'director',
redirect: from => ({
redirect: (from) => ({
name: 'directorRange',
params: {
...from.params,
@ -107,7 +107,7 @@ const routes = [
},
{
path: '/channel/:entitySlug',
redirect: from => ({
redirect: (from) => ({
name: 'channel',
params: {
...from.params,
@ -123,7 +123,7 @@ const routes = [
},
{
path: '/network/:entitySlug',
redirect: from => ({
redirect: (from) => ({
name: 'network',
params: {
...from.params,
@ -139,7 +139,7 @@ const routes = [
},
{
path: '/studio/:entitySlug',
redirect: from => ({
redirect: (from) => ({
name: 'studio',
params: {
...from.params,
@ -155,7 +155,7 @@ const routes = [
},
{
path: '/tag/:tagSlug',
redirect: from => ({
redirect: (from) => ({
name: 'tag',
params: {
...from.params,
@ -171,7 +171,7 @@ const routes = [
},
{
path: '/actors',
redirect: from => ({
redirect: (from) => ({
name: 'actors',
params: {
...from.params,
@ -229,7 +229,7 @@ const routes = [
},
{
path: '/stash/:stashId/:stashSlug',
redirect: from => ({
redirect: (from) => ({
name: 'stash',
params: {
...from.params,

12
assets/js/tags/actions.js
View File

@ -35,7 +35,7 @@ function initTagsActions(store, _router) {
name
slug
}
poster: tagsPosterByTagId {
poster: tagsPoster {
media {
id
thumbnail
@ -188,14 +188,14 @@ function initTagsActions(store, _router) {
before,
orderBy,
offset: Math.max(0, (pageNumber - 1)) * limit,
exclude: store.state.ui.tagFilter.filter(tagFilter => tagFilter !== tagSlug),
exclude: store.state.ui.tagFilter.filter((tagFilter) => tagFilter !== tagSlug),
hasAuth: !!store.state.auth.user,
userId: store.state.auth.user?.id,
});
return {
tag: curateTag(tagBySlug, null, curateRelease),
releases: tagBySlug.scenesConnection.releases.map(release => curateRelease(release)),
releases: tagBySlug.scenesConnection.releases.map((release) => curateRelease(release)),
totalCount: tagBySlug.scenesConnection.totalCount,
};
}
@ -218,7 +218,7 @@ function initTagsActions(store, _router) {
id
name
slug
poster: tagsPosterByTagId {
poster: tagsPoster {
media {
thumbnail
comment
@ -259,7 +259,7 @@ function initTagsActions(store, _router) {
limit,
});
return tags.map(tag => curateTag(tag, store.state.ui.sfw));
return tags.map((tag) => curateTag(tag, store.state.ui.sfw));
}
async function searchTags({ _commit }, {
@ -325,7 +325,7 @@ function initTagsActions(store, _router) {
minLength,
});
return tags.map(tag => curateTag(tag, store.state.ui.sfw));
return tags.map((tag) => curateTag(tag, store.state.ui.sfw));
}
async function fetchTagReleases({ _commit }, tagId) {

8
assets/js/ui/actions.js
View File

@ -70,7 +70,7 @@ function initUiActions(store, _router) {
slug
}
}
entity: alertsEntityByAlertId {
entity: alertsEntity {
entity {
id
name
@ -103,7 +103,7 @@ function initUiActions(store, _router) {
};
}
const curatedNotifications = notifications.nodes.map(notification => curateNotification(notification));
const curatedNotifications = notifications.nodes.map((notification) => curateNotification(notification));
return {
notifications: curatedNotifications,
@ -222,8 +222,8 @@ function initUiActions(store, _router) {
});
return {
releases: res?.results.map(result => curateRelease(result.release)) || [],
actors: res?.actors.map(actor => curateActor(actor)) || [],
releases: res?.results.map((result) => curateRelease(result.release)) || [],
actors: res?.actors.map((actor) => curateActor(actor)) || [],
};
}

2
assets/js/ui/observers.js
View File

@ -3,7 +3,7 @@ async function initUiObservers(store, _router) {
body.classList.add(store.state.ui.theme);
store.watch(state => state.ui.theme, (newTheme, oldTheme) => {
store.watch((state) => state.ui.theme, (newTheme, oldTheme) => {
body.classList.add(newTheme);
body.classList.remove(oldTheme);
});

19990
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

82
package.json
View File

@ -7,7 +7,7 @@
"start": "node -r source-map-support/register src/init.js",
"webpack": "webpack --env=production --mode=production",
"webpack-dev": "webpack --env=development --mode=development",
"webpack-watch": "webpack --progress --colors --watch --env=development --mode=development",
"webpack-watch": "webpack --progress --color --watch --env=development --mode=development",
"babel": "babel src --source-maps -d dist",
"babel-watch": "babel src -w --source-maps -d dist",
"build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
@ -40,38 +40,37 @@
"devDependencies": {
"@babel/cli": "^7.12.10",
"@babel/core": "^7.8.4",
"@babel/eslint-parser": "^7.16.0",
"@babel/plugin-proposal-optional-chaining": "^7.8.3",
"@babel/preset-env": "^7.8.4",
"@babel/register": "^7.8.3",
"@vue/compiler-sfc": "^3.0.4",
"autoprefixer": "^9.7.4",
"babel-eslint": "^10.1.0",
"autoprefixer": "^10.4.0",
"babel-loader": "^8.0.6",
"babel-preset-airbnb": "^3.3.2",
"css-loader": "^5.0.1",
"eslint": "^7.20.0",
"eslint-config-airbnb": "^17.1.1",
"eslint-config-airbnb-base": "^13.2.0",
"babel-preset-airbnb": "^5.0.0",
"css-loader": "^6.5.0",
"eslint": "^8.1.0",
"eslint-config-airbnb": "^18.2.1",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.20.1",
"eslint-plugin-jsx-a11y": "^6.2.3",
"eslint-plugin-react": "^7.18.3",
"eslint-plugin-vue": "^6.2.1",
"eslint-watch": "^4.0.2",
"eslint-webpack-plugin": "^2.5.2",
"mini-css-extract-plugin": "^1.3.3",
"node-sass": "^5.0.0",
"postcss-loader": "^3.0.0",
"eslint-plugin-vue": "^8.0.3",
"eslint-watch": "^7.0.0",
"eslint-webpack-plugin": "^3.1.0",
"mini-css-extract-plugin": "^2.4.3",
"node-sass": "^6.0.1",
"postcss-loader": "^6.2.0",
"raw-loader": "^4.0.2",
"sass-loader": "^11.0.1",
"style-loader": "^0.23.1",
"vue-loader": "^16.1.2",
"sass-loader": "^12.3.0",
"style-loader": "^3.3.1",
"vue-loader": "^16.8.2",
"webpack": "^5.11.0",
"webpack-cli": "^3.3.11"
"webpack-cli": "^4.9.1"
},
"dependencies": {
"@casl/ability": "^5.2.2",
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
"@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
"@graphile-contrib/pg-simplify-inflector": "^6.1.0",
"acorn": "^8.0.4",
"array-equal": "^1.0.0",
"aws-sdk": "^2.847.0",
@ -88,63 +87,60 @@
"cloudscraper": "^4.6.0",
"config": "^3.2.5",
"connect-session-knex": "^2.0.0",
"convert": "^1.6.2",
"convert": "^4.2.4",
"cookie": "^0.4.0",
"csv-stringify": "^5.3.6",
"dayjs": "^1.8.21",
"dompurify": "^2.0.11",
"ejs": "^3.0.1",
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
"express-promise-router": "^4.1.0",
"express-react-views": "^0.11.0",
"express-session": "^1.17.1",
"face-api.js": "^0.22.2",
"faker": "^5.1.0",
"file-type": "^14.1.4",
"file-type": "^16.5.3",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^7.0.1",
"graphile-utils": "^4.5.6",
"graphql": "^14.6.0",
"fs-extra": "^10.0.0",
"graphile-utils": "^4.12.2",
"graphql": "^15.4.0",
"html-entities": "^2.3.2",
"iconv-lite": "^0.5.1",
"inquirer": "^7.3.3",
"iconv-lite": "^0.6.3",
"inquirer": "^8.2.0",
"inspector-api": "^1.4.2",
"jsdom": "^16.3.0",
"knex": "^0.21.13",
"jsdom": "^18.0.0",
"knex": "^0.95.12",
"knex-migrate": "^1.7.4",
"longjohn": "^0.2.12",
"mime": "^2.4.4",
"mitt": "^2.1.0",
"mitt": "^3.0.0",
"moment": "^2.24.0",
"nanoid": "^2.1.11",
"nanoid": "^3.1.30",
"object-merge-advanced": "^12.1.0",
"object.omit": "^3.0.0",
"opn": "^5.5.0",
"opn": "^6.0.0",
"pg": "^8.5.1",
"postgraphile": "^4.10.0",
"postgraphile-plugin-connection-filter": "^1.1.3",
"postgraphile-plugin-connection-filter": "^2.2.2",
"promise-task-queue": "^1.2.0",
"prop-types": "^15.7.2",
"react": "^16.13.0",
"react-dom": "^16.13.0",
"sharp": "^0.27.2",
"sharp": "^0.29.2",
"showdown": "^1.9.1",
"source-map-support": "^0.5.16",
"template-format": "^1.2.5",
"tippy.js": "^6.3.1",
"tough-cookie": "^3.0.1",
"tty-table": "^2.8.12",
"tough-cookie": "^4.0.0",
"tunnel": "0.0.6",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.0.3",
"video.js": "^7.11.4",
"videojs-vr": "^1.7.1",
"vue": "^3.0.4",
"vue-router": "^4.0.1",
"vuex": "^4.0.0-rc.2",
"vue": "^3.2.20",
"vue-router": "^4.0.12",
"vuex": "^4.0.2",
"why-is-node-running": "^2.2.0",
"winston": "^3.2.1",
"winston-daily-rotate-file": "^4.4.2",
"yargs": "^13.3.0"
"yargs": "^17.2.1"
}
}

BIN
public/img/tags/free-use/gia_dibella_freeusefantasy.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 680 KiB

BIN
public/img/tags/free-use/lazy/gia_dibella_freeusefantasy.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.4 KiB

BIN
public/img/tags/free-use/thumbs/gia_dibella_freeusefantasy.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 35 KiB

21
seeds/04_media.js
View File

@ -900,6 +900,7 @@ const tagMedia = [
['flexible', 'lara_frost_legalporno', 'Lara Frost in NRX059', 'legalporno'],
['free-use', 'jeni_angel_brazzersexxtra', 'Jeni Angel in "Gamer Girl Threesome Action"', 'brazzersexxtra'],
['free-use', 'veruca_james_brazzersexxtra', 'Veruca James in "The Perfect Maid"', 'brazzersexxtra'],
['free-use', 'gia_dibella_freeusefantasy', 'Gia Dibella in "Learning to Freeuse"', 'freeusefantasy'],
['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9"', 'julesjordan'],
['gangbang', 'kristen_scott_julesjordan', 'Kristen Scott in "Interracial Gangbang!"', 'julesjordan'],
['gangbang', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
@ -1066,14 +1067,14 @@ const tagMedia = [
}));
/* eslint-disable max-len */
exports.seed = knex => Promise.resolve()
exports.seed = (knex) => Promise.resolve()
.then(async () => {
await upsert('media', sfw, 'id');
const tags = await knex('tags').whereIn('slug', tagMedia.map(item => item.tagSlug));
const tags = await knex('tags').whereIn('slug', tagMedia.map((item) => item.tagSlug));
const entities = await knex('entities')
.whereIn('slug', tagMedia.map(item => item.entitySlug).filter(Boolean))
.whereIn('slug', tagMedia.map((item) => item.entitySlug).filter(Boolean))
.orderBy('type', 'DESC');
const entitiesBySlug = entities.reduce((acc, entity) => ({
@ -1093,7 +1094,7 @@ exports.seed = knex => Promise.resolve()
concurrency: 20,
});
const { inserted, updated } = await upsert('media', tagMediaWithDimensions.map(media => ({
const { inserted, updated } = await upsert('media', tagMediaWithDimensions.map((media) => ({
id: media.id,
path: media.path,
thumbnail: media.thumbnail,
@ -1114,15 +1115,15 @@ exports.seed = knex => Promise.resolve()
[tagPhoto.tagSlug]: (acc[tagPhoto.tagSlug] || []).concat(tagPhoto),
}), {});
const tagPosters = Object.values(tagMediaBySlug).map(tag => tag[0]);
const tagPhotos = Object.values(tagMediaBySlug).map(tag => tag.slice(1)).flat();
const tagPosters = Object.values(tagMediaBySlug).map((tag) => tag[0]);
const tagPhotos = Object.values(tagMediaBySlug).map((tag) => tag.slice(1)).flat();
const tagPosterEntries = tagPosters.map(poster => ({
const tagPosterEntries = tagPosters.map((poster) => ({
tag_id: tagIdsBySlug[poster.tagSlug],
media_id: mediaIdsByPath[poster.path],
}));
const tagPhotoEntries = tagPhotos.map(photo => ({
const tagPhotoEntries = tagPhotos.map((photo) => ({
tag_id: tagIdsBySlug[photo.tagSlug],
media_id: mediaIdsByPath[photo.path],
}));
@ -1135,10 +1136,10 @@ exports.seed = knex => Promise.resolve()
// clean up (re)moved tag media
await Promise.all([
knex('tags_posters')
.whereNotIn('media_id', tagPosters.map(photo => photo.id))
.whereNotIn('media_id', tagPosters.map((photo) => photo.id))
.delete(),
knex('tags_photos')
.whereNotIn('media_id', tagPhotos.map(photo => photo.id))
.whereNotIn('media_id', tagPhotos.map((photo) => photo.id))
.delete(),
]);
});

2
src/.eslintrc
View File

@ -1,7 +1,7 @@
{
"extends": "airbnb-base",
"parserOptions": {
"parser": "babel-eslint",
"parser": "@babel/eslint-parser",
"sourceType": "script"
},
"rules": {

104
src/actors.js
View File

@ -124,9 +124,9 @@ function getMostFrequent(items) {
}
function getMostFrequentDate(dates) {
const year = getMostFrequent(dates.map(dateX => dateX.getFullYear()));
const month = getMostFrequent(dates.map(dateX => dateX.getMonth()));
const date = getMostFrequent(dates.map(dateX => dateX.getDate()));
const year = getMostFrequent(dates.map((dateX) => dateX.getFullYear()));
const month = getMostFrequent(dates.map((dateX) => dateX.getMonth()));
const date = getMostFrequent(dates.map((dateX) => dateX.getDate()));
if (year === null || month === null || date === null) {
return null;
@ -153,7 +153,7 @@ function toBaseActors(actorsOrNames, release) {
}
const baseActors = actorsOrNames
.filter(actorOrName => actorOrName && (typeof actorOrName === 'string' || actorOrName.name))
.filter((actorOrName) => actorOrName && (typeof actorOrName === 'string' || actorOrName.name))
.map((actorOrName) => {
const [baseName, entryId] = (actorOrName.name || actorOrName).split(':');
@ -265,7 +265,7 @@ function curateActor(actor, withDetails = false, isProfile = false) {
size: actor.avatar.size,
source: actor.avatar.source,
},
...(actor.profiles && { profiles: actor.profiles?.map(profile => curateActor(profile, true, true)) }),
...(actor.profiles && { profiles: actor.profiles?.map((profile) => curateActor(profile, true, true)) }),
}),
};
@ -285,7 +285,7 @@ function curateActorEntry(baseActor, batchId) {
}
function curateActorEntries(baseActors, batchId) {
return baseActors.map(baseActor => curateActorEntry(baseActor, batchId));
return baseActors.map((baseActor) => curateActorEntry(baseActor, batchId));
}
function curateProfileEntry(profile) {
@ -448,7 +448,7 @@ async function curateProfile(profile, actor) {
curatedProfile.scenes = toBaseReleases(profile.scenes || profile.releases, profile.entity, actor)
// attach actor to base scene, in case it was not scraped
.map((scene) => {
if (actor && !scene.actors?.find(sceneActor => slugify(sceneActor) === actor.slug || slugify(sceneActor.name) === actor.slug)) {
if (actor && !scene.actors?.find((sceneActor) => slugify(sceneActor) === actor.slug || slugify(sceneActor.name) === actor.slug)) {
return {
...scene,
actors: [actor, ...(scene.actors || [])],
@ -477,10 +477,10 @@ async function fetchProfiles(actorIdsOrNames) {
.modify((query) => {
if (actorIdsOrNames) {
query
.whereIn('actor_id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number'))
.whereIn('actor_id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((builder) => {
builder
.whereIn('actors.name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string'))
.whereIn('actors.name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('actors.entity_id');
});
}
@ -517,12 +517,12 @@ async function interpolateProfiles(actorIdsOrNames) {
...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }),
...(profile.birth_state && { state: profile.birth_state }),
...(profile.birth_city && { city: profile.birth_city }),
}].filter(location => Object.keys(location).length > 0),
}].filter((location) => Object.keys(location).length > 0),
residence: [...acc.residence || [], {
...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }),
...(profile.residence_state && { state: profile.residence_state }),
...(profile.residence_city && { city: profile.residence_city }),
}].filter(location => Object.keys(location).length > 0),
}].filter((location) => Object.keys(location).length > 0),
}), {});
const mostFrequentValues = [
@ -549,7 +549,7 @@ async function interpolateProfiles(actorIdsOrNames) {
...mostFrequentValues,
};
profile.height = getMostFrequent(valuesByProperty.height.filter(height => height > 50 && height < 300)); // remove unlikely values
profile.height = getMostFrequent(valuesByProperty.height.filter((height) => height > 50 && height < 300)); // remove unlikely values
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
@ -558,21 +558,21 @@ async function interpolateProfiles(actorIdsOrNames) {
profile.natural_boobs = profile.gender === 'male' ? null : getMostFrequent(valuesByProperty.natural_boobs);
// ensure most frequent country, city and state match up
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map(location => location.country));
const remainingOriginCountries = valuesByProperty.origin.filter(location => location.country === profile.birth_country_alpha2);
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map((location) => location.country));
const remainingOriginCountries = valuesByProperty.origin.filter((location) => location.country === profile.birth_country_alpha2);
profile.birth_state = getMostFrequent(remainingOriginCountries.map(location => location.state));
const remainingOriginStates = remainingOriginCountries.filter(location => !profile.birth_state || location.state === profile.birth_state);
profile.birth_state = getMostFrequent(remainingOriginCountries.map((location) => location.state));
const remainingOriginStates = remainingOriginCountries.filter((location) => !profile.birth_state || location.state === profile.birth_state);
profile.birth_city = getMostFrequent(remainingOriginStates.map(location => location.city));
profile.birth_city = getMostFrequent(remainingOriginStates.map((location) => location.city));
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map(location => location.country));
const remainingResidenceCountries = valuesByProperty.residence.filter(location => location.country === profile.residence_country_alpha2);
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map((location) => location.country));
const remainingResidenceCountries = valuesByProperty.residence.filter((location) => location.country === profile.residence_country_alpha2);
profile.residence_state = getMostFrequent(remainingResidenceCountries.map(location => location.state));
const remainingResidenceStates = remainingResidenceCountries.filter(location => !profile.residence_state || location.state === profile.residence_state);
profile.residence_state = getMostFrequent(remainingResidenceCountries.map((location) => location.state));
const remainingResidenceStates = remainingResidenceCountries.filter((location) => !profile.residence_state || location.state === profile.residence_state);
profile.residence_city = getMostFrequent(remainingResidenceStates.map(location => location.city));
profile.residence_city = getMostFrequent(remainingResidenceStates.map((location) => location.city));
profile.weight = getAverage(valuesByProperty.weight);
@ -580,8 +580,8 @@ async function interpolateProfiles(actorIdsOrNames) {
profile.piercings = getLongest(valuesByProperty.piercings);
profile.avatar_media_id = actorProfiles
.map(actorProfile => actorProfile.avatar)
.filter(avatar => avatar && (avatar.entropy === null || avatar.entropy > 5.5))
.map((actorProfile) => actorProfile.avatar)
.filter((avatar) => avatar && (avatar.entropy === null || avatar.entropy > 5.5))
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
return profile;
@ -598,10 +598,10 @@ async function interpolateProfiles(actorIdsOrNames) {
.modify((modifyBuilder) => {
if (actorIdsOrNames) {
modifyBuilder
.whereIn('id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number'))
.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((whereBuilder) => {
whereBuilder
.whereIn('name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string'))
.whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('entity_id');
});
}
@ -610,7 +610,7 @@ async function interpolateProfiles(actorIdsOrNames) {
.transacting(transaction);
// insert new interpolated data
const queries = interpolatedProfiles.map(profile => knex('actors')
const queries = interpolatedProfiles.map((profile) => knex('actors')
.where('id', profile.id)
.update(profile)
.transacting(transaction));
@ -621,8 +621,8 @@ async function interpolateProfiles(actorIdsOrNames) {
}
async function upsertProfiles(profiles) {
const newProfileEntries = profiles.filter(profile => !profile.update).map(profile => curateProfileEntry(profile)).filter(Boolean);
const updatingProfileEntries = profiles.filter(profile => profile.update).map(profile => curateProfileEntry(profile)).filter(Boolean);
const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
if (newProfileEntries.length > 0) {
await bulkInsert('actors_profiles', newProfileEntries);
@ -632,7 +632,7 @@ async function upsertProfiles(profiles) {
if (argv.force && updatingProfileEntries.length > 0) {
const transaction = await knex.transaction();
const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles')
const queries = updatingProfileEntries.map((profileEntry) => knex('actors_profiles')
.where('id', profileEntry.id)
.update(profileEntry)
.returning(['id', 'actor_id'])
@ -647,7 +647,7 @@ async function upsertProfiles(profiles) {
}
async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) {
const validSources = actor.entity ? sources.filter(source => source === actor.entity.slug) : sources;
const validSources = actor.entity ? sources.filter((source) => source === actor.entity.slug) : sources;
const profiles = Promise.map(validSources, async (source) => {
try {
@ -748,12 +748,12 @@ async function getActorNames(actorNames) {
)
`, [argv.actorsUpdate || new Date()]);
return actorsWithoutProfiles.rows.map(actor => actor.name);
return actorsWithoutProfiles.rows.map((actor) => actor.name);
}
async function storeProfiles(profiles) {
const profilesWithAvatarIds = await associateAvatars(profiles);
const actorIds = Array.from(new Set(profiles.map(profile => profile.id)));
const actorIds = Array.from(new Set(profiles.map((profile) => profile.id)));
await upsertProfiles(profilesWithAvatarIds);
await interpolateProfiles(actorIds);
@ -772,7 +772,7 @@ async function scrapeActors(argNames) {
fetchEntitiesBySlug(entitySlugs, 'desc'),
knex('actors')
.select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
.whereIn('actors.slug', baseActors.map(baseActor => baseActor.slug))
.whereIn('actors.slug', baseActors.map((baseActor) => baseActor.slug))
.whereNull('actors.alias_for')
.leftJoin('entities', 'entities.id', 'actors.entity_id')
.groupBy('actors.id', 'entities.id'),
@ -786,7 +786,7 @@ async function scrapeActors(argNames) {
},
}), {});
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlugAndEntryId[baseActor.slug]?.[baseActor.entryId]);
const newBaseActors = baseActors.filter((baseActor) => !existingActorEntriesBySlugAndEntryId[baseActor.slug]?.[baseActor.entryId]);
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
@ -799,7 +799,7 @@ async function scrapeActors(argNames) {
const existingProfiles = await knex('actors_profiles')
.select(knex.raw('actors_profiles.*, row_to_json(avatars) as avatar'))
.whereIn('actor_id', actors.map(actor => actor.id))
.whereIn('actor_id', actors.map((actor) => actor.id))
.leftJoin('media as avatars', 'avatars.id', 'actors_profiles.avatar_media_id');
const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({
@ -812,7 +812,7 @@ async function scrapeActors(argNames) {
const profilesPerActor = await Promise.map(
actors,
async actor => scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId),
async (actor) => scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId),
{ concurrency: 10 },
);
@ -833,7 +833,7 @@ async function scrapeActors(argNames) {
async function getOrCreateActors(baseActors, batchId) {
// WHERE IN causes stack depth error and performance issues with a large amount of values, no knex VALUES helper available
const actorValues = baseActors.map(actor => knex.raw('(:slug, :entityId, :entryId, :collisionLikely)', {
const actorValues = baseActors.map((actor) => knex.raw('(:slug, :entityId, :entryId, :collisionLikely)', {
slug: actor.slug,
entityId: actor.entity.id,
entryId: actor.entryId,
@ -867,7 +867,7 @@ async function getOrCreateActors(baseActors, batchId) {
},
}), {});
const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.entity.id]?.[baseActor.entryId]?.[baseActor.slug] && !existingActorSlugs.null?.null?.[baseActor.slug]);
const uniqueBaseActors = baseActors.filter((baseActor) => !existingActorSlugs[baseActor.entity.id]?.[baseActor.entryId]?.[baseActor.slug] && !existingActorSlugs.null?.null?.[baseActor.slug]);
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
const newActors = await bulkInsert('actors', curatedActorEntries);
@ -884,13 +884,13 @@ async function getOrCreateActors(baseActors, batchId) {
}), {});
const newActorProfiles = await Promise.all(baseActors
.filter(actor => actor.hasProfile)
.map(actor => ({
.filter((actor) => actor.hasProfile)
.map((actor) => ({
...actor,
id: newActorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || newActorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug],
}))
.filter(actor => !!actor.id)
.map(actor => curateProfile(actor)));
.filter((actor) => !!actor.id)
.map((actor) => curateProfile(actor)));
await storeProfiles(newActorProfiles);
@ -950,16 +950,16 @@ async function associatePeople(releases, batchId, type = 'actor') {
const releaseActorAssociations = Object.entries(baseActorsByReleaseId)
.map(([releaseId, releaseActors]) => releaseActors
.map(releaseActor => ({
.map((releaseActor) => ({
release_id: releaseId,
...(actorIdsByEntityIdEntryIdAndSlug[releaseActor.entity?.id]?.[releaseActor.entryId]?.[releaseActor.slug] || actorIdsByEntityIdEntryIdAndSlug.null.null[releaseActor.slug]),
})))
.flat();
const validReleaseActorAssociations = releaseActorAssociations.filter(association => association.release_id && association[personKey]);
const validReleaseActorAssociations = releaseActorAssociations.filter((association) => association.release_id && association[personKey]);
if (releaseActorAssociations.length > validReleaseActorAssociations.length) {
const invalidReleaseActorAssociations = releaseActorAssociations.filter(association => !association.release_id || !association[personKey]);
const invalidReleaseActorAssociations = releaseActorAssociations.filter((association) => !association.release_id || !association[personKey]);
logger.error(invalidReleaseActorAssociations);
}
@ -1021,15 +1021,15 @@ async function searchActors(query) {
.from(knex.raw('search_actors(?) as actors', [query]))
.limit(100);
return actors.map(actor => curateActor(actor));
return actors.map((actor) => curateActor(actor));
}
async function flushProfiles(actorIdsOrNames) {
const profiles = await fetchProfiles(actorIdsOrNames);
const actorNames = Array.from(new Set(profiles.map(profile => profile.actor.name)));
const actorNames = Array.from(new Set(profiles.map((profile) => profile.actor.name)));
const deleteCount = await knex('actors_profiles')
.whereIn('id', profiles.map(profile => profile.id))
.whereIn('id', profiles.map((profile) => profile.id))
.delete();
await interpolateProfiles(actorIdsOrNames);
@ -1050,14 +1050,14 @@ async function flushProfiles(actorIdsOrNames) {
async function deleteActors(actorIdsOrNames) {
const actors = await knex('actors')
.whereIn('id', actorIdsOrNames.filter(idOrName => typeof idOrName === 'number'))
.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((builder) => {
builder
.whereIn('name', actorIdsOrNames.filter(idOrName => typeof idOrName === 'string'))
.whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('entity_id');
});
const actorIds = actors.map(actor => actor.id);
const actorIds = actors.map((actor) => actor.id);
const sceneIds = await knex('releases_actors')
.select('releases.id')

16
src/alerts.js
View File

@ -22,15 +22,15 @@ async function addAlert(alert, sessionUser) {
.returning('id');
await Promise.all([
alert.actors?.length > 0 && bulkInsert('alerts_actors', alert.actors.map(actorId => ({
alert.actors?.length > 0 && bulkInsert('alerts_actors', alert.actors.map((actorId) => ({
alert_id: alertId,
actor_id: actorId,
})), false),
alert.tags?.length > 0 && bulkInsert('alerts_tags', alert.tags.map(tagId => ({
alert.tags?.length > 0 && bulkInsert('alerts_tags', alert.tags.map((tagId) => ({
alert_id: alertId,
tag_id: tagId,
})), false),
alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map(stashId => ({
alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map((stashId) => ({
alert_id: alertId,
stash_id: stashId,
})), false),
@ -106,20 +106,20 @@ async function notify(scenes) {
))))
GROUP BY releases.id, users.id, alerts.id;
`, {
sceneIds: scenes.map(scene => scene.id),
sceneIds: scenes.map((scene) => scene.id),
});
const notifications = releases.rows
.filter(alert => alert.notify)
.map(notification => ({
.filter((alert) => alert.notify)
.map((notification) => ({
user_id: notification.user_id,
alert_id: notification.alert_id,
scene_id: notification.scene_id,
}));
const stashes = releases.rows
.filter(release => release.stashes.length > 0)
.flatMap(release => release.stashes.map(stash => ({
.filter((release) => release.stashes.length > 0)
.flatMap((release) => release.stashes.map((stash) => ({
scene_id: release.scene_id,
stash_id: stash,
})));

59
src/app.js
View File

@ -22,6 +22,7 @@ const { flushOrphanedMedia } = require('./media');
const getFileEntries = require('./utils/file-entries');
const inspector = new Inspector();
let done = false;
function logActive() {
console.log('log active!');
@ -32,24 +33,47 @@ function logActive() {
}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
}
/*
function monitorMemory() {
logger.debug(`Memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
if (!done) {
setTimeout(() => monitorMemory(), 10000);
}
}
*/
async function stopMemorySample() {
const profile = await inspector.heap.stopSampling();
const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
await inspector.heap.disable();
fs.writeFile(filepath, JSON.stringify(profile));
await fs.writeFile(filepath, JSON.stringify(profile));
logger.info(`Saved heap sample to ${filepath}`);
}
async function startMemorySample() {
await inspector.heap.enable();
await inspector.heap.startSampling();
// monitorMemory();
logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
setTimeout(async () => {
await stopMemorySample();
if (!done) {
await startMemorySample();
}
}, 30000);
}
async function init() {
try {
if (argv.memory) {
await inspector.heap.enable();
await inspector.heap.startSampling();
logger.info('Started heap sampling');
await startMemorySample();
}
if (argv.logActive) {
@ -122,7 +146,7 @@ async function init() {
const actorNames = (argv.actors || []).concat(actorsFromFile || []);
const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.scenes).flat().filter(Boolean);
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map((actor) => actor.scenes).flat().filter(Boolean);
const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
@ -133,10 +157,10 @@ async function init() {
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const sceneMovies = deepScenes ? deepScenes.filter(scene => scene.movie).map(scene => ({ ...scene.movie, entity: scene.entity })) : [];
const sceneMovies = deepScenes ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
if (argv.report) {
@ -150,23 +174,12 @@ async function init() {
await associateMovieScenes(storedMovies, storedScenes);
}
if (argv.memory) {
await stopMemorySample();
}
knex.destroy();
} catch (error) {
logger.error(error);
if (argv.memory) {
await stopMemorySample();
}
knex.destroy();
throw error;
}
knex.destroy();
done = true;
}
module.exports = init;

8
src/argv.js
View File

@ -4,7 +4,11 @@ const config = require('config');
const yargs = require('yargs');
const moment = require('moment');
function interpretAfter(after) {
function interpretAfter(after, ignoreIfEmpty = false) {
if (!after && ignoreIfEmpty) {
return null;
}
if (!after) {
return new Date(0, 0, 0);
}
@ -313,6 +317,6 @@ const { argv } = yargs
default: 60000,
})
.coerce('after', interpretAfter)
.coerce('actors-update', interpretAfter);
.coerce('actors-update', (after) => interpretAfter(after, true));
module.exports = argv;

2
src/deep.js
View File

@ -185,7 +185,7 @@ async function scrapeReleases(baseReleases, entitiesBySlug, type) {
return Promise.map(
baseReleases,
async baseRelease => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
{ concurrency: 10 },
);
}

16
src/entities.js
View File

@ -46,7 +46,7 @@ function curateEntity(entity, includeParameters = false) {
} : {};
if (entity.tags) {
curatedEntity.tags = entity.tags.map(tag => ({
curatedEntity.tags = entity.tags.map((tag) => ({
id: tag.id,
name: tag.name,
slug: tag.slug,
@ -59,14 +59,14 @@ function curateEntity(entity, includeParameters = false) {
}
if (entity.children) {
curatedEntity.children = entity.children.map(child => curateEntity({
curatedEntity.children = entity.children.map((child) => curateEntity({
...child,
parent: curatedEntity.id ? curatedEntity : null,
}, includeParameters));
}
if (entity.included_children) {
curatedEntity.includedChildren = entity.included_children.map(child => curateEntity({
curatedEntity.includedChildren = entity.included_children.map((child) => curateEntity({
...child,
parent: curatedEntity.id ? curatedEntity : null,
}, includeParameters));
@ -79,7 +79,7 @@ function curateEntity(entity, includeParameters = false) {
}
async function curateEntities(entities, includeParameters) {
return Promise.all(entities.map(async entity => curateEntity(entity, includeParameters)));
return Promise.all(entities.map(async (entity) => curateEntity(entity, includeParameters)));
}
function urlToSiteSlug(url) {
@ -102,8 +102,8 @@ async function fetchIncludedEntities() {
includeAll: !argv.networks && !argv.channels && !config.include?.networks && !config.include?.channels,
includedNetworks: argv.networks || (!argv.channels && config.include?.networks) || [],
includedChannels: argv.channels || (!argv.networks && config.include?.channels) || [],
excludedNetworks: argv.excludeNetworks || config.exclude?.networks.filter(network => !argv.networks?.includes(network)) || [], // ignore explicitly included networks
excludedChannels: argv.excludeChannels || config.exclude?.channels.filter(channel => !argv.channels?.includes(channel)) || [], // ignore explicitly included channels
excludedNetworks: argv.excludeNetworks || config.exclude?.networks.filter((network) => !argv.networks?.includes(network)) || [], // ignore explicitly included networks
excludedChannels: argv.excludeChannels || config.exclude?.channels.filter((channel) => !argv.channels?.includes(channel)) || [], // ignore explicitly included channels
};
const rawNetworks = await knex.raw(`
@ -228,11 +228,11 @@ async function fetchEntitiesBySlug(entitySlugs, sort = 'asc') {
}
async function fetchReleaseEntities(baseReleases) {
const baseReleasesWithoutEntity = baseReleases.filter(release => release.url && !release.site && !release.entity);
const baseReleasesWithoutEntity = baseReleases.filter((release) => release.url && !release.site && !release.entity);
const entitySlugs = Array.from(new Set(
baseReleasesWithoutEntity
.map(baseRelease => urlToSiteSlug(baseRelease.url))
.map((baseRelease) => urlToSiteSlug(baseRelease.url))
.filter(Boolean),
));

2
src/logger.js
View File

@ -16,7 +16,7 @@ function logger(filepath) {
return winston.createLogger({
format: winston.format.combine(
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format(info => (info instanceof Error
winston.format((info) => (info instanceof Error
? { ...info, message: info.stack }
: { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(),
winston.format.colorize(),

38
src/media.js
View File

@ -190,7 +190,7 @@ function sortBaseTrailersByQuality(sources, role) {
function fallbackMediaToBaseMedia(rawMedia, role, metadata) {
const baseSources = rawMedia
.map(source => toBaseSource(source))
.map((source) => toBaseSource(source))
.filter(Boolean);
const sortedBaseSources = sortBaseTrailersByQuality(baseSources, role);
@ -225,12 +225,12 @@ function toBaseMedias(rawMedias, role, metadata) {
async function findSourceDuplicates(baseMedias) {
const sourceUrls = baseMedias
.map(baseMedia => baseMedia.sources.map(source => source.src))
.map((baseMedia) => baseMedia.sources.map((source) => source.src))
.flat()
.filter(Boolean);
const extractUrls = baseMedias
.map(baseMedia => baseMedia.sources.map(source => source.url))
.map((baseMedia) => baseMedia.sources.map((source) => source.url))
.flat()
.filter(Boolean);
@ -246,12 +246,12 @@ async function findSourceDuplicates(baseMedias) {
}
async function findHashDuplicates(medias) {
const hashes = medias.map(media => media.meta?.hash || media.entry?.hash).filter(Boolean);
const hashes = medias.map((media) => media.meta?.hash || media.entry?.hash).filter(Boolean);
const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
const uniqueHashMedias = medias.filter(media => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
const uniqueHashMedias = medias.filter((media) => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
if (!media.meta?.hash) {
@ -278,8 +278,8 @@ async function findHashDuplicates(medias) {
const selfUniqueHashMedias = Object.values(selfUniqueMediasByHash);
const existingHashMedias = medias
.filter(media => existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash])
.map(media => ({
.filter((media) => existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash])
.map((media) => ({
...media,
entry: existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash],
}))
@ -563,8 +563,8 @@ streamQueue.define('fetchStreamSource', async ({ source, tempFileTarget, hashStr
const video = ffmpeg(source.stream)
.format('mp4')
.outputOptions(['-movflags frag_keyframe+empty_moov'])
.on('start', cmd => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
.on('error', error => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
.on('start', (cmd) => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
.on('error', (error) => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
.pipe();
await pipeline(video, hashStream, tempFileTarget);
@ -745,7 +745,7 @@ async function storeMedias(baseMedias, options) {
const fetchedMedias = await Promise.map(
baseMedias,
async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
async (baseMedia) => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
{ concurrency: 100 }, // don't overload disk (or network, although this has its own throttling)
);
@ -753,7 +753,7 @@ async function storeMedias(baseMedias, options) {
const savedMedias = await Promise.map(
uniqueHashMedias,
async baseMedia => storeFile(baseMedia, options),
async (baseMedia) => storeFile(baseMedia, options),
{ concurrency: 100 }, // don't overload disk
);
@ -761,13 +761,13 @@ async function storeMedias(baseMedias, options) {
// overwrite files in case image processing was changed
await Promise.map(
existingHashMedias,
async baseMedia => storeFile(baseMedia, options),
async (baseMedia) => storeFile(baseMedia, options),
{ concurrency: 100 }, // don't overload disk
);
}
const newMediaWithEntries = savedMedias.filter(Boolean).map((media, index) => curateMediaEntry(media, index));
const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry);
const newMediaEntries = newMediaWithEntries.filter((media) => media.newEntry).map((media) => media.entry);
try {
await bulkInsert('media', newMediaEntries, false);
@ -851,7 +851,7 @@ async function associateAvatars(profiles) {
return profiles;
}
const profilesWithBaseMedias = profiles.map(profile => (profile.avatar
const profilesWithBaseMedias = profiles.map((profile) => (profile.avatar
? {
...profile,
avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', {
@ -862,7 +862,7 @@ async function associateAvatars(profiles) {
: profile
));
const baseMedias = profilesWithBaseMedias.map(profile => profile.avatarBaseMedia).filter(Boolean);
const baseMedias = profilesWithBaseMedias.map((profile) => profile.avatarBaseMedia).filter(Boolean);
const storedMedias = await storeMedias(baseMedias, { stats: true });
const storedMediasById = itemsByKey(storedMedias, 'id');
@ -885,13 +885,13 @@ async function associateAvatars(profiles) {
async function deleteS3Objects(media) {
const objects = media
.map(item => [
.map((item) => [
{ Key: item.path },
{ Key: item.thumbnail },
{ Key: item.lazy },
])
.flat()
.filter(item => item.Key);
.filter((item) => item.Key);
const status = await s3.deleteObjects({
Bucket: config.s3.bucket,
@ -936,7 +936,7 @@ async function flushOrphanedMedia() {
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
.delete();
await Promise.all(orphanedMedia.filter(media => !media.is_s3).map(media => Promise.all([
await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([
media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),
media.thumbnail && fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }),
media.lazy && fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }),
@ -945,7 +945,7 @@ async function flushOrphanedMedia() {
logger.info(`Removed ${orphanedMedia.length} media files from database and storage`);
if (config.s3.enabled) {
await deleteS3Objects(orphanedMedia.filter(media => media.is_s3));
await deleteS3Objects(orphanedMedia.filter((media) => media.is_s3));
}
await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });

24
src/releases.js
View File

@ -142,7 +142,7 @@ function curateRelease(release, withMedia = false, withPoster = true) {
slug: release.parent.slug,
},
},
actors: (release.actors || []).map(actor => ({
actors: (release.actors || []).map((actor) => ({
id: actor.id,
name: actor.name,
slug: actor.slug,
@ -150,12 +150,12 @@ function curateRelease(release, withMedia = false, withPoster = true) {
entityId: actor.entity_id,
aliasFor: actor.alias_for,
})),
tags: (release.tags || []).map(tag => ({
tags: (release.tags || []).map((tag) => ({
id: tag.id,
name: tag.name,
slug: tag.slug,
})),
chapters: (release.chapters || []).map(chapter => ({
chapters: (release.chapters || []).map((chapter) => ({
id: chapter.id,
index: chapter.index,
time: chapter.time,
@ -174,7 +174,7 @@ function curateRelease(release, withMedia = false, withPoster = true) {
} : null,
}),
...(withMedia && {
photos: (release.photos || []).map(photo => ({
photos: (release.photos || []).map((photo) => ({
id: photo.id,
path: photo.path,
thumbnail: release.poster.thumbnail,
@ -207,16 +207,16 @@ function curateGraphqlRelease(release) {
description: release.description || null,
duration: release.duration,
entity: release.entity,
actors: release.actors.map(actor => actor.actor),
tags: release.tags.map(tag => tag.tag),
...(release.chapters && { chapters: release.chapters.map(chapter => ({
actors: release.actors.map((actor) => actor.actor),
tags: release.tags.map((tag) => tag.tag),
...(release.chapters && { chapters: release.chapters.map((chapter) => ({
...chapter,
tags: chapter.tags.map(tag => tag.tag),
tags: chapter.tags.map((tag) => tag.tag),
poster: chapter.poster?.media || null,
photos: chapter.photos.map(photo => photo.media),
photos: chapter.photos.map((photo) => photo.media),
})) }),
poster: release.poster?.media || null,
...(release.photos && { photos: release.photos.map(photo => photo.media) }),
...(release.photos && { photos: release.photos.map((photo) => photo.media) }),
trailer: release.trailer?.media || null,
createdAt: release.createdAt,
};
@ -256,7 +256,7 @@ async function fetchScenes(limit = 100) {
limit: Math.min(limit, 10000),
});
return releases.map(release => curateGraphqlRelease(release));
return releases.map((release) => curateGraphqlRelease(release));
}
async function searchScenes(query, limit = 100, relevance = 0) {
@ -289,7 +289,7 @@ async function searchScenes(query, limit = 100, relevance = 0) {
relevance,
});
return releases.map(release => curateGraphqlRelease({ ...release.release, relevance: release.rank }));
return releases.map((release) => curateGraphqlRelease({ ...release.release, relevance: release.rank }));
}
async function deleteScenes(sceneIds) {

6
src/scrapers/adultempire.js
View File

@ -25,7 +25,7 @@ function scrapeAllTour(scenes, channel) {
release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, '');
release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY');
release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map(actor => actor.trim());
release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map((actor) => actor.trim());
const poster = query.img('.scene-img-wrapper img');
release.poster = [
@ -124,7 +124,7 @@ async function scrapeRelease({ query, html }, url, channel, baseRelease, options
};
}
release.photos = query.imgs('#dv_frames a > img').map(photo => [
release.photos = query.imgs('#dv_frames a > img').map((photo) => [
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`),
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`),
photo,
@ -301,7 +301,7 @@ async function fetchProfile(baseActor, channel, include) {
const searchRes = await http.get(`${channel.url}/search/SearchAutoComplete_Agg_ByMedia?rows=9&name_startsWith=${slugify(baseActor.name, '+')}`);
if (searchRes.ok) {
const actorResult = searchRes.body.Results.find(result => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description));
const actorResult = searchRes.body.Results.find((result) => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description));
if (actorResult) {
return fetchProfilePage(`${channel.url}${actorResult.BasicResponseGroup.id}`, channel, include);

4
src/scrapers/adulttime.js
View File

@ -22,13 +22,13 @@ async function networkFetchScene(url, site, release) {
async function fetchLatest(site, page = 1) {
const releases = await fetchApiLatest(site, page, false);
return releases.map(release => curateRelease(release, site));
return releases.map((release) => curateRelease(release, site));
}
async function fetchUpcoming(site, page = 1) {
const releases = await fetchApiUpcoming(site, page, false);
return releases.map(release => curateRelease(release, site));
return releases.map((release) => curateRelease(release, site));
}
module.exports = {

2
src/scrapers/amateurallure.js
View File

@ -34,7 +34,7 @@ function extractActors(scene) {
async function fetchLatestWrap(site, page = 1, include, preData) {
const latest = await fetchLatest(site, page, include, preData);
return latest.map(scene => extractActors(scene));
return latest.map((scene) => extractActors(scene));
}
async function fetchSceneWrap(url, channel, baseRelease, include) {

4
src/scrapers/americanpornstar.js
View File

@ -13,7 +13,7 @@ function scrapeScene({ query }, channel) {
release.description = query.cnt('.latest_update_description');
release.date = query.date('.update_date', 'MM/DD/YYYY');
release.actors = query.all('.tour_update_models a').map(actorEl => ({
release.actors = query.all('.tour_update_models a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null),
}));
@ -30,7 +30,7 @@ function scrapeScene({ query }, channel) {
poster,
];
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map(img => [
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
img.replace(/.jpg$/, '-full.jpg'),
img,
]);

10
src/scrapers/assylum.js
View File

@ -6,8 +6,8 @@ function extractActors(actorString) {
return actorString
?.replace(/.*:|\(.*\)|\d+(-|\s)year(-|\s)old|nurses?|tangled/ig, '') // remove Patient:, (date) and other nonsense
.split(/\band\b|\bvs\b|\/|,|&/ig)
.map(actor => actor.trim())
.filter(actor => !!actor && !/\banal\b|\bschool\b|\bgamer\b|\breturn\b|\bfor\b|\bare\b|\bpart\b|realdoll|bimbo|p\d+/ig.test(actor))
.map((actor) => actor.trim())
.filter((actor) => !!actor && !/\banal\b|\bschool\b|\bgamer\b|\breturn\b|\bfor\b|\bare\b|\bpart\b|realdoll|bimbo|p\d+/ig.test(actor))
|| [];
}
@ -16,7 +16,7 @@ function matchActors(actorString, models) {
return [];
}
return models.filter(model => new RegExp(model.name, 'i').test(actorString));
return models.filter((model) => new RegExp(model.name, 'i').test(actorString));
}
function scrapeLatest(scenes, site, models) {
@ -61,7 +61,7 @@ function scrapeScene({ html, qu }, url, site, include, models) {
release.tags = qu.all('.tags a', true);
release.photos = qu.imgs('.stills img').map(photoPath => `${site.url}/${photoPath}`);
release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
const posterIndex = 'splash:';
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
@ -101,7 +101,7 @@ async function fetchModels(site, page = 1, accModels = []) {
if (res.ok) {
const models = extractModels(res.item, site);
const nextPage = res.item.qa('.pagenumbers', true)
.map(pageX => Number(pageX))
.map((pageX) => Number(pageX))
.filter(Boolean) // remove << and >>
.includes(page + 1);

4
src/scrapers/aziani.js
View File

@ -46,7 +46,7 @@ function scrapeScene({ html, qu }, url) {
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
release.actors = qu.all('h5:not(.video_categories) a').map(actor => ({
release.actors = qu.all('h5:not(.video_categories) a').map((actor) => ({
name: qu.q(actor, null, true),
url: qu.url(actor, null),
}));
@ -58,7 +58,7 @@ function scrapeScene({ html, qu }, url) {
const poster = qu.img('a img');
release.poster = getFallbacks(poster);
release.photos = qu.imgs('.featured-video img', 'src0_1x').map(source => getFallbacks(source));
release.photos = qu.imgs('.featured-video img', 'src0_1x').map((source) => getFallbacks(source));
return release;
}

4
src/scrapers/badoink.js
View File

@ -30,7 +30,7 @@ function scrapeAll(scenes, channel) {
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
release.actors = query.all('.video-card-details--cast a').map(el => ({
release.actors = query.all('.video-card-details--cast a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
}));
@ -57,7 +57,7 @@ function scrapeScene({ query }, url, channel) {
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
release.actors = query.all('.video-actors a').map(el => ({
release.actors = query.all('.video-actors a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
}));

6
src/scrapers/bamvisions.js
View File

@ -22,13 +22,13 @@ function scrapeAll(scenes, site) {
if (/bts/i.test(release.title)) release.tags = ['behind the scenes'];
[release.poster, ...release.photos] = qu.all('.item-thumbs img')
.map(source => [
.map((source) => [
source.getAttribute('src0_3x'),
source.getAttribute('src0_2x'),
source.getAttribute('src0_1x'),
]
.filter(Boolean)
.map(fallback => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`)));
.map((fallback) => (/^http/.test(fallback) ? fallback : `${site.url}${fallback}`)));
release.entryId = `${formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
@ -116,7 +116,7 @@ async function scrapeProfile({ qu }, site, withScenes) {
qu.q('.profile-pic img', 'src0_3x'),
qu.q('.profile-pic img', 'src0_2x'),
qu.q('.profile-pic img', 'src0_1x'),
].filter(Boolean).map(source => (/^http/.test(source) ? source : `${site.url}${source}`));
].filter(Boolean).map((source) => (/^http/.test(source) ? source : `${site.url}${source}`));
if (withScenes) {
const actorId = qu.q('.profile-pic img', 'id')?.match(/set-target-(\d+)/)?.[1];

14
src/scrapers/bang.js
View File

@ -48,7 +48,7 @@ async function fetchPhotos(scene) {
});
if (res.ok && res.body.images) {
return res.body.images.map(image => qu.prefixUrl(image, 'https://photos.bang.com'));
return res.body.images.map((image) => qu.prefixUrl(image, 'https://photos.bang.com'));
}
return null;
@ -59,7 +59,7 @@ async function scrapeScene(scene, entity, options) {
entryId: scene.id,
title: scene.name,
description: scene.description,
tags: scene.genres.concat(scene.actions).map(genre => genre.name),
tags: scene.genres.concat(scene.actions).map((genre) => genre.name),
duration: scene.duration,
};
@ -69,19 +69,19 @@ async function scrapeScene(scene, entity, options) {
const date = new Date(scene.releaseDate);
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: genderMap[actor.gender] }));
release.actors = scene.actors.map((actor) => ({ name: actor.name, gender: genderMap[actor.gender] }));
if (scene.is4k) release.tags.push('4k');
if (scene.gay) release.tags.push('gay');
const defaultPoster = scene.screenshots.find(photo => photo.default === true);
const screens = scene.screenshots.filter(photo => photo.default === false);
const defaultPoster = scene.screenshots.find((photo) => photo.default === true);
const screens = scene.screenshots.filter((photo) => photo.default === false);
const remainingScreens = defaultPoster ? screens : screens.slice(1);
const poster = defaultPoster || screens[0];
release.poster = getScreenUrl(poster, scene);
release.photos = remainingScreens.map(photo => getScreenUrl(photo, scene));
release.photos = remainingScreens.map((photo) => getScreenUrl(photo, scene));
if (options?.includePhotos) {
const photos = await fetchPhotos(scene);
@ -399,7 +399,7 @@ async function fetchProfile({ name: actorName }, context, include) {
});
if (res.ok) {
const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase());
const actor = res.body.hits.hits.find((hit) => hit._source.name.toLowerCase() === actorName.toLowerCase());
if (actor) {
return scrapeProfile(actor._source, context.entity, include);

3
src/scrapers/boobpedia.js
View File

@ -8,10 +8,9 @@ function scrapeProfile(html) {
const profile = {};
const bio = qu.all('.infobox tr[valign="top"]')
.map(detail => qu.all(detail, 'td', true))
.map((detail) => qu.all(detail, 'td', true))
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
/* unreliable, see: Syren De Mer
const catlinks = qa('#mw-normal-catlinks a', true);
const isTrans = catlinks.some(link => link.match(/shemale|transgender/i));

8
src/scrapers/bradmontana.js
View File

@ -26,7 +26,7 @@ function scrapeAll(scenes) {
release.entryId = new URL(release.url).pathname.match(/\/videos\/([\w-]+)/)[1];
release.title = query.cnt('.title') || query.q('img', 'title');
release.actors = subtitle.slice(subtitle.indexOf(':') + 1).split(',').map(actor => actor.trim()).filter(Boolean);
release.actors = subtitle.slice(subtitle.indexOf(':') + 1).split(',').map((actor) => actor.trim()).filter(Boolean);
release.poster = query.img('.thumb img');
@ -48,13 +48,13 @@ function scrapeScene({ query, html }, url, channel) {
const dataString = query.html('.yoast-schema-graph');
const data = dataString && JSON.parse(dataString)['@graph'];
const pageData = data.find(item => item['@type'] === 'WebPage');
const imageData = data.find(item => item['@type'] === 'ImageObject');
const pageData = data.find((item) => item['@type'] === 'WebPage');
const imageData = data.find((item) => item['@type'] === 'ImageObject');
release.entryId = new URL(url).pathname.match(/\/videos\/([\w-]+)/)[1];
release.title = query.cnt('.video .title h1')
|| data.find(item => item['@type'] === 'BreadcrumbList')?.itemListElement.slice(-1)[0].item.name
|| data.find((item) => item['@type'] === 'BreadcrumbList')?.itemListElement.slice(-1)[0].item.name
|| pageData?.name.slice(0, pageData.name.lastIndexOf('-')).trim();
release.description = query.cnt('.video .descript');

4
src/scrapers/cherrypimps.js
View File

@ -66,7 +66,7 @@ function scrapeProfile({ query }) {
const profile = {};
const keys = query.all('.model-descr_line:not(.model-descr_rait) p.text span', true);
const values = query.all('.model-descr_line:not(.model-descr_rait) p.text').map(el => query.text(el));
const values = query.all('.model-descr_line:not(.model-descr_rait) p.text').map((el) => query.text(el));
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)\s*cm\)/)?.[1]);
@ -100,7 +100,7 @@ function scrapeProfile({ query }) {
profile.piercings = bio.piercings;
}
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim());
const avatar = query.q('.model-img img');
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;

2
src/scrapers/cumlouder.js
View File

@ -43,7 +43,7 @@ function scrapeScene({ query }, channel, html) {
release.date = date;
release.datePrecision = precision;
release.actors = query.all('.sub-video .pornstar-link').map(el => ({
release.actors = query.all('.sub-video .pornstar-link').map((el) => ({
name: query.cnt(el, null),
url: query.url(el, null, 'href', { origin: 'https://www.cumlouder.com' }),
}));

10
src/scrapers/ddfnetwork.js
View File

@ -52,10 +52,10 @@ async function scrapeScene({ query }, url, channel) {
release.poster = query.poster() || query.poster('dl8-video') || query.img('#videoBlock img');
release.photos = query.urls('.photo-slider-guest .card a');
release.trailer = query.all('source[type="video/mp4"]').map(trailer => ({
release.trailer = query.all('source[type="video/mp4"]').map((trailer) => ({
src: trailer.src,
quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null,
vr: channel.tags?.some(tag => tag.slug === 'vr'),
vr: channel.tags?.some((tag) => tag.slug === 'vr'),
}));
return release;
@ -63,7 +63,7 @@ async function scrapeScene({ query }, url, channel) {
async function fetchActorReleases(urls) {
// DDF Network and DDF Network Stream list all scenes, exclude
const sources = urls.filter(url => !/ddfnetwork/.test(url));
const sources = urls.filter((url) => !/ddfnetwork/.test(url));
const releases = await Promise.all(sources.map(async (url) => {
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)');
@ -79,10 +79,10 @@ async function fetchActorReleases(urls) {
}
async function scrapeProfile({ query }, _url, actorName) {
const keys = query.all('.about-title', true).map(key => slugify(key, '_'));
const keys = query.all('.about-title', true).map((key) => slugify(key, '_'));
const values = query.all('.about-info').map((el) => {
if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', ');
return Array.from(el.children, (child) => child.textContent.trim()).join(', ');
}
return el.textContent.trim();

2
src/scrapers/dogfart.js
View File

@ -47,7 +47,7 @@ function scrapeLatest(html, site, filter = true) {
const entryId = `${site.slug}_${pathname.split('/')[4]}`;
const title = element.querySelector('.scene-title').textContent;
const actors = title.split(/[,&]|\band\b/).map(actor => actor.replace(/BTS/i, '').trim());
const actors = title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim());
const poster = `https:${element.querySelector('img').src}`;
const teaser = sceneLinkElement.dataset.preview_clip_url;

8
src/scrapers/dorcel.js
View File

@ -12,7 +12,7 @@ function scrapeAll(scenes, channel) {
release.title = query.cnt('.title');
release.actors = query.all('.actors a').map(actorEl => ({
release.actors = query.all('.actors a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }),
}));
@ -40,7 +40,7 @@ function scrapeScene({ query }, url, channel) {
release.date = query.date('.publish_date', 'MMMM DD, YYYY');
release.duration = query.dur('.duration');
release.actors = query.all('.actress a').map(actorEl => ({
release.actors = query.all('.actress a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }),
}));
@ -91,7 +91,7 @@ function scrapeMovie({ query, el }, url, channel) {
release.duration = query.dur('.duration');
release.actors = query.all('.actors .actor').map(actorEl => ({
release.actors = query.all('.actors .actor').map((actorEl) => ({
name: query.cnt(actorEl, '.name'),
url: query.url(actorEl, 'a', 'href', { origin: channel.url }),
avatar: query.sourceSet(actorEl, '.thumbnail img', 'data-srcset'),
@ -99,7 +99,7 @@ function scrapeMovie({ query, el }, url, channel) {
release.poster = query.sourceSet('.banner', 'data-src')?.[0];
release.covers = [query.all(query.el('.cover').parentElement, 'source')
?.map(coverEl => query.sourceSet(coverEl, null, 'data-srcset'))
?.map((coverEl) => query.sourceSet(coverEl, null, 'data-srcset'))
.flat()
.sort((coverA, coverB) => {
const resA = Number(coverA.match(/_(\d{3,})_/)?.[1]);

14
src/scrapers/elevatedx.js
View File

@ -53,7 +53,7 @@ function getImageWithFallbacks(q, selector, site, el) {
q(selector, 'src0_1x'),
];
return sources.filter(Boolean).map(src => `${site.parameters?.media || site.url}${src}`);
return sources.filter(Boolean).map((src) => `${site.parameters?.media || site.url}${src}`);
}
function scrapeAllClassic(scenes, channel) {
@ -107,7 +107,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
if (channel.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
if (channel.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
return null;
}
@ -143,7 +143,7 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)');
release.actors = query.all('.models-list-thumbs a').map(el => ({
release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.cnt(el, 'span'),
avatar: getImageWithFallbacks(query.q, 'img', entity, el),
url: query.url(el, null),
@ -164,8 +164,8 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
if (stars) release.stars = Number(stars);
if (entity.type === 'network') {
const channelRegExp = new RegExp(entity.children.map(channel => channel.parameters?.match || channel.name).join('|'), 'i');
const channel = release.tags.find(tag => channelRegExp.test(tag));
const channelRegExp = new RegExp(entity.children.map((channel) => channel.parameters?.match || channel.name).join('|'), 'i');
const channel = release.tags.find((tag) => channelRegExp.test(tag));
if (channel) {
release.channel = slugify(channel, '');
@ -199,8 +199,8 @@ async function scrapeProfile({ query }, entity, parameters) {
avatarEl.getAttribute('src0'),
avatarEl.getAttribute('src'),
]
.filter(avatar => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map(avatar => qu.prefixUrl(avatar, entity.url));
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map((avatar) => qu.prefixUrl(avatar, entity.url));
if (avatarSources.length) profile.avatar = avatarSources;
}

6
src/scrapers/famedigital.js
View File

@ -18,7 +18,7 @@ function extractLowArtActors(release) {
const actors = release.title
.replace(/solo/i, '')
.split(/,|\band\b/ig)
.map(actor => actor.trim());
.map((actor) => actor.trim());
return {
...release,
@ -32,7 +32,7 @@ async function networkFetchLatest(site, page = 1) {
const releases = await fetchLatest(site, page);
if (site.slug === 'lowartfilms') {
return releases.map(release => extractLowArtActors(release));
return releases.map((release) => extractLowArtActors(release));
}
return releases;
@ -76,7 +76,7 @@ async function fetchClassicProfile(actorName, { site }) {
if (!pornstarsRes.ok) return null;
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find(el => slugify(el.textContent) === actorSlug)
.find((el) => slugify(el.textContent) === actorSlug)
?.value;
if (actorPath) {

4
src/scrapers/firstanalquest.js
View File

@ -14,7 +14,7 @@ function scrapeAllA(scenes, channel) {
release.date = query.date('.thumb-added, .date', ['MMM D, YYYY', 'MMMM DD, YYYY'], /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.thumb-duration');
release.actors = query.all('.thumb-models a, .models a').map(actorEl => ({
release.actors = query.all('.thumb-models a, .models a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }),
}));
@ -70,7 +70,7 @@ function scrapeSceneA({ query }, url, channel) {
release.duration = query.dur('.media-body li span, .duration');
release.actors = query.all('.media-body a[href*="models/"], .models a').map(actorEl => ({
release.actors = query.all('.media-body a[href*="models/"], .models a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }),
}));

4
src/scrapers/freeones.js
View File

@ -9,7 +9,7 @@ function scrapeProfile(html, actorName) {
const { document } = new JSDOM(html).window;
const profile = { name: actorName };
const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), el => decodeURI(el.href)).reduce((acc, item) => {
const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), (el) => decodeURI(el.href)).reduce((acc, item) => {
const keyMatch = item.match(/\[\w+\]/);
if (keyMatch) {
@ -52,7 +52,7 @@ function scrapeProfile(html, actorName) {
if (bio.height) profile.height = Number(bio.height.split(',')[0]);
if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]);
profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), el => el.href);
profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), (el) => el.href);
const avatar = document.querySelector('.profile-image-large img').src;
if (!avatar.match('placeholder')) profile.avatar = { src: avatar, credit: null };

28
src/scrapers/gamma.js
View File

@ -33,7 +33,7 @@ async function fetchApiCredentials(referer, site) {
const res = await http.get(referer);
const body = res.body.toString();
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
const apiLine = body.split('\n').find((bodyLine) => bodyLine.match('apiKey'));
if (!apiLine) {
throw new Error(`No Gamma API key found for ${referer}`);
@ -169,7 +169,7 @@ async function getThumbs(entryId, site, parameters) {
});
if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) {
return res.body.results[0].hits[0].set_pictures.map(img => ([
return res.body.results[0].hits[0].set_pictures.map((img) => ([
`https://transform.gammacdn.com/photo_set${img.thumb_path}`,
`https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`,
]));
@ -214,7 +214,7 @@ async function scrapeApiReleases(json, site) {
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.director = scene.directors[0]?.name || null;
release.actors = scene.actors.map(actor => ({
release.actors = scene.actors.map((actor) => ({
entryId: actor.actor_id,
name: actor.name,
gender: actor.gender,
@ -226,7 +226,7 @@ async function scrapeApiReleases(json, site) {
}));
release.tags = scene.master_categories
.concat(scene.categories?.map(category => category.name))
.concat(scene.categories?.map((category) => category.name))
.filter(Boolean); // some categories don't have a name
const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
@ -272,7 +272,7 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
[release.likes, release.dislikes] = $(element).find('.value')
.toArray()
.map(value => Number($(value).text()));
.map((value) => Number($(value).text()));
const posterEl = $(element).find('.imgLink img, .tlcImageItem');
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
@ -327,13 +327,13 @@ async function scrapeScene(html, url, site, baseRelease, mobileHtml, options) {
const actors = data?.actor || data2?.actor;
if (actors) {
release.actors = actors.map(actor => ({
release.actors = actors.map((actor) => ({
name: actor.name,
gender: actor.gender,
}));
}
const hasTrans = release.actors?.some(actor => actor.gender === 'shemale');
const hasTrans = release.actors?.some((actor) => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
@ -420,7 +420,7 @@ async function scrapeSceneApi(data, site, options) {
release.duration = data.length;
release.date = new Date(data.date * 1000) || qu.parseDate(data.release_date, 'YYYY-MM-DD');
release.actors = data.actors.map(actor => ({
release.actors = data.actors.map((actor) => ({
entryId: actor.actor_id,
name: actor.name,
gender: actor.gender,
@ -429,7 +429,7 @@ async function scrapeSceneApi(data, site, options) {
: qu.prefixUrl(`/en/pornstar/${actor.url_name}/${data.actor_id}`, site.url),
}));
release.tags = data.categories.map(category => category.name);
release.tags = data.categories.map((category) => category.name);
if (data.pictures) {
release.poster = [
@ -501,7 +501,7 @@ async function scrapeMovie({ query, html }, window, url, entity, options) {
release.date = qu.extractDate(data.dvdReleaseDate);
release.title = data.dvdName;
release.actors = data.dvdActors.map(actor => ({ name: actor.actorName, entryId: actor.actorId }));
release.actors = data.dvdActors.map((actor) => ({ name: actor.actorName, entryId: actor.actorId }));
release.tags = query.cnts('.dvdCol a');
release.scenes = scrapeAll(html, entity, entity.url);
@ -602,9 +602,9 @@ function scrapeApiProfile(data, releases, siteSlug) {
if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
const avatarPaths = Object.values(data.pictures).reverse();
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map(avatarPath => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map((avatarPath) => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
if (releases) profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
if (releases) profile.releases = releases.map((release) => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
return profile;
}
@ -723,7 +723,7 @@ function getDeepUrl(url, site, baseRelease, mobile) {
const filter = new Set(['en', 'video', 'scene', site.slug, site.parent.slug]);
const pathname = baseRelease?.path || new URL(url).pathname
.split('/')
.filter(component => !filter.has(component))
.filter((component) => !filter.has(component))
.join('/'); // reduce to scene ID and title slug
const sceneId = baseRelease?.entryId || pathname.match(/\/(\d+)\//)?.[1];
@ -863,7 +863,7 @@ async function fetchApiProfile({ name: actorName }, context, include) {
});
if (res.status === 200 && res.body.results[0].hits.length > 0) {
const actorData = res.body.results[0].hits.find(actor => slugify(actor.name) === slugify(actorName));
const actorData = res.body.results[0].hits.find((actor) => slugify(actor.name) === slugify(actorName));
if (actorData) {
const actorScenes = include.releases && await fetchActorScenes(actorData.name, apiUrl, siteSlug);

6
src/scrapers/hitzefrei.js
View File

@ -42,13 +42,13 @@ function scrapeScene({ query }, url) {
release.duration = query.dur('.content-metas span:nth-child(2)');
release.likes = query.number('.content-metas span:nth-child(6)');
release.actors = query.all('.model-thumb img').map(el => ({
release.actors = query.all('.model-thumb img').map((el) => ({
name: query.q(el, null, 'alt'),
avatar: query.img(el, null, 'src'),
}));
release.poster = query.poster('.content-video video');
release.photos = query.urls('#photo-carousel a').map(photo => [
release.photos = query.urls('#photo-carousel a').map((photo) => [
photo.replace('/full', ''),
photo,
photo.replace('/full', '/thumbs'),
@ -135,7 +135,7 @@ async function fetchProfile(baseActor, entity, include) {
});
if (searchRes.ok) {
const actor = searchRes.body.find(result => result.type === 'model' && result.title === baseActor.name);
const actor = searchRes.body.find((result) => result.type === 'model' && result.title === baseActor.name);
if (actor) {
const actorRes = await qu.get(actor.url);

2
src/scrapers/hookuphotshot.js
View File

@ -21,7 +21,7 @@ function scrapeAll(scenes) {
avatar: [
avatarEl.src.replace(/-\d+x\d+/, ''),
avatarEl.src,
].map(src => ({ src, interval: 1000, concurrency: 1 })),
].map((src) => ({ src, interval: 1000, concurrency: 1 })),
}),
};
}).concat({

14
src/scrapers/hush.js
View File

@ -53,7 +53,7 @@ function getImageWithFallbacks(q, selector, site, el) {
q(selector, 'src0_1x'),
];
return sources.filter(Boolean).map(src => `${site.parameters?.media || site.url}${src}`);
return sources.filter(Boolean).map((src) => `${site.parameters?.media || site.url}${src}`);
}
function scrapeAll(scenes, channel) {
@ -108,7 +108,7 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
if (site.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
if (site.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
return null;
}
@ -132,7 +132,7 @@ function scrapeScene({ html, query }, channel, url) {
const poster = qu.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
[release.poster, ...release.photos] = [poster, ...query.imgs('.item-thumb img', 'src0_1x', { origin: channel.url })]
.map(src => src && [
.map((src) => src && [
src.replace('-1x', '-3x'),
src.replace('-1x', '-2x'),
src,
@ -161,7 +161,7 @@ function scrapeSceneT1({ html, query }, site, url, baseRelease) {
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)');
release.actors = query.all('.models-list-thumbs a').map(el => ({
release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.q(el, 'span', true),
avatar: getImageWithFallbacks(query.q, 'img', site, el),
}));
@ -180,8 +180,8 @@ function scrapeSceneT1({ html, query }, site, url, baseRelease) {
if (stars) release.stars = Number(stars);
if (site.type === 'network') {
const channelRegExp = new RegExp(site.children.map(channel => channel.parameters?.match || channel.name).join('|'), 'i');
const channel = release.tags.find(tag => channelRegExp.test(tag));
const channelRegExp = new RegExp(site.children.map((channel) => channel.parameters?.match || channel.name).join('|'), 'i');
const channel = release.tags.find((tag) => channelRegExp.test(tag));
if (channel) {
release.channel = slugify(channel, '');
@ -290,7 +290,7 @@ async function scrapeProfile({ query, el }, channel, options) {
if (bio.piercings && /yes/i.test(bio.piercings)) profile.hasPiercings = true;
if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false;
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim());
profile.social = [bio.onlyfans, bio.twitter, bio.instagram].filter(Boolean);

4
src/scrapers/insex.js
View File

@ -26,7 +26,7 @@ function scrapeLatest(scenes, site) {
release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
}
release.actors = actors.map(actor => actor.trim());
release.actors = actors.map((actor) => actor.trim());
const description = query.q('.articleCopyText', true);
if (description) release.description = description.slice(0, description.lastIndexOf('('));
@ -81,7 +81,7 @@ function scrapeScene({ query }, site) {
release.title = title.trim();
release.description = query.q('.articleCopyText', true);
release.actors = actors.map(actor => actor.trim());
release.actors = actors.map((actor) => actor.trim());
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
release.duration = query.dur('.articlePostDateText a:nth-child(2)');

2
src/scrapers/inthecrack.js
View File

@ -117,7 +117,7 @@ function scrapeProfile({ query }, actorName, actorAvatar, channel, releasesFromS
profile.releases = releasesFromScene?.[profile.name] || scrapeProfileScenes(qu.initAll(query.all('.Models li')), actorName, channel);
// avatar is the poster of a scene, find scene and use its high quality poster instead
const avatarRelease = profile.releases.find(release => new URL(release.poster[1]).pathname === new URL(actorAvatar).pathname);
const avatarRelease = profile.releases.find((release) => new URL(release.poster[1]).pathname === new URL(actorAvatar).pathname);
profile.avatar = avatarRelease?.poster[0];
return profile;

12
src/scrapers/jayrock.js
View File

@ -13,7 +13,7 @@ async function fetchActors(entryId, channel, { token, time }) {
const res = await http.get(url);
if (res.statusCode === 200 && res.body.status === true) {
return Object.values(res.body.response.collection).map(actor => Object.values(actor.modelId.collection)[0].stageName);
return Object.values(res.body.response.collection).map((actor) => Object.values(actor.modelId.collection)[0].stageName);
}
return [];
@ -46,7 +46,7 @@ function scrapeLatest(items, channel) {
release.title = query.cnt('h5 a');
[release.poster, ...release.photos] = query.imgs('.screenshot').map(src => [
[release.poster, ...release.photos] = query.imgs('.screenshot').map((src) => [
// unnecessarily large
// src.replace(/\/\d+/, 3840),
// src.replace(/\/\d+/, '/2000'),
@ -99,7 +99,7 @@ function scrapeScene({ query, html }, url, channel) {
[release.poster, ...release.photos] = [poster]
.concat(photos)
.filter(Boolean)
.map(src => [
.map((src) => [
src.replace(/\/(\d+)\/\d+/, '/$1/1500'),
src.replace(/\/(\d+)\/\d+/, '/$1/1000'),
src,
@ -128,8 +128,8 @@ async function scrapeSceneApi(scene, channel, tokens, deep) {
release.date = new Date(scene.sites.collection[scene.id].publishDate);
release.poster = scene._resources.primary[0].url;
if (scene.tags) release.tags = Object.values(scene.tags.collection).map(tag => tag.alias);
if (scene._resources.base) release.photos = scene._resources.base.map(resource => resource.url);
if (scene.tags) release.tags = Object.values(scene.tags.collection).map((tag) => tag.alias);
if (scene._resources.base) release.photos = scene._resources.base.map((resource) => resource.url);
if (deep) {
// don't make external requests during update scraping, as this would happen for every scene on the page
@ -149,7 +149,7 @@ async function scrapeSceneApi(scene, channel, tokens, deep) {
}
function scrapeLatestApi(scenes, site, tokens) {
return Promise.map(scenes, async scene => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
return Promise.map(scenes, async (scene) => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
}
async function fetchToken(channel) {

2
src/scrapers/jesseloadsmonsterfacials.js
View File

@ -33,7 +33,7 @@ function scrapeLatest(scenes, dates, site) {
const poster = qu.img('img[src*="photos/"][width="400"]');
release.poster = `${site.url}/visitors/${poster}`;
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map(source => `${site.url}/visitors/${source}`);
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
return release;
});

14
src/scrapers/julesjordan.js
View File

@ -55,7 +55,7 @@ async function getPhotosLegacy(entryId, site, type = 'highres', page = 1) {
// don't add first URL to pages to prevent unnecessary duplicate request
const photos = scrapePhotos(html, type);
const pages = Array.from(new Set($('.page_numbers a').toArray().map(el => $(el).attr('href'))));
const pages = Array.from(new Set($('.page_numbers a').toArray().map((el) => $(el).attr('href'))));
const otherPhotos = pages
? await Promise.map(pages, async (pageX) => {
@ -84,7 +84,7 @@ async function getPhotos(entryId, site, type = 'highres', page = 1) {
const res = await http.get(albumUrl);
const html = res.body.toString();
const sourceLines = html.split(/\n/).filter(line => line.match(/ptx\["\w+"\]/));
const sourceLines = html.split(/\n/).filter((line) => line.match(/ptx\["\w+"\]/));
const sources = sourceLines.reduce((acc, sourceLine) => {
const quality = sourceLine.match(/\["\w+"\]/)[0].slice(2, -2);
const sourceStart = sourceLine.match(/\/trial|\/tour|\/content/);
@ -261,7 +261,7 @@ async function scrapeScene({ html, query }, url, site, include) {
}
if (include.trailer && site.slug !== 'manuelferrara') {
const trailerLines = html.split('\n').filter(line => /movie\["trailer\w*"\]\[/i.test(line));
const trailerLines = html.split('\n').filter((line) => /movie\["trailer\w*"\]\[/i.test(line));
if (trailerLines.length) {
release.trailer = trailerLines.map((trailerLine) => {
@ -307,7 +307,7 @@ function scrapeMovie({ el, query }, url, site) {
const scenes = scrapeAll(sceneQus, site);
const curatedScenes = scenes
?.map(scene => ({ ...scene, movie }))
?.map((scene) => ({ ...scene, movie }))
.sort((sceneA, sceneB) => sceneA.date - sceneB.date);
movie.date = curatedScenes?.[0].date;
@ -354,13 +354,13 @@ function scrapeProfile(html, url, actorName, entity) {
avatarEl.getAttribute('src0'),
avatarEl.getAttribute('src'),
]
.filter(avatar => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map(avatar => qu.prefixUrl(avatar, entity.url));
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map((avatar) => qu.prefixUrl(avatar, entity.url));
if (avatarSources.length) profile.avatar = avatarSources;
}
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href);
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), (el) => el.href);
return profile;
}

4
src/scrapers/karups.js
View File

@ -35,7 +35,7 @@ function scrapeScene({ query }, url) {
release.title = query.cnt('.title');
release.date = query.date('.date .content', 'MMM Do, YYYY');
release.actors = query.all('.models .content a').map(modelEl => ({
release.actors = query.all('.models .content a').map((modelEl) => ({
name: query.cnt(modelEl),
url: query.url(modelEl, null),
}));
@ -76,7 +76,7 @@ async function fetchProfile(baseActor, entity) {
return searchRes.status;
}
const actorUrl = searchRes.items.find(item => slugify(item.query.cnt('.title')) === baseActor.slug)?.query.url('a');
const actorUrl = searchRes.items.find((item) => slugify(item.query.cnt('.title')) === baseActor.slug)?.query.url('a');
if (!actorUrl) {
return null;

4
src/scrapers/kellymadison.js
View File

@ -53,7 +53,7 @@ function scrapeLatest(scenes, site) {
}
return release;
}).filter(scene => scene);
}).filter((scene) => scene);
}
async function scrapeScene({ query, html }, url, baseRelease, channel, session) {
@ -100,7 +100,7 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session)
const trailerInfoRes = await http.post(trailerInfoUrl, null, { session });
if (trailerInfoRes.ok && trailerInfoRes.body.sources?.length > 0) {
release.trailer = trailerInfoRes.body.sources.map(trailer => ({
release.trailer = trailerInfoRes.body.sources.map((trailer) => ({
src: trailer.src,
type: trailer.type,
/* unreliable, sometimes actual video is 720p

8
src/scrapers/killergram.js
View File

@ -4,11 +4,11 @@ const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
function scrapeAll({ query }) {
const urls = query.urls('td > a:not([href*=joinnow])').map(pathname => `http://killergram.com/${encodeURI(pathname)}`);
const urls = query.urls('td > a:not([href*=joinnow])').map((pathname) => `http://killergram.com/${encodeURI(pathname)}`);
const posters = query.imgs('td > a img');
const titles = query.all('.episodeheadertext', true);
const actors = query.all('.episodetextinfo:nth-child(3)').map(el => query.all(el, 'a', true));
const channels = query.all('.episodetextinfo:nth-child(2) a', true).map(channel => slugify(channel, ''));
const actors = query.all('.episodetextinfo:nth-child(3)').map((el) => query.all(el, 'a', true));
const channels = query.all('.episodetextinfo:nth-child(2) a', true).map((channel) => slugify(channel, ''));
if ([urls.length, posters.length, titles.length, actors.length, channels.length].every((value, index, array) => value === array[0])) { // make sure every set has the same number of items
const releases = urls.map((url, index) => ({
@ -51,7 +51,7 @@ function scrapeScene({ query, html }, url) {
}
async function fetchActorReleases({ query }, url, remainingPages, actorName, accReleases = []) {
const releases = scrapeAll({ query }).filter(release => release.actors.includes(actorName));
const releases = scrapeAll({ query }).filter((release) => release.actors.includes(actorName));
if (remainingPages.length > 0) {
const { origin, pathname, searchParams } = new URL(url);

8
src/scrapers/kink.js
View File

@ -19,7 +19,7 @@ function scrapeAll(scenes) {
release.stars = query.q('.average-rating', 'data-rating') / 10;
release.poster = query.img('.adimage');
release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map(photo => [
release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map((photo) => [
photo.replace('410/', '830/'),
photo,
]);
@ -40,13 +40,13 @@ async function scrapeScene({ query }, url) {
release.description = query.q('.description-text', true);
release.date = query.date('.shoot-date', 'MMMM DD, YYYY');
release.actors = query.all('.names a', true).map(actor => actor.replace(/,\s*/, ''));
release.actors = query.all('.names a', true).map((actor) => actor.replace(/,\s*/, ''));
release.director = query.q('.director-name', true);
release.photos = query.imgs('.gallery .thumb img, #gallerySlider .gallery-img', 'data-image-file');
release.poster = query.poster();
release.tags = query.all('.tag-list a[href*="/tag"]', true).map(tag => tag.replace(/,\s*/, ''));
release.tags = query.all('.tag-list a[href*="/tag"]', true).map((tag) => tag.replace(/,\s*/, ''));
const trailer = query.q('.player span[data-type="trailer-src"]', 'data-url');
@ -154,7 +154,7 @@ async function fetchProfile({ name: actorName }, entity, include) {
const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
if (searchRes.ok) {
const actorItem = searchRes.items.find(item => item.query.exists(`.model-link img[alt="${actorName}"]`));
const actorItem = searchRes.items.find((item) => item.query.exists(`.model-link img[alt="${actorName}"]`));
if (actorItem) {
const actorPath = actorItem.query.url('.model-link');

8
src/scrapers/legalporno.js
View File

@ -126,7 +126,7 @@ async function scrapeScene(html, url, site, useGallery) {
'1080p': 1080,
};
release.trailer = data.clip.qualities.map(trailer => ({
release.trailer = data.clip.qualities.map((trailer) => ({
src: trailer.src,
type: trailer.type,
quality: qualityMap[trailer.quality] || trailer.quality,
@ -147,10 +147,10 @@ async function scrapeProfile(html, _url, actorName) {
};
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
const entries = Array.from(document.querySelectorAll('.model--description tr'), el => el.textContent.replace(/\n/g, '').split(':'));
const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
const bio = entries
.filter(entry => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
.filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
profile.birthPlace = bio.Nationality;
@ -184,7 +184,7 @@ async function fetchProfile({ name: actorName }) {
const res = await http.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
const data = res.body;
const result = data.terms.find(item => item.type === 'model');
const result = data.terms.find((item) => item.type === 'model');
if (result) {
const bioRes = await http.get(result.url);

6
src/scrapers/littlecapricedreams.js
View File

@ -65,7 +65,7 @@ async function fetchPhotos(url) {
const res = await qu.get(url, '.et_post_gallery');
if (res.ok) {
return res.item.query.urls('a').map(imgUrl => ({
return res.item.query.urls('a').map((imgUrl) => ({
src: imgUrl,
referer: url,
}));
@ -89,14 +89,14 @@ async function scrapeScene({ query }, url, channel, include) {
release.date = query.date('.vid_date', 'MMMM D, YYYY');
release.duration = query.dur('.vid_length');
release.actors = query.all('.vid_infos a[href*="author/"]').map(actorEl => ({
release.actors = query.all('.vid_infos a[href*="author/"]').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null),
}));
release.tags = query.cnts('.vid_infos a[rel="tag"]');
const posterData = data['@graph']?.find(item => item['@type'] === 'ImageObject');
const posterData = data['@graph']?.find((item) => item['@type'] === 'ImageObject');
const poster = posterData?.url
|| query.q('meta[property="og:image"]', 'content')

2
src/scrapers/mikeadriano.js
View File

@ -20,7 +20,7 @@ function scrapeAll(scenes) {
release.duration = query.dur('.total-time');
const [poster, ...primaryPhotos] = query.imgs('a img');
const secondaryPhotos = query.styles('.thumb-top, .thumb-bottom, .thumb-mouseover', 'background-image').map(style => style.match(/url\((.*)\)/)[1]);
const secondaryPhotos = query.styles('.thumb-top, .thumb-bottom, .thumb-mouseover', 'background-image').map((style) => style.match(/url\((.*)\)/)[1]);
release.poster = poster;
release.photos = primaryPhotos.concat(secondaryPhotos);

44
src/scrapers/mindgeek.js
View File

@ -14,14 +14,14 @@ const { inchesToCm, lbsToKg } = require('../utils/convert');
function getThumbs(scene) {
if (scene.images.poster) {
return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
.filter(img => typeof img === 'object') // remove alternateText property
.map(image => image.xl.url);
.filter((img) => typeof img === 'object') // remove alternateText property
.map((image) => image.xl.url);
}
if (scene.images.card_main_rect) {
return scene.images.card_main_rect
.concat(scene.images.card_secondary_rect || [])
.map(image => image.xl.url.replace('.thumb', ''));
.map((image) => image.xl.url.replace('.thumb', ''));
}
return [];
@ -29,14 +29,14 @@ function getThumbs(scene) {
function getVideos(data) {
const teaserSources = data.videos.mediabook?.files;
const trailerSources = data.children.find(child => child.type === 'trailer')?.videos.full?.files;
const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
const teaser = teaserSources && Object.values(teaserSources).map(source => ({
const teaser = teaserSources && Object.values(teaserSources).map((source) => ({
src: source.urls.view,
quality: parseInt(source.format, 10),
}));
const trailer = trailerSources && Object.values(trailerSources).map(source => ({
const trailer = trailerSources && Object.values(trailerSources).map((source) => ({
src: source.urls.view,
quality: parseInt(source.format, 10),
}));
@ -59,8 +59,8 @@ function scrapeLatestX(data, site, filterChannel) {
release.date = new Date(data.dateReleased);
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map(tag => tag.name);
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map((tag) => tag.name);
[release.poster, ...release.photos] = getThumbs(data);
@ -69,15 +69,15 @@ function scrapeLatestX(data, site, filterChannel) {
if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({
release.chapters = data.timeTags?.map((chapter) => ({
time: chapter.startTime,
duration: chapter.endTime - chapter.startTime,
tags: [chapter.name],
}));
if ((site.parameters?.extract === true && data.collections.length > 0) // release should not belong to any channel
|| (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) // release should belong to specific channel
|| (filterChannel && !data.collections?.some(collection => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes
|| (typeof site.parameters?.extract === 'string' && !data.collections.some((collection) => collection.shortName === site.parameters.extract)) // release should belong to specific channel
|| (filterChannel && !data.collections?.some((collection) => collection.id === site.parameters?.siteId))) { // used to separate upcoming Brazzers scenes
return {
...release,
exclude: true,
@ -88,11 +88,11 @@ function scrapeLatestX(data, site, filterChannel) {
}
async function scrapeLatest(items, site, filterChannel) {
const latestReleases = items.map(data => scrapeLatestX(data, site, filterChannel));
const latestReleases = items.map((data) => scrapeLatestX(data, site, filterChannel));
return {
scenes: latestReleases.filter(scene => !scene.exclude),
unextracted: latestReleases.filter(scene => scene.exclude),
scenes: latestReleases.filter((scene) => !scene.exclude),
unextracted: latestReleases.filter((scene) => scene.exclude),
};
}
@ -108,8 +108,8 @@ function scrapeScene(data, url, _site, networkName) {
release.date = new Date(data.dateReleased);
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map(tag => tag.name);
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map((tag) => tag.name);
[release.poster, ...release.photos] = getThumbs(data);
@ -118,7 +118,7 @@ function scrapeScene(data, url, _site, networkName) {
if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({
release.chapters = data.timeTags?.map((chapter) => ({
time: chapter.startTime,
duration: chapter.endTime - chapter.startTime,
tags: [chapter.name],
@ -213,18 +213,18 @@ function scrapeProfile(data, html, releases = [], networkName) {
|| data.images.card_main_rect[0].xs?.url;
}
const birthdate = query.all('li').find(el => /Date of Birth/.test(el.textContent));
const birthdate = query.all('li').find((el) => /Date of Birth/.test(el.textContent));
if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');
if (data.tags.some(tag => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
if (data.tags.some((tag) => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
profile.naturalBoobs = true;
}
if (data.tags.some(tag => /boob type/i.test(tag.category) && /enhanced/i.test(tag.name))) {
if (data.tags.some((tag) => /boob type/i.test(tag.category) && /enhanced/i.test(tag.name))) {
profile.naturalBoobs = false;
}
profile.releases = releases.map(release => scrapeScene(release, null, null, networkName));
profile.releases = releases.map((release) => scrapeScene(release, null, null, networkName));
return profile;
}
@ -325,7 +325,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
});
if (res.statusCode === 200) {
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
const actorData = res.body.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
if (actorData) {
const actorUrl = `https://www.${entity.slug}.com/${entity.parameters?.actorPath || 'model'}/${actorData.id}/${actorSlug}`;

4
src/scrapers/naughtyamerica.js
View File

@ -69,7 +69,7 @@ function scrapeScene(html, url, site) {
const posterPath = $('video, dl8-video').attr('poster') || $('img.start-card').attr('src');
const poster = posterPath && `https:${posterPath}`;
const photos = $('.contain-scene-images.desktop-only a').map((index, el) => $(el).attr('href')).toArray().filter(Boolean).map(photo => `https:${photo}`);
const photos = $('.contain-scene-images.desktop-only a').map((index, el) => $(el).attr('href')).toArray().filter(Boolean).map((photo) => `https:${photo}`);
const trailerEl = $('source');
const trailerSrc = trailerEl.attr('src');
@ -120,7 +120,7 @@ async function scrapeProfile(html) {
const releases = query.urls('.scene-item > a:first-child');
const otherPages = query.urls('.pagination a:not([rel=next]):not([rel=prev])');
const olderReleases = await Promise.all(otherPages.map(async page => fetchActorReleases(page)));
const olderReleases = await Promise.all(otherPages.map(async (page) => fetchActorReleases(page)));
profile.releases = releases.concat(olderReleases.flat());

8
src/scrapers/nubiles.js
View File

@ -78,7 +78,7 @@ async function scrapeScene({ query }, url, site) {
release.tags = query.all('.categories a', true);
release.poster = query.poster() || query.img('.fake-video-player img');
release.trailer = query.all('source').map(source => ({
release.trailer = query.all('source').map((source) => ({
src: source.src,
quality: Number(source.getAttribute('res')),
}));
@ -106,11 +106,11 @@ function scrapeProfile({ query }, _actorName, origin) {
profile.residencePlace = bio.location;
profile.height = heightToCm(bio.height);
[profile.bust, profile.waist, profile.hip] = bio.figure.split('-').map(v => Number(v) || v);
[profile.bust, profile.waist, profile.hip] = bio.figure.split('-').map((v) => Number(v) || v);
profile.avatar = query.img('.model-profile img');
const releases = query.all('.content-grid-item').filter(el => /video\//.test(query.url(el, '.img-wrapper a'))); // filter out photos
const releases = query.all('.content-grid-item').filter((el) => /video\//.test(query.url(el, '.img-wrapper a'))); // filter out photos
profile.releases = scrapeAll(query.initAll(releases), null, origin);
return profile;
@ -143,7 +143,7 @@ async function fetchProfile({ name: actorName }, { site }) {
if (!resModels.ok) return resModels.status;
const modelPath = resModels.item.qu.all('.content-grid-item a.title').find(el => slugify(el.textContent) === slugify(actorName));
const modelPath = resModels.item.qu.all('.content-grid-item a.title').find((el) => slugify(el.textContent) === slugify(actorName));
if (modelPath) {
const modelUrl = `${origin}${modelPath}`;

2
src/scrapers/pascalssubsluts.js
View File

@ -26,7 +26,7 @@ function scrapeAll(months, channel, year) {
gender: 'female',
url: query.url('a.video-pop-up', 'data-modellink', { origin: `${channel.url}/submissive` }),
}]
.filter(actor => !/lockdown/i.test(actor.name))
.filter((actor) => !/lockdown/i.test(actor.name))
.concat({
name: 'Pascal White',
gender: 'male',

12
src/scrapers/perfectgonzo.js
View File

@ -17,21 +17,21 @@ function extractMaleModelsFromTags(tagContainer) {
return [];
}
const tagEls = Array.from(tagContainer.childNodes, node => ({ type: node.nodeType, text: node.textContent.trim() })).filter(node => node.text.length > 0);
const modelLabelIndex = tagEls.findIndex(node => node.text === 'Male Models');
const tagEls = Array.from(tagContainer.childNodes, (node) => ({ type: node.nodeType, text: node.textContent.trim() })).filter((node) => node.text.length > 0);
const modelLabelIndex = tagEls.findIndex((node) => node.text === 'Male Models');
if (modelLabelIndex > -1) {
const nextLabelIndex = tagEls.findIndex((node, index) => index > modelLabelIndex && node.type === 3);
const maleModels = tagEls.slice(modelLabelIndex + 1, nextLabelIndex);
return maleModels.map(model => model.text);
return maleModels.map((model) => model.text);
}
return [];
}
async function extractChannelFromPhoto(photo, channel) {
const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug);
const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map((child) => child.slug);
const channelMatch = photo.match(new RegExp(siteSlugs.join('|')));
if (channelMatch) {
@ -52,7 +52,7 @@ async function scrapeLatest(scenes, site) {
const slug = new URL(release.url).pathname.split('/')[2];
release.entryId = getHash(`${site.slug}${slug}${release.date.toISOString()}`);
release.actors = release.title.split('&').map(actor => actor.trim());
release.actors = release.title.split('&').map((actor) => actor.trim());
[release.poster, ...release.photos] = query.imgs('.bloc-link img');
@ -78,7 +78,7 @@ async function scrapeScene({ query }, site, url) {
const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
release.photos = query.all('.bxslider_pics img').map(el => el.dataset.original || el.src);
release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
release.poster = query.poster();
const trailer = query.trailer();

4
src/scrapers/pierrewoodman.js
View File

@ -67,7 +67,7 @@ function scrapeScene({ query, html }, url, entity) {
release.description = query.cnt('.info_container .description');
release.date = query.date('.info_container .info_line:nth-child(1)', 'YYYY-MM-DD') || query.date('.description', 'DD MMMM YYYY', /\d{1,2} \w+ \d{4}/);
release.actors = query.all('.girl_item, .starring .item').map(actorEl => mapActor(actorEl, query, entity));
release.actors = query.all('.girl_item, .starring .item').map((actorEl) => mapActor(actorEl, query, entity));
release.duration = query.duration('.infos .description');
@ -81,7 +81,7 @@ function scrapeScene({ query, html }, url, entity) {
release.tags = query.cnts('.tags a:not(.more_tag)');
release.poster = removeImageBorder(html.match(/image: "(.*?)"/)?.[1]);
release.trailer = html.match(/url: "(.*mp4.*)"/g)?.map(src => ({
release.trailer = html.match(/url: "(.*mp4.*)"/g)?.map((src) => ({
src: src.match(/"(.*)"/)?.[1],
quality: Number(src.match(/[-/](\d+)p/)?.[1]),
}));

2
src/scrapers/porncz.js
View File

@ -40,7 +40,7 @@ function scrapeScene({ query }, url, channel) {
release.date = date;
release.datePrecision = precision;
release.actors = query.cnts(details.actors, 'a').map(actor => capitalize(actor, { uncapitalize: true }));
release.actors = query.cnts(details.actors, 'a').map((actor) => capitalize(actor, { uncapitalize: true }));
release.duration = query.duration(details.duration);
release.tags = query.cnts(details.genres, 'a');

6
src/scrapers/porndoe.js
View File

@ -13,7 +13,7 @@ function scrapeAll(scenes) {
release.title = query.cnt('[class*="item-title"] a') || query.q('.bottom .link', 'title');
release.date = query.date('[class*="item-date"]', 'MMM DD, YYYY');
release.actors = query.all('[class*="item-actors"] a').map(el => ({
release.actors = query.all('[class*="item-actors"] a').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
}));
@ -44,7 +44,7 @@ function scrapeScene({ query }, url) {
release.description = query.meta('name=description') || query.q('read-even-more', true);
release.date = query.date('.h5-published', 'MMM DD, YYYY', /\w{3} \d{1,2}, \d{4}/);
release.actors = query.all('.video-top-details .actors a[href*="/models"]').map(el => ({
release.actors = query.all('.video-top-details .actors a[href*="/models"]').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
}));
@ -53,7 +53,7 @@ function scrapeScene({ query }, url) {
release.tags = query.all('.video-top-details a[href*="/categories"], .video-top-details a[href*="/tags"]', true);
release.poster = query.img('.poster img') || query.meta('itemprop=thumbnailUrl');
release.photos = query.imgs('#gallery-thumbs [class*="thumb"]', 'data-bg').slice(1).map(photo => [ // first image is poster
release.photos = query.imgs('#gallery-thumbs [class*="thumb"]', 'data-bg').slice(1).map((photo) => [ // first image is poster
photo.replace('512x288', '1472x828'),
photo,
]);

4
src/scrapers/pornhub.js
View File

@ -16,7 +16,7 @@ const hairMap = {
async function scrapeProfile(html, _url, actorName) {
const { document } = new JSDOM(html).window;
const entries = Array.from(document.querySelectorAll('.infoPiece'), el => el.textContent.replace(/\n|\t/g, '').split(':'));
const entries = Array.from(document.querySelectorAll('.infoPiece'), (el) => el.textContent.replace(/\n|\t/g, '').split(':'));
const bio = entries.reduce((acc, [key, value]) => (key ? { ...acc, [key.trim()]: value.trim() } : acc), {});
const profile = {
@ -47,7 +47,7 @@ async function scrapeProfile(html, _url, actorName) {
if (bio.Tattoos) profile.hasTattoos = bio.Tattoos === 'Yes';
if (avatarEl && !/default\//.test(avatarEl.src)) profile.avatar = avatarEl.src;
profile.social = Array.from(document.querySelectorAll('.socialList a'), el => el.href).filter(link => link !== 'https://www.twitter.com/'); // PH links to Twitter itself for some reason
profile.social = Array.from(document.querySelectorAll('.socialList a'), (el) => el.href).filter((link) => link !== 'https://www.twitter.com/'); // PH links to Twitter itself for some reason
return profile;
}

14
src/scrapers/score.js
View File

@ -9,7 +9,7 @@ function scrapePhotos(html) {
const { qis } = ex(html, '#photos-page');
const photos = qis('img');
return photos.map(photo => [
return photos.map((photo) => [
photo
.replace('x_800', 'x_xl')
.replace('_tn', ''),
@ -76,22 +76,22 @@ async function scrapeScene(html, url, site) {
release.actors = qu.all('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true);
if (release.actors.length === 0) {
const actorEl = qu.all('.stat').find(stat => /Featuring/.test(stat.textContent));
const actorEl = qu.all('.stat').find((stat) => /Featuring/.test(stat.textContent));
const actorString = qu.text(actorEl);
release.actors = actorString?.split(/,\band\b|,/g).map(actor => actor.trim()) || [];
release.actors = actorString?.split(/,\band\b|,/g).map((actor) => actor.trim()) || [];
}
if (release.actors.length === 0 && site.parameters?.actors) release.actors = site.parameters.actors;
release.tags = qu.all('a[href*=tag]', true);
const dateEl = qu.all('.value').find(el => /\w+ \d+\w+, \d{4}/.test(el.textContent));
const dateEl = qu.all('.value').find((el) => /\w+ \d+\w+, \d{4}/.test(el.textContent));
release.date = qu.date(dateEl, null, 'MMMM Do, YYYY')
|| qu.date('.date', 'MMMM Do, YYYY', /\w+ \d{1,2}\w+, \d{4}/)
|| qu.date('.info .holder', 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/);
const durationEl = qu.all('value').find(el => /\d{1,3}:\d{2}/.test(el.textContent));
const durationEl = qu.all('value').find((el) => /\d{1,3}:\d{2}/.test(el.textContent));
release.duration = qu.dur(durationEl);
release.poster = qu.poster('video') || qu.img('.flowplayer img') || html.match(/posterImage: '(.*\.jpg)'/)?.[1] || null; // _800.jpg is larger than _xl.jpg in landscape
@ -100,7 +100,7 @@ async function scrapeScene(html, url, site) {
if (photosUrl) {
release.photos = await fetchPhotos(photosUrl);
} else {
release.photos = qu.imgs('img[src*=ThumbNails], .p-photos .tn img').map(photo => [
release.photos = qu.imgs('img[src*=ThumbNails], .p-photos .tn img').map((photo) => [
photo.replace('_tn', ''),
photo,
]);
@ -126,7 +126,7 @@ async function scrapeScene(html, url, site) {
function scrapeModels(html, actorName) {
const { qa } = ex(html);
const model = qa('.model a').find(link => link.title === actorName);
const model = qa('.model a').find((link) => link.title === actorName);
return model?.href || null;
}

10
src/scrapers/spizoo.js
View File

@ -15,7 +15,7 @@ function scrapeAll(scenes) {
release.entryId = getEntryId(release.url);
release.title = query.cnt('.title-label a');
release.actors = query.all('.update_models a').map(el => ({
release.actors = query.all('.update_models a').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
}));
@ -37,7 +37,7 @@ function scrapeScene({ query }, url) {
release.description = query.cnt('#sceneInfo .description');
release.actors = query.all('#sceneInfo .data-others a[href*="/models"]').map(el => ({
release.actors = query.all('#sceneInfo .data-others a[href*="/models"]').map((el) => ({
name: query.el(el, null, 'title'),
url: query.url(el, null),
}));
@ -50,8 +50,8 @@ function scrapeScene({ query }, url) {
release.poster = [poster, poster?.replace(/imgw=\w+/, 'imgw=680')];
release.photos = query.imgs('.photos-holder img')
.filter(src => new URL(src).pathname !== posterPathname)
.map(src => [
.filter((src) => new URL(src).pathname !== posterPathname)
.map((src) => [
src.replace(/imgw=\d+/, 'imgw=1284'),
src,
]);
@ -74,7 +74,7 @@ function scrapeProfileScenes(scenes) {
release.description = query.cnt('.model-update-description');
release.actors = query.all('.model-labels a').map(el => ({
release.actors = query.all('.model-labels a').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
}));

8
src/scrapers/teamskeet.js
View File

@ -13,7 +13,7 @@ function getChannelSlug(channelName, entity) {
}
const channelSlug = slugify(channelName, '', { removePunctuation: true });
const channel = entity.children.find(child => new RegExp(channelSlug).test(child.slug));
const channel = entity.children.find((child) => new RegExp(channelSlug).test(child.slug));
return channel?.slug || null;
}
@ -27,8 +27,8 @@ function scrapeScene(scene, channel) {
release.title = scene.title;
release.date = qu.extractDate(scene.publishedDate);
release.actors = scene.models?.map(model => model.modelName) || [];
release.actors = scene.models?.map(model => ({
release.actors = scene.models?.map((model) => model.modelName) || [];
release.actors = scene.models?.map((model) => ({
name: model.modelName,
avatar: `https://images.mylfcdn.net/tsv4/model/profiles/${slugify(model.modelName, '_')}.jpg`,
url: `${channel.url}/models/www.mylf.com/models/${model.modelId}`,
@ -113,7 +113,7 @@ function scrapeProfile(actor, entity) {
}
profile.avatar = actor.img;
profile.scenes = actor.movies?.map(scene => scrapeScene(scene, entity));
profile.scenes = actor.movies?.map((scene) => scrapeScene(scene, entity));
return profile;
}

10
src/scrapers/teencoreclub.js
View File

@ -25,15 +25,15 @@ function scrapeAll(scenes, entity) {
release.date = moment.utc(scene.year, 'YYYY').toDate();
release.datePrecision = 'year';
release.actors = scene.actors.map(actor => ({
release.actors = scene.actors.map((actor) => ({
name: actor.name.trim(),
avatar: actor.image || null,
})).filter(actor => actor.name && slugify(actor.name) !== 'amateur-girl');
})).filter((actor) => actor.name && slugify(actor.name) !== 'amateur-girl');
release.duration = scene.duration;
release.stars = scene.video_rating_score;
[release.poster, ...release.photos] = scene.screenshots.map(url => prefixUrl(url));
[release.poster, ...release.photos] = scene.screenshots.map((url) => prefixUrl(url));
if (scene.is_gay) {
release.tags = ['gay'];
@ -64,7 +64,7 @@ async function scrapeScene({ query }, url) {
release.description = query.q('.detail-description', true);
release.duration = query.dur('.detail-meta li:first-child');
const actors = [query.q('.detail-hero-title h1', true)?.trim()].filter(name => name && slugify(name) !== 'amateur-girl');
const actors = [query.q('.detail-hero-title h1', true)?.trim()].filter((name) => name && slugify(name) !== 'amateur-girl');
if (actors.length > 0) {
release.actors = actors;
@ -143,7 +143,7 @@ async function fetchProfile({ name: actorName }, { entity }, include) {
const res = await http.get(`https://teencoreclub.com/api/actors?query=${actorName}`);
if (res.ok) {
const actor = res.body.data.find(item => slugify(item.name) === slugify(actorName));
const actor = res.body.data.find((item) => slugify(item.name) === slugify(actorName));
if (actor) {
return scrapeProfile(actor, entity, include);

8
src/scrapers/teenmegaworld.js
View File

@ -14,7 +14,7 @@ function scrapeAll(scenes, channel) {
release.title = query.cnt('.title');
release.date = query.date('time', 'MMMM D, YYYY');
release.actors = query.all('.actors a').map(el => ({
release.actors = query.all('.actors a').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
}));
@ -29,7 +29,7 @@ function scrapeAll(scenes, channel) {
const siteId = query.url('.site a', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
if (siteId) {
release.channel = network.children.find(child => child.parameters.siteId.toString() === siteId)?.slug;
release.channel = network.children.find((child) => child.parameters.siteId.toString() === siteId)?.slug;
}
return release;
@ -48,7 +48,7 @@ function scrapeScene({ query }, url, channel) {
release.date = query.date('.title-line .date', 'MMMM D, YYYY');
release.duration = query.number('.dur') * 60;
release.actors = query.all('.site a[href*="/models"]').map(el => ({
release.actors = query.all('.site a[href*="/models"]').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
}));
@ -63,7 +63,7 @@ function scrapeScene({ query }, url, channel) {
const siteId = query.url('.site a[href*="site[]"]', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
if (siteId) {
release.channel = network.children.find(child => child.parameters.siteId.toString() === siteId)?.slug;
release.channel = network.children.find((child) => child.parameters.siteId.toString() === siteId)?.slug;
}
return release;

6
src/scrapers/topwebmodels.js
View File

@ -20,7 +20,7 @@ function scrapeSceneX(scene) {
release.date = new Date(scene.release_date);
release.actors = scene.models
.map(actor => (/&/.test(actor.name)
.map((actor) => (/&/.test(actor.name)
? actor.name.split(/\s*&\s*/)
: {
name: actor.name,
@ -31,7 +31,7 @@ function scrapeSceneX(scene) {
.flat();
release.stars = scene.rating;
release.tags = scene.tags.map(tag => tag.name);
release.tags = scene.tags.map((tag) => tag.name);
if (mime.getType(scene.thumb) === 'image/gif') {
release.teaser = scene.thumb;
@ -128,7 +128,7 @@ async function fetchProfile(baseActor, entity, options) {
return searchRes.status;
}
const actor = searchRes.body.models.items.find(model => slugify(model.name) === slugify(baseActor.name));
const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
if (actor) {
return scrapeProfile(actor, options);

4
src/scrapers/traxxx.js
View File

@ -217,7 +217,7 @@ function gender() {
}
function actors(release) {
const length = release.tags.some(tag => ['dp', 'dap', 'gangbang'].includes(tag))
const length = release.tags.some((tag) => ['dp', 'dap', 'gangbang'].includes(tag))
? Math.floor(Math.random() * 6) + 3
: Math.floor(Math.random() * 3) + 2;
@ -254,7 +254,7 @@ async function fetchLatest(entity, page, options) {
// const poster = 'sfw/kittens/thumbs/iNEXVlX-RLs.jpeg';
release.poster = `http://${config.web.host}:${config.web.port}/img/${poster}?id=${nanoid()}`; // ensure source is unique
release.photos = photos.map(photo => `http://${config.web.host}:${config.web.port}/img/${photo}?id=${nanoid()}`);
release.photos = photos.map((photo) => `http://${config.web.host}:${config.web.port}/img/${photo}?id=${nanoid()}`);
}
release.tags = await knex('tags')

8
src/scrapers/vivid.js
View File

@ -17,7 +17,7 @@ function scrapeLatestNative(scenes, site) {
release.date = ed(scene.release_date, 'YYYY-MM-DD');
release.duration = parseInt(scene.runtime, 10) * 60;
release.actors = scene.cast?.map(actor => ({
release.actors = scene.cast?.map((actor) => ({
name: actor.stagename,
gender: actor.gender.toLowerCase(),
avatar: actor.placard,
@ -38,10 +38,10 @@ function scrapeSceneNative({ html, q, qa }, url, _site) {
release.title = q('.scene-h2-heading', true);
release.description = q('.indie-model-p', true);
const dateString = qa('h5').find(el => /Released/.test(el.textContent)).textContent;
const dateString = qa('h5').find((el) => /Released/.test(el.textContent)).textContent;
release.date = ed(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
const duration = qa('h5').find(el => /Runtime/.test(el.textContent)).textContent;
const duration = qa('h5').find((el) => /Runtime/.test(el.textContent)).textContent;
const [hours, minutes] = duration.match(/\d+/g);
if (minutes) release.duration = (hours * 3600) + (minutes * 60);
@ -111,7 +111,7 @@ async function fetchSceneWrapper(url, site, release) {
});
if (searchRes.statusCode === 200 && searchRes.body.code === 200) {
const sceneMatch = searchRes.body.responseData.find(item => slugify(item.name) === slugify(scene.title));
const sceneMatch = searchRes.body.responseData.find((item) => slugify(item.name) === slugify(scene.title));
if (sceneMatch) {
return {

18
src/scrapers/vixen.js
View File

@ -15,7 +15,7 @@ const genderMap = {
function getPosterFallbacks(poster) {
return poster
.filter(image => /landscape/i.test(image.name))
.filter((image) => /landscape/i.test(image.name))
.sort((imageA, imageB) => imageB.height - imageA.height)
.map((image) => {
const sources = [image.src, image.highdpi?.['2x'], image.highdpi?.['3x']];
@ -23,7 +23,7 @@ function getPosterFallbacks(poster) {
return image.height === 1080 ? sources : sources.reverse();
})
.flat()
.map(src => ({
.map((src) => ({
src,
expectType: {
'binary/octet-stream': 'image/jpeg',
@ -33,8 +33,8 @@ function getPosterFallbacks(poster) {
function getTeaserFallbacks(teaser) {
return teaser
.filter(video => /landscape/i.test(video.name))
.map(video => ({
.filter((video) => /landscape/i.test(video.name))
.map((video) => ({
src: video.src,
type: video.type,
quality: Number(String(video.height).replace('353', '360')),
@ -44,7 +44,7 @@ function getTeaserFallbacks(teaser) {
function getAvatarFallbacks(avatar) {
return avatar
.sort((imageA, imageB) => imageB.height - imageA.height)
.map(image => [image.highdpi?.['3x'], image.highdpi?.['2x'], image.src])
.map((image) => [image.highdpi?.['3x'], image.highdpi?.['2x'], image.src])
.flat();
}
@ -149,7 +149,7 @@ async function getPhotos(url) {
});
const state = htmlRes?.window.__APOLLO_STATE__;
const key = Object.values(state.ROOT_QUERY).find(query => query?.__ref)?.__ref;
const key = Object.values(state.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
const data = state[key];
console.log(data);
@ -158,7 +158,7 @@ async function getPhotos(url) {
return [];
}
return data.carousel.slice(1).map(photo => photo.main?.[0].src).filter(Boolean);
return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
}
function scrapeAll(scenes, site, origin) {
@ -191,7 +191,7 @@ function scrapeUpcoming(scene, site) {
release.title = scene.targetUrl
.slice(1)
.split('-')
.map(component => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
.map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
.join(' ');
release.url = `${site.url}/videos${scene.targetUrl}`;
@ -243,7 +243,7 @@ async function scrapeScene(data, url, site, baseRelease, options) {
const trailer = await getTrailer(scene, site, url);
if (trailer) release.trailer = trailer;
release.chapters = data.video.chapters?.video.map(chapter => ({
release.chapters = data.video.chapters?.video.map((chapter) => ({
tags: [chapter.title],
time: chapter.seconds,
}));

2
src/scrapers/vogov.js
View File

@ -131,7 +131,7 @@ function scrapeScene(html, url) {
release.actors = qu.all('.info-video-models a', true);
release.tags = qu.all('.info-video-category a', true);
release.photos = qu.urls('.swiper-wrapper .swiper-slide a').map(source => source.replace('.jpg/', '.jpg'));
release.photos = qu.urls('.swiper-wrapper .swiper-slide a').map((source) => source.replace('.jpg/', '.jpg'));
release.poster = qu.meta('meta[property="og:image"]');
if (!release.poster) {

8
src/scrapers/wankzvr.js
View File

@ -22,7 +22,7 @@ async function getTrailerUrl(release, channel, request) {
});
if (res.ok) {
const trailers = res.body.streams.map(trailer => ({
const trailers = res.body.streams.map((trailer) => ({
src: trailer.url,
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
vr: true,
@ -47,7 +47,7 @@ function scrapeAll(scenes, channel) {
release.title = query.cnt('.card__h');
release.date = query.date('.card__date', 'D MMMM, YYYY');
release.actors = query.all('.card__links a').map(el => ({
release.actors = query.all('.card__links a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
}));
@ -82,14 +82,14 @@ async function scrapeScene({ query }, url, channel, baseRelease, options, reques
release.date = query.date('.detail__date', 'D MMMM, YYYY');
release.duration = query.number('.time') * 60;
release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map(el => ({
release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
}));
release.tags = query.cnts('.tag-list .tag').concat(query.cnts('.detail__specs-list .detail__specs-item'));
release.photos = query.all('.photo-strip__slide').map(el => ([
release.photos = query.all('.photo-strip__slide').map((el) => ([
qu.query.img(el, null, 'data-src'),
qu.query.img(el, 'img', 'src'),
]));

8
src/scrapers/whalemember.js
View File

@ -18,7 +18,7 @@ function scrapeLatest(html, site) {
release.entryId = scene.dataset.videoId;
release.title = scene.querySelector('.card-title').textContent;
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent);
release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent);
// slow CDN?
const poster = scene.querySelector('.single-image').dataset.src;
@ -32,7 +32,7 @@ function scrapeLatest(html, site) {
concurrency: 1,
};
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), (el) => ({
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
referer: site.url,
attempts: 5,
@ -63,7 +63,7 @@ function scrapeScene(html, site, url) {
release.url = url;
release.title = scene.querySelector('.t2019-stitle').textContent.trim();
release.description = scene.querySelector('#t2019-description').textContent.trim();
release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), el => el.textContent);
release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), (el) => el.textContent);
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
@ -75,7 +75,7 @@ function scrapeScene(html, site, url) {
}
// unreliable CDN
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => ({
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), (el) => ({
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
referer: site.url,
attempts: 5,

4
src/sites.js
View File

@ -39,7 +39,7 @@ function curateSite(site, includeParameters = false) {
}
async function curateSites(sites, includeParameters) {
return Promise.all(sites.map(async site => curateSite(site, includeParameters)));
return Promise.all(sites.map(async (site) => curateSite(site, includeParameters)));
}
function destructConfigNetworks(networks = []) {
@ -165,7 +165,7 @@ async function fetchIncludedSites() {
async function fetchSites(queryObject) {
const sites = await knex('sites')
.where(builder => whereOr(queryObject, 'sites', builder))
.where((builder) => whereOr(queryObject, 'sites', builder))
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',

2
src/stashes.js
View File

@ -58,7 +58,7 @@ async function fetchStashes(domain, itemId, sessionUser) {
})
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
return stashes.map(stash => curateStash(stash));
return stashes.map((stash) => curateStash(stash));
}
async function createStash(newStash, sessionUser) {

38
src/store-releases.js
View File

@ -17,7 +17,7 @@ const { notify } = require('./alerts');
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
const slugBase = release.title
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map(actor => actor.name).join(' ')}`)
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
|| null;
@ -74,11 +74,11 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce
}
async function attachChannelEntities(releases) {
const releasesWithoutEntity = releases.filter(release => release.channel && (!release.entity || release.entity.type === 'network'));
const releasesWithoutEntity = releases.filter((release) => release.channel && (!release.entity || release.entity.type === 'network'));
const channelEntities = await knex('entities')
.select(knex.raw('entities.*, row_to_json(parents) as parent'))
.whereIn('entities.slug', releasesWithoutEntity.map(release => release.channel))
.whereIn('entities.slug', releasesWithoutEntity.map((release) => release.channel))
.where('entities.type', 'channel')
.leftJoin('entities AS parents', 'parents.id', 'entities.parent_id');
@ -108,7 +108,7 @@ async function attachChannelEntities(releases) {
}
async function attachStudios(releases) {
const studioSlugs = releases.map(release => release.studio).filter(Boolean);
const studioSlugs = releases.map((release) => release.studio).filter(Boolean);
const studios = await knex('entities')
.whereIn('slug', studioSlugs)
@ -186,7 +186,7 @@ function filterInternalDuplicateReleases(releases) {
}, {});
return Object.values(releasesByEntityIdAndEntryId)
.map(entityReleases => Object.values(entityReleases))
.map((entityReleases) => Object.values(entityReleases))
.flat();
}
@ -194,11 +194,11 @@ async function filterDuplicateReleases(releases) {
const internalUniqueReleases = filterInternalDuplicateReleases(releases);
const duplicateReleaseEntries = await knex('releases')
.whereIn(['entry_id', 'entity_id'], internalUniqueReleases.map(release => [release.entryId, release.entity.id]))
.whereIn(['entry_id', 'entity_id'], internalUniqueReleases.map((release) => [release.entryId, release.entity.id]))
.orWhereIn(['entry_id', 'entity_id'], internalUniqueReleases
// scene IDs shared across network, mark as duplicate so scene can be updated with channel if only available on release day (i.e. Perv City)
.filter(release => release.entity.parent?.parameters?.networkEntryIds)
.map(release => [release.entryId, release.entity.parent.id]));
.filter((release) => release.entity.parent?.parameters?.networkEntryIds)
.map((release) => [release.entryId, release.entity.parent.id]));
const duplicateReleasesByEntityIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
if (!acc[release.entity_id]) acc[release.entity_id] = {};
@ -207,10 +207,10 @@ async function filterDuplicateReleases(releases) {
return acc;
}, {});
const duplicateReleases = internalUniqueReleases.filter(release => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
const duplicateReleases = internalUniqueReleases.filter((release) => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|| duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
const uniqueReleases = internalUniqueReleases.filter((release) => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
&& !duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
return {
@ -263,7 +263,7 @@ async function updateSceneSearch(releaseIds) {
async function storeChapters(releases) {
const chapters = releases
.map(release => release.chapters?.map((chapter, index) => ({
.map((release) => release.chapters?.map((chapter, index) => ({
releaseId: release.id,
index: index + 1,
time: chapter.time,
@ -278,7 +278,7 @@ async function storeChapters(releases) {
.filter(Boolean)
.sort((chapterA, chapterB) => chapterA.time - chapterB.time);
const curatedChapterEntries = chapters.map(chapter => ({
const curatedChapterEntries = chapters.map((chapter) => ({
index: chapter.index,
time: chapter.time,
duration: chapter.duration,
@ -297,7 +297,7 @@ async function storeChapters(releases) {
},
}), {});
const chaptersWithId = chapters.map(chapter => ({
const chaptersWithId = chapters.map((chapter) => ({
...chapter,
id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.index],
}));
@ -316,13 +316,13 @@ async function storeScenes(releases) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const releasesWithChannels = await attachChannelEntities(releases);
const releasesWithBaseActors = releasesWithChannels.map(release => ({ ...release, actors: toBaseActors(release.actors) }));
const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) }));
const releasesWithStudios = await attachStudios(releasesWithBaseActors);
// uniqueness is entity ID + entry ID, filter uniques after adding entities
const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId)));
const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId)));
const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries);
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
@ -355,13 +355,13 @@ async function storeScenes(releases) {
]);
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
await updateSceneSearch(releasesWithId.map(release => release.id));
await updateSceneSearch(releasesWithId.map((release) => release.id));
// media is more error-prone, associate separately
await associateReleaseMedia(releasesWithId);
if (argv.sceneActors && actors) {
await scrapeActors(actors.map(actor => actor.name));
await scrapeActors(actors.map((actor) => actor.name));
}
logger.info(`Stored ${storedReleaseEntries.length}, updated ${updated.rowCount} releases`);
@ -446,12 +446,12 @@ async function storeMovies(movies) {
const { uniqueReleases } = await filterDuplicateReleases(movies);
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const curatedMovieEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie')));
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
const moviesWithId = attachReleaseIds(movies, storedMovies);
await updateMovieSearch(moviesWithId.map(movie => movie.id));
await updateMovieSearch(moviesWithId.map((movie) => movie.id));
await associateReleaseMedia(moviesWithId, 'movie');
return moviesWithId;

26
src/tags.js
View File

@ -32,7 +32,7 @@ function curateTag(tag) {
priority: tag.priority,
group: curateTag(tag.group),
aliasFor: curateTag(tag.alias),
aliases: (tag.aliases || []).map(aliasTag => curateTag(aliasTag)),
aliases: (tag.aliases || []).map((aliasTag) => curateTag(aliasTag)),
};
if (tag.poster) {
@ -40,7 +40,7 @@ function curateTag(tag) {
}
if (tag.photos) {
curatedTag.photos = tag.photos.map(photo => curateTagMedia(photo));
curatedTag.photos = tag.photos.map((photo) => curateTagMedia(photo));
}
return curatedTag;
@ -75,13 +75,13 @@ function withRelations(queryBuilder, withMedia) {
async function matchReleaseTags(releases) {
const rawTags = releases
.map(release => release.tags).flat()
.map((release) => release.tags).flat()
.filter(Boolean);
const casedTags = [...new Set(
rawTags
.concat(rawTags.map(tag => tag.toLowerCase()))
.concat(rawTags.map(tag => tag.toUpperCase())),
.concat(rawTags.map((tag) => tag.toLowerCase()))
.concat(rawTags.map((tag) => tag.toUpperCase())),
)];
const tagEntries = await knex('tags')
@ -98,7 +98,7 @@ async function matchReleaseTags(releases) {
}
async function getEntityTags(releases) {
const entityIds = releases.map(release => release.entity?.id).filter(Boolean);
const entityIds = releases.map((release) => release.entity?.id).filter(Boolean);
const entityTags = await knex('entities_tags').whereIn('entity_id', entityIds);
const entityTagIdsByEntityId = entityTags.reduce((acc, entityTag) => {
@ -117,12 +117,12 @@ async function getEntityTags(releases) {
function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type) {
const tagAssociations = releases
.map((release) => {
const entityTagIds = entityTagIdsByEntityId[release.entity?.id]?.map(tag => ({ id: tag.id, origin: tag.name })) || [];
const entityTagIds = entityTagIdsByEntityId[release.entity?.id]?.map((tag) => ({ id: tag.id, origin: tag.name })) || [];
const releaseTags = release.tags?.filter(Boolean) || [];
const releaseTagsWithIds = releaseTags.every(tag => typeof tag === 'number')
const releaseTagsWithIds = releaseTags.every((tag) => typeof tag === 'number')
? releaseTags // obsolete scraper returned pre-matched tags
: releaseTags.map(tag => ({
: releaseTags.map((tag) => ({
id: tagIdsBySlug[slugify(tag)],
original: tag,
}));
@ -133,7 +133,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit
.concat(entityTagIds)
.filter(Boolean),
)]
.map(tag => ({
.map((tag) => ({
[`${type}_id`]: release.id,
tag_id: tag.id,
original_tag: tag.original,
@ -161,7 +161,7 @@ async function associateReleaseTags(releases, type = 'release') {
async function fetchTag(tagId) {
const tag = await knex('tags')
.modify(queryBuilder => withRelations(queryBuilder, true))
.modify((queryBuilder) => withRelations(queryBuilder, true))
.where((builder) => {
if (Number(tagId)) {
builder.where('tags.id', tagId);
@ -179,10 +179,10 @@ async function fetchTag(tagId) {
async function fetchTags(limit = 100) {
const tags = await knex('tags')
.modify(queryBuilder => withRelations(queryBuilder, false))
.modify((queryBuilder) => withRelations(queryBuilder, false))
.limit(limit);
return tags.map(tag => curateTag(tag));
return tags.map((tag) => curateTag(tag));
}
module.exports = {

24
src/updates.js
View File

@ -28,8 +28,8 @@ function mapReleasesToEntityIdAndEntryId(acc, release) {
function filterLocalUniqueReleases(releases, accReleases) {
const localDuplicateReleasesBySiteIdAndEntryId = accReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
const localUniqueReleases = releases.filter(release => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
const localDuplicateReleases = releases.filter(release => localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
const localUniqueReleases = releases.filter((release) => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
const localDuplicateReleases = releases.filter((release) => localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
return {
localUniqueReleases,
@ -39,7 +39,7 @@ function filterLocalUniqueReleases(releases, accReleases) {
async function filterUniqueReleases(releases) {
const releaseIdentifiers = releases
.map(release => [release.entity.id, release.entryId]);
.map((release) => [release.entity.id, release.entryId]);
const duplicateReleaseEntries = await knex('releases')
.select(knex.raw('releases.*, row_to_json(entities) as entity'))
@ -55,13 +55,13 @@ async function filterUniqueReleases(releases) {
.orWhere(knex.raw('updated_at - date > INTERVAL \'1 day\'')); // scene was updated after the release date, no updates expected
});
const duplicateReleases = duplicateReleaseEntries.map(release => curateRelease(release));
const duplicateReleases = duplicateReleaseEntries.map((release) => curateRelease(release));
const duplicateReleasesByEntityIdAndEntryId = duplicateReleases.reduce(mapReleasesToEntityIdAndEntryId, {});
const internalUniqueReleasesByEntityIdAndEntryId = releases.reduce((acc, release) => mapReleasesToEntityIdAndEntryId(acc, release), {});
const internalUniqueReleases = Object.values(internalUniqueReleasesByEntityIdAndEntryId).map(releasesByEntryId => Object.values(releasesByEntryId)).flat();
const internalUniqueReleases = Object.values(internalUniqueReleasesByEntityIdAndEntryId).map((releasesByEntryId) => Object.values(releasesByEntryId)).flat();
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
const uniqueReleases = internalUniqueReleases.filter((release) => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
return { uniqueReleases, duplicateReleases };
}
@ -83,7 +83,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
return accReleases.length + pageReleases.length < argv.last;
}
if (!pageReleases.concat(unextracted).every(release => !!release.date)) { // some scenes don't have dates
if (!pageReleases.concat(unextracted).every((release) => !!release.date)) { // some scenes don't have dates
return accReleases.length + pageReleases.length < argv.missingDateLimit;
}
@ -124,8 +124,8 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
return accReleases;
}
const validPageReleases = pageReleases.filter(release => release?.entryId); // filter out empty and unidentified releases
const pageReleasesWithEntity = validPageReleases.map(release => ({ ...release, entity: release.entity || entity }));
const validPageReleases = pageReleases.filter((release) => release?.entryId); // filter out empty and unidentified releases
const pageReleasesWithEntity = validPageReleases.map((release) => ({ ...release, entity: release.entity || entity }));
if (pageReleases.length > validPageReleases.length) {
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
@ -140,10 +140,10 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const releases = await scrapeReleasesPage(argv.page || 1, []);
const hasDates = releases.every(release => !!release.date);
const hasDates = releases.every((release) => !!release.date);
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|| (hasDates && releases.filter(release => moment(release.date).isAfter(argv.after)))
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|| releases.slice(0, Math.max(argv.missingDateLimit, 0));
const { uniqueReleases, duplicateReleases } = argv.force
@ -280,7 +280,7 @@ async function fetchUpdates() {
const scrapedNetworks = await Promise.map(
includedNetworks,
async networkEntity => (networkEntity.parameters?.sequential
async (networkEntity) => (networkEntity.parameters?.sequential
? scrapeNetworkSequential(networkEntity)
: scrapeNetworkParallel(networkEntity)),
{ concurrency: 5 },

2
src/users.js
View File

@ -18,7 +18,7 @@ function curateUser(user) {
identityVerified: user.identity_verified,
ability,
createdAt: user.created_at,
stashes: user.stashes?.filter(Boolean).map(stash => curateStash(stash)) || [],
stashes: user.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [],
};
return curatedUser;

4
src/utils/bulk-insert.js
View File

@ -22,13 +22,13 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
const chunked = chunk(items, chunkSize);
const queries = chunked
.map(chunkItems => knex.raw(updated || ':query RETURNING *;', {
.map((chunkItems) => knex.raw(updated || ':query RETURNING *;', {
query: knex(table).insert(chunkItems),
}).transacting(transaction));
const responses = await Promise.all(queries);
return responses.flat().map(response => response.rows).flat();
return responses.flat().map((response) => response.rows).flat();
});
}

2
src/utils/capitalize.js
View File

@ -7,7 +7,7 @@ function capitalize(string, { trim = true, uncapitalize = false } = {}) {
const capitalized = string
.split(/\s+/)
.map(component => `${component.charAt(0).toUpperCase()}${uncapitalize ? component.slice(1).toLowerCase() : component.slice(1)}`)
.map((component) => `${component.charAt(0).toUpperCase()}${uncapitalize ? component.slice(1).toLowerCase() : component.slice(1)}`)
.join(' ');
return trim ? capitalized.trim() : capitalized;

2
src/utils/convert.js
View File

@ -62,7 +62,7 @@ function convertManyApi(input, to) {
const curatedInput = input
.replace('\'', 'ft')
.replace(/"|''/, 'in')
.replace(/\d+ft\s*\d+\s*$/, match => `${match}in`); // height without any inch symbol
.replace(/\d+ft\s*\d+\s*$/, (match) => `${match}in`); // height without any inch symbol
return Math.round(convertMany(curatedInput).to(to)) || null;
}

2
src/utils/file-entries.js
View File

@ -8,7 +8,7 @@ async function getFileEntries(location) {
}
const file = await fs.promises.readFile(location, 'utf-8');
const entries = file.split(/\n/).map(entry => entry.trim()).filter(Boolean);
const entries = file.split(/\n/).map((entry) => entry.trim()).filter(Boolean);
return entries;
}

2
src/utils/media.js
View File

@ -27,7 +27,7 @@ async function fetchSource(link) {
const tempFileStream = fs.createWriteStream(tempFilePath);
const hashStream = new PassThrough();
hashStream.on('data', chunk => hasher.write(chunk));
hashStream.on('data', (chunk) => hasher.write(chunk));
try {
const res = await http.get(link, null, {

20
src/utils/qu.js
View File

@ -102,7 +102,7 @@ function all(context, selector, attrArg, applyTrim = true) {
const attr = attrArg === true ? 'textContent' : attrArg;
if (attr) {
return Array.from(context.querySelectorAll(selector), el => q(el, null, attr, applyTrim));
return Array.from(context.querySelectorAll(selector), (el) => q(el, null, attr, applyTrim));
}
return Array.from(context.querySelectorAll(selector));
@ -155,7 +155,7 @@ function jsons(context, selector) {
function htmls(context, selector) {
const els = all(context, selector, null, true);
return els.map(el => el.innerHTML);
return els.map((el) => el.innerHTML);
}
function texts(context, selector, applyTrim = true, filter = true) {
@ -163,8 +163,8 @@ function texts(context, selector, applyTrim = true, filter = true) {
if (!el) return null;
const nodes = Array.from(el.childNodes)
.filter(node => node.nodeName === '#text')
.map(node => (applyTrim ? trim(node.textContent) : node.textContent));
.filter((node) => node.nodeName === '#text')
.map((node) => (applyTrim ? trim(node.textContent) : node.textContent));
return filter ? nodes.filter(Boolean) : nodes;
}
@ -272,7 +272,7 @@ function images(context, selector = 'img', attr, { origin, protocol = 'https' }
const imageEls = all(context, selector, attribute);
return imageEls.map(imageEl => prefixUrl(imageEl, origin, protocol));
return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol));
}
function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
@ -289,7 +289,7 @@ function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https
function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
const urlEls = all(context, selector, attr);
return attr ? urlEls.map(urlEl => prefixUrl(urlEl, origin, protocol)) : urlEls;
return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls;
}
function sourceSet(context, selector, attr = 'srcset', options = {}) {
@ -330,7 +330,7 @@ function sourceSet(context, selector, attr = 'srcset', options = {}) {
return sources;
}
return sources.map(source => source.url);
return sources.map((source) => source.url);
}
function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
@ -348,7 +348,7 @@ function video(context, selector = 'source', attr = 'src', { origin, protocol =
function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
const trailerEls = all(context, selector, attr);
return attr ? trailerEls.map(trailerEl => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
}
function duration(context, selector, match, attr = 'textContent') {
@ -499,11 +499,11 @@ function init(context, selector, window) {
function initAll(context, selector, window) {
if (Array.isArray(context)) {
return context.map(element => init(element, null, window));
return context.map((element) => init(element, null, window));
}
return Array.from(context.querySelectorAll(selector))
.map(element => init(element, null, window));
.map((element) => init(element, null, window));
}
function extract(htmlValue, selector, options) {

Some files were not shown because too many files have changed in this diff Show More