36 Commits

Author SHA1 Message Date
DebaucheryLibrarian
283c44d2fc 1.250.44 2026-03-20 01:59:25 +01:00
DebaucheryLibrarian
a7df43eb55 Added Pascal's Subsluts affiliate. 2026-03-20 01:59:22 +01:00
DebaucheryLibrarian
7e227a4ea5 1.250.43 2026-03-17 23:54:35 +01:00
DebaucheryLibrarian
1c8df6415d Added Jawked to Karups, set up affiliate links. 2026-03-17 23:54:32 +01:00
DebaucheryLibrarian
0beb54312a Updating Manticore scenes after tag reassociation. 2026-03-16 04:58:46 +01:00
DebaucheryLibrarian
95d68fa966 1.250.42 2026-03-16 04:43:04 +01:00
DebaucheryLibrarian
50e7b1a437 Separated spitroast from MFM tag, added Hardwerk DP tags. 2026-03-16 04:43:02 +01:00
DebaucheryLibrarian
6cad394e88 Removed stray console log. 2026-03-16 02:50:31 +01:00
DebaucheryLibrarian
186f9660c3 Increased dangerous query bindings threshold. 2026-03-15 22:30:55 +01:00
DebaucheryLibrarian
ef7c24ab47 1.250.41 2026-03-15 21:33:16 +01:00
DebaucheryLibrarian
1b6aaafe10 Using batch insert module for media, calculating chunk size based on item size. 2026-03-15 21:33:14 +01:00
DebaucheryLibrarian
31aa1118e7 1.250.40 2026-03-15 20:07:53 +01:00
DebaucheryLibrarian
74d03b7483 Using browser for Nubiles, added She's Breeding Material. 2026-03-15 20:07:51 +01:00
DebaucheryLibrarian
40ea7eb80a 1.250.39 2026-03-15 17:42:08 +01:00
DebaucheryLibrarian
0d30115ad5 Refreshing entity slug cache in seeds. Added Hardwerk to Radical. 2026-03-15 17:42:07 +01:00
DebaucheryLibrarian
0560fac1ff 1.250.38 2026-03-13 05:14:45 +01:00
DebaucheryLibrarian
108bf3b168 Integrated manticore stash sync tool. 2026-03-13 05:14:42 +01:00
DebaucheryLibrarian
155e235246 Fixed Aylo specifying wrong host for media. 2026-03-10 05:54:00 +01:00
DebaucheryLibrarian
bff665c6ec 1.250.37 2026-03-10 04:41:36 +01:00
DebaucheryLibrarian
c7111329dc Improved knex error reporting. 2026-03-10 04:41:30 +01:00
DebaucheryLibrarian
d7c1c0ae5c 1.250.36 2026-03-09 05:36:21 +01:00
DebaucheryLibrarian
ea298d7edb Fixed Aylo scraper ignoring session configuration. 2026-03-09 05:36:17 +01:00
DebaucheryLibrarian
99dfcae920 1.250.35 2026-03-08 04:03:57 +01:00
DebaucheryLibrarian
24cba1e1fa Deleting flushed scenes from manticore. 2026-03-08 04:03:55 +01:00
DebaucheryLibrarian
076bdad310 1.250.34 2026-03-06 04:25:55 +01:00
DebaucheryLibrarian
d432d291dd Added See Him Solo to Hussie Pass, added HP affiliates. 2026-03-06 04:25:51 +01:00
DebaucheryLibrarian
220f7e787d 1.250.33 2026-03-05 02:00:53 +01:00
DebaucheryLibrarian
f1caa77e4b Added scene tags table to manticore scenes tool. 2026-03-05 02:00:43 +01:00
DebaucheryLibrarian
ff633436cb 1.250.32 2026-03-04 02:53:21 +01:00
DebaucheryLibrarian
6860072a51 Added database support for actor-specific scene tags. 2026-03-04 02:53:17 +01:00
DebaucheryLibrarian
2c7b4cfc22 1.250.31 2026-03-04 01:57:39 +01:00
DebaucheryLibrarian
7d9e1be8d4 Added Lesbian Factor. 2026-03-04 01:57:33 +01:00
DebaucheryLibrarian
00db4b1b5b 1.250.30 2026-03-03 23:47:50 +01:00
DebaucheryLibrarian
9f1cf1575a Added ASG Max channel parameters. 2026-03-03 23:47:48 +01:00
DebaucheryLibrarian
4f13e4ed28 1.250.29 2026-03-03 23:11:33 +01:00
DebaucheryLibrarian
9805aa7b5b Added Deep Inside to Disruptive Films. Added Sodomy Squad affiliate. 2026-03-03 23:11:32 +01:00
25 changed files with 1303 additions and 384 deletions

View File

@@ -27,7 +27,7 @@
"require-await": "off",
"no-param-reassign": ["error", {
"props": true,
"ignorePropertyModificationsFor": ["state", "acc", "req"]
"ignorePropertyModificationsFor": ["state", "acc", "req", "error"]
}]
},
"globals": {

View File

@@ -188,6 +188,8 @@ module.exports = {
'wishescumtrue',
// hentaied
'somegore',
// digital playground
'digitalplayground', // no longer updates, produces a bunch of garbage for some reason
],
networks: [
// dummy network for testing
@@ -228,6 +230,12 @@ module.exports = {
'store2.psmcdn.net', // Team Skeet API
],
},
remote: {
enabled: false,
address: 'http://10.0.0.1:3333',
key: null,
methods: ['browser'],
},
cloudflare: {
enable: false,
auto: true, // try bypass when CF challenge is detected
@@ -279,7 +287,7 @@ module.exports = {
trailerQuality: [540, 720, 960, 480, 1080, 360, 320, 1440, 1600, 1920, 2160, 270, 240, 180],
limit: 25, // max number of photos per release
attempts: 2,
flushOrphaned: true,
flushOrphaned: false,
flushWindow: 1000,
streams: {
enabled: true, // fetch streams

View File

@@ -0,0 +1,21 @@
exports.up = async function(knex) {
await knex.schema.alterTable('releases_tags', (table) => {
table.integer('actor_id')
.references('id')
.inTable('actors');
table.dropUnique(['tag_id', 'release_id']);
});
await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
};
exports.down = async function(knex) {
await knex.schema.alterTable('releases_tags', (table) => {
table.dropColumn('actor_id');
table.unique(['tag_id', 'release_id']);
});
await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
};

660
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "traxxx",
"version": "1.250.28",
"version": "1.250.44",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "traxxx",
"version": "1.250.28",
"version": "1.250.44",
"license": "ISC",
"dependencies": {
"@aws-sdk/client-s3": "^3.458.0",
@@ -94,7 +94,7 @@
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.37",
"undici": "^5.28.1",
"unprint": "^0.18.35",
"unprint": "^0.19.4",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3",
"video.js": "^8.6.1",
@@ -7112,6 +7112,33 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/call-bound": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz",
"integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"get-intrinsic": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/callsites": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -8290,6 +8317,30 @@
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/dotenv": {
"version": "17.3.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.3.1.tgz",
"integrity": "sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA==",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/eastasianwidth": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
@@ -8576,6 +8627,22 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-errors": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
"integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-iterator-helpers": {
"version": "1.0.15",
"resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.0.15.tgz",
@@ -8603,6 +8670,17 @@
"integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==",
"dev": true
},
"node_modules/es-object-atoms": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
"dependencies": {
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-set-tostringtag": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.2.tgz",
@@ -10256,14 +10334,23 @@
}
},
"node_modules/get-intrinsic": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz",
"integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==",
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"has-proto": "^1.0.1",
"has-symbols": "^1.0.3",
"hasown": "^2.0.0"
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -10277,6 +10364,18 @@
"node": ">=8.0.0"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/get-stdin": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz",
@@ -10468,11 +10567,11 @@
}
},
"node_modules/gopd": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
"integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
"dependencies": {
"get-intrinsic": "^1.1.3"
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -10707,9 +10806,9 @@
}
},
"node_modules/has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"engines": {
"node": ">= 0.4"
},
@@ -10743,9 +10842,9 @@
"dev": true
},
"node_modules/hasown": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
"integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
"integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
"dependencies": {
"function-bind": "^1.1.2"
},
@@ -12891,6 +12990,14 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -16034,9 +16141,12 @@
}
},
"node_modules/object-inspect": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
"integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
"version": "1.13.4",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
"integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
@@ -18450,6 +18560,55 @@
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/router": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz",
"integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==",
"optional": true,
"dependencies": {
"debug": "^4.4.0",
"depd": "^2.0.0",
"is-promise": "^4.0.0",
"parseurl": "^1.3.3",
"path-to-regexp": "^8.0.0"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/router/node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
"optional": true,
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/router/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"optional": true
},
"node_modules/router/node_modules/path-to-regexp": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz",
"integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==",
"optional": true,
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/rrweb-cssom": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz",
@@ -18982,13 +19141,68 @@
}
},
"node_modules/side-channel": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
"integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
"integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==",
"dependencies": {
"call-bind": "^1.0.0",
"get-intrinsic": "^1.0.2",
"object-inspect": "^1.9.0"
"es-errors": "^1.3.0",
"object-inspect": "^1.13.3",
"side-channel-list": "^1.0.0",
"side-channel-map": "^1.0.1",
"side-channel-weakmap": "^1.0.2"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-list": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz",
"integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==",
"dependencies": {
"es-errors": "^1.3.0",
"object-inspect": "^1.13.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-map": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz",
"integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==",
"dependencies": {
"call-bound": "^1.0.2",
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.5",
"object-inspect": "^1.13.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-weakmap": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz",
"integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==",
"dependencies": {
"call-bound": "^1.0.2",
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.5",
"object-inspect": "^1.13.3",
"side-channel-map": "^1.0.1"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -20385,18 +20599,19 @@
}
},
"node_modules/unprint": {
"version": "0.18.35",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.35.tgz",
"integrity": "sha512-oTCBE8pGzfTFlSb0QbYv/ctICTmcU/K81gOPfchn+efLHu48hq1S3582JHvwXAXCjiRKZYatJlEFzUTXVtfuvA==",
"license": "ISC",
"version": "0.19.4",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.19.4.tgz",
"integrity": "sha512-n+s9eNeVPmJO4Qs3qiyCcoH6SE/jAnnYJDOa18OA5//jFrEppRnt9bbwbGTXYnt7ZIsDMXU6ZZBf5crOCHu6+g==",
"dependencies": {
"bottleneck": "^2.19.5",
"cookie": "^1.1.1",
"deepmerge": "^4.2.2",
"dotenv": "^17.3.1",
"eslint": "^8.17.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-config-airbnb-base": "^15.0.0",
"jsdom": "^17.0.0",
"minimist": "^1.2.8",
"moment-timezone": "^0.5.34",
"object-hash": "^3.0.0",
"patchright": "^1.56.1",
@@ -20404,6 +20619,9 @@
"srcset": "^4.0.0",
"tunnel": "^0.0.6",
"undici": "^7.18.2"
},
"optionalDependencies": {
"express": "^5.2.1"
}
},
"node_modules/unprint/node_modules/@tootallnate/once": {
@@ -20414,6 +20632,72 @@
"node": ">= 6"
}
},
"node_modules/unprint/node_modules/accepts": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz",
"integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==",
"optional": true,
"dependencies": {
"mime-types": "^3.0.0",
"negotiator": "^1.0.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/unprint/node_modules/body-parser": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz",
"integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==",
"optional": true,
"dependencies": {
"bytes": "^3.1.2",
"content-type": "^1.0.5",
"debug": "^4.4.3",
"http-errors": "^2.0.0",
"iconv-lite": "^0.7.0",
"on-finished": "^2.4.1",
"qs": "^6.14.1",
"raw-body": "^3.0.1",
"type-is": "^2.0.1"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/body-parser/node_modules/iconv-lite": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
"integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
"optional": true,
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/content-disposition": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz",
"integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==",
"optional": true,
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/cookie": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz",
@@ -20426,6 +20710,15 @@
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/cookie-signature": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz",
"integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==",
"optional": true,
"engines": {
"node": ">=6.6.0"
}
},
"node_modules/unprint/node_modules/cssstyle": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",
@@ -20486,6 +20779,22 @@
"node": ">=12"
}
},
"node_modules/unprint/node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/unprint/node_modules/domexception": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz",
@@ -20506,6 +20815,97 @@
"node": ">=8"
}
},
"node_modules/unprint/node_modules/encodeurl": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
"integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"optional": true,
"engines": {
"node": ">= 0.8"
}
},
"node_modules/unprint/node_modules/express": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz",
"integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==",
"optional": true,
"dependencies": {
"accepts": "^2.0.0",
"body-parser": "^2.2.1",
"content-disposition": "^1.0.0",
"content-type": "^1.0.5",
"cookie": "^0.7.1",
"cookie-signature": "^1.2.1",
"debug": "^4.4.0",
"depd": "^2.0.0",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"etag": "^1.8.1",
"finalhandler": "^2.1.0",
"fresh": "^2.0.0",
"http-errors": "^2.0.0",
"merge-descriptors": "^2.0.0",
"mime-types": "^3.0.0",
"on-finished": "^2.4.1",
"once": "^1.4.0",
"parseurl": "^1.3.3",
"proxy-addr": "^2.0.7",
"qs": "^6.14.0",
"range-parser": "^1.2.1",
"router": "^2.2.0",
"send": "^1.1.0",
"serve-static": "^2.2.0",
"statuses": "^2.0.1",
"type-is": "^2.0.1",
"vary": "^1.1.2"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/express/node_modules/cookie": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
"optional": true,
"engines": {
"node": ">= 0.6"
}
},
"node_modules/unprint/node_modules/finalhandler": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz",
"integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==",
"optional": true,
"dependencies": {
"debug": "^4.4.0",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"on-finished": "^2.4.1",
"parseurl": "^1.3.3",
"statuses": "^2.0.1"
},
"engines": {
"node": ">= 18.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/fresh": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz",
"integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==",
"optional": true,
"engines": {
"node": ">= 0.8"
}
},
"node_modules/unprint/node_modules/html-encoding-sniffer": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz",
@@ -20517,6 +20917,26 @@
"node": ">=10"
}
},
"node_modules/unprint/node_modules/http-errors": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
"integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
"optional": true,
"dependencies": {
"depd": "~2.0.0",
"inherits": "~2.0.4",
"setprototypeof": "~1.2.0",
"statuses": "~2.0.2",
"toidentifier": "~1.0.1"
},
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/http-proxy-agent": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz",
@@ -20603,6 +21023,66 @@
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz",
"integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g=="
},
"node_modules/unprint/node_modules/media-typer": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz",
"integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==",
"optional": true,
"engines": {
"node": ">= 0.8"
}
},
"node_modules/unprint/node_modules/merge-descriptors": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz",
"integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==",
"optional": true,
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/unprint/node_modules/mime-db": {
"version": "1.54.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz",
"integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==",
"optional": true,
"engines": {
"node": ">= 0.6"
}
},
"node_modules/unprint/node_modules/mime-types": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz",
"integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==",
"optional": true,
"dependencies": {
"mime-db": "^1.54.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/unprint/node_modules/negotiator": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz",
"integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==",
"optional": true,
"engines": {
"node": ">= 0.6"
}
},
"node_modules/unprint/node_modules/object-hash": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
@@ -20616,6 +21096,52 @@
"resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
"integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw=="
},
"node_modules/unprint/node_modules/qs": {
"version": "6.15.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz",
"integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==",
"optional": true,
"dependencies": {
"side-channel": "^1.1.0"
},
"engines": {
"node": ">=0.6"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/unprint/node_modules/raw-body": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz",
"integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==",
"optional": true,
"dependencies": {
"bytes": "~3.1.2",
"http-errors": "~2.0.1",
"iconv-lite": "~0.7.0",
"unpipe": "~1.0.0"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/unprint/node_modules/raw-body/node_modules/iconv-lite": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
"integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
"optional": true,
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/saxes": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",
@@ -20627,6 +21153,60 @@
"node": ">=10"
}
},
"node_modules/unprint/node_modules/send": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz",
"integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==",
"optional": true,
"dependencies": {
"debug": "^4.4.3",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"etag": "^1.8.1",
"fresh": "^2.0.0",
"http-errors": "^2.0.1",
"mime-types": "^3.0.2",
"ms": "^2.1.3",
"on-finished": "^2.4.1",
"range-parser": "^1.2.1",
"statuses": "^2.0.2"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/serve-static": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz",
"integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==",
"optional": true,
"dependencies": {
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"parseurl": "^1.3.3",
"send": "^1.2.0"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unprint/node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
"optional": true,
"engines": {
"node": ">= 0.8"
}
},
"node_modules/unprint/node_modules/tr46": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz",
@@ -20638,6 +21218,20 @@
"node": ">=8"
}
},
"node_modules/unprint/node_modules/type-is": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz",
"integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==",
"optional": true,
"dependencies": {
"content-type": "^1.0.5",
"media-typer": "^1.1.0",
"mime-types": "^3.0.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/unprint/node_modules/undici": {
"version": "7.18.2",
"resolved": "https://registry.npmjs.org/undici/-/undici-7.18.2.tgz",

View File

@@ -1,6 +1,6 @@
{
"name": "traxxx",
"version": "1.250.28",
"version": "1.250.44",
"description": "All the latest porn releases in one place",
"main": "src/app.js",
"scripts": {
@@ -153,7 +153,7 @@
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.37",
"undici": "^5.28.1",
"unprint": "^0.18.35",
"unprint": "^0.19.4",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3",
"video.js": "^8.6.1",

View File

@@ -815,6 +815,10 @@ const tags = [
description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
group: 'group',
},
{
name: 'spitroast',
slug: 'spitroast',
},
{
name: 'military',
slug: 'military',
@@ -1893,6 +1897,14 @@ const aliases = [
name: 'double penetration (dp)',
for: 'dp',
},
{
name: 'double penetration ass pussy',
for: 'dp',
},
{
name: 'double penetration mouth pussy',
for: 'spitroast',
},
{
name: 'double penetration - dp',
for: 'dp',
@@ -2355,11 +2367,6 @@ const aliases = [
name: 'spit',
for: 'saliva',
},
{
name: 'spitroast',
for: 'mfm',
secondary: true,
},
{
name: 'spoon',
for: 'spooning',
@@ -3019,7 +3026,7 @@ const priorities = [ // higher index is higher priority
['facial', 'swallowing', 'creampie', 'anal-creampie', 'oral-creampie', 'cum-in-mouth', 'throatpie'],
['lesbian', 'rough', 'milf', 'male-focus', 'bdsm', 'oil'],
['threesome', 'mfm', 'mff', 'trainbang', 'pissing'],
['anal', 'bukkake'],
['anal', 'bukkake', 'spitroast'],
['dp', 'dap', 'triple-penetration', 'tap', 'dvp', 'tvp', 'airtight'],
['blowbang', 'orgy'],
['gangbang'],

View File

@@ -2,6 +2,14 @@
const omit = require('object.omit');
const upsert = require('../src/utils/upsert');
const redis = require('../src/redis');
const entityPrefixes = {
channel: '',
network: '_',
studio: '*',
info: '@',
};
const grandParentNetworks = [
{
@@ -905,80 +913,88 @@ const networks = [
},
];
exports.seed = (knex) => Promise.resolve()
.then(async () => {
await Promise.all([].concat(grandParentNetworks, parentNetworks, networks).map(async (network) => {
if (network.rename) {
return knex('entities')
.where({
type: network.type || 'network',
slug: network.rename,
})
.update('slug', network.slug);
}
exports.seed = async (knex) => {
await Promise.all([].concat(grandParentNetworks, parentNetworks, networks).map(async (network) => {
if (network.rename) {
return knex('entities')
.where({
type: network.type || 'network',
slug: network.rename,
})
.update('slug', network.slug);
}
return null;
}).filter(Boolean));
return null;
}).filter(Boolean));
const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => (omit({ ...network, type: 'network' }, 'rename'))), ['slug', 'type'], knex);
const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => (omit({ ...network, type: 'network' }, 'rename'))), ['slug', 'type'], knex);
const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const parentNetworksWithGrandParent = parentNetworks.map((network) => ({
slug: network.slug,
name: network.name,
type: network.type || 'network',
alias: network.alias,
url: network.url,
description: network.description,
has_logo: network.hasLogo ?? true,
showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
parameters: network.parameters || null,
options: network.options,
parent_id: grandParentNetworksBySlug[network.parent] || null,
}));
const parentNetworksWithGrandParent = parentNetworks.map((network) => ({
slug: network.slug,
name: network.name,
type: network.type || 'network',
alias: network.alias,
url: network.url,
description: network.description,
has_logo: network.hasLogo ?? true,
showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
parameters: network.parameters || null,
options: network.options,
parent_id: grandParentNetworksBySlug[network.parent] || null,
}));
const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex);
const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex);
const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const networksWithParent = networks.map((network) => ({
slug: network.slug,
name: network.name,
type: network.type || 'network',
alias: network.alias,
url: network.url,
description: network.description,
has_logo: network.hasLogo ?? true,
showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
parameters: network.parameters || null,
options: network.options,
parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null,
}));
const networksWithParent = networks.map((network) => ({
slug: network.slug,
name: network.name,
type: network.type || 'network',
alias: network.alias,
url: network.url,
description: network.description,
has_logo: network.hasLogo ?? true,
showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
parameters: network.parameters || null,
options: network.options,
parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null,
}));
const networkEntries = await upsert('entities', networksWithParent, ['slug', 'type'], knex);
const networkEntries = await upsert('entities', networksWithParent, ['slug', 'type'], knex);
const networkIdsBySlug = [].concat(
grandParentNetworkEntries.inserted,
grandParentNetworkEntries.updated,
parentNetworkEntries.inserted,
parentNetworkEntries.updated,
networkEntries.inserted,
networkEntries.updated,
).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const networkIdsBySlug = [].concat(
grandParentNetworkEntries.inserted,
grandParentNetworkEntries.updated,
parentNetworkEntries.inserted,
parentNetworkEntries.updated,
networkEntries.inserted,
networkEntries.updated,
).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean);
const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean);
const tagEntries = await knex('tags').whereIn('slug', tagSlugs);
const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {});
const tagEntries = await knex('tags').whereIn('slug', tagSlugs);
const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {});
const tagAssociations = networks
.map((network) => (network.tags
? network.tags.map((tagSlug) => ({
entity_id: networkIdsBySlug[network.slug],
tag_id: tagIdsBySlug[tagSlug],
inherit: true,
}))
: []))
.flat();
const tagAssociations = networks
.map((network) => (network.tags
? network.tags.map((tagSlug) => ({
entity_id: networkIdsBySlug[network.slug],
tag_id: tagIdsBySlug[tagSlug],
inherit: true,
}))
: []))
.flat();
await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
});
await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
const entities = await knex('entities').select('id', 'slug', 'type');
await redis.connect();
await redis.del('traxxx:entities:id_by_slug');
await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [`${entityPrefixes[entity.type]}${entity.slug}`, entity.id]));
await redis.disconnect();
};

View File

@@ -1,4 +1,12 @@
const upsert = require('../src/utils/upsert');
const redis = require('../src/redis');
const entityPrefixes = {
channel: '',
network: '_',
studio: '*',
info: '@',
};
/* eslint-disable max-len */
const sites = [
@@ -609,6 +617,9 @@ const sites = [
slug: 'asgmaxoriginals',
url: 'https://www.asgmax.com/en/channel/asgmaxoriginals',
parent: 'asgmax',
parameters: {
queryChannel: 'asgmaxoriginals',
},
},
{
name: 'ASG Max Films',
@@ -616,6 +627,9 @@ const sites = [
url: 'https://www.asgmax.com/en/channel/asgmaxfilms',
parent: 'asgmax',
hasLogo: false,
parameters: {
queryChannel: 'asgmaxfilms',
},
},
{
name: 'ASG International',
@@ -623,6 +637,9 @@ const sites = [
url: 'https://www.asgmax.com/en/channel/asginternational',
parent: 'asgmax',
hasLogo: false,
parameters: {
queryChannel: 'asginternational',
},
},
{
name: 'ASG Massage',
@@ -631,6 +648,9 @@ const sites = [
parent: 'asgmax',
tags: ['massage'],
hasLogo: false,
parameters: {
queryChannel: 'asgmassage',
},
},
{
name: 'ASG Auditions',
@@ -639,6 +659,9 @@ const sites = [
parent: 'asgmax',
tags: ['audition'],
hasLogo: false,
parameters: {
queryChannel: 'asgauditions',
},
},
{
name: 'ASG Free Use',
@@ -647,6 +670,9 @@ const sites = [
parent: 'asgmax',
tags: ['free-use'],
hasLogo: false,
parameters: {
queryChannel: 'asgfreeuse',
},
},
{
name: 'Exeter Hill College',
@@ -655,6 +681,9 @@ const sites = [
parent: 'asgmax',
hasLogo: false,
tags: ['animated'],
parameters: {
queryChannel: 'asgexeterhillcollege',
},
},
// ASG MAX INDEPENDENT
{
@@ -1070,6 +1099,17 @@ const sites = [
queryChannel: 'asgmaxtruemale',
},
},
{
name: 'Deep Inside',
slug: 'deepinside',
url: 'https://www.asgmax.com/en/channel/asgmaxdeepinside',
parent: 'disruptivefilms',
tags: ['gay'],
hasLogo: false,
parameters: {
queryChannel: 'asgmaxdeepinside',
},
},
// AMATEUR ALLURE
{
name: 'Amateur Allure',
@@ -5422,6 +5462,12 @@ const sites = [
referer: 'https://www.girlsway.com',
},
},
{
slug: 'lesbianfactor',
name: 'Lesbian Factor',
url: 'https://www.lesbianfactor.com',
parent: 'girlsway',
},
// HITZEFREI
{
slug: 'unleashed',
@@ -5532,6 +5578,16 @@ const sites = [
latest: 'https://seehimfuck.com',
},
},
{
slug: 'seehimsolo',
name: 'See Him Solo',
url: 'https://seehimsolo.com',
tags: ['male-focus', 'solo'],
parent: 'hussiepass',
parameters: {
latest: 'https://seehimsolo.com/categories/movies-2/{page}/latest/',
},
},
{
slug: 'interracialpovs',
name: 'Interracial POVs',
@@ -6116,7 +6172,8 @@ const sites = [
slug: 'privatecollection',
rename: 'karupsprivatecollection',
alias: ['kpc'],
url: 'https://www.karups.com/site/kpc/', // trailing slash required
url: 'https://www.karupspc.com',
// url: 'https://www.karups.com/site/kpc/', // trailing slash required
parent: 'karups',
},
{
@@ -6124,7 +6181,8 @@ const sites = [
slug: 'hometownamateurs',
rename: 'karupshometownamateurs',
alias: ['kha'],
url: 'https://www.karups.com/site/kha/',
url: 'https://www.karupsha.com',
// url: 'https://www.karups.com/site/kha/',
parent: 'karups',
},
{
@@ -6132,7 +6190,8 @@ const sites = [
slug: 'olderwomen',
rename: 'karupsolderwomen',
alias: ['kow'],
url: 'https://www.karups.com/site/kow/',
url: 'https://www.karupsow.com',
// url: 'https://www.karups.com/site/kow/',
parent: 'karups',
},
{
@@ -6143,6 +6202,14 @@ const sites = [
independent: true,
tags: ['gay'],
},
{
name: 'Jawked',
slug: 'jawked',
url: 'https://www.jawked.com',
parent: 'karups',
independent: true,
tags: ['gay'],
},
// KELLY MADISON MEDIA / 5K / 8K
{
slug: 'teenfidelity',
@@ -9381,9 +9448,9 @@ const sites = [
parent: 'nubiles',
},
{
slug: 'caughtmycoach',
name: 'Caught My Coach',
url: 'https://caughtmycoach.com',
slug: 'shesbreedingmaterial',
name: 'She\'s Breeding Material',
url: 'https://shesbreedingmaterial.com',
parent: 'nubiles',
},
// PASCALS SUBSLUTS
@@ -10506,15 +10573,28 @@ const sites = [
siteAsSerie: true,
},
},
{
name: 'Hardwerk',
slug: 'hardwerk',
url: 'https://hardwerk.com',
independent: true,
parent: 'radical',
parameters: {
endpoint: 'jC4SrjH8YVDtRejiA0PMx',
videos: 'films',
actors: 'performers',
},
},
// REALITY KINGS
{
name: 'Look At Her Now',
url: 'https://www.lookathernow.com',
description: 'Look At Her Now brings you best HD reality porn videos every week. Check out these girls before and after they get some rough pounding.',
parameters: { native: true },
// parameters: { siteId: 300 },
slug: 'lookathernow',
parent: 'realitykings',
parameters: {
siteId: 364,
},
},
{
name: 'We Live Together',
@@ -15491,195 +15571,86 @@ sites.reduce((acc, site) => {
}, new Set());
/* eslint-disable max-len */
exports.seed = (knex) => Promise.resolve()
.then(async () => {
await Promise.all(sites.map(async (channel) => {
if (channel.rename) {
await knex('entities')
.where({
type: channel.type || 'channel',
slug: channel.rename,
})
.update('slug', channel.slug);
exports.seed = async (knex) => {
await Promise.all(sites.map(async (channel) => {
if (channel.rename) {
await knex('entities')
.where({
type: channel.type || 'channel',
slug: channel.rename,
})
.update('slug', channel.slug);
return;
return;
}
if (channel.delete) {
await knex('entities')
.where({
type: channel.type || 'channel',
slug: channel.slug,
})
.delete();
}
}).filter(Boolean));
const networks = await knex('entities')
.where('type', 'network')
.orWhereNull('parent_id');
const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const tags = await knex('tags').select('*').whereNull('alias_for');
const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({
slug: site.slug,
name: site.name,
name_stylized: site.style,
type: site.type || 'channel',
alias: site.alias,
description: site.description,
url: site.url,
parameters: site.parameters || null,
options: site.options,
parent_id: networksMap[site.parent] || null,
priority: site.priority || 0,
independent: !!site.independent,
visible: site.visible,
showcased: site.showcased,
has_logo: site.hasLogo === undefined ? true : site.hasLogo,
}));
const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex);
const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const tagAssociations = sites.map((site) => (site.tags && !site.delete
? site.tags.map((tagSlug) => {
const tag = tagsMap[tagSlug];
if (!tag) {
console.warn(`Tag ${tagSlug} for ${site.slug} does not exist`);
}
if (channel.delete) {
await knex('entities')
.where({
type: channel.type || 'channel',
slug: channel.slug,
})
.delete();
}
}).filter(Boolean));
return {
entity_id: sitesMap[site.slug],
tag_id: tagsMap[tagSlug],
inherit: true,
};
})
: []
)).flat();
const networks = await knex('entities')
.where('type', 'network')
.orWhereNull('parent_id');
await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const entities = await knex('entities').select('id', 'slug', 'type');
const tags = await knex('tags').select('*').whereNull('alias_for');
const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
await redis.connect();
const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({
slug: site.slug,
name: site.name,
name_stylized: site.style,
type: site.type || 'channel',
alias: site.alias,
description: site.description,
url: site.url,
parameters: site.parameters || null,
options: site.options,
parent_id: networksMap[site.parent] || null,
priority: site.priority || 0,
independent: !!site.independent,
visible: site.visible,
showcased: site.showcased,
has_logo: site.hasLogo === undefined ? true : site.hasLogo,
}));
await redis.del('traxxx:entities:id_by_slug');
await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [`${entityPrefixes[entity.type]}${entity.slug}`, entity.id]));
const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex);
const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const tagAssociations = sites.map((site) => (site.tags && !site.delete
? site.tags.map((tagSlug) => {
const tag = tagsMap[tagSlug];
if (!tag) {
console.warn(`Tag ${tagSlug} for ${site.slug} does not exist`);
}
return {
entity_id: sitesMap[site.slug],
tag_id: tagsMap[tagSlug],
inherit: true,
};
})
: []
)).flat();
return upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
});
await redis.disconnect();
};
exports.sites = sites;
/*
'X-Art' => 'xart',
'met-art' => 'metart',
'18og' => '18OnlyGirls',
'a1o1' => 'Asian1on1',
'add' => 'ManualAddActors',
'analb' => 'AnalBeauty',
'bgonzo' => 'BangGonzo',
'btlbd' => 'BigTitsLikeBigDicks',
'bjf' => 'BlowjobFridays',
'cws' => 'CzechWifeSwap',
'Daughter' => 'DaughterSwap',
'Daughters' => 'DaughterSwap',
'dc' => 'DorcelVision',
'dpg' => 'DigitalPlayground',
'dsw' => 'DaughterSwap',
'faq' => 'FirstAnalQuest',
'ft' => 'FastTimes',
'fittingroom' => 'Fitting-Room',
'gbcp' => 'GangbangCreampie',
'hart' => 'Hegre',
'hegre-art' => 'Hegre',
'kha' => 'KarupsHA',
'kow' => 'KarupsOW',
'kpc' => 'KarupsPC',
'la' => 'LatinAdultery',
'lcd' => 'LittleCaprice',
'lhf' => 'LoveHerFeet',
'littlecapricedreams' => 'Little Caprice Dreams',
'maj' => 'ManoJob',
'mfl' => 'Mofos',
'mj' => 'ManoJob',
'mpov' => 'MrPOV',
'naughtyamericavr' => 'NaughtyAmerica',
'news' => 'NewSensations',
'ps' => 'PropertySex',
'sart' => 'SexArt',
'sbj' => 'StreetBlowjobs',
'sislove' => 'SisLovesMe',
'tds' => 'TheDickSuckers',
'these' => 'TheStripperExperience',
'tlc' => 'TeensLoveCream',
'tle' => 'TheLifeErotic',
'tog' => 'TonightsGirlfriend',
'wowg' => 'WowGirls',
'wy' => 'WebYoung',
'itc' => 'InTheCrack',
"abbw" => "AbbyWinters",
"abme" => "AbuseMe",
"ana" => "AnalAngels",
"atke" => "ATKExotics",
"atkg" => "ATKGalleria",
"atkgfs" => "ATKGirlfriends",
"atkh" => "ATKHairy",
"aktp" => "ATKPetites",
"ba" => "Beauty-Angels",
"bna" => "BrandNew",
"bam" => "BruceAndMorgan",
"bcast" => "BrutalCastings",
"bd" => "BrutalDildos",
"bpu" => "BrutalPickups",
"cza" => "CzhecAmateurs",
"czbb" => "CzechBangBus",
"czb" => "CzechBitch",
"cc" => "CzechCasting",
"czc" => "CzechCouples",
"czestro" => "CzechEstrogenolit",
"czf" => "CzechFantasy",
"czgb" => "CzechGangBang",
"cgfs" => "CzechGFS",
"czharem" => "CzechHarem",
"czm" => "CzechMassage",
"czo" => "CzechOrgasm",
"czps" => "CzechPawnShop",
"css" => "CzechStreets",
"cztaxi" => "CzechTaxi",
"czt" => "CzechTwins",
"dts" => "DeepThroatSirens",
"doan" => "DiaryOfANanny",
"ds" => "DungeonSex",
"ffr" => "FacialsForever",
"ff" => "FilthyFamily",
"fbbg" => "FirstBGG",
"fs" => "FuckStudies",
"tfcp" => "FullyClothedPissing",
"gdp" => "GirlsDoPorn",
"Harmony" => "HarmonyVision",
"hletee" => "HelplessTeens",
"jlmf" => "JessieLoadsMonsterFacials",
"lang" => "LANewGirl",
"mmp" => "MMPNetwork",
"mbc" => "MyBabysittersClub",
"nvg" => "NetVideoGirls",
"oo" => "Only-Opaques",
"os" => "Only-Secretaries",
"oss" => "OnlySilAndSatin",
"psus" => "PascalsSubSluts",
"psp" => "PorsntarsPunishment",
"pdmqfo" => "QuestForOrgasm",
"sed" => "SexualDisgrace",
"sislov" => "SisLovesMe",
"tslw" => "SlimeWave",
"stre" => "StrictRestraint",
"t18" => "Taboo18",
"tsma" => "TeenSexMania",
"tsm" => "TeenSexMovs",
"ttw" => "TeensInTheWoods",
"tgw" => "ThaiGirlsWild",
"taob" => "TheArtOfBlowJob",
"trwo" => "TheRealWorkout",
"tt" => "TryTeens",
"vp" => "VIPissy",
"wrh" => "WeAreHairy",
"yt" => "YoungThroats",
];
*/

View File

@@ -208,6 +208,11 @@ const affiliates = [
url: 'https://www.g2buddy.com/disruptivefilms/go.php?pr=9&su=2&si=119&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'sodomysquad',
url: 'https://www.g2buddy.com/sodomysquad/go.php?pr=9&su=2&si=137&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
// gamma > ags max > next door studios
// excluded affiliate links that link back to main site and don't seem to track properly
{
@@ -714,6 +719,11 @@ const affiliates = [
url: 'https://register.join-toughlovex.com/track/MzAwMDA5NzkuMy43Ni4xOTcuMC4wLjAuMC4w',
comment: 'rev share',
},
{
channel: 'hardwerk',
url: 'https://register.hardwerk.com/track/MzAwMDA5NzkuMy4xNTEuMzM5LjAuMC4wLjAuMA',
comment: 'rev share',
},
// radical > topwebmodels
{
network: 'topwebmodels',
@@ -940,6 +950,83 @@ const affiliates = [
query: 'ref=4c331ef6',
},
},
// POV Porn Cash / HussiePass
{
network: 'hussiepass',
url: 'https://secure.hussiepass.com/track/MTk0NS4xLjUuNy4wLjAuMC4wLjA',
comment: '50% revshare',
parameters: {
// hussiepass website does not show network scenes
channelScenes: false,
},
},
{
channel: 'povpornstars',
url: 'https://join.povpornstars.com/track/MTk0NS4xLjMuNS4wLjAuMC4wLjA',
comment: '50% revshare',
},
{
channel: 'interracialpovs',
url: 'https://join.interracialpovs.com/track/MTk0NS4xLjYuOC4wLjAuMC4wLjA',
comment: '50% revshare',
},
{
channel: 'ravebunnys',
url: 'https://secure.ravebunnys.com/track/MTk0NS4xLjExLjI5LjAuMC4wLjAuMA',
comment: '50% revshare',
},
{
channel: 'hotandtatted',
url: 'https://join.hotandtatted.com/track/MTk0NS4xLjEwLjEyLjAuMC4wLjAuMA',
comment: '50% revshare',
},
{
channel: 'seehimfuck',
url: 'https://join.seehimfuck.com/track/MTk0NS4xLjcuOS4wLjAuMC4wLjA',
comment: '50% revshare',
},
{
channel: 'seehimsolo',
url: 'https://join.seehimsolo.com/track/MTk0NS4xLjguMTAuMC4wLjAuMC4w',
comment: '50% revshare',
},
// karups
{
network: 'karups',
url: 'https://secure.karups.com/track/MjAwMTAwMS4xLjEuMS4wLjAuMC4wLjA',
comment: 'revshare',
},
{
channel: 'hometownamateurs',
url: 'https://secure.karupsha.com/track/MjAwMTAwMS4xLjMuMy4wLjAuMC4wLjA',
comment: 'revshare',
},
{
channel: 'olderwomen',
url: 'https://secure.karupsow.com/track/MjAwMTAwMS4xLjQuNC4wLjAuMC4wLjA',
comment: 'revshare',
},
{
channel: 'privatecollection',
url: 'https://secure.karupspc.com/track/MjAwMTAwMS4xLjIuMi4wLjAuMC4wLjA',
comment: 'revshare',
},
{
channel: 'boyfun',
url: 'https://secure.boyfun.com/track/MjAwMTAwMS4xLjUuNS4wLjAuMC4wLjA',
comment: 'revshare',
},
{
channel: 'jawked',
url: 'https://secure.jawked.com/track/MjAwMTAwMS4xLjExLjExLjAuMC4wLjAuMA',
comment: 'revshare',
},
// ARL cash
{
channel: 'pascalssubsluts',
url: 'https://join.pascalssubsluts.com/track/MTc2Ny4xOC41LjkuMC4wLjAuMC4w',
comment: 'revshare',
},
// etc
{
network: 'bang',

View File

@@ -31,13 +31,14 @@ const getFileEntries = require('./utils/file-entries');
const inspector = new Inspector();
let done = false;
unprint.options({
const unprintOptions = {
logErrors: false,
timeout: argv.requestTimeout,
userAgent: 'traxxx',
browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
apiUserAgent: 'traxxx',
clientRetirement: config.bypass.browser.clientRetirement,
// remote: config.bypass.remote,
limits: {
...config.limits,
default: {
@@ -47,10 +48,12 @@ unprint.options({
browser: config.limits.browser,
},
proxy: config.proxy,
});
};
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.isProxied ? ' proxied' : ''}${event.isBrowser ? ' browser' : ''}) ${event.url}`));
unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.isProxied ? 'proxied ' : ''}${event.isBrowser ? 'browser ' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`));
unprint.options(unprintOptions);
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.isRemote ? ' remote' : ''}${event.isProxied ? ' proxied' : ''}${event.isBrowser ? ' browser' : ''}) ${event.url}`));
unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.isRemote ? ' remote' : ''}${event.isProxied ? 'proxied ' : ''}${event.isBrowser ? 'browser ' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`));
unprint.on('browserOpen', (event) => logger.debug(`Unprint opened browsers ${event.keys} (${event.active}/${config.bypass.browser.clientRetirement} active, ${event.clients} clients)`));
unprint.on('browserClose', (event) => logger.debug(`Unprint closed${event.retired ? ' retired' : ''} browsers ${event.keys} (${event.active}/${config.bypass.browser.clientRetirement} active, ${event.clients} clients)`));
@@ -121,6 +124,14 @@ async function testProxy() {
throw new Error(`Proxy is offline (${res.status})`);
}
}
if (unprintOptions.remote.enable) {
await unprint.post(`${unprintOptions.remote.address}/options`, unprintOptions, {
headers: {
'unprint-key': unprintOptions.remote.key,
},
});
}
}
async function init() {

View File

@@ -3,7 +3,7 @@
const config = require('config');
const knex = require('knex');
module.exports = knex({
const knexInstance = knex({
client: 'pg',
connection: config.database.owner,
pool: config.database.pool,
@@ -11,3 +11,23 @@ module.exports = knex({
asyncStackTraces: process.env.NODE_ENV === 'development',
// debug: process.env.NODE_ENV === 'development',
});
knexInstance.on('query', function onQuery(query) {
const bindingCount = query.bindings?.length ?? 0;
if (bindingCount > 50000) {
const error = new Error(`[knex] Dangerous query: ${bindingCount} bindings detected: ${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`);
Error.captureStackTrace(error, onQuery);
// console.error(error);
throw error; // optionally hard-fail so you get a real stack trace
}
});
knexInstance.on('query-error', (error, query) => {
error.knexSql = `${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`;
error.knexBindingCount = query.bindings?.length;
});
module.exports = knexInstance;

View File

@@ -23,7 +23,7 @@ const logger = require('./logger')(__filename);
const argv = require('./argv');
const knex = require('./knex');
const http = require('./utils/http');
const bulkInsert = require('./utils/bulk-insert');
const batchInsert = require('./utils/batch-insert');
const chunk = require('./utils/chunk');
const { get } = require('./utils/qu');
const { fetchEntityReleaseIds } = require('./entity-releases');
@@ -647,6 +647,7 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
const res = await http.get(source.src, {
limits: 'media',
headers: {
host: new URL(source.src).hostname,
...(source.referer && { referer: source.referer }),
...(source.host && { host: source.host }),
},
@@ -923,7 +924,7 @@ async function storeMedias(baseMedias, options) {
const newMediaEntries = newMediaWithEntries.filter((media) => media.newEntry).map((media) => media.entry);
try {
await bulkInsert('media', newMediaEntries, false);
await batchInsert('media', newMediaEntries, { confict: false });
return [...newMediaWithEntries, ...existingHashMedias];
} catch (error) {
@@ -992,11 +993,11 @@ async function associateReleaseMedia(releases, type = 'release') {
.filter(Boolean);
if (associations.length > 0) {
await bulkInsert(`${type}s_${role}`, associations, false);
await batchInsert(`${type}s_${role}`, associations, { conflict: false });
}
} catch (error) {
if (error.entries) {
logger.error(util.inspect(error.entries, null, null, { color: true }));
logger.error(util.inspect(error.entries.slice(0, 2), null, null, { color: true }), `${Math.min(error.entries.length, 2)} of ${error.length}`);
}
logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'})`);

View File

@@ -140,6 +140,7 @@ module.exports = {
purgatoryx: radical,
topwebmodels: radical,
lucidflix: radical,
hardwerk: radical,
// hush / hussiepass
eyeontheguy: hush,
hushpass: hush,

View File

@@ -148,7 +148,6 @@ function scrapeRelease(data, url, channel, networkName, options) {
[release.poster, ...release.photos] = getThumbs(data).map((src) => ({
src,
referer: url,
host: 'mediavault-private-fl.project1content.com',
}));
const { teaser, trailer } = getVideos(data);
@@ -274,7 +273,7 @@ async function fetchLatest(site, page = 1, options) {
return null;
}
const { instanceToken } = options.beforeNetwork?.instanceToken
const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
? options.beforeNetwork
: await getSession(site, options.parameters, url);

View File

@@ -36,7 +36,7 @@ function scrapeAll(scenes) {
}
async function fetchLatest(channel, page) {
const res = await unprint.get(new URL(`./videos/page${page}.html`, channel.url).href, { // some sites require a trailing slash, join paths properly
const res = await unprint.get(new URL(`./videos/page${page}.html`, channel.url).href, { // some sites require a trailing slash, join paths properly; don't use origin in case channel path is used
selectAll: '.listing-videos .item',
cookies: {
warningHidden: 'hide',

View File

@@ -70,8 +70,7 @@ function scrapeAll(scenes, entity) {
async function fetchLatest(site, page = 1) {
const url = `${site.url}/video/gallery/${(page - 1) * 12}`; // /0 redirects back to /
const res = await unprint.get(url, {
interface: 'request',
const res = await unprint.browser(url, {
selectAll: '.content-grid-item',
});
@@ -86,9 +85,8 @@ async function fetchUpcoming(site) {
if (site.parameters?.upcoming) {
const url = `${site.url}/video/upcoming`;
const res = await unprint.get(url, {
const res = await unprint.browser(url, {
selectAll: '.content-grid-item',
interface: 'request',
});
if (res.ok) {
@@ -139,9 +137,7 @@ async function scrapeScene({ query }, { url, entity, include }) {
}
async function fetchScene(url, entity, _baseRelease, include) {
const res = await unprint.get(url, {
interface: 'request',
});
const res = await unprint.browser(url);
if (res.ok) {
return scrapeScene(res.context, { url, entity, include });
@@ -185,9 +181,7 @@ async function findModel(actor, entity) {
const url = `${origin}/model/alpha/${firstLetter}`;
const resModels = await unprint.get(url, {
interface: 'request',
});
const resModels = await unprint.browser(url);
if (!resModels.ok) {
return resModels.status;
@@ -217,9 +211,7 @@ async function fetchProfile(actor, { entity }) {
const model = await findModel(actor, entity);
if (model) {
const resModel = await unprint.get(model.url, {
interface: 'request',
});
const resModel = await unprint.browser(model.url);
if (resModel.ok) {
return scrapeProfile(resModel.context, model.avatar);

View File

@@ -14,8 +14,10 @@ function scrapeAll(months, channel, year) {
return unprint.initAll(scenes).map(({ query }) => {
const release = {};
release.url = query.url('a.video-pop-up', { origin: `${channel.origin}/submissive/` });
release.entryId = new URL(release.url).searchParams.get('id');
const videoUrl = query.url('a.video-pop-up', { origin: `${channel.origin}/submissive/` });
release.entryId = new URL(videoUrl).searchParams.get('id');
release.forceDeep = true;
release.title = query.content('.updates-item-title h4');
@@ -53,20 +55,24 @@ async function fetchLatest(channel, page = 1) {
return res.status;
}
function scrapeScene({ html }, url) {
function scrapeScene({ html }, baseRelease) {
const release = {};
release.entryId = new URL(url).searchParams.get('id');
release.entryId = baseRelease.entryId;
release.trailer = html.match(/file: '(.*)'/)[1];
return release;
}
async function fetchScene(url, channel) {
const res = await unprint.get(url);
async function fetchScene(_url, channel, baseRelease) {
if (!baseRelease.entryId) {
return null;
}
const res = await unprint.get(`${channel.origin}/submissive/player-load.php?id=${baseRelease.entryId}`);
if (res.ok) {
return scrapeScene(res.context, url, channel);
return scrapeScene(res.context, baseRelease);
}
return res.status;

View File

@@ -215,7 +215,7 @@ function scrapeProfile(data, channel, scenes, parameters) {
async function fetchProfile(actor, { channel, parameters }) {
const endpoint = await fetchEndpoint(channel);
const res = await http.get(`${channel.url}/_next/data/${endpoint}/models/${actor.slug}.json?slug=${actor.slug}`);
const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
if (res.ok && res.body.pageProps?.model) {
return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);

View File

@@ -4,6 +4,7 @@ const logger = require('./logger')(__filename);
const knex = require('./knex');
const { fetchEntityReleaseIds } = require('./entities');
const { updateSceneSearch } = require('./update-search');
const slugify = require('./utils/slugify');
const batchInsert = require('./utils/batch-insert');
@@ -199,6 +200,8 @@ async function reassociateTagEntries(tagEntries, rematch) {
tag_id: matchedTags[slugify(tagEntry.original_tag)],
})).filter((tagEntry) => tagEntry.tag_id);
const sceneIds = Array.from(new Set(updatedTagEntries.map((tagEntry) => tagEntry.release_id))).filter(Boolean);
if (updatedTagEntries.length > 0) {
const trx = await knex.transaction();
@@ -212,10 +215,13 @@ async function reassociateTagEntries(tagEntries, rematch) {
})), {
conflict: false,
transaction: trx,
commit: true,
});
await updateSceneSearch(sceneIds);
}
logger.info(`Updated ${updatedTagEntries.length} tags in ${new Set(updatedTagEntries.map((tagEntry) => tagEntry.release_id)).size} scenes`);
logger.info(`Updated ${updatedTagEntries.length} tags in ${sceneIds.length} scenes`);
}
async function reassociateReleaseTags(rawSceneIds, rematch) {

15
src/tools/huge-query.js Normal file
View File

@@ -0,0 +1,15 @@
'use strict';
const knex = require('../knex');
async function init() {
const data = Array.from({ length: 100_000 }, (value, index) => ({
id: `test_affiliate_${index}`,
}));
await knex('affiliates').insert(data);
console.log('Done!');
}
init();

View File

@@ -41,7 +41,7 @@ async function fetchScenes() {
studios.name as studio_name,
grandparents.id as parent_network_id,
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
@@ -136,6 +136,14 @@ async function init() {
dupe_index int
)`);
await utilsApi.sql('drop table if exists scenes_tags');
await utilsApi.sql(`create table scenes_tags (
id int,
scene_id int,
tag_id int,
actor_id int
)`);
console.log('Recreated scenes table');
console.log('Fetching scenes from primary database');
@@ -143,49 +151,62 @@ async function init() {
console.log('Fetched scenes from primary database');
const docs = scenes.map((scene) => {
const docs = scenes.flatMap((scene) => {
const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
return {
replace: {
index: 'scenes',
id: scene.id,
doc: {
title: scene.title || undefined,
title_filtered: filteredTitle || undefined,
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
created_at: Math.round(scene.created_at.getTime() / 1000),
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
is_showcased: scene.showcased,
entry_id: scene.entry_id || undefined,
shoot_id: scene.shoot_id || undefined,
channel_id: scene.channel_id,
channel_slug: scene.channel_slug,
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
network_id: scene.network_id || undefined,
network_slug: scene.network_slug || undefined,
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
studio_id: scene.studio_id || undefined,
studio_slug: scene.studio_slug || undefined,
studio_name: scene.studio_name || undefined,
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
actor_ids: scene.actors.map((actor) => actor.f1),
actors: scene.actors.map((actor) => actor.f2).join(),
tag_ids: scene.tags.map((tag) => tag.f1),
tags: flatTags.join(' '),
movie_ids: scene.movies.map((movie) => movie.f1),
movies: scene.movies.map((movie) => movie.f2).join(' '),
serie_ids: scene.series.map((serie) => serie.f1),
series: scene.series.map((serie) => serie.f2).join(' '),
fingerprints: scene.fingerprints.join(' '),
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
stashed: scene.stashed || 0,
dupe_index: scene.dupe_index || 0,
return [
{
replace: {
index: 'scenes',
id: scene.id,
doc: {
title: scene.title || undefined,
title_filtered: filteredTitle || undefined,
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
created_at: Math.round(scene.created_at.getTime() / 1000),
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
is_showcased: scene.showcased,
entry_id: scene.entry_id || undefined,
shoot_id: scene.shoot_id || undefined,
channel_id: scene.channel_id,
channel_slug: scene.channel_slug,
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
network_id: scene.network_id || undefined,
network_slug: scene.network_slug || undefined,
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
studio_id: scene.studio_id || undefined,
studio_slug: scene.studio_slug || undefined,
studio_name: scene.studio_name || undefined,
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
actor_ids: scene.actors.map((actor) => actor.f1),
actors: scene.actors.map((actor) => actor.f2).join(),
tag_ids: scene.tags.map((tag) => tag.f1),
tags: flatTags.join(' '),
movie_ids: scene.movies.map((movie) => movie.f1),
movies: scene.movies.map((movie) => movie.f2).join(' '),
serie_ids: scene.series.map((serie) => serie.f1),
series: scene.series.map((serie) => serie.f2).join(' '),
fingerprints: scene.fingerprints.join(' '),
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
stashed: scene.stashed || 0,
dupe_index: scene.dupe_index || 0,
},
},
},
};
...scene.tags.map((tag) => ({
replace: {
index: 'scenes_tags',
// id: scene.id,
doc: {
scene_id: scene.id,
tag_id: tag.f1,
actor_id: tag.f5,
},
},
})),
];
});
// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {

View File

@@ -0,0 +1,88 @@
'use strict';
const config = require('config');
const manticore = require('manticoresearch');
const knex = require('../knex');
const chunk = require('../utils/chunk');
const mantiClient = new manticore.ApiClient();
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
const utilsApi = new manticore.UtilsApi(mantiClient);
const indexApi = new manticore.IndexApi(mantiClient);
async function syncStashes(domain = 'scene') {
await utilsApi.sql(`truncate table ${domain}s_stashed`);
const stashes = await knex(`stashes_${domain}s`)
.select(
`stashes_${domain}s.id as stashed_id`,
`stashes_${domain}s.${domain}_id`,
'stashes.id as stash_id',
'stashes.user_id as user_id',
`stashes_${domain}s.created_at as created_at`,
)
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
await chain;
const stashDocs = stashChunk.map((stash) => ({
replace: {
index: `${domain}s_stashed`,
id: stash.stashed_id,
doc: {
[`${domain}_id`]: stash[`${domain}_id`],
stash_id: stash.stash_id,
user_id: stash.user_id,
created_at: Math.round(stash.created_at.getTime() / 1000),
},
},
}));
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
}, Promise.resolve());
}
async function init() {
await utilsApi.sql('drop table if exists scenes_stashed');
await utilsApi.sql(`create table if not exists scenes_stashed (
scene_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await utilsApi.sql('drop table if exists movies_stashed');
await utilsApi.sql(`create table if not exists movies_stashed (
movie_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await utilsApi.sql('drop table if exists actors_stashed');
await utilsApi.sql(`create table if not exists actors_stashed (
actor_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await syncStashes('scene');
await syncStashes('actor');
await syncStashes('movie');
console.log('Done!');
knex.destroy();
}
init();

View File

@@ -16,14 +16,14 @@ async function updateManticoreStashedScenes(docs) {
await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
await chain;
const sceneIds = docsChunk.map((doc) => doc.replace.id);
const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
const stashes = await knex('stashes_scenes')
.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
.whereIn('scene_id', sceneIds);
const stashDocs = docsChunk.flatMap((doc) => {
const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
if (sceneStashes.length === 0) {
@@ -50,6 +50,25 @@ async function updateManticoreStashedScenes(docs) {
if (stashDocs.length > 0) {
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
}
const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
if (deleteSceneIds.length > 0) {
await indexApi.callDelete({
index: 'scenes_stashed',
query: {
bool: {
must: [
{
in: {
scene_id: deleteSceneIds,
},
},
],
},
},
});
}
}, Promise.resolve());
}
@@ -128,9 +147,20 @@ async function updateManticoreSceneSearch(releaseIds) {
studios.showcased
`, releaseIds && [releaseIds]);
// console.log(scenes.rows);
const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
const docs = releaseIds.map((sceneId) => {
const scene = scenesById[sceneId];
if (!scene) {
return {
delete: {
index: 'scenes',
id: sceneId,
},
};
}
const docs = scenes.rows.map((scene) => {
const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
@@ -291,7 +321,20 @@ async function updateManticoreMovieSearch(movieIds) {
movies_covers.*
`, movieIds && [movieIds]);
const docs = movies.rows.map((movie) => {
const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
const docs = movieIds.map((movieId) => {
const movie = moviesById[movieId];
if (!movie) {
return {
delete: {
index: 'movies',
id: movieId,
},
};
}
const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
id: tag.f1,
name: tag.f2,

View File

@@ -4,11 +4,12 @@ const knex = require('../knex');
const chunk = require('./chunk');
const logger = require('../logger')(__filename);
const chunkTarget = 50_000; // PostgreSQL allows 65,535 binding parameters, allow for a bit of margin
// improved version of bulkInsert
async function batchInsert(table, items, {
conflict = true,
update = false,
chunkSize = 1000,
concurrent = false,
transaction,
commit = false,
@@ -17,6 +18,10 @@ async function batchInsert(table, items, {
throw new Error('No table specified for batch insert');
}
if (conflict && update) {
throw new Error('Batch insert conflict must specify columns, or update must be disabled');
}
if (!Array.isArray(items)) {
throw new Error('Batch insert items are not an array');
}
@@ -25,8 +30,20 @@ async function batchInsert(table, items, {
return [];
}
const chunks = chunk(items, chunkSize);
// PostgreSQL's bindings limit applies to individual values, so item size needs to be taken into account
const itemSize = items.reduce((acc, item) => Math.max(acc, Object.keys(item).length), 0);
if (itemSize === 0) {
throw new Error('Batch insert items are empty');
}
const chunks = chunk(items, Math.floor(chunkTarget / itemSize));
const conflicts = [].concat(conflict).filter((column) => typeof column === 'string'); // conflict might be 'true'
if (conflicts.length > 0 && !update) {
throw new Error('Batch insert conflict columns must be specified together with update');
}
const trx = transaction || await knex.transaction();
try {
@@ -49,12 +66,6 @@ async function batchInsert(table, items, {
.onConflict(conflicts)
.merge();
}
throw new Error('Batch insert conflict columns must be specified together with update');
}
if (conflict && update) {
throw new Error('Batch insert conflict must specify columns, or update must be disabled');
}
// error on any conflict

View File

@@ -153,6 +153,7 @@ const actors = [
{ entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'lucidflix', name: 'Ava Amira', fields: ['avatar', 'description', 'gender'] },
{ entity: 'hardwerk', name: 'Luna Silver', fields: ['avatar', 'gender'] },
// wankz
{ entity: 'wankzvr', name: 'Melody Marks', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'measurements', 'age'] },
{ entity: 'milfvr', name: 'Ember Snow', fields: ['avatar', 'gender', 'description', 'measurements', 'birthPlace', 'height', 'age'] },