Compare commits
152 Commits
4dfda71a94
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0560fac1ff | ||
|
|
108bf3b168 | ||
|
|
155e235246 | ||
|
|
bff665c6ec | ||
|
|
c7111329dc | ||
|
|
d7c1c0ae5c | ||
|
|
ea298d7edb | ||
|
|
99dfcae920 | ||
|
|
24cba1e1fa | ||
|
|
076bdad310 | ||
|
|
d432d291dd | ||
|
|
220f7e787d | ||
|
|
f1caa77e4b | ||
|
|
ff633436cb | ||
|
|
6860072a51 | ||
|
|
2c7b4cfc22 | ||
|
|
7d9e1be8d4 | ||
|
|
00db4b1b5b | ||
|
|
9f1cf1575a | ||
|
|
4f13e4ed28 | ||
|
|
9805aa7b5b | ||
|
|
0cc6ebc305 | ||
|
|
016c24af28 | ||
|
|
2158550091 | ||
|
|
68ddc8cb78 | ||
|
|
bc5693e44a | ||
|
|
7276d90629 | ||
|
|
1a1af95a10 | ||
|
|
bcb7a56588 | ||
|
|
16648d50f6 | ||
|
|
062dc0e75e | ||
|
|
42effd53fc | ||
|
|
3a3403bb1f | ||
|
|
6fb4989256 | ||
|
|
9750ca4b79 | ||
|
|
0500f7eda8 | ||
|
|
19beff7dbc | ||
|
|
dfe1b84992 | ||
|
|
3d3b544cb4 | ||
|
|
65fa6027ee | ||
|
|
b3a0ba72eb | ||
|
|
f3e2143b45 | ||
|
|
d289f95d3d | ||
|
|
d8b41ec9b5 | ||
|
|
05f7d8b814 | ||
|
|
c2fc09fdaa | ||
|
|
8a7210a3b9 | ||
|
|
e029ca7fd0 | ||
|
|
ffcfae69d5 | ||
|
|
dcaee01ce8 | ||
|
|
7561a4577e | ||
|
|
98b735dbae | ||
|
|
d2daed788c | ||
|
|
23257745a7 | ||
|
|
156954553d | ||
|
|
eb20af14a6 | ||
|
|
ae247c7a91 | ||
|
|
d49e6ef488 | ||
|
|
2b20d98ee0 | ||
|
|
b8cf6a3e71 | ||
|
|
af57f412c9 | ||
|
|
3696b81e69 | ||
|
|
5b6fefd43b | ||
|
|
a863ab888d | ||
|
|
209a81ef71 | ||
|
|
bd91dcbc77 | ||
|
|
b89f25405a | ||
|
|
198f08cb3a | ||
|
|
febaac3865 | ||
|
|
f82167656b | ||
|
|
6e20d7d216 | ||
|
|
612a489cdf | ||
|
|
db2e5b2da4 | ||
|
|
d81310ed25 | ||
|
|
ec86aa9286 | ||
|
|
5d58ddcd49 | ||
|
|
c515c8aeb3 | ||
|
|
debf92afd7 | ||
|
|
601f930324 | ||
|
|
e77ced44c7 | ||
|
|
9f37f54634 | ||
|
|
dc7f325d13 | ||
|
|
35c941488e | ||
|
|
fc32843c5a | ||
|
|
26b31fb10a | ||
|
|
9aa6c9c6c5 | ||
|
|
855a15bc73 | ||
|
|
3329661135 | ||
|
|
791bd6bf27 | ||
|
|
d6be985c4b | ||
|
|
7286846308 | ||
|
|
81dfce8b3d | ||
|
|
aff0e27c55 | ||
|
|
68fe786cb7 | ||
|
|
9a0b0a8989 | ||
|
|
60b8271e4f | ||
|
|
a52042b56c | ||
|
|
7a3dac865e | ||
|
|
74e0fb721d | ||
|
|
ba366df7a5 | ||
|
|
d4e6082d2e | ||
|
|
ea325b8ec5 | ||
|
|
41b1f39752 | ||
|
|
c75c3e3ed9 | ||
|
|
ee495a5cde | ||
|
|
b52e871cfe | ||
|
|
0fc725873e | ||
|
|
db14657101 | ||
|
|
5524efb3ba | ||
|
|
1397349058 | ||
|
|
25cac4d0ab | ||
|
|
5a282cc372 | ||
|
|
3e6592d1f3 | ||
|
|
b95e2fadf7 | ||
|
|
e3b922da6c | ||
|
|
9bf42ff6c0 | ||
|
|
ba127ee53d | ||
|
|
3ee73d2f77 | ||
|
|
4e68a69e02 | ||
|
|
570be66a91 | ||
|
|
0092aab579 | ||
|
|
78e9088ee5 | ||
|
|
cdb4644b42 | ||
|
|
f80f73d0d4 | ||
|
|
e70acdfe79 | ||
|
|
75c5a37567 | ||
|
|
9aa7fd77c4 | ||
|
|
6b768b6828 | ||
|
|
0f44b4d410 | ||
|
|
3368e2b343 | ||
|
|
8e77a5ef46 | ||
|
|
29dc94d77d | ||
|
|
03b039c937 | ||
|
|
f4f1afdf3b | ||
|
|
fefb165274 | ||
|
|
a3633c2f9f | ||
|
|
a007593390 | ||
|
|
f31c69f255 | ||
|
|
9a088599e6 | ||
|
|
01341b80d6 | ||
|
|
fec70db20b | ||
|
|
7bbb7cb43d | ||
|
|
e7a1ad42f5 | ||
|
|
be90982420 | ||
|
|
ce94456494 | ||
|
|
fe3fcd0741 | ||
|
|
5ca7537e24 | ||
|
|
35449ef906 | ||
|
|
bebebf8447 | ||
|
|
101c84763a | ||
|
|
36ca313a89 | ||
|
|
d07d3c2c8d |
@@ -27,7 +27,7 @@
|
||||
"require-await": "off",
|
||||
"no-param-reassign": ["error", {
|
||||
"props": true,
|
||||
"ignorePropertyModificationsFor": ["state", "acc", "req"]
|
||||
"ignorePropertyModificationsFor": ["state", "acc", "req", "error"]
|
||||
}]
|
||||
},
|
||||
"globals": {
|
||||
|
||||
@@ -188,6 +188,8 @@ module.exports = {
|
||||
'wishescumtrue',
|
||||
// hentaied
|
||||
'somegore',
|
||||
// digital playground
|
||||
'digitalplayground', // no longer updates, produces a bunch of garbage for some reason
|
||||
],
|
||||
networks: [
|
||||
// dummy network for testing
|
||||
@@ -196,144 +198,7 @@ module.exports = {
|
||||
'forbondage',
|
||||
],
|
||||
},
|
||||
profiles: [
|
||||
[
|
||||
'evilangel',
|
||||
'famedigital',
|
||||
'devilsfilm',
|
||||
'roccosiffredi',
|
||||
],
|
||||
[
|
||||
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), (sometimes) Burning Angel and Wicked have their own assets
|
||||
'xempire',
|
||||
'blowpass',
|
||||
],
|
||||
[
|
||||
// MindGeek; Mile High Media has its own assets
|
||||
'brazzers',
|
||||
'realitykings',
|
||||
'mofos',
|
||||
'digitalplayground',
|
||||
'twistys',
|
||||
'babes',
|
||||
'fakehub',
|
||||
'sexyhub',
|
||||
'metrohd',
|
||||
'iconmale',
|
||||
'men',
|
||||
'transangels',
|
||||
],
|
||||
'wicked',
|
||||
'burningangel',
|
||||
'milehighmedia',
|
||||
[
|
||||
'vixen',
|
||||
'tushy',
|
||||
'blacked',
|
||||
'tushyraw',
|
||||
'blackedraw',
|
||||
'deeper',
|
||||
],
|
||||
[
|
||||
// Nubiles
|
||||
'nubiles',
|
||||
'nubilesporn',
|
||||
'deeplush',
|
||||
'brattysis',
|
||||
'nfbusty',
|
||||
'anilos',
|
||||
'hotcrazymess',
|
||||
'thatsitcomshow',
|
||||
],
|
||||
'21sextury',
|
||||
'dogfartnetwork',
|
||||
'adultempire',
|
||||
'julesjordan',
|
||||
'dorcelclub',
|
||||
'bang',
|
||||
'pervcity',
|
||||
'kink',
|
||||
'peternorth',
|
||||
'naughtyamerica',
|
||||
'cherrypimps',
|
||||
'pimpxxx',
|
||||
'18vr',
|
||||
'babevr',
|
||||
'badoinkvr',
|
||||
'realvr',
|
||||
'vrcosplayx',
|
||||
'teamskeet',
|
||||
'mylf',
|
||||
'spermmania',
|
||||
[
|
||||
'letsdoeit',
|
||||
'mamacitaz',
|
||||
'forbondage',
|
||||
'amateureuro',
|
||||
'vipsexvault',
|
||||
'transbella',
|
||||
],
|
||||
[
|
||||
'hussiepass',
|
||||
'hushpass',
|
||||
'interracialpass',
|
||||
'interracialpovs',
|
||||
'povpornstars',
|
||||
'seehimfuck',
|
||||
'eyeontheguy',
|
||||
],
|
||||
[
|
||||
// Full Porn Network
|
||||
'analized',
|
||||
'hergape',
|
||||
'jamesdeen',
|
||||
'dtfsluts',
|
||||
'analbbc',
|
||||
'analviolation',
|
||||
'baddaddypov',
|
||||
'girlfaction',
|
||||
'homemadeanalwhores',
|
||||
'mugfucked',
|
||||
'onlyprince',
|
||||
'pervertgallery',
|
||||
'povperverts',
|
||||
],
|
||||
'wankzvr',
|
||||
'milfvr',
|
||||
'tranzvr',
|
||||
'topwebmodels',
|
||||
'pascalssubsluts',
|
||||
'kellymadison',
|
||||
'5kporn',
|
||||
'private',
|
||||
'bangbros',
|
||||
'hitzefrei',
|
||||
'porncz',
|
||||
'czechav',
|
||||
'angelogodshackoriginal',
|
||||
'littlecapricedreams',
|
||||
'missyx',
|
||||
'gangbangcreampie',
|
||||
'gloryholesecrets',
|
||||
'aziani',
|
||||
[
|
||||
'firstanalquest',
|
||||
'doubleviewcasting',
|
||||
],
|
||||
[
|
||||
'silverstonedvd',
|
||||
'silviasaint',
|
||||
],
|
||||
[
|
||||
'analvids',
|
||||
'pornworld',
|
||||
],
|
||||
'pierrewoodman',
|
||||
'score',
|
||||
'boobpedia',
|
||||
'pornhub',
|
||||
'freeones',
|
||||
],
|
||||
profiles: null,
|
||||
interpolation: {
|
||||
excludeAvatarCredits: [ // never allow
|
||||
'Pierre Woodman',
|
||||
@@ -416,7 +281,7 @@ module.exports = {
|
||||
trailerQuality: [540, 720, 960, 480, 1080, 360, 320, 1440, 1600, 1920, 2160, 270, 240, 180],
|
||||
limit: 25, // max number of photos per release
|
||||
attempts: 2,
|
||||
flushOrphaned: true,
|
||||
flushOrphaned: false,
|
||||
flushWindow: 1000,
|
||||
streams: {
|
||||
enabled: true, // fetch streams
|
||||
|
||||
31
migrations/20260222055254_unique_origin_tags.js
Normal file
31
migrations/20260222055254_unique_origin_tags.js
Normal file
@@ -0,0 +1,31 @@
|
||||
exports.up = async (knex) => {
|
||||
// dedupe
|
||||
await knex.raw(`
|
||||
DELETE
|
||||
FROM releases_tags
|
||||
WHERE ctid IN
|
||||
(
|
||||
SELECT ctid
|
||||
FROM(
|
||||
SELECT
|
||||
*,
|
||||
ctid,
|
||||
row_number() OVER (PARTITION BY release_id, original_tag ORDER BY ctid)
|
||||
FROM releases_tags
|
||||
)s
|
||||
WHERE row_number >= 2
|
||||
)
|
||||
`);
|
||||
|
||||
await knex.schema.alterTable('releases_tags', (table) => {
|
||||
table.increments('id');
|
||||
table.unique(['release_id', 'original_tag']);
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async (knex) => {
|
||||
await knex.schema.alterTable('releases_tags', (table) => {
|
||||
table.dropColumn('id');
|
||||
table.dropUnique(['release_id', 'original_tag']);
|
||||
});
|
||||
};
|
||||
23
migrations/20260301042453_foot_float.js
Normal file
23
migrations/20260301042453_foot_float.js
Normal file
@@ -0,0 +1,23 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('actors', (table) => {
|
||||
table.decimal('foot')
|
||||
.alter();
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('actors_profiles', (table) => {
|
||||
table.decimal('foot')
|
||||
.alter();
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('actors', (table) => {
|
||||
table.integer('foot')
|
||||
.alter();
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('actors_profiles', (table) => {
|
||||
table.integer('foot')
|
||||
.alter();
|
||||
});
|
||||
};
|
||||
13
migrations/20260302222545_series_alt_descriptions.js
Normal file
13
migrations/20260302222545_series_alt_descriptions.js
Normal file
@@ -0,0 +1,13 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('series', (table) => {
|
||||
table.specificType('alt_descriptions', 'text ARRAY');
|
||||
table.json('attributes');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('series', (table) => {
|
||||
table.dropColumn('alt_descriptions');
|
||||
table.dropColumn('attributes');
|
||||
});
|
||||
};
|
||||
21
migrations/20260304020542_scene_actor_tags.js
Normal file
21
migrations/20260304020542_scene_actor_tags.js
Normal file
@@ -0,0 +1,21 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('releases_tags', (table) => {
|
||||
table.integer('actor_id')
|
||||
.references('id')
|
||||
.inTable('actors');
|
||||
|
||||
table.dropUnique(['tag_id', 'release_id']);
|
||||
});
|
||||
|
||||
await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('releases_tags', (table) => {
|
||||
table.dropColumn('actor_id');
|
||||
|
||||
table.unique(['tag_id', 'release_id']);
|
||||
});
|
||||
|
||||
await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
|
||||
};
|
||||
18
package-lock.json
generated
18
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.248.51",
|
||||
"version": "1.250.38",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "traxxx",
|
||||
"version": "1.248.51",
|
||||
"version": "1.250.38",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.458.0",
|
||||
@@ -94,7 +94,7 @@
|
||||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.18.31",
|
||||
"unprint": "^0.18.35",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
@@ -18822,6 +18822,11 @@
|
||||
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
|
||||
"integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw=="
|
||||
},
|
||||
"node_modules/set-cookie-parser": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.0.1.tgz",
|
||||
"integrity": "sha512-n7Z7dXZhJbwuAHhNzkTti6Aw9QDDjZtm3JTpTGATIdNzdQz5GuFs22w90BcvF4INfnrL5xrX3oGsuqO5Dx3A1Q=="
|
||||
},
|
||||
"node_modules/set-function-length": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz",
|
||||
@@ -20380,9 +20385,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/unprint": {
|
||||
"version": "0.18.31",
|
||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.31.tgz",
|
||||
"integrity": "sha512-7z4kC7eyyyRUeGBGc+lByRv1WqWP1NkO5Fwh6RwOs60x6HW3mffYUxa2R02fjpUOsEaPRukZwEsikP5jOA/JAA==",
|
||||
"version": "0.18.35",
|
||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.35.tgz",
|
||||
"integrity": "sha512-oTCBE8pGzfTFlSb0QbYv/ctICTmcU/K81gOPfchn+efLHu48hq1S3582JHvwXAXCjiRKZYatJlEFzUTXVtfuvA==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"bottleneck": "^2.19.5",
|
||||
@@ -20395,6 +20400,7 @@
|
||||
"moment-timezone": "^0.5.34",
|
||||
"object-hash": "^3.0.0",
|
||||
"patchright": "^1.56.1",
|
||||
"set-cookie-parser": "^3.0.1",
|
||||
"srcset": "^4.0.0",
|
||||
"tunnel": "^0.0.6",
|
||||
"undici": "^7.18.2"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.248.51",
|
||||
"version": "1.250.38",
|
||||
"description": "All the latest porn releases in one place",
|
||||
"main": "src/app.js",
|
||||
"scripts": {
|
||||
@@ -153,7 +153,7 @@
|
||||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.18.31",
|
||||
"unprint": "^0.18.35",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
|
||||
@@ -537,6 +537,10 @@ const tags = [
|
||||
name: 'enhanced butt',
|
||||
slug: 'enhanced-butt',
|
||||
},
|
||||
{
|
||||
name: 'extreme insertion',
|
||||
slug: 'extreme-insertion',
|
||||
},
|
||||
{
|
||||
name: 'facefucking',
|
||||
slug: 'facefucking',
|
||||
@@ -834,6 +838,11 @@ const tags = [
|
||||
slug: 'natural-boobs',
|
||||
group: 'body',
|
||||
},
|
||||
{
|
||||
name: 'natural butt',
|
||||
slug: 'natural-butt',
|
||||
group: 'body',
|
||||
},
|
||||
{
|
||||
name: 'nipple clamps',
|
||||
slug: 'nipple-clamps',
|
||||
@@ -954,6 +963,11 @@ const tags = [
|
||||
slug: 'pyjamas',
|
||||
group: 'clothing',
|
||||
},
|
||||
{
|
||||
name: 'rave',
|
||||
slug: 'rave',
|
||||
group: 'clothing',
|
||||
},
|
||||
{
|
||||
name: 'redhead',
|
||||
slug: 'redhead',
|
||||
@@ -1612,6 +1626,10 @@ const aliases = [
|
||||
name: 'big tits d-dd cup',
|
||||
for: 'big-boobs',
|
||||
},
|
||||
{
|
||||
name: 'busty',
|
||||
for: 'big-boobs',
|
||||
},
|
||||
{
|
||||
name: 'busty - big boobs',
|
||||
for: 'big-boobs',
|
||||
@@ -2155,6 +2173,18 @@ const aliases = [
|
||||
name: 'natural tits',
|
||||
for: 'natural-boobs',
|
||||
},
|
||||
{
|
||||
name: 'natural butt',
|
||||
for: 'natural-butt',
|
||||
},
|
||||
{
|
||||
name: 'natural ass',
|
||||
for: 'natural-butt',
|
||||
},
|
||||
{
|
||||
name: 'real ass',
|
||||
for: 'natural-butt',
|
||||
},
|
||||
{
|
||||
name: 'oiled',
|
||||
for: 'oil',
|
||||
@@ -2994,6 +3024,8 @@ const priorities = [ // higher index is higher priority
|
||||
['blowbang', 'orgy'],
|
||||
['gangbang'],
|
||||
['gay', 'transsexual', 'bisexual', 'hentai'],
|
||||
['pissing'],
|
||||
['compilation', 'bts'],
|
||||
].reduce((acc, slugs, index) => {
|
||||
slugs.forEach((slug) => { acc[slug] = index; });
|
||||
|
||||
|
||||
@@ -9,6 +9,9 @@ const grandParentNetworks = [
|
||||
name: 'Gamma Entertainment',
|
||||
url: 'https://www.gammaentertainment.com',
|
||||
alias: ['gammaentertainment'],
|
||||
options: {
|
||||
preferNetwork: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'hush',
|
||||
@@ -56,7 +59,6 @@ const parentNetworks = [
|
||||
url: 'https://www.21sextury.com',
|
||||
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
mobile: 'https://m.dpfanatics.com/en/video',
|
||||
},
|
||||
parent: 'gamma',
|
||||
@@ -105,7 +107,6 @@ const parentNetworks = [
|
||||
url: 'https://www.asgmax.com',
|
||||
parent: 'gamma',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
scene: 'https://www.asgmax.com/en/video/asgmax',
|
||||
},
|
||||
},
|
||||
@@ -187,7 +188,6 @@ const networks = [
|
||||
url: 'https://www.21sextreme.com',
|
||||
description: 'Welcome to 21Sextreme.com, your portal to fisting porn, old and young lesbians, horny grannies & extreme BDSM featuring the best Euro & American Pornstars',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
mobile: 'https://m.dpfanatics.com/en/video',
|
||||
},
|
||||
parent: '21sextury',
|
||||
@@ -198,7 +198,6 @@ const networks = [
|
||||
url: 'https://www.21naturals.com',
|
||||
description: 'Welcome to 21Naturals.com, the porn network featuring the hottest pornstars from all over the world in all natural porn and erotic sex videos. Watch thousands of girls with natural tits',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
mobile: 'https://m.dpfanatics.com/en/video',
|
||||
},
|
||||
parent: '21sextury',
|
||||
@@ -228,7 +227,6 @@ const networks = [
|
||||
description: 'Adult Time is a premium streaming service for adults! Watch adult movies, series, and channels from the top names in the industry.',
|
||||
parent: 'gamma',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
referer: 'https://freetour.adulttime.com/en/join',
|
||||
// scene: false,
|
||||
},
|
||||
@@ -320,7 +318,6 @@ const networks = [
|
||||
url: 'https://www.blowpass.com',
|
||||
description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
referer: 'https://www.blowpass.com',
|
||||
},
|
||||
parent: 'gamma',
|
||||
@@ -373,9 +370,6 @@ const networks = [
|
||||
url: 'https://www.evilangel.com',
|
||||
description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
|
||||
parent: 'gamma',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'exploitedx',
|
||||
@@ -419,7 +413,7 @@ const networks = [
|
||||
description: 'The world famous Dogfart Interracial series. Online since 1996, we have the largest collection of Interracial videos, pictures and content on the web.',
|
||||
parent: 'dfxtra',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
profileReferer: 'https://www.dogfartnetwork.com',
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -432,9 +426,6 @@ const networks = [
|
||||
slug: 'fantasymassage',
|
||||
name: 'Fantasy Massage',
|
||||
url: 'https://www.fantasymassage.com',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
parent: 'gamma',
|
||||
},
|
||||
{
|
||||
@@ -444,7 +435,6 @@ const networks = [
|
||||
description: 'Watch and download thousands of the best porn videos at FameDigital.com, the largest porn network on the web! The hottest teens, MILFs and more pornstars are all here!',
|
||||
parameters: {
|
||||
mobile: 'https://m.dpfanatics.com/en/video',
|
||||
layout: 'api',
|
||||
},
|
||||
parent: 'gamma',
|
||||
},
|
||||
@@ -464,7 +454,6 @@ const networks = [
|
||||
url: 'https://www.filthykings.com',
|
||||
parent: 'gamma',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
queryChannel: true,
|
||||
scene: 'https://www.filthykings.com/en/video/filthykings',
|
||||
referer: 'https://www.filthykings.com',
|
||||
@@ -491,6 +480,7 @@ const networks = [
|
||||
slug: 'fullpornnetwork',
|
||||
name: 'Full Porn Network',
|
||||
url: 'https://www.fullpornnetwork.com',
|
||||
showcased: false,
|
||||
description: 'FullPornNetwork.com is the latest and greatest for one stop shop porn sites. Check out the expanding library of the multi-site network. All of fan\'s favorite content from ANALIZED.COM, DTFsluts.com, YourMomDoesPorn.com and many more. Give die hard porn fans access to an array of premium content available in 4k and 1080p. Full access included streaming hd and unlimited downloads. Be exclusive, be a member to FullPornNetwork.com Today.',
|
||||
},
|
||||
{
|
||||
@@ -504,9 +494,6 @@ const networks = [
|
||||
name: 'Girlsway',
|
||||
url: 'https://www.girlsway.com',
|
||||
description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
parent: 'gamma',
|
||||
},
|
||||
{
|
||||
@@ -554,7 +541,6 @@ const networks = [
|
||||
slug: 'jayrock',
|
||||
name: 'JayRock Productions',
|
||||
url: 'http://jayrockcontent.com',
|
||||
parent: 'gamma',
|
||||
},
|
||||
{
|
||||
slug: 'julesjordan',
|
||||
@@ -702,6 +688,9 @@ const networks = [
|
||||
name: 'Nubiles',
|
||||
url: 'https://www.nubiles.com',
|
||||
description: 'Welcome to the teen megasite that started it all! Browse our massive HD collection of fresh legal hotties at Nubiles.net.',
|
||||
options: {
|
||||
forceDeepUpdate: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'perfectgonzo',
|
||||
@@ -755,7 +744,6 @@ const networks = [
|
||||
description: 'PureTaboo.com is the ultimate site for family taboo porn, featuring submissive teens & virgins in rough sex videos in ultra 4k HD.',
|
||||
parent: 'gamma',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
scene: 'https://www.puretaboo.com/en/video',
|
||||
referer: 'https://www.puretaboo.com',
|
||||
},
|
||||
@@ -775,6 +763,7 @@ const networks = [
|
||||
slug: 'score',
|
||||
name: 'SCORE',
|
||||
url: 'https://www.scorepass.com',
|
||||
showcased: false,
|
||||
},
|
||||
{
|
||||
slug: 'sexyhub',
|
||||
@@ -852,7 +841,7 @@ const networks = [
|
||||
description: 'Home of the Kim Kardashian Sex Tape, Porn Parodies, and over 30,000 XXX Movies from The World Leader In Adult Entertainment.',
|
||||
parent: 'gamma',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
sceneMovies: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -878,10 +867,8 @@ const networks = [
|
||||
description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
|
||||
parent: 'gamma',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
sceneMovies: false,
|
||||
actorScenes: 'https://www.xempire.com/en/videos/xempire/latest/{page}/All-Categories/0{actorPath}',
|
||||
actorAvailableOnSites: ['allblackx', 'darkx', 'eroticax', 'hardx', 'lesbianx', 'xempire', 'xempirepartners'],
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -889,9 +876,6 @@ const networks = [
|
||||
name: 'Zero Tolerance',
|
||||
alias: ['ztod'],
|
||||
url: 'https://www.zerotolerancefilms.com',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
parent: 'gamma',
|
||||
},
|
||||
// ASG MAX
|
||||
@@ -948,7 +932,8 @@ exports.seed = (knex) => Promise.resolve()
|
||||
description: network.description,
|
||||
has_logo: network.hasLogo ?? true,
|
||||
showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
|
||||
parameters: network.parameters,
|
||||
parameters: network.parameters || null,
|
||||
options: network.options,
|
||||
parent_id: grandParentNetworksBySlug[network.parent] || null,
|
||||
}));
|
||||
|
||||
@@ -964,7 +949,8 @@ exports.seed = (knex) => Promise.resolve()
|
||||
description: network.description,
|
||||
has_logo: network.hasLogo ?? true,
|
||||
showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
|
||||
parameters: network.parameters,
|
||||
parameters: network.parameters || null,
|
||||
options: network.options,
|
||||
parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null,
|
||||
}));
|
||||
|
||||
|
||||
@@ -579,7 +579,6 @@ const sites = [
|
||||
tags: ['gay'],
|
||||
independent: true,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
scene: 'https://www.chaosmen.com/en/video/chaosmen',
|
||||
},
|
||||
},
|
||||
@@ -601,7 +600,6 @@ const sites = [
|
||||
parent: 'gamma',
|
||||
independent: true,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
scene: 'https://www.tabooheat.com/en/video/tabooheat',
|
||||
},
|
||||
},
|
||||
@@ -611,6 +609,9 @@ const sites = [
|
||||
slug: 'asgmaxoriginals',
|
||||
url: 'https://www.asgmax.com/en/channel/asgmaxoriginals',
|
||||
parent: 'asgmax',
|
||||
parameters: {
|
||||
queryChannel: 'asgmaxoriginals',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'ASG Max Films',
|
||||
@@ -618,6 +619,9 @@ const sites = [
|
||||
url: 'https://www.asgmax.com/en/channel/asgmaxfilms',
|
||||
parent: 'asgmax',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
queryChannel: 'asgmaxfilms',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'ASG International',
|
||||
@@ -625,6 +629,9 @@ const sites = [
|
||||
url: 'https://www.asgmax.com/en/channel/asginternational',
|
||||
parent: 'asgmax',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
queryChannel: 'asginternational',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'ASG Massage',
|
||||
@@ -633,6 +640,9 @@ const sites = [
|
||||
parent: 'asgmax',
|
||||
tags: ['massage'],
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
queryChannel: 'asgmassage',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'ASG Auditions',
|
||||
@@ -641,6 +651,9 @@ const sites = [
|
||||
parent: 'asgmax',
|
||||
tags: ['audition'],
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
queryChannel: 'asgauditions',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'ASG Free Use',
|
||||
@@ -649,6 +662,9 @@ const sites = [
|
||||
parent: 'asgmax',
|
||||
tags: ['free-use'],
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
queryChannel: 'asgfreeuse',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Exeter Hill College',
|
||||
@@ -657,6 +673,9 @@ const sites = [
|
||||
parent: 'asgmax',
|
||||
hasLogo: false,
|
||||
tags: ['animated'],
|
||||
parameters: {
|
||||
queryChannel: 'asgexeterhillcollege',
|
||||
},
|
||||
},
|
||||
// ASG MAX INDEPENDENT
|
||||
{
|
||||
@@ -1020,12 +1039,12 @@ const sites = [
|
||||
{
|
||||
name: 'Disruptive Films',
|
||||
slug: 'disruptivefilms',
|
||||
delete: true,
|
||||
url: 'https://www.disruptivefilms.com',
|
||||
parent: 'disruptivefilms',
|
||||
tags: ['gay'],
|
||||
parameters: {
|
||||
queryChannel: 'asgmaxdisruptivefilms',
|
||||
filterChannel: 'asgmaxdisruptivefilms',
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1072,6 +1091,17 @@ const sites = [
|
||||
queryChannel: 'asgmaxtruemale',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Deep Inside',
|
||||
slug: 'deepinside',
|
||||
url: 'https://www.asgmax.com/en/channel/asgmaxdeepinside',
|
||||
parent: 'disruptivefilms',
|
||||
tags: ['gay'],
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
queryChannel: 'asgmaxdeepinside',
|
||||
},
|
||||
},
|
||||
// AMATEUR ALLURE
|
||||
{
|
||||
name: 'Amateur Allure',
|
||||
@@ -1325,7 +1355,6 @@ const sites = [
|
||||
seriesId: 268,
|
||||
// areaId: 3,
|
||||
areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
|
||||
blockId: 114458,
|
||||
scene: 'https://aziani.com',
|
||||
},
|
||||
},
|
||||
@@ -1338,9 +1367,7 @@ const sites = [
|
||||
parameters: {
|
||||
// areaId: 2,
|
||||
areaId: '4f0a134f-8a1b-4bcb-8013-7e02fac4f61d',
|
||||
blockId: 114064,
|
||||
seriesId: 107,
|
||||
modelBlockId: 114129,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1351,10 +1378,9 @@ const sites = [
|
||||
independent: true,
|
||||
parameters: {
|
||||
// areaId: 11,
|
||||
videos: '/creampiled-videos',
|
||||
areaId: '29c61dae-db14-419b-93a4-d016b928dee9',
|
||||
blockId: 115990,
|
||||
seriesId: 436,
|
||||
modelBlockId: 115345,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1366,7 +1392,6 @@ const sites = [
|
||||
parameters: {
|
||||
// areaId: 8,
|
||||
areaId: 'ae0a26fe-7f08-433d-bb04-a9b6f358c48e',
|
||||
blockId: 116531,
|
||||
seriesId: 395,
|
||||
},
|
||||
},
|
||||
@@ -1378,7 +1403,6 @@ const sites = [
|
||||
parameters: {
|
||||
// areaId: 3,
|
||||
areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
|
||||
blockId: 114458,
|
||||
seriesId: 105,
|
||||
scene: 'https://aziani.com',
|
||||
},
|
||||
@@ -1392,7 +1416,6 @@ const sites = [
|
||||
parameters: {
|
||||
// areaId: 3,
|
||||
areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
|
||||
blockId: 114458,
|
||||
seriesId: 106,
|
||||
scene: 'https://aziani.com',
|
||||
},
|
||||
@@ -1433,7 +1456,6 @@ const sites = [
|
||||
independent: true,
|
||||
parameters: {
|
||||
scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie',
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1445,7 +1467,6 @@ const sites = [
|
||||
independent: true,
|
||||
parameters: {
|
||||
scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets',
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
/* different layout
|
||||
@@ -1458,7 +1479,6 @@ const sites = [
|
||||
independent: true,
|
||||
parameters: {
|
||||
scene: 'https://www.portagloryhole.com/scenes',
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
*/
|
||||
@@ -2692,9 +2712,6 @@ const sites = [
|
||||
url: 'https://www.biphoria.com',
|
||||
independent: true,
|
||||
tags: ['bisexual'],
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
parent: 'gamma',
|
||||
},
|
||||
// BLOWPASS
|
||||
@@ -3187,9 +3204,6 @@ const sites = [
|
||||
alias: ['burna'],
|
||||
url: 'https://www.burningangel.com',
|
||||
independent: true,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
parent: 'gamma',
|
||||
},
|
||||
// CHERRY PIMPS
|
||||
@@ -3835,7 +3849,6 @@ const sites = [
|
||||
parent: 'gamma',
|
||||
independent: true,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
deep: 'https://www.diabolic.com/en/video/diabolic',
|
||||
actors: 'https://www.diabolic.com/en/pornstar/view/{slug}/{id}',
|
||||
},
|
||||
@@ -3907,9 +3920,6 @@ const sites = [
|
||||
name: 'DFXtra',
|
||||
url: 'https://www.dfxtra.com',
|
||||
parent: 'dfxtra',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'dfxtraoriginals',
|
||||
@@ -3917,9 +3927,6 @@ const sites = [
|
||||
url: 'https://www.dfxtra.com',
|
||||
parent: 'dfxtra',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'dfxtracompilations',
|
||||
@@ -3927,9 +3934,6 @@ const sites = [
|
||||
url: 'https://www.dfxtra.com',
|
||||
parent: 'dfxtra',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'dfxbigbangz',
|
||||
@@ -3937,9 +3941,6 @@ const sites = [
|
||||
url: 'https://www.dfxtra.com',
|
||||
parent: 'dfxtra',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'dfxsolemates',
|
||||
@@ -3947,9 +3948,6 @@ const sites = [
|
||||
url: 'https://www.dfxtra.com',
|
||||
parent: 'dfxtra',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'cheatingwithmyex',
|
||||
@@ -3957,9 +3955,6 @@ const sites = [
|
||||
url: 'https://www.dfxtra.com',
|
||||
parent: 'dfxtra',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
},
|
||||
// DFXTRA DOGFART
|
||||
{
|
||||
@@ -5459,6 +5454,12 @@ const sites = [
|
||||
referer: 'https://www.girlsway.com',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'lesbianfactor',
|
||||
name: 'Lesbian Factor',
|
||||
url: 'https://www.lesbianfactor.com',
|
||||
parent: 'girlsway',
|
||||
},
|
||||
// HITZEFREI
|
||||
{
|
||||
slug: 'unleashed',
|
||||
@@ -5565,6 +5566,19 @@ const sites = [
|
||||
url: 'https://seehimfuck.com',
|
||||
tags: ['male-focus'],
|
||||
parent: 'hussiepass',
|
||||
parameters: {
|
||||
latest: 'https://seehimfuck.com',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'seehimsolo',
|
||||
name: 'See Him Solo',
|
||||
url: 'https://seehimsolo.com',
|
||||
tags: ['male-focus', 'solo'],
|
||||
parent: 'hussiepass',
|
||||
parameters: {
|
||||
latest: 'https://seehimsolo.com/categories/movies-2/{page}/latest/',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'interracialpovs',
|
||||
@@ -5576,14 +5590,28 @@ const sites = [
|
||||
{
|
||||
slug: 'povpornstars',
|
||||
name: 'POV Pornstars',
|
||||
url: 'http://www.povpornstars.com',
|
||||
url: 'https://www.povpornstars.com',
|
||||
tags: ['pov'],
|
||||
parent: 'hussiepass',
|
||||
parameters: {
|
||||
latest: 'http://www.povpornstars.com/tour/categories/movies_%d_d.html',
|
||||
profile: 'http://www.povpornstars.com/tour/models/%s.html',
|
||||
latest: 'https://www.povpornstars.com/tour/categories/movies_{page}_d.html',
|
||||
profile: 'https://www.povpornstars.com/tour/models/{actor}.html',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'ravebunnys',
|
||||
name: 'Rave Bunnys',
|
||||
url: 'https://ravebunnys.com',
|
||||
tags: ['rave'],
|
||||
parent: 'hussiepass',
|
||||
},
|
||||
{
|
||||
slug: 'hotandtatted',
|
||||
name: 'Hot and Tatted',
|
||||
url: 'https://hotandtatted.com',
|
||||
tags: ['tattoos'],
|
||||
parent: 'hussiepass',
|
||||
},
|
||||
// HUSH PASS
|
||||
{
|
||||
slug: 'shotherfirst',
|
||||
@@ -5591,7 +5619,7 @@ const sites = [
|
||||
url: 'https://shotherfirst.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/shot-her-first_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/shot-her-first_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5602,7 +5630,7 @@ const sites = [
|
||||
url: 'https://whitezilla.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/whitezilla_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/whitezilla_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5613,7 +5641,7 @@ const sites = [
|
||||
url: 'https://frathousefuckfest.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/frat-house-fuck-fest_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/frat-house-fuck-fest_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5624,7 +5652,7 @@ const sites = [
|
||||
url: 'https://freakyfirsttimers.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/freaky-first-timers_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/freaky-first-timers_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5635,7 +5663,7 @@ const sites = [
|
||||
url: 'https://milfinvaders.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/milf-invaders_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/milf-invaders_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5646,7 +5674,7 @@ const sites = [
|
||||
url: 'https://housewivesneedcash.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/housewives-need-cash_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/housewives-need-cash_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5657,7 +5685,7 @@ const sites = [
|
||||
url: 'https://bubblebuttbonanza.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/bubble-butt-bonanza_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/bubble-butt-bonanza_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5668,7 +5696,7 @@ const sites = [
|
||||
url: 'https://suburbansexparty.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/suburban-sex-party_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/suburban-sex-party_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5679,7 +5707,7 @@ const sites = [
|
||||
url: 'https://buttnakedinthestreets.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/ButtNakedInStreets_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/ButtNakedInStreets_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
match: 'Butt Naked In Streets',
|
||||
t1: true,
|
||||
@@ -5691,7 +5719,7 @@ const sites = [
|
||||
url: 'https://muffbumperpatrol.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/muff-bumper-patrol_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/muff-bumper-patrol_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5702,7 +5730,7 @@ const sites = [
|
||||
url: 'https://biggathananigga.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/bigga-than-a-nigga_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/bigga-than-a-nigga_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5713,7 +5741,7 @@ const sites = [
|
||||
url: 'https://bachelorpartyfuckfest.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/bachelor-party-fuck-fest_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/bachelor-party-fuck-fest_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5724,7 +5752,7 @@ const sites = [
|
||||
url: 'https://teencumdumpsters.com',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/teen-cum-dumpsters_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/teen-cum-dumpsters_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5734,7 +5762,7 @@ const sites = [
|
||||
name: 'POV Hunnies',
|
||||
parent: 'hushpass',
|
||||
parameters: {
|
||||
latest: 'https://hushpass.com/t1/categories/POVHunnies_%d_d.html',
|
||||
latest: 'https://hushpass.com/t1/categories/POVHunnies_{page}_d.html',
|
||||
media: 'https://hushpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5872,7 +5900,7 @@ const sites = [
|
||||
tags: ['interracial'],
|
||||
parent: 'interracialpass',
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/2-big-to-be-true_%d_d.html',
|
||||
latest: 'https://www.interracialpass.com/t1/categories/2-big-to-be-true_{page}_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5884,7 +5912,7 @@ const sites = [
|
||||
tags: ['interracial'],
|
||||
parent: 'interracialpass',
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/abominable-black-man_%d_d.html',
|
||||
latest: 'https://www.interracialpass.com/t1/categories/abominable-black-man_{page}_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5896,7 +5924,7 @@ const sites = [
|
||||
parent: 'interracialpass',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/BootyAnnihilation_%d_d.html',
|
||||
latest: 'https://www.interracialpass.com/t1/categories/BootyAnnihilation_{page}_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5908,7 +5936,7 @@ const sites = [
|
||||
tags: ['interracial'],
|
||||
parent: 'interracialpass',
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/daddys-worst-nightmare_%d_d.html',
|
||||
latest: 'https://www.interracialpass.com/t1/categories/daddys-worst-nightmare_{page}_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5920,7 +5948,7 @@ const sites = [
|
||||
tags: ['interracial'],
|
||||
parent: 'interracialpass',
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/monster-cock-fuck-fest_%d_d.html',
|
||||
latest: 'https://www.interracialpass.com/t1/categories/monster-cock-fuck-fest_{page}_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5932,7 +5960,7 @@ const sites = [
|
||||
tags: ['interracial'],
|
||||
parent: 'interracialpass',
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/my-daughters-fucking-a-black-dude_%d_d.html',
|
||||
latest: 'https://www.interracialpass.com/t1/categories/my-daughters-fucking-a-black-dude_{page}_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5944,7 +5972,7 @@ const sites = [
|
||||
tags: ['interracial'],
|
||||
parent: 'interracialpass',
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/my-moms-fucking-blackzilla_%d_d.html',
|
||||
latest: 'https://www.interracialpass.com/t1/categories/my-moms-fucking-blackzilla_{page}_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
t1: true,
|
||||
},
|
||||
@@ -5956,7 +5984,7 @@ const sites = [
|
||||
tags: ['interracial'],
|
||||
parent: 'interracialpass',
|
||||
parameters: {
|
||||
latest: 'https://www.interracialpass.com/t1/categories/my-wifes-first-monster-cock_%d_d.html',
|
||||
latest: 'https://www.interracialpass.com/t1/categories/my-wifes-first-monster-cock_{page}_d.html',
|
||||
media: 'https://www.interracialpass.com',
|
||||
match: 'My Wifes First Monster Cock',
|
||||
t1: true,
|
||||
@@ -6069,7 +6097,6 @@ const sites = [
|
||||
url: 'https://cospimps.com',
|
||||
parent: 'jayrock',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
/* Gamma scenes are out of date
|
||||
referer: 'https://www.21sextury.com',
|
||||
scene: false,
|
||||
@@ -7279,6 +7306,29 @@ const sites = [
|
||||
showcased: true,
|
||||
hasLogo: false,
|
||||
visible: true,
|
||||
// merged into Christian Clay's studio, but that started rereleasing from other channels
|
||||
},
|
||||
{
|
||||
name: 'Adventure Teens',
|
||||
slug: 'adventureteens',
|
||||
url: 'https://www.analvids.com/studios/deeplove',
|
||||
alias: ['deeplove'],
|
||||
parent: 'analvids',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
studioId: 1007,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Lancelot Styles',
|
||||
slug: 'lancelotstyles',
|
||||
url: 'https://www.analvids.com/studios/latinhot_studio',
|
||||
alias: ['latinhot studio', 'latin hot studio'],
|
||||
parent: 'analvids',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
studioId: 698,
|
||||
},
|
||||
},
|
||||
// LETS DOE IT
|
||||
{
|
||||
@@ -8959,7 +9009,7 @@ const sites = [
|
||||
{
|
||||
slug: 'afterschool',
|
||||
name: 'After School',
|
||||
url: 'https://www.naughtyamerica.com/site/after-schooll', // sic
|
||||
url: 'https://www.naughtyamerica.com/site/after-school',
|
||||
parent: 'naughtyamericavr',
|
||||
},
|
||||
{
|
||||
@@ -10509,10 +10559,11 @@ const sites = [
|
||||
name: 'Look At Her Now',
|
||||
url: 'https://www.lookathernow.com',
|
||||
description: 'Look At Her Now brings you best HD reality porn videos every week. Check out these girls before and after they get some rough pounding.',
|
||||
parameters: { native: true },
|
||||
// parameters: { siteId: 300 },
|
||||
slug: 'lookathernow',
|
||||
parent: 'realitykings',
|
||||
parameters: {
|
||||
siteId: 364,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'We Live Together',
|
||||
@@ -11763,6 +11814,7 @@ const sites = [
|
||||
slug: 'scoreland',
|
||||
url: 'https://www.scoreland.com',
|
||||
parent: 'score',
|
||||
showcased: true,
|
||||
parameters: {
|
||||
path: '/big-boob-videos',
|
||||
},
|
||||
@@ -13349,9 +13401,11 @@ const sites = [
|
||||
name: 'Sic Flics',
|
||||
styled: 'SicFlics',
|
||||
url: 'https://www.sicflics.com',
|
||||
tags: ['extreme-insertion'],
|
||||
parameters: {
|
||||
layout: 'mobile', // has scene links
|
||||
actorTags: {
|
||||
// no dedicated actor fields, these tags are used instead
|
||||
maria: 'Queen Maria',
|
||||
crazywifeslut: 'CrazyWifeSlut',
|
||||
kassiopeia: 'Kassiopeia',
|
||||
@@ -15000,12 +15054,12 @@ const sites = [
|
||||
name: 'Vivid Celeb',
|
||||
url: 'https://www.vividceleb.com',
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2018-03-25'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'vivid',
|
||||
name: 'Vivid',
|
||||
url: 'https://www.vivid.com/en/videos/sites/vivid',
|
||||
parent: 'vivid',
|
||||
},
|
||||
{
|
||||
slug: 'thebrats',
|
||||
@@ -15022,122 +15076,72 @@ const sites = [
|
||||
{
|
||||
slug: 'nineteen',
|
||||
name: 'Nineteen',
|
||||
url: 'http://www.nineteen.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/nineteen',
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2019-01-23'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'nastystepfamily',
|
||||
name: 'Nasty Step Family',
|
||||
url: 'http://www.nastystepfamily.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/nastystepfamily',
|
||||
tags: ['family'],
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2019-01-29'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'girlswhofuckgirls',
|
||||
name: 'Girls Who Fuck Girls',
|
||||
url: 'http://www.girlswhofuckgirls.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/girlswhofuckgirls',
|
||||
tags: ['lesbian'],
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2019-05-21'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'petited',
|
||||
name: 'Petited',
|
||||
url: 'http://www.petited.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/petited',
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2019-01-28'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'orgytrain',
|
||||
name: 'Orgy Train',
|
||||
url: 'http://www.orgytrain.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/orgytrain',
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2019-01-09'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'momisamilf',
|
||||
name: 'Mom Is A MILF',
|
||||
url: 'http://www.momisamilf.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/momisamilf',
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2019-01-25'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'blackwhitefuckfest',
|
||||
name: 'Black White Fuck Fest',
|
||||
url: 'http://www.blackwhitefuckfest.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/blackwhitefuckfest',
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2019-01-30'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: '65inchhugeasses',
|
||||
name: '65 Inch Huge Asses',
|
||||
url: 'http://www.65inchhugeasses.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/65inchhugeasses',
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2019-05-18'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'brandnewfaces',
|
||||
name: 'Brand New Faces',
|
||||
url: 'http://www.brandnewfaces.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/brandnewfaces',
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2018-02-28'),
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'vividclassic',
|
||||
name: 'Vivid Classic',
|
||||
url: 'http://www.vividclassic.com',
|
||||
url: 'https://www.vivid.com/en/videos/sites/vividclassic',
|
||||
parent: 'vivid',
|
||||
},
|
||||
{
|
||||
slug: 'tsdivas',
|
||||
name: 'TS Divas',
|
||||
url: 'https://www.vivid.com/en/videos/sites/tsdivas',
|
||||
tags: ['transsexual'],
|
||||
hasLogo: false,
|
||||
parent: 'vivid',
|
||||
parameters: {
|
||||
referer: 'https://www.thebrats.com',
|
||||
deep: 'https://www.thebrats.com/en/video',
|
||||
scene: false,
|
||||
lastNative: new Date('2016-06-29'),
|
||||
},
|
||||
},
|
||||
// VIXEN
|
||||
{
|
||||
@@ -15441,9 +15445,6 @@ const sites = [
|
||||
url: 'https://www.wicked.com',
|
||||
description: 'Welcome to the new Wicked.com! Watch over 25 years of Wicked Pictures\' brand of award-winning porn for couples and women in 4k HD movies & xxx videos',
|
||||
independent: true,
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
},
|
||||
parent: 'gamma',
|
||||
},
|
||||
// XEMPIRE
|
||||
@@ -15580,9 +15581,9 @@ exports.seed = (knex) => Promise.resolve()
|
||||
alias: site.alias,
|
||||
description: site.description,
|
||||
url: site.url,
|
||||
parameters: site.parameters,
|
||||
parameters: site.parameters || null,
|
||||
options: site.options,
|
||||
parent_id: networksMap[site.parent],
|
||||
parent_id: networksMap[site.parent] || null,
|
||||
priority: site.priority || 0,
|
||||
independent: !!site.independent,
|
||||
visible: site.visible,
|
||||
|
||||
@@ -208,6 +208,11 @@ const affiliates = [
|
||||
url: 'https://www.g2buddy.com/disruptivefilms/go.php?pr=9&su=2&si=119&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'sodomysquad',
|
||||
url: 'https://www.g2buddy.com/sodomysquad/go.php?pr=9&su=2&si=137&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
// gamma > ags max > next door studios
|
||||
// excluded affiliate links that link back to main site and don't seem to track properly
|
||||
{
|
||||
@@ -494,6 +499,92 @@ const affiliates = [
|
||||
scene: false, // redirects to Adult Time
|
||||
},
|
||||
},
|
||||
// gamma > vivid
|
||||
{
|
||||
network: 'vivid',
|
||||
url: 'https://www.g2fame.com/vivid/go.php?pr=8&su=2&si=330&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
parameters: {
|
||||
scene: false, // redirects to homepage
|
||||
},
|
||||
},
|
||||
{
|
||||
channel: 'wheretheboysarent',
|
||||
url: 'https://www.g2fame.com/wheretheboysarent/go.php?pr=8&su=2&si=368&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'thebrats',
|
||||
url: 'https://www.g2fame.com/thebrats/go.php?pr=8&su=2&si=369&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
// gamma > zero tolerance
|
||||
{
|
||||
network: 'zerotolerance',
|
||||
url: 'https://www.g2fame.com/zerotolerancefilms/go.php?pr=8&su=2&si=507&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'zerotolerancefilms',
|
||||
url: 'https://www.g2fame.com/zerotolerancefilms/go.php?pr=8&su=2&si=507&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: '3rddegreefilms',
|
||||
url: 'https://www.g2fame.com/3rddegreefilms/go.php?pr=8&su=2&si=537&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'addicted2girls',
|
||||
url: 'https://www.g2fame.com/addicted2girls/go.php?pr=8&su=2&si=477&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'genderxfilms',
|
||||
url: 'https://www.g2fame.com/genderxfilms/go.php?pr=8&su=2&si=397&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'gangbangcreampie',
|
||||
url: 'https://www.g2fame.com/gangbangcreampie/go.php?pr=8&su=2&si=656&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'gloryholesecrets',
|
||||
url: 'https://www.g2fame.com/gloryholesecrets/go.php?pr=8&su=2&si=655&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'tabooheat',
|
||||
url: 'https://www.g2fame.com/tabooheat/go.php?pr=8&su=2&si=552&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'wicked',
|
||||
url: 'https://www.g2fame.com/wicked/go.php?pr=8&su=2&si=371&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
// gamma > independent channels
|
||||
{
|
||||
channel: 'biphoria',
|
||||
url: 'https://www.g2fame.com/biphoria/go.php?pr=8&su=2&si=418&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'burningangel',
|
||||
url: 'https://www.g2fame.com/burningangel/go.php?pr=8&su=2&si=174&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'chaosmen',
|
||||
url: 'https://www.g2fame.com/chaosmen/go.php?pr=8&su=2&si=608&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
{
|
||||
channel: 'diabolic',
|
||||
url: 'https://www.g2fame.com/diabolic/go.php?pr=8&su=2&si=523&ad=277470&pa=index&ar=&buffer=',
|
||||
comment: 'per signup',
|
||||
},
|
||||
// kelly madison / 8k
|
||||
{
|
||||
network: 'kellymadison',
|
||||
@@ -847,6 +938,53 @@ const affiliates = [
|
||||
channel: 'xangels',
|
||||
url: 'https://secure.x-angels.com/track/MzAxNjcxLjUuMzguODEuMC4wLjAuMC4w',
|
||||
},
|
||||
// Snow Valley / Tsunami
|
||||
{
|
||||
network: 'snowvalley',
|
||||
parameters: {
|
||||
query: 'ref=4c331ef6',
|
||||
},
|
||||
},
|
||||
// POV Porn Cash / HussiePass
|
||||
{
|
||||
network: 'hussiepass',
|
||||
url: 'https://secure.hussiepass.com/track/MTk0NS4xLjUuNy4wLjAuMC4wLjA',
|
||||
comment: '50% revshare',
|
||||
parameters: {
|
||||
// hussiepass website does not show network scenes
|
||||
channelScenes: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
channel: 'povpornstars',
|
||||
url: 'https://join.povpornstars.com/track/MTk0NS4xLjMuNS4wLjAuMC4wLjA',
|
||||
comment: '50% revshare',
|
||||
},
|
||||
{
|
||||
channel: 'interracialpovs',
|
||||
url: 'https://join.interracialpovs.com/track/MTk0NS4xLjYuOC4wLjAuMC4wLjA',
|
||||
comment: '50% revshare',
|
||||
},
|
||||
{
|
||||
channel: 'ravebunnys',
|
||||
url: 'https://secure.ravebunnys.com/track/MTk0NS4xLjExLjI5LjAuMC4wLjAuMA',
|
||||
comment: '50% revshare',
|
||||
},
|
||||
{
|
||||
channel: 'hotandtatted',
|
||||
url: 'https://join.hotandtatted.com/track/MTk0NS4xLjEwLjEyLjAuMC4wLjAuMA',
|
||||
comment: '50% revshare',
|
||||
},
|
||||
{
|
||||
channel: 'seehimfuck',
|
||||
url: 'https://join.seehimfuck.com/track/MTk0NS4xLjcuOS4wLjAuMC4wLjA',
|
||||
comment: '50% revshare',
|
||||
},
|
||||
{
|
||||
channel: 'seehimsolo',
|
||||
url: 'https://join.seehimsolo.com/track/MTk0NS4xLjguMTAuMC4wLjAuMC4w',
|
||||
comment: '50% revshare',
|
||||
},
|
||||
// etc
|
||||
{
|
||||
network: 'bang',
|
||||
@@ -855,6 +993,11 @@ const affiliates = [
|
||||
},
|
||||
comment: '$25 per signup',
|
||||
},
|
||||
{
|
||||
channel: 'sicflics',
|
||||
url: 'http://refer.ccbill.com/cgi-bin/clicks.cgi?CA=921613-0002&PA=2647683',
|
||||
comment: '40% rebill',
|
||||
},
|
||||
];
|
||||
|
||||
const bannerTags = {
|
||||
|
||||
@@ -17,11 +17,12 @@ const domPurify = DOMPurify(window);
|
||||
// const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
const redis = require('./redis');
|
||||
const scrapers = require('./scrapers/scrapers').actors;
|
||||
const actorScrapers = require('./scrapers/scrapers').actors;
|
||||
|
||||
const argv = require('./argv');
|
||||
const include = require('./utils/argv-include')(argv);
|
||||
const bulkInsert = require('./utils/bulk-insert');
|
||||
const batchInsert = require('./utils/batch-insert');
|
||||
const chunk = require('./utils/chunk');
|
||||
const logger = require('./logger')(__filename);
|
||||
|
||||
@@ -46,6 +47,7 @@ const commonContext = {
|
||||
slugify,
|
||||
omit,
|
||||
unprint,
|
||||
batchInsert,
|
||||
};
|
||||
|
||||
const hairColors = {
|
||||
@@ -349,6 +351,7 @@ function curateProfileEntry(profile) {
|
||||
tattoos: profile.tattoos,
|
||||
blood_type: profile.bloodType,
|
||||
avatar_media_id: profile.avatarMediaId || null,
|
||||
updated_at: knex.raw('DEFAULT'), // default should be NOW(), this will update the column
|
||||
};
|
||||
|
||||
return curatedProfileEntry;
|
||||
@@ -438,35 +441,35 @@ async function curateProfile(profile, actor) {
|
||||
|| null;
|
||||
|
||||
curatedProfile.dateOfDeath = Number.isNaN(Number(profile.dateOfDeath)) ? null : profile.dateOfDeath;
|
||||
curatedProfile.age = Number(profile.age) || null;
|
||||
curatedProfile.age = Math.round(profile.age) || null;
|
||||
|
||||
curatedProfile.height = Number(profile.height) || profile.height?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.weight = Number(profile.weight) || profile.weight?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.shoeSize = Number(profile.shoeSize) || profile.shoeSize?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.height = Math.round(profile.height || profile.height?.match?.(/\d+/)?.[0]) || null;
|
||||
curatedProfile.weight = Math.round(profile.weight || profile.weight?.match?.(/\d+/)?.[0]) || null;
|
||||
|
||||
// separate measurement values
|
||||
curatedProfile.cup = profile.cup || (typeof profile.bust === 'string' && profile.bust?.match?.(/[a-zA-Z]+/)?.[0]) || null;
|
||||
curatedProfile.bust = Number(profile.bust) || profile.bust?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.waist = Number(profile.waist) || profile.waist?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.hip = Number(profile.hip) || profile.hip?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.bust = Math.round(profile.bust || profile.bust?.match?.(/\d+/)?.[0]) || null;
|
||||
curatedProfile.waist = Math.round(profile.waist || profile.waist?.match?.(/\d+/)?.[0]) || null;
|
||||
curatedProfile.hip = Math.round(profile.hip || profile.hip?.match?.(/\d+/)?.[0]) || null;
|
||||
|
||||
curatedProfile.leg = Number(profile.leg) || profile.leg?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.thigh = Number(profile.thigh) || profile.thigh?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.foot = Number(profile.foot) || profile.foot?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.leg = Math.round(profile.leg || profile.leg?.match?.(/\d+/)?.[0]) || null;
|
||||
curatedProfile.thigh = Math.round(profile.thigh || profile.thigh?.match?.(/\d+/)?.[0]) || null;
|
||||
curatedProfile.foot = Number(profile.foot || profile.foot?.match?.(/\d+/)?.[0]) || null;
|
||||
curatedProfile.shoeSize = Number(profile.shoeSize || profile.shoeSize?.match?.(/\d+/)?.[0]) || null;
|
||||
|
||||
// combined measurement value
|
||||
// ExCoGi uses x, Jules Jordan has spaces between the dashes, SpermMenia/Cum Buffet sometimes misses cup
|
||||
const measurements = profile.measurements?.match(/(\d+)([a-z]+)?(?:\s*[-x]\s*(\d+)\s*[-x]\s*(\d+))?/i);
|
||||
|
||||
if (measurements) {
|
||||
curatedProfile.bust = Number(measurements[1]) || null;
|
||||
curatedProfile.bust = Math.round(measurements[1]) || null;
|
||||
curatedProfile.cup = measurements[2] || null;
|
||||
curatedProfile.waist = Number(measurements[3]) || null;
|
||||
curatedProfile.hip = Number(measurements[4]) || null;
|
||||
curatedProfile.waist = Math.round(measurements[3]) || null;
|
||||
curatedProfile.hip = Math.round(measurements[4]) || null;
|
||||
}
|
||||
|
||||
curatedProfile.penisLength = Number(profile.penisLength) || profile.penisLength?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.penisGirth = Number(profile.penisGirth) || profile.penisGirth?.match?.(/\d+/)?.[0] || null;
|
||||
curatedProfile.penisLength = Math.round(profile.penisLength || profile.penisLength?.match?.(/\d+/)?.[0]) || null;
|
||||
curatedProfile.penisGirth = Math.round(profile.penisGirth || profile.penisGirth?.match?.(/\d+/)?.[0]) || null;
|
||||
|
||||
curatedProfile.isCircumcised = getBoolean(profile.isCircumcised);
|
||||
curatedProfile.naturalBoobs = getBoolean(profile.naturalBoobs);
|
||||
@@ -544,7 +547,7 @@ async function curateProfile(profile, actor) {
|
||||
|
||||
async function insertProfiles(newProfiles) {
|
||||
if (newProfiles.length > 0) {
|
||||
const entries = await bulkInsert('actors_profiles', newProfiles);
|
||||
const entries = await batchInsert('actors_profiles', newProfiles);
|
||||
|
||||
logger.info(`Saved ${newProfiles.length} actor profiles`);
|
||||
|
||||
@@ -606,10 +609,7 @@ async function upsertProfiles(profiles) {
|
||||
}));
|
||||
|
||||
if (avatars.length > 0) {
|
||||
await knex('actors_avatars')
|
||||
.insert(avatars)
|
||||
.onConflict()
|
||||
.ignore();
|
||||
await batchInsert('actors_avatars', avatars, { conflict: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -624,7 +624,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
|
||||
try {
|
||||
const entity = entitiesBySlug[scraperSlug] || null;
|
||||
|
||||
const scraper = scrapers[scraperSlug];
|
||||
const scraper = actorScrapers[scraperSlug];
|
||||
const layoutScraper = resolveLayoutScraper(entity, scraper);
|
||||
|
||||
if (!layoutScraper?.fetchProfile) {
|
||||
@@ -759,7 +759,8 @@ function curateSocials(socials, platformsByHostname) {
|
||||
|
||||
async function associateSocials(profiles) {
|
||||
const { platformsByHostname } = await actorsCommon;
|
||||
const profileEntries = await knex('actors_profiles').whereIn(['actor_id', 'entity_id'], profiles.map((profile) => [profile.actorId, profile.entity.id]));
|
||||
const profileEntryChunks = await Promise.all(chunk(profiles).map((profilesChunk) => knex('actors_profiles').whereIn(['actor_id', 'entity_id'], profilesChunk.map((profile) => [profile.actorId, profile.entity.id]))));
|
||||
const profileEntries = profileEntryChunks.flat();
|
||||
|
||||
const profileEntriesByActorIdAndEntityId = profileEntries.reduce((acc, profileEntry) => {
|
||||
if (!acc[profileEntry.actor_id]) {
|
||||
@@ -784,16 +785,14 @@ async function associateSocials(profiles) {
|
||||
return;
|
||||
}
|
||||
|
||||
await knex('actors_socials')
|
||||
.insert(curateSocials(profile.social, platformsByHostname).map((social) => ({
|
||||
platform: social.platform,
|
||||
handle: social.handle,
|
||||
url: social.url,
|
||||
actor_id: profile.actorId,
|
||||
// profile_id: profileId,
|
||||
})))
|
||||
.onConflict()
|
||||
.ignore();
|
||||
await batchInsert('actors_socials', curateSocials(profile.social, platformsByHostname).map((social) => ({
|
||||
platform: social.platform,
|
||||
handle: social.handle,
|
||||
url: social.url,
|
||||
actor_id: profile.actorId,
|
||||
})), {
|
||||
conflict: false,
|
||||
});
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
@@ -832,11 +831,11 @@ async function scrapeActors(argNames) {
|
||||
|
||||
logger.info(`Scraping profiles for ${actorNames.length} actors`);
|
||||
|
||||
const sources = argv.profileSources || config.profiles || Object.keys(scrapers.actors);
|
||||
const sources = argv.profileSources || config.profiles || Object.keys(actorScrapers);
|
||||
const entitySlugs = sources.flat();
|
||||
|
||||
const [entitiesBySlug, existingActorEntries] = await Promise.all([
|
||||
fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'], prefer: argv.prefer || 'channel' }),
|
||||
fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'], prefer: argv.prefer || 'options' }),
|
||||
knex('actors')
|
||||
.select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
|
||||
.whereIn('actors.slug', baseActors.map((baseActor) => baseActor.slug))
|
||||
|
||||
26
src/app.js
26
src/app.js
@@ -24,7 +24,8 @@ const { updateSceneSearch, updateMovieSearch } = require('./update-search');
|
||||
const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
|
||||
const { flushEntities } = require('./entities');
|
||||
const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases');
|
||||
const { flushOrphanedMedia } = require('./media');
|
||||
const { flushOrphanedMedia, detachReleaseMedia, detachEntityReleaseMedia } = require('./media');
|
||||
const { reassociateEntityReleaseTags, reassociateReleaseTags, reassociateOriginalTags } = require('./tags');
|
||||
const getFileEntries = require('./utils/file-entries');
|
||||
|
||||
const inspector = new Inspector();
|
||||
@@ -159,7 +160,8 @@ async function init() {
|
||||
}
|
||||
|
||||
if (argv.flushNetworks || argv.flushChannels) {
|
||||
await flushEntities(argv.flushNetworks, argv.flushChannels);
|
||||
// inject flushOrphanedMedia to prevent circular dependency with entity media flush
|
||||
await flushEntities(argv.flushNetworks, argv.flushChannels, flushOrphanedMedia);
|
||||
}
|
||||
|
||||
if (argv.flushBatches) {
|
||||
@@ -186,10 +188,30 @@ async function init() {
|
||||
await deleteMovies(argv.deleteMovies);
|
||||
}
|
||||
|
||||
if (argv.originalTags) {
|
||||
await reassociateOriginalTags(argv.originalTags, argv.rematchTags);
|
||||
}
|
||||
|
||||
if (argv.releaseTags) {
|
||||
await reassociateReleaseTags(argv.releaseTags, argv.rematchTags);
|
||||
}
|
||||
|
||||
if (argv.networkReleaseTags || argv.channelReleaseTags) {
|
||||
await reassociateEntityReleaseTags(argv.networkReleaseTags, argv.channelReleaseTags, argv.rematchTags);
|
||||
}
|
||||
|
||||
if (argv.flushOrphanedMedia) {
|
||||
await flushOrphanedMedia();
|
||||
}
|
||||
|
||||
if (argv.detachReleaseMedia) {
|
||||
await detachReleaseMedia(argv.detachReleaseMedia);
|
||||
}
|
||||
|
||||
if (argv.detachNetworkMedia || argv.detachChannelMedia) {
|
||||
await detachEntityReleaseMedia(argv.detachNetworkMedia, argv.detachChannelMedia);
|
||||
}
|
||||
|
||||
if (argv.request) {
|
||||
const res = await http[argv.requestMethod](argv.request);
|
||||
|
||||
|
||||
53
src/argv.js
53
src/argv.js
@@ -349,7 +349,32 @@ const { argv } = yargs
|
||||
describe: 'Remove files from storage when flushing media.',
|
||||
type: 'boolean',
|
||||
alias: 'flush-files',
|
||||
default: true,
|
||||
})
|
||||
.option('detach-channel-media', {
|
||||
describe: 'Remove media files from channel scenes.',
|
||||
type: 'array',
|
||||
})
|
||||
.option('detach-network-media', {
|
||||
describe: 'Remove media files from network scenes.',
|
||||
type: 'array',
|
||||
})
|
||||
.option('detach-release-media', {
|
||||
describe: 'Remove media files from network scenes.',
|
||||
type: 'array',
|
||||
alias: ['detach-scene-media'],
|
||||
})
|
||||
.option('detach-media-domains', {
|
||||
describe: 'Only detach these types of media.',
|
||||
type: 'array',
|
||||
default: [
|
||||
'posters',
|
||||
'photos',
|
||||
'caps',
|
||||
'trailers',
|
||||
'teasers',
|
||||
'covers',
|
||||
],
|
||||
alias: ['detach-media'],
|
||||
})
|
||||
.option('flush-channels', {
|
||||
describe: 'Delete all scenes and movies from channels.',
|
||||
@@ -407,6 +432,32 @@ const { argv } = yargs
|
||||
type: 'array',
|
||||
alias: ['delete-movie', 'remove-movies', 'remove-movies'],
|
||||
})
|
||||
.option('original-tags', {
|
||||
describe: 'Reassociate original tag names',
|
||||
type: 'array',
|
||||
alias: ['tags'],
|
||||
})
|
||||
.option('release-tags', {
|
||||
describe: 'Reassociate tags for scene IDs',
|
||||
type: 'array',
|
||||
alias: ['scene-tags'],
|
||||
})
|
||||
.option('channel-release-tags', {
|
||||
describe: 'Reassociate tags for all channel releases',
|
||||
type: 'array',
|
||||
alias: ['channel-scene-tags', 'channel-tags'],
|
||||
})
|
||||
.option('network-release-tags', {
|
||||
describe: 'Reassociate tags for all network releases',
|
||||
type: 'array',
|
||||
alias: ['network-scene-tags', 'network-tags'],
|
||||
})
|
||||
.option('rematch-tags', {
|
||||
describe: 'Reassociate tags that are already associated',
|
||||
type: 'boolean',
|
||||
alias: 'rematch',
|
||||
default: false,
|
||||
})
|
||||
.option('request', {
|
||||
describe: 'Make an arbitrary HTTP request',
|
||||
type: 'string',
|
||||
|
||||
@@ -69,6 +69,7 @@ async function fetchUnprintScene(scraper, url, entity, baseRelease, options, typ
|
||||
entity,
|
||||
baseRelease,
|
||||
headers: res.headers,
|
||||
cookies: res.cookies,
|
||||
include,
|
||||
beforeFetchScenes: options.beforeFetchScenes,
|
||||
parameters: options.parameters,
|
||||
|
||||
@@ -7,8 +7,9 @@ const logger = require('./logger')(__filename);
|
||||
const argv = require('./argv');
|
||||
const knex = require('./knex');
|
||||
const { deleteScenes, deleteMovies, deleteSeries } = require('./releases');
|
||||
const { flushOrphanedMedia } = require('./media');
|
||||
const { resolveScraper, resolveLayoutScraper } = require('./scrapers/resolve');
|
||||
const { fetchEntityReleaseIds } = require('./entity-releases');
|
||||
const getRecursiveParameters = require('./utils/get-recursive-parameters');
|
||||
|
||||
function getRecursiveParent(entity) {
|
||||
if (!entity) {
|
||||
@@ -257,13 +258,18 @@ async function fetchEntitiesBySlug(entitySlugs, options = { prefer: 'channel', a
|
||||
entitySlugs: entitySlugs.filter((slug) => !slug.includes('.')),
|
||||
entityHosts: entitySlugs.filter((slug) => slug.includes('.')).map((hostname) => `%${hostname}`),
|
||||
entityTypes: options.types || ['channel', 'network'],
|
||||
sort: knex.raw(options.prefer === 'channel' ? 'asc' : 'desc'),
|
||||
sort: knex.raw(options.prefer === 'channel' || options.prefer === 'options' ? 'asc' : 'desc'),
|
||||
});
|
||||
|
||||
// channel entity will overwrite network entity
|
||||
// by default channel entity will overwrite network entity
|
||||
const entitiesBySlug = entities.rows.reduce((accEntities, { entity }) => {
|
||||
const host = urlToHostname(entity.url);
|
||||
const curatedEntity = accEntities[entity.slug] || accEntities[host] || curateEntity(entity, true);
|
||||
const entityOptions = getRecursiveParameters(entity, 'options');
|
||||
const accEntity = accEntities[entity.slug] || accEntities[host];
|
||||
|
||||
const curatedEntity = !accEntity || (options.prefer === 'options' && entity.type === 'network' && entityOptions.preferNetwork)
|
||||
? curateEntity(entity, true)
|
||||
: accEntity;
|
||||
|
||||
return {
|
||||
...accEntities,
|
||||
@@ -368,81 +374,10 @@ async function searchEntities(query, type, limit) {
|
||||
return curateEntities(entities);
|
||||
}
|
||||
|
||||
async function flushEntities(networkSlugs = [], channelSlugs = []) {
|
||||
async function flushEntities(networkSlugs = [], channelSlugs = [], flushOrphanedMedia) {
|
||||
const { sceneIds, movieIds, serieIds } = await fetchEntityReleaseIds(networkSlugs, channelSlugs);
|
||||
const entitySlugs = networkSlugs.concat(channelSlugs).join(', ');
|
||||
|
||||
const entityQuery = knex
|
||||
.withRecursive('selected_entities', knex.raw(`
|
||||
SELECT entities.*
|
||||
FROM entities
|
||||
WHERE
|
||||
entities.slug = ANY(:networkSlugs)
|
||||
AND entities.type = 'network'
|
||||
OR (entities.slug = ANY(:channelSlugs)
|
||||
AND entities.type = 'channel')
|
||||
UNION ALL
|
||||
SELECT entities.*
|
||||
FROM entities
|
||||
INNER JOIN selected_entities ON selected_entities.id = entities.parent_id
|
||||
`, {
|
||||
networkSlugs,
|
||||
channelSlugs,
|
||||
}));
|
||||
|
||||
const sceneIds = await entityQuery
|
||||
.clone()
|
||||
.select('releases.id')
|
||||
.distinct('releases.id')
|
||||
.from('selected_entities')
|
||||
.leftJoin('releases', 'releases.entity_id', 'selected_entities.id')
|
||||
.whereNotNull('releases.id')
|
||||
.modify((builder) => {
|
||||
if (argv.flushAfter) {
|
||||
builder.where('effective_date', '>=', argv.flushAfter);
|
||||
}
|
||||
|
||||
if (argv.flushBefore) {
|
||||
builder.where('effective_date', '<=', argv.flushBefore);
|
||||
}
|
||||
})
|
||||
.pluck('releases.id');
|
||||
|
||||
const movieIds = await entityQuery
|
||||
.clone()
|
||||
.select('movies.id')
|
||||
.distinct('movies.id')
|
||||
.from('selected_entities')
|
||||
.leftJoin('movies', 'movies.entity_id', 'selected_entities.id')
|
||||
.whereNotNull('movies.id')
|
||||
.modify((builder) => {
|
||||
if (argv.flushAfter) {
|
||||
builder.where('effective_date', '>=', argv.flushAfter);
|
||||
}
|
||||
|
||||
if (argv.flushBefore) {
|
||||
builder.where('effective_date', '<=', argv.flushBefore);
|
||||
}
|
||||
})
|
||||
.pluck('movies.id');
|
||||
|
||||
const serieIds = await entityQuery
|
||||
.clone()
|
||||
.select('series.id')
|
||||
.distinct('series.id')
|
||||
.from('selected_entities')
|
||||
.leftJoin('series', 'series.entity_id', 'selected_entities.id')
|
||||
.whereNotNull('series.id')
|
||||
.modify((builder) => {
|
||||
if (argv.flushAfter) {
|
||||
builder.where('date', '>=', argv.flushAfter);
|
||||
}
|
||||
|
||||
if (argv.flushBefore) {
|
||||
builder.where('date', '<=', argv.flushBefore);
|
||||
}
|
||||
})
|
||||
.pluck('series.id');
|
||||
|
||||
if (sceneIds.length === 0 && movieIds.length === 0 && serieIds.length === 0) {
|
||||
logger.info(`No scenes, movies or series found to remove for ${entitySlugs}`);
|
||||
return;
|
||||
@@ -479,6 +414,7 @@ module.exports = {
|
||||
fetchIncludedEntities,
|
||||
fetchReleaseEntities,
|
||||
fetchEntitiesBySlug,
|
||||
fetchEntityReleaseIds,
|
||||
fetchEntity,
|
||||
fetchEntities,
|
||||
getRecursiveParent,
|
||||
|
||||
88
src/entity-releases.js
Normal file
88
src/entity-releases.js
Normal file
@@ -0,0 +1,88 @@
|
||||
'use strict';
|
||||
|
||||
const knex = require('./knex');
|
||||
const argv = require('./argv');
|
||||
|
||||
async function fetchEntityReleaseIds(networkSlugs = [], channelSlugs = []) {
|
||||
const entityQuery = knex
|
||||
.withRecursive('selected_entities', knex.raw(`
|
||||
SELECT entities.*
|
||||
FROM entities
|
||||
WHERE
|
||||
entities.slug = ANY(:networkSlugs)
|
||||
AND entities.type = 'network'
|
||||
OR (entities.slug = ANY(:channelSlugs)
|
||||
AND entities.type = 'channel')
|
||||
UNION ALL
|
||||
SELECT entities.*
|
||||
FROM entities
|
||||
INNER JOIN selected_entities ON selected_entities.id = entities.parent_id
|
||||
`, {
|
||||
networkSlugs,
|
||||
channelSlugs,
|
||||
}));
|
||||
|
||||
const sceneIds = await entityQuery
|
||||
.clone()
|
||||
.select('releases.id')
|
||||
.distinct('releases.id')
|
||||
.from('selected_entities')
|
||||
.leftJoin('releases', 'releases.entity_id', 'selected_entities.id')
|
||||
.whereNotNull('releases.id')
|
||||
.modify((builder) => {
|
||||
if (argv.flushAfter) {
|
||||
builder.where('effective_date', '>=', argv.flushAfter);
|
||||
}
|
||||
|
||||
if (argv.flushBefore) {
|
||||
builder.where('effective_date', '<=', argv.flushBefore);
|
||||
}
|
||||
})
|
||||
.pluck('releases.id');
|
||||
|
||||
const movieIds = await entityQuery
|
||||
.clone()
|
||||
.select('movies.id')
|
||||
.distinct('movies.id')
|
||||
.from('selected_entities')
|
||||
.leftJoin('movies', 'movies.entity_id', 'selected_entities.id')
|
||||
.whereNotNull('movies.id')
|
||||
.modify((builder) => {
|
||||
if (argv.flushAfter) {
|
||||
builder.where('effective_date', '>=', argv.flushAfter);
|
||||
}
|
||||
|
||||
if (argv.flushBefore) {
|
||||
builder.where('effective_date', '<=', argv.flushBefore);
|
||||
}
|
||||
})
|
||||
.pluck('movies.id');
|
||||
|
||||
const serieIds = await entityQuery
|
||||
.clone()
|
||||
.select('series.id')
|
||||
.distinct('series.id')
|
||||
.from('selected_entities')
|
||||
.leftJoin('series', 'series.entity_id', 'selected_entities.id')
|
||||
.whereNotNull('series.id')
|
||||
.modify((builder) => {
|
||||
if (argv.flushAfter) {
|
||||
builder.where('date', '>=', argv.flushAfter);
|
||||
}
|
||||
|
||||
if (argv.flushBefore) {
|
||||
builder.where('date', '<=', argv.flushBefore);
|
||||
}
|
||||
})
|
||||
.pluck('series.id');
|
||||
|
||||
return {
|
||||
sceneIds,
|
||||
movieIds,
|
||||
serieIds,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchEntityReleaseIds,
|
||||
};
|
||||
22
src/knex.js
22
src/knex.js
@@ -3,7 +3,7 @@
|
||||
const config = require('config');
|
||||
const knex = require('knex');
|
||||
|
||||
module.exports = knex({
|
||||
const knexInstance = knex({
|
||||
client: 'pg',
|
||||
connection: config.database.owner,
|
||||
pool: config.database.pool,
|
||||
@@ -11,3 +11,23 @@ module.exports = knex({
|
||||
asyncStackTraces: process.env.NODE_ENV === 'development',
|
||||
// debug: process.env.NODE_ENV === 'development',
|
||||
});
|
||||
|
||||
knexInstance.on('query', function onQuery(query) {
|
||||
const bindingCount = query.bindings?.length ?? 0;
|
||||
|
||||
if (bindingCount > 10000) {
|
||||
const error = new Error(`[knex] Dangerous query: ${bindingCount} bindings detected: ${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`);
|
||||
|
||||
Error.captureStackTrace(error, onQuery);
|
||||
// console.error(error);
|
||||
|
||||
throw error; // optionally hard-fail so you get a real stack trace
|
||||
}
|
||||
});
|
||||
|
||||
knexInstance.on('query-error', (error, query) => {
|
||||
error.knexSql = `${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`;
|
||||
error.knexBindingCount = query.bindings?.length;
|
||||
});
|
||||
|
||||
module.exports = knexInstance;
|
||||
|
||||
@@ -9,7 +9,7 @@ require('winston-daily-rotate-file');
|
||||
const args = require('./argv');
|
||||
|
||||
function logger(filepath) {
|
||||
const root = filepath.match(/src[/\\]|dist[/\\]/);
|
||||
const root = filepath.match(/src[/\\]|dist[/\\]|tests[/\\]/);
|
||||
const filename = filepath.slice(root.index + root[0].length)
|
||||
.replace(path.extname(filepath), '');
|
||||
|
||||
|
||||
30
src/media.js
30
src/media.js
@@ -26,6 +26,7 @@ const http = require('./utils/http');
|
||||
const bulkInsert = require('./utils/bulk-insert');
|
||||
const chunk = require('./utils/chunk');
|
||||
const { get } = require('./utils/qu');
|
||||
const { fetchEntityReleaseIds } = require('./entity-releases');
|
||||
|
||||
// const pipeline = util.promisify(stream.pipeline);
|
||||
const streamQueue = taskQueue();
|
||||
@@ -646,6 +647,7 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
|
||||
const res = await http.get(source.src, {
|
||||
limits: 'media',
|
||||
headers: {
|
||||
host: new URL(source.src).hostname,
|
||||
...(source.referer && { referer: source.referer }),
|
||||
...(source.host && { host: source.host }),
|
||||
},
|
||||
@@ -1159,8 +1161,36 @@ async function flushOrphanedMedia(stage = 1) {
|
||||
}
|
||||
}
|
||||
|
||||
async function detachReleaseMedia(rawSceneIds) {
|
||||
const sceneIds = rawSceneIds.map((sceneId) => Number(sceneId)).filter(Boolean);
|
||||
|
||||
await argv.detachMediaDomains.reduce(async (chain, domain) => {
|
||||
await chain;
|
||||
|
||||
const mediaEntries = await knex(`releases_${domain}`).whereIn('release_id', sceneIds);
|
||||
|
||||
await knex(`releases_${domain}`)
|
||||
.whereIn('release_id', sceneIds)
|
||||
.delete();
|
||||
|
||||
logger.info(`Removed ${mediaEntries.length} ${domain} from ${new Set(mediaEntries.map((mediaEntry) => mediaEntry.release_id)).size} scenes`);
|
||||
}, Promise.resolve());
|
||||
|
||||
if (argv.flushOrphanedMedia !== false) {
|
||||
await flushOrphanedMedia();
|
||||
}
|
||||
}
|
||||
|
||||
async function detachEntityReleaseMedia(networkSlugs = [], channelSlugs = []) {
|
||||
const { sceneIds } = await fetchEntityReleaseIds(networkSlugs, channelSlugs);
|
||||
|
||||
await detachReleaseMedia(sceneIds);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
associateAvatars,
|
||||
associateReleaseMedia,
|
||||
flushOrphanedMedia,
|
||||
detachReleaseMedia,
|
||||
detachEntityReleaseMedia,
|
||||
};
|
||||
|
||||
@@ -148,6 +148,8 @@ module.exports = {
|
||||
interracialpovs: hush,
|
||||
povpornstars: hush,
|
||||
seehimfuck: hush,
|
||||
ravebunnys: hush,
|
||||
hotandtatted: hush,
|
||||
// wankzvr
|
||||
wankzvr,
|
||||
tranzvr: wankzvr,
|
||||
|
||||
@@ -57,7 +57,7 @@ function getCovers(images, target = 'cover') {
|
||||
}
|
||||
|
||||
function getVideos(data) {
|
||||
const teaserSources = data.videos.mediabook?.files;
|
||||
const teaserSources = data.videos?.mediabook?.files;
|
||||
const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
|
||||
|
||||
const teaser = teaserSources && Object.values(teaserSources).map((source) => ({
|
||||
@@ -84,7 +84,7 @@ function scrapeLatestX(data, site, filterChannel, options) {
|
||||
|
||||
release.url = `${basepath}/${data.id}/${slugify(release.title)}`; // spartanId doesn't work in URLs
|
||||
release.date = new Date(data.dateReleased);
|
||||
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
|
||||
release.duration = data.videos?.mediabook?.length > 1 ? data.videos.mediabook.length : null;
|
||||
|
||||
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
|
||||
release.tags = data.tags.map((tag) => tag.name);
|
||||
@@ -127,6 +127,10 @@ async function scrapeLatest(items, site, filterChannel, options) {
|
||||
}
|
||||
|
||||
function scrapeRelease(data, url, channel, networkName, options) {
|
||||
if (Array.isArray(data)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const release = {};
|
||||
|
||||
const { title, description } = data;
|
||||
@@ -136,7 +140,7 @@ function scrapeRelease(data, url, channel, networkName, options) {
|
||||
release.description = description;
|
||||
|
||||
release.date = new Date(data.dateReleased);
|
||||
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
|
||||
release.duration = data.videos?.mediabook?.length > 1 ? data.videos.mediabook.length : null;
|
||||
|
||||
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
|
||||
release.tags = data.tags.map((tag) => tag.name);
|
||||
@@ -144,7 +148,6 @@ function scrapeRelease(data, url, channel, networkName, options) {
|
||||
[release.poster, ...release.photos] = getThumbs(data).map((src) => ({
|
||||
src,
|
||||
referer: url,
|
||||
host: 'mediavault-private-fl.project1content.com',
|
||||
}));
|
||||
|
||||
const { teaser, trailer } = getVideos(data);
|
||||
@@ -270,7 +273,7 @@ async function fetchLatest(site, page = 1, options) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { instanceToken } = options.beforeNetwork?.instanceToken
|
||||
const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
|
||||
? options.beforeNetwork
|
||||
: await getSession(site, options.parameters, url);
|
||||
|
||||
|
||||
@@ -71,10 +71,32 @@ function scrapeScene(data, channel, parameters) {
|
||||
return release;
|
||||
}
|
||||
|
||||
async function getBlockId(slug, dataSource, entity, parameters) {
|
||||
const res = await unprint.get(`https://azianistudios.com/tour_api.php/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
|
||||
headers: {
|
||||
Referer: entity.url,
|
||||
'x-nats-cms-area-id': parameters.areaId,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.data.success) {
|
||||
// unsure how the blocks differ exactly, but type set_view is missing directors for some reason
|
||||
return res.data.blocks?.find((block) => ['navigation', 'html'].includes(block.settings.type))?.cms_block_id || null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1, { parameters }) {
|
||||
const blockId = await getBlockId(parameters.videos || '/videos', { page }, channel, parameters);
|
||||
|
||||
if (!blockId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const query = new URLSearchParams({
|
||||
cms_area_id: parameters.areaId,
|
||||
cms_block_id: parameters.blockId,
|
||||
cms_block_id: blockId,
|
||||
count: 100,
|
||||
start: (page - 1) * 100,
|
||||
orderby: 'published_desc',
|
||||
@@ -106,10 +128,20 @@ async function fetchLatest(channel, page = 1, { parameters }) {
|
||||
async function fetchScene(url, entity, _baseRelease, { parameters }) {
|
||||
const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
|
||||
|
||||
if (!entryId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const blockId = await getBlockId('/video/:id', entryId, entity, parameters);
|
||||
|
||||
if (!blockId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const query = new URLSearchParams({
|
||||
cms_set_ids: entryId,
|
||||
cms_area_id: parameters.areaId,
|
||||
cms_block_id: parameters.blockId,
|
||||
cms_block_id: blockId,
|
||||
content: 1,
|
||||
orderby: 'published_desc',
|
||||
content_type: 'video',
|
||||
@@ -172,7 +204,7 @@ function scrapeProfile(data, entity, parameters) {
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile({ url }, { entity, parameters }) {
|
||||
async function fetchProfile({ name, url }, { entity, parameters }) {
|
||||
if (!url) {
|
||||
// no easy search option
|
||||
return null;
|
||||
@@ -180,9 +212,19 @@ async function fetchProfile({ url }, { entity, parameters }) {
|
||||
|
||||
const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
|
||||
|
||||
if (!actorId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const blockId = await getBlockId('/model/:id', { models: name, id: actorId }, entity, parameters);
|
||||
|
||||
if (!blockId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const query = new URLSearchParams({
|
||||
cms_data_value_ids: actorId,
|
||||
cms_block_id: entity.parameters.modelBlockId || entity.parameters.blockId,
|
||||
cms_block_id: blockId,
|
||||
cms_data_type_id: 4,
|
||||
}).toString();
|
||||
|
||||
|
||||
@@ -41,7 +41,12 @@ function getAvatarFallback(url) {
|
||||
return [
|
||||
`${origin}${pathname}`,
|
||||
url,
|
||||
];
|
||||
].map((src) => ({
|
||||
src,
|
||||
expectType: {
|
||||
'application/octet-stream': 'image/jpeg',
|
||||
},
|
||||
}));
|
||||
} catch (_error) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
'use strict';
|
||||
|
||||
const { ex } = require('../utils/q');
|
||||
const http = require('../utils/http');
|
||||
const unprint = require('unprint');
|
||||
|
||||
function scrapeProfile(html) {
|
||||
const { qu } = ex(html); /* eslint-disable-line object-curly-newline */
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function scrapeProfile({ query }) {
|
||||
const profile = {};
|
||||
|
||||
const bio = qu.all('.infobox tr[valign="top"]')
|
||||
.map((detail) => qu.all(detail, 'td', true))
|
||||
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
|
||||
const bio = query.all('.infobox tr[valign="top"]')
|
||||
.map((detail) => unprint.query.contents(detail, 'td'))
|
||||
.reduce((acc, [key, value]) => ({ ...acc, [slugify(key, '_')]: value }), {});
|
||||
|
||||
/* unreliable, see: Syren De Mer
|
||||
const catlinks = qa('#mw-normal-catlinks a', true);
|
||||
@@ -17,71 +17,51 @@ function scrapeProfile(html) {
|
||||
profile.gender = isTrans ? 'transsexual' : 'female';
|
||||
*/
|
||||
|
||||
profile.birthdate = qu.date('.bday', 'YYYY-MM-DD');
|
||||
profile.dateOfBirth = query.date('.bday', 'YYYY-MM-DD');
|
||||
|
||||
profile.description = qu.q('#mw-content-text > p', true);
|
||||
profile.description = query.content('.mw-parser-output > p');
|
||||
|
||||
if (bio.Born) profile.birthPlace = bio.Born.slice(bio.Born.lastIndexOf(')') + 1);
|
||||
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
|
||||
profile.placeOfBirth = bio.born;
|
||||
profile.ethnicity = bio.ethnicity;
|
||||
profile.nationality = bio.nationality.split(',')[0];
|
||||
|
||||
if (bio.Measurements) {
|
||||
const measurements = bio.Measurements
|
||||
.match(/\d+(\w+)?-\d+-\d+/g)
|
||||
?.slice(-1)[0] // allow for both '34C-25-36' and '86-64-94 cm / 34-25-37 in'
|
||||
.split('-');
|
||||
profile.measurements = bio.measurements;
|
||||
|
||||
// account for measuemrents being just e.g. '32EE'
|
||||
if (measurements) {
|
||||
const [bust, waist, hip] = measurements;
|
||||
|
||||
if (/[a-zA-Z]/.test(bust)) profile.bust = bust; // only use bust if cup size is included
|
||||
|
||||
profile.waist = Number(waist);
|
||||
profile.hip = Number(hip);
|
||||
}
|
||||
|
||||
if (/^\d+\w+$/.test(bio.Measurements)) profile.bust = bio.Measurements;
|
||||
if (bio.bra_cup_size) {
|
||||
[profile.bust] = bio.bra_cup_size.match(/^\d+\w+/) || [];
|
||||
}
|
||||
|
||||
if (bio.Bra_cup_size) {
|
||||
const bust = bio.Bra_cup_size.match(/^\d+\w+/);
|
||||
if (bust) [profile.bust] = bust;
|
||||
}
|
||||
if (/enhanced/i.test(bio.boobs)) profile.naturalBoobs = false;
|
||||
if (/natural/i.test(bio.boobs)) profile.naturalBoobs = true;
|
||||
|
||||
if (bio.Boobs === 'Enhanced') profile.naturalBoobs = false;
|
||||
if (bio.Boobs === 'Natural') profile.naturalBoobs = true;
|
||||
if (bio.height) profile.height = Number(bio.height.match(/\d+\.\d+/g).slice(-1)[0]) * 100;
|
||||
if (bio.weight) profile.weight = Number(bio.weight.match(/\d+/g)[1]);
|
||||
|
||||
if (bio.Height) profile.height = Number(bio.Height.match(/\d+\.\d+/g).slice(-1)[0]) * 100;
|
||||
if (bio.Weight) profile.weight = Number(bio.Weight.match(/\d+/g)[1]);
|
||||
if (bio.eye_color) profile.eyes = bio.eye_color;
|
||||
if (bio.hair) [profile.hairColor] = bio.hair.split(/(?=[A-Z])/); // field concatted to BrunetteLong
|
||||
|
||||
if (bio.Eye_color) profile.eyes = bio.Eye_color;
|
||||
if (bio.Hair) [profile.hair] = bio.Hair.split(',');
|
||||
if (bio.blood_group) profile.blood = bio.blood_group;
|
||||
if (bio.also_known_as) profile.aliases = bio.also_known_as.split(', ');
|
||||
|
||||
if (bio.Blood_group) profile.blood = bio.Blood_group;
|
||||
if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', ');
|
||||
|
||||
const avatarThumbPath = qu.img('.image img');
|
||||
const avatarThumbPath = query.img('.infobox .mw-file-description img');
|
||||
|
||||
if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) {
|
||||
const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', '');
|
||||
|
||||
profile.avatar = {
|
||||
src: `http://www.boobpedia.com${avatarPath}`,
|
||||
credit: null,
|
||||
};
|
||||
profile.avatar = unprint.prefixUrl(avatarPath, 'https://www.boobpedia.com');
|
||||
}
|
||||
|
||||
profile.social = qu.urls('.infobox a.external');
|
||||
profile.socials = query.urls('.infobox a.external');
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }) {
|
||||
const actorSlug = actorName.replace(/\s+/, '_');
|
||||
const res = await http.get(`http://www.boobpedia.com/boobs/${actorSlug}`);
|
||||
const actorSlug = slugify(actorName, '_', { lower: false });
|
||||
const res = await unprint.get(`https://www.boobpedia.com/boobs/${actorSlug}`);
|
||||
|
||||
if (res.statusCode === 200) {
|
||||
return scrapeProfile(res.body.toString());
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.context);
|
||||
}
|
||||
|
||||
return null;
|
||||
|
||||
@@ -81,7 +81,7 @@ function scrapeProfile({ query }, _entity) {
|
||||
|
||||
const bio = Object.fromEntries(keys.map((key, index) => [slugify(key, '_'), values[index]]));
|
||||
|
||||
profile.age = Number(bio.age) || null;
|
||||
profile.age = unprint.extractNumber(bio.age);
|
||||
profile.height = convert(bio.height, 'cm');
|
||||
profile.measurements = bio.measurements;
|
||||
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const {
|
||||
fetchLatest,
|
||||
fetchApiLatest,
|
||||
fetchUpcoming,
|
||||
fetchApiUpcoming,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
fetchApiProfile,
|
||||
scrapeAll,
|
||||
} = require('./gamma');
|
||||
|
||||
const { get } = require('../utils/qu');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function extractLowArtActors(release) {
|
||||
const actors = release.title
|
||||
.replace(/solo/i, '')
|
||||
.split(/,|\band\b/ig)
|
||||
.map((actor) => actor.trim());
|
||||
|
||||
return {
|
||||
...release,
|
||||
actors,
|
||||
};
|
||||
}
|
||||
|
||||
async function networkFetchLatest(site, page = 1) {
|
||||
if (site.parameters?.api) return fetchApiLatest(site, page, false);
|
||||
|
||||
const releases = await fetchLatest(site, page);
|
||||
|
||||
if (site.slug === 'lowartfilms') {
|
||||
return releases.map((release) => extractLowArtActors(release));
|
||||
}
|
||||
|
||||
return releases;
|
||||
}
|
||||
|
||||
async function networkFetchScene(url, site) {
|
||||
const release = await fetchScene(url, site);
|
||||
|
||||
if (site.slug === 'lowartfilms') {
|
||||
return extractLowArtActors(release);
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function networkFetchUpcoming(site, page = 1) {
|
||||
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
|
||||
|
||||
return fetchUpcoming(site, page);
|
||||
}
|
||||
|
||||
function getActorReleasesUrl(actorPath, page = 1) {
|
||||
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
|
||||
}
|
||||
|
||||
function scrapeClassicProfile({ qu, html }, site) {
|
||||
const profile = {};
|
||||
|
||||
profile.avatar = qu.img('.actorPicture');
|
||||
profile.releases = scrapeAll(html, null, site.url, false);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchClassicProfile(actorName, { site }) {
|
||||
const actorSlug = slugify(actorName);
|
||||
|
||||
const url = `${site.url}/en/pornstars`;
|
||||
const pornstarsRes = await get(url);
|
||||
|
||||
if (!pornstarsRes.ok) return null;
|
||||
|
||||
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
|
||||
.find((el) => slugify(el.textContent) === actorSlug)
|
||||
?.value;
|
||||
|
||||
if (actorPath) {
|
||||
const actorUrl = `${site.url}${actorPath}`;
|
||||
const res = await get(actorUrl);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeClassicProfile(res.item, site);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function networkFetchProfile({ name: actorName }, context, include) {
|
||||
const profile = await ((context.site.parameters?.api && fetchApiProfile(actorName, context, include))
|
||||
|| (context.site.parameters?.classic && include.scenes && fetchClassicProfile(actorName, context, include)) // classic profiles only have scenes, no bio
|
||||
|| fetchProfile({ name: actorName }, context, true, getActorReleasesUrl, include));
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest: networkFetchLatest,
|
||||
fetchProfile: networkFetchProfile,
|
||||
fetchScene: networkFetchScene,
|
||||
fetchUpcoming: networkFetchUpcoming,
|
||||
};
|
||||
@@ -91,11 +91,13 @@ function scrapeProfile({ query }, { url }) {
|
||||
profile.description = [query.content('.model-bio-text, #performer-description'), bio.funfact].filter(Boolean).join(' ');
|
||||
profile.aliases = bio.alias?.split(/[,\n]/).map((alias) => alias.trim());
|
||||
|
||||
profile.age = parseInt(bio.age, 10) || null;
|
||||
profile.age = unprint.extractNumber(bio.age);
|
||||
profile.dateOfBirth = unprint.extractDate(bio.age, 'MM/DD/YYYY');
|
||||
profile.measurements = bio.measurements;
|
||||
profile.height = Number(bio.height?.match(/(\d+)\s*cm/)?.[1]) || convert(bio.height, 'cm');
|
||||
|
||||
console.log(profile);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,20 +1,21 @@
|
||||
'use strict';
|
||||
|
||||
const util = require('util');
|
||||
const unprint = require('unprint');
|
||||
const format = require('template-format');
|
||||
|
||||
const qu = require('../utils/q');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { feetInchesToCm, inchesToCm } = require('../utils/convert');
|
||||
const tryUrls = require('../utils/try-urls');
|
||||
const { convert } = require('../utils/convert');
|
||||
|
||||
function deriveEntryId(release) {
|
||||
if (release.date && release.url) {
|
||||
const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
|
||||
|
||||
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
|
||||
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
|
||||
}
|
||||
|
||||
if (release.date && release.title) {
|
||||
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
|
||||
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
|
||||
}
|
||||
|
||||
return null;
|
||||
@@ -22,7 +23,7 @@ function deriveEntryId(release) {
|
||||
|
||||
function extractPoster(posterPath, site, baseRelease) {
|
||||
if (posterPath && !/400.jpg/.test(posterPath)) {
|
||||
const poster = `${site.parameters?.media || site.url}${posterPath}`;
|
||||
const poster = unprint.prefixUrl(posterPath, site.parameters?.media || site.url);
|
||||
const posterSources = [
|
||||
poster,
|
||||
// upscaled
|
||||
@@ -40,38 +41,38 @@ function extractPoster(posterPath, site, baseRelease) {
|
||||
return [baseRelease?.poster || null, []];
|
||||
}
|
||||
|
||||
function getImageWithFallbacks(q, selector, site, el) {
|
||||
function getImageWithFallbacks(query, selector, site, el) {
|
||||
const sources = el
|
||||
? [
|
||||
q(el, selector, 'src0_3x'),
|
||||
q(el, selector, 'src0_2x'),
|
||||
q(el, selector, 'src0_1x'),
|
||||
unprint.query.attribute(el, selector, 'src0_3x'),
|
||||
unprint.query.attribute(el, selector, 'src0_2x'),
|
||||
unprint.query.attribute(el, selector, 'src0_1x'),
|
||||
]
|
||||
: [
|
||||
q(selector, 'src0_3x'),
|
||||
q(selector, 'src0_2x'),
|
||||
q(selector, 'src0_1x'),
|
||||
query.attribute(selector, 'src0_3x'),
|
||||
query.attribute(selector, 'src0_2x'),
|
||||
query.attribute(selector, 'src0_1x'),
|
||||
];
|
||||
|
||||
return sources.filter(Boolean).map((src) => `${site.parameters?.media || site.url}${src}`);
|
||||
return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url));
|
||||
}
|
||||
|
||||
function scrapeAll(scenes, channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.title = query.q('h4 a', true);
|
||||
release.title = query.content('h4 a');
|
||||
release.url = query.url('a');
|
||||
|
||||
release.date = query.date('.date', 'YYYY-MM-DD');
|
||||
release.duration = query.duration('.time');
|
||||
|
||||
const count = query.number('a img', null, 'cnt');
|
||||
const count = query.number('a img', { attribute: 'cnt' });
|
||||
|
||||
[release.poster, ...release.photos] = Array.from({ length: count }, (value, index) => [
|
||||
query.img('a img', `src${index}_3x`, { origin: channel.url }),
|
||||
query.img('a img', `src${index}_2x`, { origin: channel.url }),
|
||||
query.img('a img', `src${index}_1x`, { origin: channel.url }),
|
||||
[release.poster, ...release.photos] = Array.from({ length: count }, (_value, index) => [
|
||||
query.img('a img', { attribute: `src${index}_3x`, origin: channel.url }),
|
||||
query.img('a img', { attribute: `src${index}_2x`, origin: channel.url }),
|
||||
query.img('a img', { attribute: `src${index}_1x`, origin: channel.url }),
|
||||
]);
|
||||
|
||||
release.stars = query.count('img[src*="star_full"]') + (query.count('img[src*="star_half"]') * 0.5);
|
||||
@@ -85,18 +86,18 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
|
||||
release.title = query.attribute('h4 a', 'title') || query.content('h4 a');
|
||||
release.url = query.url('h4 a');
|
||||
|
||||
release.date = query.date('.more-info-div', 'MMM D, YYYY');
|
||||
release.duration = query.dur('.more-info-div');
|
||||
release.duration = query.duration('.more-info-div');
|
||||
|
||||
if (/bts|behind the scenes/i.test(release.title)) release.tags = ['behind the scenes'];
|
||||
|
||||
const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
|
||||
const posterPath = query.attribute('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
|
||||
|
||||
if (posterPath) {
|
||||
const poster = /^http/.test(posterPath) ? posterPath : `${site.parameters?.media || site.url}${posterPath}`;
|
||||
const poster = unprint.prefixUrl(posterPath, site.parameters?.media || site.url);
|
||||
|
||||
release.poster = [
|
||||
poster.replace('-1x', '-3x'),
|
||||
@@ -117,19 +118,40 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1, _include, { uniqueReleases = [], duplicateReleases = [] }) {
|
||||
const url = (site.parameters?.latest && format(site.parameters.latest, { page }))
|
||||
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|
||||
|| `${site.url}/categories/movies_${page}_d.html`;
|
||||
|
||||
const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .updateItem' });
|
||||
|
||||
if (!res.ok) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
if (site.parameters?.t1) {
|
||||
return scrapeAllT1(res.context, site, [...uniqueReleases, ...duplicateReleases]);
|
||||
}
|
||||
|
||||
return scrapeAll(res.context, site, uniqueReleases);
|
||||
}
|
||||
|
||||
function scrapeScene({ html, query }, channel, url) {
|
||||
const release = { url }; // url used for entry ID
|
||||
|
||||
release.title = query.cnt('.videoDetails h3');
|
||||
release.description = query.cnt('.videoDetails p');
|
||||
release.title = query.content('.videoDetails h3, .videoDetails h1');
|
||||
release.description = query.content('.videoDetails p');
|
||||
|
||||
release.date = query.date('.videoInfo p', ['MM/DD/YYYY', 'YYYY-MM-DD']);
|
||||
release.duration = Number(query.cnt('.videoInfo p:nth-of-type(2)')?.match(/(\d+) min/i)?.[1]) * 60;
|
||||
release.duration = Number(query.content('.videoInfo p:nth-of-type(2)')?.match(/(\d+) min/i)?.[1]) * 60;
|
||||
|
||||
release.actors = query.cnts('.update_models a');
|
||||
release.actors = query.all('.update_models a').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: channel.origin }),
|
||||
}));
|
||||
|
||||
const posterPath = html.match(/poster="([\w-/.]+)"/)?.[1];
|
||||
const poster = qu.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
|
||||
const poster = unprint.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
|
||||
|
||||
[release.poster, ...release.photos] = [poster, ...query.imgs('.item-thumb img', 'src0_1x', { origin: channel.url })]
|
||||
.map((src) => src && [
|
||||
@@ -141,10 +163,10 @@ function scrapeScene({ html, query }, channel, url) {
|
||||
const trailerPath = html.match(/\/trailers?\/.*.mp4/);
|
||||
|
||||
if (trailerPath) {
|
||||
release.trailer = qu.prefixUrl(trailerPath, channel.parameters?.media || channel.url);
|
||||
release.trailer = unprint.prefixUrl(trailerPath, channel.parameters?.media || channel.url);
|
||||
}
|
||||
|
||||
release.tags = query.cnts('.featuring a[href*="categories/"]');
|
||||
release.tags = query.contents('.featuring a[href*="categories/"]');
|
||||
release.stars = query.count('.stars img[src*="star_full"]') + (query.count('.stars img[src*="star_half"]') * 0.5);
|
||||
|
||||
release.entryId = deriveEntryId(release);
|
||||
@@ -155,29 +177,34 @@ function scrapeScene({ html, query }, channel, url) {
|
||||
function scrapeSceneT1({ html, query }, site, url, baseRelease) {
|
||||
const release = { url };
|
||||
|
||||
release.title = query.q('.trailer-section-head .section-title', true);
|
||||
release.title = query.content('.trailer-section-head .section-title');
|
||||
release.description = query.text('.row .update-info-block');
|
||||
|
||||
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
release.duration = query.dur('.update-info-row:nth-child(2)');
|
||||
release.duration = query.duration('.update-info-row:nth-child(2)');
|
||||
|
||||
release.actors = query.all('.models-list-thumbs a').map((el) => ({
|
||||
name: query.q(el, 'span', true),
|
||||
avatar: getImageWithFallbacks(query.q, 'img', site, el),
|
||||
name: unprint.query.content(el, 'span'),
|
||||
url: unprint.query.url(el, null),
|
||||
avatar: getImageWithFallbacks(query, 'img', site, el),
|
||||
}));
|
||||
|
||||
release.tags = query.all('.tags a', true);
|
||||
release.tags = query.contents('.tags a');
|
||||
|
||||
// const posterPath = html.match(/poster="(.*\.jpg)/)?.[1];
|
||||
const posterPath = query.q('.player-thumb img', 'src0_1x');
|
||||
const posterPath = query.img('.player-thumb img', { attribute: 'src0_1x' });
|
||||
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
|
||||
|
||||
[release.poster, release.photos] = extractPoster(posterPath, site, baseRelease);
|
||||
|
||||
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
|
||||
if (trailer && /^http/.test(trailer)) release.trailer = { src: trailer, referer: url };
|
||||
else if (trailer) release.trailer = { src: `${site.parameters?.media || site.url}${trailer}`, referer: url };
|
||||
if (trailer) {
|
||||
release.trailer = {
|
||||
src: unprint.prefixUrl(trailer, site.parameters?.media || site.url),
|
||||
referer: url,
|
||||
};
|
||||
}
|
||||
|
||||
const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
|
||||
if (stars) release.stars = Number(stars);
|
||||
release.stars = query.number('.update-rating');
|
||||
|
||||
if (site.type === 'network') {
|
||||
const channelRegExp = new RegExp(site.children.map((channel) => channel.parameters?.match || channel.name).join('|'), 'i');
|
||||
@@ -188,16 +215,99 @@ function scrapeSceneT1({ html, query }, site, url, baseRelease) {
|
||||
}
|
||||
}
|
||||
|
||||
// release.entryId = q('.player-thumb img', 'id')?.match(/set-target-(\d+)/)[1];
|
||||
release.entryId = deriveEntryId(release);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfileT1({ el, query }, site) {
|
||||
const profile = {};
|
||||
async function fetchScene(url, site, baseRelease) {
|
||||
const res = await unprint.get(url);
|
||||
|
||||
const bio = query.all('.detail-div + .detail-div p, .detail-div p', true).reduce((acc, info) => {
|
||||
if (!res.ok) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
if (site.parameters?.t1) {
|
||||
return scrapeSceneT1(res.context, site, url, baseRelease);
|
||||
}
|
||||
|
||||
return scrapeScene(res.context, site, url, baseRelease);
|
||||
}
|
||||
|
||||
async function fetchActorScenes({ query }, channel, accScenes = []) {
|
||||
const scenes = scrapeAll(unprint.initAll(query.all('.item-video')), channel);
|
||||
const nextPage = query.url('.next a');
|
||||
|
||||
if (nextPage) {
|
||||
const res = await unprint.get(nextPage);
|
||||
|
||||
if (res.ok) {
|
||||
return fetchActorScenes(res.context, channel, scenes.concat(accScenes));
|
||||
}
|
||||
}
|
||||
|
||||
return accScenes.concat(scenes);
|
||||
}
|
||||
|
||||
async function scrapeProfile({ query }, url, channel, options) {
|
||||
const profile = { url };
|
||||
|
||||
const bio = query.all('.stats li').reduce((acc, bioEl) => {
|
||||
const key = unprint.query.content(bioEl, 'strong');
|
||||
const value = unprint.query.url(bioEl, null) || unprint.query.text(bioEl);
|
||||
|
||||
return {
|
||||
...acc,
|
||||
[slugify(key, '_')]: value,
|
||||
};
|
||||
}, {});
|
||||
|
||||
if (bio.date_of_birth) profile.dateOfBirth = unprint.extractDate(bio.date_of_birth, 'MMMM D, YYYY');
|
||||
if (bio.birthplace) profile.birthPlace = bio.birthplace;
|
||||
if (bio.fun_fact) profile.description = bio.fun_fact;
|
||||
|
||||
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
|
||||
|
||||
if (bio.height) profile.height = convert(bio.height, 'cm');
|
||||
if (bio.weight) profile.weight = convert(bio.weight.match(/(\d+)\s*lb/i)?.[1], 'lb', 'kg');
|
||||
if (bio.shoe_size && !/unknown/i.test(bio.shoe_size)) profile.foot = unprint.extractNumber(bio.shoe_size);
|
||||
|
||||
profile.measurements = bio.measurements;
|
||||
|
||||
if (bio.penis_length) profile.penisLength = Number(bio.penis_length.match(/(\d+)\s*cm/i)?.[1] || convert(bio.penis_length.match(/(\d+\.?\d+)\s*in/i)?.[1], 'cm')) || null;
|
||||
if (bio.penis_girth) profile.penisGirth = Number(bio.penis_girth.match(/(\d+)\s*cm/i)?.[1] || convert(bio.penis_girth.match(/(\d+\.?\d+)\s*in/i)?.[1], 'cm')) || null;
|
||||
if (bio.circumcised && /yes/i.test(bio.circumcised)) profile.isCircumcised = true;
|
||||
if (bio.circumcised && /no/i.test(bio.circumcised)) profile.isCircumcised = false;
|
||||
|
||||
if (bio.natural_breasts && /yes/i.test(bio.natural_breasts)) profile.naturalBoobs = true;
|
||||
if (bio.natural_breasts && /no/i.test(bio.natural_breasts)) profile.naturalBoobs = false;
|
||||
|
||||
if (bio.tattoos && /(yes)|(some)|(many)/i.test(bio.tattoos)) profile.hasTattoos = true;
|
||||
if (bio.tattoos && /no/i.test(bio.tattoos)) profile.hasTattoos = false;
|
||||
if (bio.piercings && /(yes)|(some)|(many)/i.test(bio.piercings)) profile.hasPiercings = true;
|
||||
if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false;
|
||||
|
||||
if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim()).filter((alias) => !/known/i.test(alias)); // filter out "No known aliases"
|
||||
|
||||
profile.socials = [bio.onlyfans, bio.twitter, bio.instagram, bio.domain].filter(Boolean);
|
||||
|
||||
profile.avatar = [
|
||||
query.img('.profile-pic img', { attribute: 'src0_3x', origin: channel.url }),
|
||||
query.img('.profile-pic img', { attribute: 'src0_2x', origin: channel.url }),
|
||||
query.img('.profile-pic img', { attribute: 'src0_1x', origin: channel.url }),
|
||||
];
|
||||
|
||||
if (options.includeActorScenes) {
|
||||
profile.releases = await fetchActorScenes({ query }, channel);
|
||||
}
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
function scrapeProfileT1({ query }, url, site) {
|
||||
const profile = { url };
|
||||
|
||||
const bio = query.contents('.detail-div + .detail-div p, .detail-div p').reduce((acc, info) => {
|
||||
const [key, value] = info.split(':');
|
||||
|
||||
if (!value) return acc;
|
||||
@@ -215,142 +325,49 @@ function scrapeProfileT1({ el, query }, site) {
|
||||
|
||||
const heightMetric = bio.height?.match(/(\d{3})(\b|c)/);
|
||||
const heightImperial = bio.height?.match(/\d{1}(\.\d)?/g);
|
||||
if (heightMetric) profile.height = Number(heightMetric[1]);
|
||||
if (heightImperial) profile.height = feetInchesToCm(Number(heightImperial[0]), Number(heightImperial[1]));
|
||||
|
||||
profile.avatar = getImageWithFallbacks(query.q, '.img-div img', site);
|
||||
if (heightMetric) {
|
||||
profile.height = Number(heightMetric[1]);
|
||||
}
|
||||
|
||||
const qReleases = qu.initAll(el, '.item-video');
|
||||
if (heightImperial) {
|
||||
profile.height = convert(`${heightImperial[0]}' ${heightImperial[1]}"`, 'cm');
|
||||
}
|
||||
|
||||
profile.avatar = getImageWithFallbacks(query, '.img-div img', site);
|
||||
|
||||
const qReleases = unprint.initAll(query.all('.item-video'));
|
||||
profile.releases = scrapeAllT1(qReleases, site);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchActorScenes({ query, el }, channel, accScenes = []) {
|
||||
const scenes = scrapeAll(qu.initAll(el, '.item-video'), channel);
|
||||
const nextPage = query.url('.next a');
|
||||
|
||||
if (nextPage) {
|
||||
const res = await qu.get(nextPage);
|
||||
|
||||
if (res.ok) {
|
||||
return fetchActorScenes(res.item, channel, scenes.concat(accScenes));
|
||||
}
|
||||
}
|
||||
|
||||
return accScenes.concat(scenes);
|
||||
}
|
||||
|
||||
async function scrapeProfile({ query, el }, channel, options) {
|
||||
const profile = {};
|
||||
|
||||
const bio = query.all('.stats li').reduce((acc, bioEl) => {
|
||||
const key = query.cnt(bioEl, 'strong');
|
||||
const value = query.url(bioEl) || query.text(bioEl);
|
||||
|
||||
return {
|
||||
...acc,
|
||||
[slugify(key, '_')]: value,
|
||||
};
|
||||
}, {});
|
||||
|
||||
if (bio.date_of_birth) profile.dateOfBirth = qu.extractDate(bio.date_of_birth, 'MMMM D, YYYY');
|
||||
if (bio.birthplace) profile.birthPlace = bio.birthplace;
|
||||
if (bio.fun_fact) profile.description = bio.fun_fact;
|
||||
|
||||
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
|
||||
|
||||
if (bio.height) profile.height = Number(bio.height.match(/^\d{2,3}/)?.[0]);
|
||||
if (bio.weight) profile.weight = Number(bio.weight.match(/^\d{2,3}/)?.[0]);
|
||||
if (bio.shoe_size) profile.foot = Number(bio.shoe_size);
|
||||
|
||||
profile.measurements = bio.measurements;
|
||||
|
||||
if (bio.penis_length) profile.penisLength = Number(bio.penis_length.match(/(\d+)\s*cm/i)?.[1] || inchesToCm(bio.penis_length.match(/(\d+\.?\d+)\s*in/i)?.[1])) || null;
|
||||
if (bio.penis_girth) profile.penisGirth = Number(bio.penis_girth.match(/(\d+)\s*cm/i)?.[1] || inchesToCm(bio.penis_girth.match(/(\d+\.?\d+)\s*in/i)?.[1])) || null;
|
||||
if (bio.circumcised && /yes/i.test(bio.circumcised)) profile.isCircumcised = true;
|
||||
if (bio.circumcised && /no/i.test(bio.circumcised)) profile.isCircumcised = false;
|
||||
|
||||
if (bio.natural_breasts && /yes/i.test(bio.natural_breasts)) profile.naturalBoobs = true;
|
||||
if (bio.natural_breasts && /no/i.test(bio.natural_breasts)) profile.naturalBoobs = false;
|
||||
|
||||
if (bio.tattoos && /(yes)|(some)|(many)/i.test(bio.tattoos)) profile.hasTattoos = true;
|
||||
if (bio.tattoos && /no/i.test(bio.tattoos)) profile.hasTattoos = false;
|
||||
if (bio.piercings && /(yes)|(some)|(many)/i.test(bio.piercings)) profile.hasPiercings = true;
|
||||
if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false;
|
||||
|
||||
if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim());
|
||||
|
||||
profile.socials = [bio.onlyfans, bio.twitter, bio.instagram, bio.domain].filter(Boolean);
|
||||
|
||||
profile.avatar = [
|
||||
query.img('.profile-pic img', 'src0_3x', { origin: channel.url }),
|
||||
query.img('.profile-pic img', 'src0_2x', { origin: channel.url }),
|
||||
query.img('.profile-pic img', 'src0_1x', { origin: channel.url }),
|
||||
];
|
||||
|
||||
if (options.includeActorScenes) {
|
||||
profile.releases = await fetchActorScenes({ query, el }, channel);
|
||||
}
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1, include, { uniqueReleases = [], duplicateReleases = [] }) {
|
||||
const url = (site.parameters?.latest && util.format(site.parameters.latest, page))
|
||||
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|
||||
|| `${site.url}/categories/movies_${page}_d.html`;
|
||||
|
||||
const res = await qu.getAll(url, '.modelfeature, .item-video, .updateItem');
|
||||
|
||||
if (!res.ok) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
if (site.parameters?.t1) {
|
||||
return scrapeAllT1(res.items, site, [...uniqueReleases, ...duplicateReleases]);
|
||||
}
|
||||
|
||||
return scrapeAll(res.items, site, uniqueReleases);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site, baseRelease) {
|
||||
const res = await qu.get(url);
|
||||
|
||||
if (!res.ok) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
if (site.parameters?.t1) {
|
||||
return scrapeSceneT1(res.item, site, url, baseRelease);
|
||||
}
|
||||
|
||||
return scrapeScene(res.item, site, url, baseRelease);
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }, { channel }, options) {
|
||||
const actorSlugA = slugify(actorName, '');
|
||||
async function fetchProfile({ name: actorName, url: actorUrl }, { channel }, options) {
|
||||
const actorSlugA = slugify(actorName, '', { lower: false });
|
||||
const actorSlugB = slugify(actorName);
|
||||
|
||||
const t1 = channel.parameters?.t1 ? 't1/' : '';
|
||||
|
||||
const res1 = channel.parameters?.profile
|
||||
? await qu.get(util.format(channel.parameters.profile, actorSlugA))
|
||||
: await qu.get(`${channel.url}/${t1}models/${actorSlugA}.html`, null, null, { followRedirects: false });
|
||||
|
||||
const res = (res1.ok && res1)
|
||||
|| (channel.parameters?.profile && await qu.get(util.format(channel.parameters.profile, actorSlugB)))
|
||||
|| await qu.get(`${channel.url}/${t1}models/${actorSlugB}.html`, null, null, { followRedirects: false });
|
||||
const { res, url } = await tryUrls([
|
||||
actorUrl,
|
||||
...channel.parameters?.profile ? [
|
||||
format(channel.parameters.profile, { actor: actorSlugA }),
|
||||
format(channel.parameters.profile, { actor: actorSlugB }),
|
||||
] : [
|
||||
`${channel.url}/${t1}models/${actorSlugA}.html`,
|
||||
`${channel.url}/${t1}models/${actorSlugB}.html`,
|
||||
],
|
||||
], { followRedirects: false });
|
||||
|
||||
if (!res.ok) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
if (channel.parameters?.t1) {
|
||||
return scrapeProfileT1(res.item, channel);
|
||||
return scrapeProfileT1(res.context, url, channel);
|
||||
}
|
||||
|
||||
return scrapeProfile(res.item, channel, options);
|
||||
return scrapeProfile(res.context, url, channel, options);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
|
||||
const logger = require('../logger');
|
||||
const { fetchApiLatest } = require('./gamma');
|
||||
const qu = require('../utils/qu');
|
||||
const http = require('../utils/http');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
async function fetchActors(entryId, channel, { token, time }) {
|
||||
const url = `${channel.url}/sapi/${token}/${time}/model.getModelContent?_method=model.getModelContent&tz=1&fields[0]=modelId.stageName&fields[1]=_last&fields[2]=modelId.upsellLink&fields[3]=modelId.upsellText&limit=25&transitParameters[contentId]=${entryId}`;
|
||||
const res = await http.get(url);
|
||||
|
||||
if (res.statusCode === 200 && res.body.status === true) {
|
||||
return Object.values(res.body.response.collection).map((actor) => Object.values(actor.modelId.collection)[0].stageName);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function fetchTrailerLocation(entryId, channel) {
|
||||
const url = `${channel.url}/api/download/${entryId}/hd1080/stream`;
|
||||
|
||||
try {
|
||||
const res = await http.get(url, {
|
||||
followRedirects: false,
|
||||
});
|
||||
|
||||
if (res.statusCode === 302) {
|
||||
return res.headers.location;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`${channel.name}: Unable to fetch trailer at '${url}': ${error.message}`);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function scrapeLatest(items, channel) {
|
||||
return items.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('h5 a', null, { origin: channel.url });
|
||||
release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
|
||||
|
||||
release.title = query.cnt('h5 a');
|
||||
|
||||
[release.poster, ...release.photos] = query.imgs('.screenshot').map((src) => [
|
||||
// unnecessarily large
|
||||
// src.replace(/\/\d+/, 3840),
|
||||
// src.replace(/\/\d+/, '/2000'),
|
||||
src.replace(/\/\d+/, '/1500'),
|
||||
src.replace(/\/\d+/, '/1000'),
|
||||
src,
|
||||
]);
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ query, html }, url, channel) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1];
|
||||
|
||||
release.title = query.cnt('h1.description');
|
||||
release.actors = query
|
||||
.all('.video-performer')
|
||||
.map((actorEl) => {
|
||||
const actorUrl = query.url(actorEl, 'a', 'href', { origin: channel.url });
|
||||
const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
|
||||
const avatar = query.img(actorEl, 'img:not([data-bgsrc*="not-available"])', 'data-bgsrc');
|
||||
|
||||
return {
|
||||
name: query.cnt(actorEl, '.video-performer-name'),
|
||||
gender: 'female',
|
||||
avatar: avatar && [
|
||||
avatar.replace(/\/actor\/(\d+)/, '/actor/500'),
|
||||
avatar,
|
||||
],
|
||||
url: actorUrl,
|
||||
entryId,
|
||||
};
|
||||
})
|
||||
.concat({ name: 'Jay Rock', gender: 'male' });
|
||||
|
||||
release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
release.duration = query.number('.release-date:last-child') * 60;
|
||||
|
||||
release.studio = query.cnt('.studio span:nth-child(2)');
|
||||
release.director = query.text('.director');
|
||||
|
||||
release.tags = query.cnts('.tags a');
|
||||
|
||||
const poster = html.match(/url\((https.+\.jpg)\)/)?.[1];
|
||||
const photos = query.imgs('#moreScreenshots img');
|
||||
|
||||
[release.poster, ...release.photos] = [poster]
|
||||
.concat(photos)
|
||||
.filter(Boolean)
|
||||
.map((src) => [
|
||||
src.replace(/\/(\d+)\/\d+/, '/$1/1500'),
|
||||
src.replace(/\/(\d+)\/\d+/, '/$1/1000'),
|
||||
src,
|
||||
]);
|
||||
|
||||
const videoId = html.match(/item: (\d+)/)?.[1];
|
||||
|
||||
if (videoId) {
|
||||
release.trailer = { stream: `https://trailer.adultempire.com/hls/trailer/${videoId}/master.m3u8` };
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function scrapeSceneApi(scene, channel, tokens, deep) {
|
||||
const release = {
|
||||
entryId: scene.id,
|
||||
title: scene.title,
|
||||
duration: scene.length,
|
||||
meta: {
|
||||
tokens, // attach tokens to reduce number of requests required for deep fetching
|
||||
},
|
||||
};
|
||||
|
||||
release.url = `${channel.url}/scene/${release.entryId}/${slugify(release.title, { encode: true })}`;
|
||||
release.date = new Date(scene.sites.collection[scene.id].publishDate);
|
||||
release.poster = scene._resources.primary[0].url;
|
||||
|
||||
if (scene.tags) release.tags = Object.values(scene.tags.collection).map((tag) => tag.alias);
|
||||
if (scene._resources.base) release.photos = scene._resources.base.map((resource) => resource.url);
|
||||
|
||||
if (deep) {
|
||||
// don't make external requests during update scraping, as this would happen for every scene on the page
|
||||
const [actors, trailer] = await Promise.all([
|
||||
fetchActors(release.entryId, channel, tokens),
|
||||
fetchTrailerLocation(release.entryId, channel),
|
||||
]);
|
||||
|
||||
release.actors = actors;
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = { src: trailer, quality: 1080 };
|
||||
}
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeLatestApi(scenes, site, tokens) {
|
||||
return Promise.map(scenes, async (scene) => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
|
||||
}
|
||||
|
||||
async function fetchToken(channel) {
|
||||
const res = await http.get(channel.url);
|
||||
const html = res.body.toString();
|
||||
|
||||
const time = html.match(/"aet":\d+/)[0].split(':')[1];
|
||||
const ah = html.match(/"ah":"[\w-]+"/)[0].split(':')[1].slice(1, -1);
|
||||
const token = ah.split('').reverse().join('');
|
||||
|
||||
return { time, token };
|
||||
}
|
||||
|
||||
async function fetchLatestApi(channel, page = 1) {
|
||||
const { time, token } = await fetchToken(channel);
|
||||
|
||||
// transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory
|
||||
const url = `${channel.url}/sapi/${token}/${time}/content.load?limit=50&offset=${(page - 1) * 50}&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`;
|
||||
const res = await http.get(url);
|
||||
|
||||
if (res.ok && res.body.status) {
|
||||
return scrapeLatestApi(res.body.response.collection, channel, { time, token });
|
||||
}
|
||||
|
||||
return res.ok ? res.body.status : res.status;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1, options, preData) {
|
||||
if (channel.parameters?.useGamma) {
|
||||
return fetchApiLatest(channel, page, preData, options, false);
|
||||
}
|
||||
|
||||
const res = await qu.getAll(`https://jayspov.net/jays-pov-updates.html?view=list&page=${page}`, '.item-grid-list-view > .grid-item');
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatest(res.items, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchSceneApi(url, channel, baseRelease) {
|
||||
const { time, token } = baseRelease?.meta.tokens || await fetchToken(channel); // use attached tokens when deep fetching
|
||||
const { pathname } = new URL(url);
|
||||
const entryId = pathname.split('/')[2];
|
||||
|
||||
const apiUrl = `${channel.url}/sapi/${token}/${time}/content.load?filter[id][fields][0]=id&filter[id][values][0]=${entryId}&transitParameters[v1]=ykYa8ALmUD&transitParameters[preset]=scene`;
|
||||
const res = await http.get(apiUrl);
|
||||
|
||||
if (res.ok && res.body.status) {
|
||||
return scrapeSceneApi(res.body.response.collection[0], channel, { time, token }, true);
|
||||
}
|
||||
|
||||
return res.ok ? res.body.status : res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel) {
|
||||
const res = await qu.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, url, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
api: {
|
||||
fetchLatest: fetchLatestApi,
|
||||
fetchScene: fetchSceneApi,
|
||||
},
|
||||
};
|
||||
@@ -125,12 +125,12 @@ async function fetchScene(sceneUrl, entity, _baseRelease, { parameters }) {
|
||||
function scrapeProfile({ query }, url) {
|
||||
const profile = { url };
|
||||
|
||||
const bio = Object.fromEntries(query.all('ul[class*="HeroModel_list"] li').map((bioEl) => [
|
||||
const bio = Object.fromEntries(query.all('ul[class*=HeroModel][class*="__list"] li').map((bioEl) => [
|
||||
slugify(unprint.query.content(bioEl, 'p:first-child'), '_'),
|
||||
unprint.query.content(bioEl, 'p:last-child'),
|
||||
]));
|
||||
|
||||
profile.description = query.content('p[class*="HeroModel_text"]');
|
||||
profile.description = query.content('p[class*=HeroModel][class*="__text"]');
|
||||
profile.birthPlace = bio.place_of_birth;
|
||||
|
||||
profile.dateOfBirth = unprint.extractDate(bio.date_of_birth, ['MMMM DD, YYYY', 'MM/DD/YYYY']);
|
||||
@@ -163,7 +163,7 @@ function scrapeProfile({ query }, url) {
|
||||
profile.piercings = profile.hasPiercings ? bio.piercings : null;
|
||||
}
|
||||
|
||||
profile.avatar = query.sourceSet('picture[class*="modelImage"] img');
|
||||
profile.avatar = query.sourceSet('picture[class*="__modelImage"] img');
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ const czechav = require('./czechav');
|
||||
const modelmedia = require('./modelmedia');
|
||||
const dorcel = require('./dorcel');
|
||||
const fabulouscash = require('./fabulouscash');
|
||||
// const famedigital = require('./famedigital');
|
||||
const firstanalquest = require('./firstanalquest');
|
||||
const elevatedx = require('./elevatedx');
|
||||
const exploitedx = require('./exploitedx');
|
||||
@@ -31,7 +30,6 @@ const hush = require('./hush');
|
||||
const innofsin = require('./innofsin');
|
||||
const insex = require('./insex');
|
||||
const inthecrack = require('./inthecrack');
|
||||
const jayrock = require('./jayrock');
|
||||
const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials');
|
||||
const julesjordan = require('./julesjordan');
|
||||
const karups = require('./karups');
|
||||
@@ -74,7 +72,6 @@ const tokyohot = require('./tokyohot');
|
||||
// const topwebmodels = require('./topwebmodels');
|
||||
const traxxx = require('./traxxx');
|
||||
const virtualtaboo = require('./virtualtaboo');
|
||||
const vivid = require('./vivid');
|
||||
const vixen = require('./vixen');
|
||||
const wankzvr = require('./wankzvr');
|
||||
const whalemember = require('./whalemember');
|
||||
@@ -138,7 +135,6 @@ module.exports = {
|
||||
insex,
|
||||
interracialpass: hush,
|
||||
inthecrack,
|
||||
jayrock,
|
||||
jerkaoke: modelmedia,
|
||||
jesseloadsmonsterfacials,
|
||||
julesjordan,
|
||||
@@ -180,7 +176,6 @@ module.exports = {
|
||||
traxxx,
|
||||
vipsexvault: porndoe,
|
||||
virtualtaboo,
|
||||
vivid,
|
||||
vixen,
|
||||
wankzvr,
|
||||
westcoastproductions: adultempire,
|
||||
|
||||
@@ -164,7 +164,8 @@ function scrapeScene({ query }, url) {
|
||||
release.tags = query.contents('.p-desc a[href*="tag/"], .desc a[href*="tag/"]');
|
||||
|
||||
const style = query.content('.vp style');
|
||||
const poster = query.img('#videos_page-page .item-img img') || style?.match(/background-image: url\('(http[\w.:/_-]+)'\);/)?.[1];
|
||||
// const poster = query.img('#videos_page-page .item-img img') || style?.match(/background-image: url\('(http[\w.:/_-]+)'\);/)?.[1];
|
||||
const poster = style?.match(/background-image: url\('(http[\w.:/_-]+)'\);/)?.[1];
|
||||
const fallbackPoster = resizeSrc(query.img('meta[itemprop="image"]', { attribute: 'content' })); // usually a different image
|
||||
|
||||
const photos = query.all('.gallery .thumb').map((imgEl) => {
|
||||
|
||||
@@ -103,7 +103,7 @@ function scrapeProfile({ query }, url, entity) {
|
||||
profile.hairColor = bio.hair;
|
||||
profile.eyes = bio.eyes;
|
||||
|
||||
profile.avatar = query.img('.model-profile-image-picture source', { origin: entity.oorigin, attribute: 'srcset' })
|
||||
profile.avatar = query.img('.model-profile-image-picture source', { origin: entity.origin, attribute: 'srcset' })
|
||||
|| query.img('.model-profile-image-picture img', { origin: entity.origin });
|
||||
|
||||
profile.scenes = scrapeAll(unprint.initAll(query.all('.video-list .thumb')), entity);
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
/* eslint-disable no-unused-vars */
|
||||
const { get, ed } = require('../utils/q');
|
||||
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
|
||||
const http = require('../utils/http');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function scrapeLatestNative(scenes, site) {
|
||||
return scenes.map((scene) => {
|
||||
const release = {};
|
||||
|
||||
release.entryId = scene.id;
|
||||
release.url = `${site.url}${scene.url}`;
|
||||
|
||||
release.title = scene.name;
|
||||
release.date = ed(scene.release_date, 'YYYY-MM-DD');
|
||||
release.duration = parseInt(scene.runtime, 10) * 60;
|
||||
|
||||
release.actors = scene.cast?.map((actor) => ({
|
||||
name: actor.stagename,
|
||||
gender: actor.gender.toLowerCase(),
|
||||
avatar: actor.placard,
|
||||
})) || [];
|
||||
|
||||
release.stars = Number(scene.rating);
|
||||
release.poster = scene.placard_800 || scene.placard;
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeSceneNative({ html, q, qa }, url, _site) {
|
||||
const release = { url };
|
||||
|
||||
release.entryId = new URL(url).pathname.split('/')[2]; // eslint-disable-line prefer-destructuring
|
||||
|
||||
release.title = q('.scene-h2-heading', true);
|
||||
release.description = q('.indie-model-p', true);
|
||||
|
||||
const dateString = qa('h5').find((el) => /Released/.test(el.textContent)).textContent;
|
||||
release.date = ed(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
|
||||
const duration = qa('h5').find((el) => /Runtime/.test(el.textContent)).textContent;
|
||||
const [hours, minutes] = duration.match(/\d+/g);
|
||||
|
||||
if (minutes) release.duration = (hours * 3600) + (minutes * 60);
|
||||
else release.duration = hours * 60; // scene shorter that 1hr, hour match are minutes
|
||||
|
||||
release.actors = qa('h4 a[href*="/stars"], h4 a[href*="/celebs"]', true);
|
||||
release.tags = qa('h5 a[href*="/categories"]', true);
|
||||
|
||||
const [poster, trailer] = html.match(/https:\/\/content.vivid.com(.*)(.jpg|.mp4)/g);
|
||||
release.poster = poster;
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = {
|
||||
src: trailer,
|
||||
};
|
||||
}
|
||||
|
||||
const channel = q('h5 a[href*="/sites"]', true);
|
||||
if (channel) release.channel = channel.replace(/\.\w+/, '');
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchLatestNative(site, page = 1) {
|
||||
if (site.parameters?.useGamma) {
|
||||
return fetchApiLatest(site, page);
|
||||
}
|
||||
|
||||
const apiUrl = `${site.url}/videos/api/?limit=50&offset=${(page - 1) * 50}&sort=datedesc`;
|
||||
const res = await http.get(apiUrl, {
|
||||
decodeJSON: true,
|
||||
});
|
||||
|
||||
if (res.statusCode === 200 && res.body.code === 200) {
|
||||
return scrapeLatestNative(res.body.responseData, site);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchUpcomingNative(site) {
|
||||
if (site.parameters?.useGamma) {
|
||||
return fetchApiUpcoming(site);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchSceneNative(url, site, release) {
|
||||
if (site.parameters?.useGamma) {
|
||||
return fetchScene(url, site, release);
|
||||
}
|
||||
|
||||
const res = await get(url);
|
||||
|
||||
return res.ok ? scrapeSceneNative(res.item, url, site) : res.status;
|
||||
}
|
||||
|
||||
async function fetchSceneWrapper(url, site, release) {
|
||||
const scene = await fetchScene(url, site, release);
|
||||
|
||||
if (scene.date - new Date(site.parameters?.lastNative) <= 0) {
|
||||
// scene is probably still available on Vivid site, use search API to get URL and original date
|
||||
const searchUrl = `${site.url}/videos/api/?limit=10&sort=datedesc&search=${encodeURI(scene.title)}`;
|
||||
const searchRes = await http.get(searchUrl, {
|
||||
decodeJSON: true,
|
||||
});
|
||||
|
||||
if (searchRes.statusCode === 200 && searchRes.body.code === 200) {
|
||||
const sceneMatch = searchRes.body.responseData.find((item) => slugify(item.name) === slugify(scene.title));
|
||||
|
||||
if (sceneMatch) {
|
||||
return {
|
||||
...scene,
|
||||
url: `${site.url}${sceneMatch.url}`,
|
||||
date: ed(sceneMatch.release_date, 'YYYY-MM-DD'),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return scene;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest: fetchApiLatest,
|
||||
fetchProfile: fetchApiProfile,
|
||||
fetchUpcoming: fetchApiUpcoming,
|
||||
fetchScene: fetchSceneWrapper,
|
||||
};
|
||||
@@ -502,6 +502,7 @@ async function fetchScene(url, channel, baseRelease, options) {
|
||||
}
|
||||
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
useBrowser: !!options.parameters?.useBrowser,
|
||||
});
|
||||
|
||||
|
||||
@@ -1,68 +1,43 @@
|
||||
'use strict';
|
||||
|
||||
const qu = require('../utils/qu');
|
||||
const http = require('../utils/http');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
async function getTrailerUrl(release, channel, request) {
|
||||
const csrfToken = request.cookie.match('csrfst=(.*?);')?.[1];
|
||||
|
||||
if (!csrfToken) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const res = await http.post(`${channel.url}/ajax/player-config.json`, {
|
||||
item_id: release.entryId,
|
||||
}, {
|
||||
headers: {
|
||||
'X-CSRF-Token': csrfToken,
|
||||
},
|
||||
session: request.session,
|
||||
encodeJSON: false,
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
const trailers = res.body.streams.map((trailer) => ({
|
||||
src: trailer.url,
|
||||
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
|
||||
vr: true,
|
||||
}));
|
||||
|
||||
return {
|
||||
trailers,
|
||||
poster: qu.prefixUrl(res.body.poster, res.body.thumbCDN),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function scrapeAll(scenes, channel) {
|
||||
function scrapeAll(scenes, channel, url) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('a', 'href', { origin: channel.url });
|
||||
release.url = query.url('a', { origin: channel.origin });
|
||||
release.entryId = new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
|
||||
|
||||
release.title = query.cnt('.card__h');
|
||||
release.date = query.date('.card__date', 'D MMMM, YYYY');
|
||||
release.title = query.content('.card__h');
|
||||
release.date = query.date('.card__date', 'D MMMM, YYYY', { match: null });
|
||||
|
||||
release.actors = query.all('.card__links a').map((el) => ({
|
||||
name: qu.query.cnt(el),
|
||||
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
||||
release.actors = query.all('.card__links a').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: channel.url }),
|
||||
}));
|
||||
|
||||
const poster = query.srcset('picture source[type="image/jpeg"]', 'data-srcset')
|
||||
|| query.srcset('picture source[type="image/jpeg"]', 'srcset')
|
||||
|| query.srcset('.video__cover', 'srcset');
|
||||
const poster = query.sourceSet('picture source[type="image/jpeg"]', 'data-srcset')
|
||||
|| query.sourceSet('picture source[type="image/jpeg"]', 'srcset')
|
||||
|| query.sourceSet('.video__cover', 'srcset');
|
||||
|
||||
if (poster?.[0]) {
|
||||
release.poster = [
|
||||
poster[0].replace(/small|tiny/, 'large'),
|
||||
...poster,
|
||||
];
|
||||
].map((src) => ({
|
||||
src,
|
||||
referer: url,
|
||||
}));
|
||||
|
||||
release.teaser = poster[0].replace(/\b(cover|hero|\d+)\/[a-z0-9_]+\.[a-z]+$/i, 'roll.webm'); // actually how site generates teaser URL
|
||||
const teaser = poster[0].replace(/\b(cover|hero|\d+)\/[a-z0-9_]+\.[a-z]+$/i, 'roll.webm'); // actually how site generates teaser URL
|
||||
|
||||
release.teaser = {
|
||||
src: teaser,
|
||||
referer: url,
|
||||
};
|
||||
}
|
||||
|
||||
release.channel = channel.slug; // avoid being assigned to WankzVR network
|
||||
@@ -71,31 +46,90 @@ function scrapeAll(scenes, channel) {
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene({ query }, url, channel, baseRelease, options, request) {
|
||||
async function fetchLatest(channel, page) {
|
||||
const url = `${channel.url}/videos?o=d&p=${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '.layout__content > .cards-list .card' }); // .cards-list is also used for hidden upcoming scenes
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.context, channel, url);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function getTrailerUrl(release, channel, cookies, referer) {
|
||||
const csrfToken = cookies.csrfst;
|
||||
|
||||
if (!csrfToken) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const res = await unprint.post(`${channel.url}/ajax/player-config.json`, {
|
||||
item_id: release.entryId,
|
||||
}, {
|
||||
form: true,
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
'X-CSRF-Token': csrfToken,
|
||||
},
|
||||
cookies,
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
const trailers = res.data.streams.map((trailer) => ({
|
||||
src: trailer.url,
|
||||
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
|
||||
vr: true,
|
||||
referer,
|
||||
}));
|
||||
|
||||
const poster = unprint.prefixUrl(res.data.poster, res.data.thumbCDN);
|
||||
|
||||
return {
|
||||
trailers,
|
||||
poster: poster && {
|
||||
src: poster,
|
||||
referer,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrapeScene({ query }, { url, entity, include, cookies }) {
|
||||
const release = {};
|
||||
const data = query.json('script[type="application/ld+json"]');
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
|
||||
|
||||
release.title = query.cnt('.detail__title');
|
||||
release.description = query.cnt('.detail__txt');
|
||||
release.title = query.content('.detail__title');
|
||||
release.description = query.content('.detail__txt');
|
||||
|
||||
release.date = query.date('.detail__date', 'D MMMM, YYYY');
|
||||
release.date = query.date('.detail__date', 'D MMMM, YYYY', { match: null });
|
||||
release.duration = query.number('.time') * 60;
|
||||
|
||||
release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map((el) => ({
|
||||
name: qu.query.cnt(el),
|
||||
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
||||
name: unprint.query.content(el),
|
||||
url: unprint.query.url(el, null, { origin: entity.origin }),
|
||||
}));
|
||||
|
||||
release.tags = query.cnts('.tag-list .tag').concat(query.cnts('.detail__specs-list .detail__specs-item'));
|
||||
release.tags = query.contents('.tag-list .tag').concat(query.contents('.detail__specs-list .detail__specs-item'));
|
||||
|
||||
release.photos = query.all('.photo-strip__slide').map((el) => ([
|
||||
qu.query.img(el, null, 'data-src'),
|
||||
qu.query.img(el, 'img', 'src'),
|
||||
]));
|
||||
unprint.query.img(el, null, 'data-src'),
|
||||
unprint.query.img(el, 'img'),
|
||||
].map((src) => ({ src, referer: url }))));
|
||||
|
||||
if (options.includePosters || options.includeTrailers) {
|
||||
const { trailers, poster } = await getTrailerUrl(release, channel, request);
|
||||
if (data?.thumbnailUrl) {
|
||||
release.poster = [
|
||||
data.thumbnailUrl.replace(/small|tiny/, 'large'),
|
||||
data.thumbnailUrl,
|
||||
].map((src) => ({ src, referer: url }));
|
||||
}
|
||||
|
||||
if (include.trailers || (!release.poster && include.poster)) {
|
||||
const { trailers, poster } = await getTrailerUrl(release, entity, cookies, url) || {};
|
||||
|
||||
release.trailer = trailers;
|
||||
release.poster = poster;
|
||||
@@ -105,17 +139,17 @@ async function scrapeScene({ query }, url, channel, baseRelease, options, reques
|
||||
}
|
||||
|
||||
async function fetchActorScenes({ query }, url, entity, page = 1, accScenes = []) {
|
||||
const scenes = scrapeAll(qu.initAll(query.all('.cards-list .card')), entity);
|
||||
const scenes = scrapeAll(unprint.initAll(query.all('.cards-list .card')), entity);
|
||||
const hasNextPage = !query.exists('.pagenav__link.inactive');
|
||||
|
||||
if (hasNextPage) {
|
||||
const { origin, pathname, searchParams } = new URL(url);
|
||||
searchParams.set('p', page + 1);
|
||||
|
||||
const res = await qu.get(`${origin}${pathname}?${searchParams}`);
|
||||
const res = await unprint.get(`${origin}${pathname}?${searchParams}`);
|
||||
|
||||
if (res.ok) {
|
||||
return fetchActorScenes(res.item, url, entity, page + 1, accScenes.concat(scenes));
|
||||
return fetchActorScenes(res.context, url, entity, page + 1, accScenes.concat(scenes));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,20 +161,24 @@ async function scrapeProfile({ query }, url, entity, options) {
|
||||
|
||||
const bio = query.all('.person__meta__item').reduce((acc, el) => ({
|
||||
...acc,
|
||||
[slugify(qu.query.cnt(el, '.person__meta__label'))]: qu.query.text(el),
|
||||
[slugify(unprint.query.content(el, '.person__meta__label'))]: unprint.query.text(el),
|
||||
}), {});
|
||||
|
||||
profile.description = query.cnt('.person__content');
|
||||
profile.description = query.content('.person__content');
|
||||
|
||||
profile.gender = entity.slug === 'tranzvr' ? 'transsexual' : 'female';
|
||||
profile.age = Number(bio.age) || null;
|
||||
|
||||
profile.birthPlace = bio.birthplace;
|
||||
|
||||
profile.height = parseInt(bio.height, 10);
|
||||
// height shown in imperial with cm between brackets when requested from North American IP, but only in cm for European IPs
|
||||
profile.height = unprint.extractNumber(bio.height, { match: /(\d+)cm/, matchIndex: 1 });
|
||||
profile.measurements = bio.measurements;
|
||||
|
||||
profile.avatar = query.srcset('.person__avatar img');
|
||||
profile.avatar = query.sourceSet('.person__avatar img').map((src) => ({
|
||||
src,
|
||||
referer: url,
|
||||
}));
|
||||
|
||||
if (options.includeActorScenes) {
|
||||
profile.scenes = await fetchActorScenes({ query }, url, entity);
|
||||
@@ -149,22 +187,12 @@ async function scrapeProfile({ query }, url, entity, options) {
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page) {
|
||||
const res = await qu.getAll(`${channel.url}/videos?o=d&p=${page}`, '.cards-list .card');
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.items, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchProfile(baseActor, { entity }, options) {
|
||||
const url = `${entity.url}/${baseActor.slug}`;
|
||||
const res = await qu.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.item, url, entity, options);
|
||||
return scrapeProfile(res.context, url, entity, options);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
@@ -174,5 +202,4 @@ module.exports = {
|
||||
fetchLatest,
|
||||
scrapeScene,
|
||||
fetchProfile,
|
||||
deprecated: true,
|
||||
};
|
||||
|
||||
67
src/tags.js
67
src/tags.js
@@ -1,8 +1,12 @@
|
||||
'use strict';
|
||||
|
||||
const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
|
||||
const { fetchEntityReleaseIds } = require('./entities');
|
||||
|
||||
const slugify = require('./utils/slugify');
|
||||
const bulkInsert = require('./utils/bulk-insert');
|
||||
const batchInsert = require('./utils/batch-insert');
|
||||
|
||||
function curateTagMedia(media) {
|
||||
if (!media) {
|
||||
@@ -73,10 +77,9 @@ function withRelations(queryBuilder, withMedia) {
|
||||
}
|
||||
}
|
||||
|
||||
async function matchReleaseTags(releases) {
|
||||
const tags = releases
|
||||
.map((release) => release.tags).flat()
|
||||
.map((tag) => tag?.trim().match(/[a-z0-9]+/ig)?.join(' ').toLowerCase())
|
||||
async function matchTags(rawTags) {
|
||||
const tags = rawTags
|
||||
.map((tag) => tag?.trim().match(/[a-z0-9()]+/ig)?.join(' ').toLowerCase())
|
||||
.filter(Boolean);
|
||||
|
||||
const tagEntries = await knex('tags')
|
||||
@@ -153,12 +156,12 @@ async function associateReleaseTags(releases, type = 'release') {
|
||||
return;
|
||||
}
|
||||
|
||||
const tagIdsBySlug = await matchReleaseTags(releases);
|
||||
const tagIdsBySlug = await matchTags(releases.flatMap((release) => release.tags));
|
||||
const entityTagIdsByEntityId = await getEntityTags(releases);
|
||||
|
||||
const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type);
|
||||
|
||||
await bulkInsert(`${type}s_tags`, tagAssociations, false);
|
||||
await batchInsert(`${type}s_tags`, tagAssociations, { conflict: false });
|
||||
}
|
||||
|
||||
async function fetchTag(tagId) {
|
||||
@@ -187,8 +190,58 @@ async function fetchTags(limit = 100) {
|
||||
return tags.map((tag) => curateTag(tag));
|
||||
}
|
||||
|
||||
async function reassociateTagEntries(tagEntries, rematch) {
|
||||
const updatableTagEntries = tagEntries.filter((tagEntry) => (!tagEntry.tag_id || rematch) && tagEntry.original_tag);
|
||||
const matchedTags = await matchTags(Array.from(new Set(updatableTagEntries.map((tagEntry) => tagEntry.original_tag))));
|
||||
|
||||
const updatedTagEntries = updatableTagEntries.map((tagEntry) => ({
|
||||
...tagEntry,
|
||||
tag_id: matchedTags[slugify(tagEntry.original_tag)],
|
||||
})).filter((tagEntry) => tagEntry.tag_id);
|
||||
|
||||
if (updatedTagEntries.length > 0) {
|
||||
const trx = await knex.transaction();
|
||||
|
||||
await trx('releases_tags')
|
||||
.whereIn('id', updatedTagEntries.map((tagEntry) => tagEntry.id))
|
||||
.delete();
|
||||
|
||||
await batchInsert('releases_tags', updatedTagEntries.map((tagEntry) => ({
|
||||
...tagEntry,
|
||||
id: undefined,
|
||||
})), {
|
||||
conflict: false,
|
||||
transaction: trx,
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(`Updated ${updatedTagEntries.length} tags in ${new Set(updatedTagEntries.map((tagEntry) => tagEntry.release_id)).size} scenes`);
|
||||
}
|
||||
|
||||
async function reassociateReleaseTags(rawSceneIds, rematch) {
|
||||
const sceneIds = rawSceneIds.map((sceneId) => Number(sceneId)).filter(Boolean);
|
||||
const tagEntries = await knex('releases_tags').whereIn('release_id', sceneIds);
|
||||
|
||||
await reassociateTagEntries(tagEntries, rematch);
|
||||
}
|
||||
|
||||
async function reassociateEntityReleaseTags(networkSlugs = [], channelSlugs = [], rematch = false) {
|
||||
const { sceneIds } = await fetchEntityReleaseIds(networkSlugs, channelSlugs);
|
||||
|
||||
await reassociateReleaseTags(sceneIds, rematch);
|
||||
}
|
||||
|
||||
async function reassociateOriginalTags(originalTags, rematch) {
|
||||
const tagEntries = await knex('releases_tags').whereIn(knex.raw('lower(releases_tags.original_tag)'), originalTags.map((originalTag) => originalTag.toLowerCase()));
|
||||
|
||||
await reassociateTagEntries(tagEntries, rematch);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
associateReleaseTags,
|
||||
fetchTag,
|
||||
fetchTags,
|
||||
reassociateEntityReleaseTags,
|
||||
reassociateReleaseTags,
|
||||
reassociateOriginalTags,
|
||||
};
|
||||
|
||||
64
src/tools/batch-test.js
Normal file
64
src/tools/batch-test.js
Normal file
@@ -0,0 +1,64 @@
|
||||
'use strict';
|
||||
|
||||
const knex = require('../knex');
|
||||
const batchInsert = require('../utils/batch-insert');
|
||||
|
||||
async function createTestTable() {
|
||||
const tableExists = await knex.schema.hasTable('batch_test');
|
||||
|
||||
if (tableExists) {
|
||||
// await knex('batch_test').delete();
|
||||
return;
|
||||
}
|
||||
|
||||
await knex.schema.createTable('batch_test', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.string('name')
|
||||
.unique();
|
||||
|
||||
table.integer('age');
|
||||
table.text('location');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
}
|
||||
|
||||
async function init() {
|
||||
await createTestTable();
|
||||
|
||||
const transaction = await knex.transaction();
|
||||
|
||||
const entries = await batchInsert('batch_test', [
|
||||
{
|
||||
name: 'John',
|
||||
age: 18,
|
||||
location: 'Home',
|
||||
},
|
||||
{
|
||||
name: 'Jack',
|
||||
age: 38,
|
||||
location: 'Work',
|
||||
},
|
||||
{
|
||||
name: 'James',
|
||||
age: 35,
|
||||
location: 'Club',
|
||||
},
|
||||
], {
|
||||
conflict: 'name',
|
||||
update: true,
|
||||
transaction,
|
||||
commit: false,
|
||||
});
|
||||
|
||||
await transaction.commit();
|
||||
console.log('ENTRIES', entries);
|
||||
|
||||
// await knex.schema.dropTable('batch_test');
|
||||
await knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
136
src/tools/gamma_banners.js
Normal file
136
src/tools/gamma_banners.js
Normal file
@@ -0,0 +1,136 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const fs = require('fs');
|
||||
const { Readable } = require('stream');
|
||||
const { pipeline } = require('stream/promises');
|
||||
|
||||
const knex = require('../knex');
|
||||
const argv = require('../argv');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
const apiUrl = 'https://vjoc5ygk89-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20JavaScript%20(3.33.0)%3B%20Browser%20(lite)%3B%20react%20(16.8.6)%3B%20react-instantsearch%20(5.7.0)%3B%20JS%20Helper%20(2.28.1)&x-algolia-application-id=VJOC5YGK89&x-algolia-api-key=c5546bdfb4d3f31daf49ed3bb1463561';
|
||||
|
||||
async function fetchBanners() {
|
||||
const res = await unprint.post(
|
||||
apiUrl,
|
||||
{
|
||||
requests: [
|
||||
{
|
||||
indexName: 'creatives',
|
||||
params: new URLSearchParams({
|
||||
hitsPerPage: 1000,
|
||||
maxValuesPerFacet: 100,
|
||||
page: 0,
|
||||
filters: '(ProgramType:Legacy OR ProgramType:Internal) AND NOT OverlayActive:false',
|
||||
facets: '["SceneActors","SceneCategories","ProgramName","Size","Niche","MediaExt","SiteTag","OverlayName"]',
|
||||
facetFilters: `[["SiteTag:${argv.site}"],["MediaExt:jpg", "MediaExt:png", "MediaExt:gif"]]`,
|
||||
}).toString(),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
'content-type': 'application/x-www-form-urlencoded',
|
||||
referer: 'https://creatives.gammae.com/',
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (res.ok && res.data.results[0]) {
|
||||
return res.data.results[0].hits;
|
||||
}
|
||||
|
||||
console.error(`Failed API request (${res.status}): ${res.body}`);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function matchTags(rawTags) {
|
||||
if (!rawTags) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const tags = rawTags
|
||||
.map((tag) => tag?.trim().match(/[a-z0-9()]+/ig)?.join(' ').toLowerCase())
|
||||
.filter(Boolean);
|
||||
|
||||
const tagEntries = await knex('tags')
|
||||
.select('tags.slug', 'aliases.slug as alias_slug')
|
||||
.whereIn(knex.raw('lower(tags.name)'), tags)
|
||||
.leftJoin('tags as aliases', 'aliases.id', 'tags.alias_for')
|
||||
.orderByRaw('CASE WHEN tags.alias_for IS NOT NULL THEN aliases.priority ELSE tags.priority END DESC');
|
||||
|
||||
return tagEntries.map((tagEntry) => tagEntry.alias_slug || tagEntry.slug);
|
||||
}
|
||||
|
||||
async function init() {
|
||||
const banners = await fetchBanners();
|
||||
|
||||
if (!banners) {
|
||||
return;
|
||||
}
|
||||
|
||||
await banners.reduce(async (chain, banner) => {
|
||||
await chain;
|
||||
|
||||
const channel = slugify(banner.SiteTag, '');
|
||||
const url = unprint.prefixUrl(banner.MediaLocation || banner.CreativeURL, 'https://cdn.banhq.com');
|
||||
|
||||
if (!url) {
|
||||
console.log('No URL found');
|
||||
console.log(banner);
|
||||
return;
|
||||
}
|
||||
|
||||
const tags = await matchTags([
|
||||
...banner.Tags?.map((tag) => tag.Value) || [],
|
||||
...banner.SceneCategories || [],
|
||||
banner.Niche,
|
||||
].filter(Boolean));
|
||||
|
||||
const fileTags = tags.slice(0, 4).join('_');
|
||||
const fileActors = banner.SceneActors?.slice(0, 2).map((actor) => slugify(actor, '_')).join('_');
|
||||
|
||||
// tags are unreliable and describe entire scene, not banner, don't include by default
|
||||
const segments = [channel, banner.Width, banner.Height, banner.MediaID, argv.actors?.[0] !== false && fileActors].filter(Boolean);
|
||||
const filename = `${segments.join('_')}${argv.tags && argv.tags ? `-${fileTags}` : ''}.${banner.MediaExt || 'jpg'}`;
|
||||
|
||||
const filepath = `/tmp/gamma/${channel}/${filename}`;
|
||||
|
||||
if (argv.inspect) {
|
||||
console.log(banner);
|
||||
}
|
||||
|
||||
if (argv.preview) {
|
||||
console.log(`Preview ${url}: ${filepath}`);
|
||||
return;
|
||||
}
|
||||
|
||||
await fs.promises.mkdir(`/tmp/gamma/${channel}`, { recursive: true });
|
||||
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
|
||||
if (res.ok && res.body) {
|
||||
const writer = fs.createWriteStream(filepath);
|
||||
|
||||
await pipeline(Readable.fromWeb(res.body), writer);
|
||||
|
||||
if (argv.actors) {
|
||||
console.log(`Saved ${url} to ${filepath}`);
|
||||
} else {
|
||||
console.log(`Saved ${url} to ${filepath}, actors ${banner.SceneActors?.join(', ') || ''}`);
|
||||
}
|
||||
} else {
|
||||
console.log(`Failed to fetch ${url} (${res.status})`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(`Failed to fetch ${url}: ${error.message}`);
|
||||
}
|
||||
}, Promise.resolve());
|
||||
|
||||
await knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
15
src/tools/huge-query.js
Normal file
15
src/tools/huge-query.js
Normal file
@@ -0,0 +1,15 @@
|
||||
'use strict';
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
async function init() {
|
||||
const data = Array.from({ length: 100_000 }, (value, index) => ({
|
||||
id: `test_affiliate_${index}`,
|
||||
}));
|
||||
|
||||
await knex('affiliates').insert(data);
|
||||
|
||||
console.log('Done!');
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -41,7 +41,7 @@ async function fetchScenes() {
|
||||
studios.name as studio_name,
|
||||
grandparents.id as parent_network_id,
|
||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
||||
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
|
||||
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
|
||||
COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
|
||||
@@ -136,6 +136,14 @@ async function init() {
|
||||
dupe_index int
|
||||
)`);
|
||||
|
||||
await utilsApi.sql('drop table if exists scenes_tags');
|
||||
await utilsApi.sql(`create table scenes_tags (
|
||||
id int,
|
||||
scene_id int,
|
||||
tag_id int,
|
||||
actor_id int
|
||||
)`);
|
||||
|
||||
console.log('Recreated scenes table');
|
||||
console.log('Fetching scenes from primary database');
|
||||
|
||||
@@ -143,49 +151,62 @@ async function init() {
|
||||
|
||||
console.log('Fetched scenes from primary database');
|
||||
|
||||
const docs = scenes.map((scene) => {
|
||||
const docs = scenes.flatMap((scene) => {
|
||||
const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
|
||||
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
|
||||
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
|
||||
|
||||
return {
|
||||
replace: {
|
||||
index: 'scenes',
|
||||
id: scene.id,
|
||||
doc: {
|
||||
title: scene.title || undefined,
|
||||
title_filtered: filteredTitle || undefined,
|
||||
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
|
||||
created_at: Math.round(scene.created_at.getTime() / 1000),
|
||||
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
|
||||
is_showcased: scene.showcased,
|
||||
entry_id: scene.entry_id || undefined,
|
||||
shoot_id: scene.shoot_id || undefined,
|
||||
channel_id: scene.channel_id,
|
||||
channel_slug: scene.channel_slug,
|
||||
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
|
||||
network_id: scene.network_id || undefined,
|
||||
network_slug: scene.network_slug || undefined,
|
||||
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
|
||||
studio_id: scene.studio_id || undefined,
|
||||
studio_slug: scene.studio_slug || undefined,
|
||||
studio_name: scene.studio_name || undefined,
|
||||
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
|
||||
actor_ids: scene.actors.map((actor) => actor.f1),
|
||||
actors: scene.actors.map((actor) => actor.f2).join(),
|
||||
tag_ids: scene.tags.map((tag) => tag.f1),
|
||||
tags: flatTags.join(' '),
|
||||
movie_ids: scene.movies.map((movie) => movie.f1),
|
||||
movies: scene.movies.map((movie) => movie.f2).join(' '),
|
||||
serie_ids: scene.series.map((serie) => serie.f1),
|
||||
series: scene.series.map((serie) => serie.f2).join(' '),
|
||||
fingerprints: scene.fingerprints.join(' '),
|
||||
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
|
||||
stashed: scene.stashed || 0,
|
||||
dupe_index: scene.dupe_index || 0,
|
||||
return [
|
||||
{
|
||||
replace: {
|
||||
index: 'scenes',
|
||||
id: scene.id,
|
||||
doc: {
|
||||
title: scene.title || undefined,
|
||||
title_filtered: filteredTitle || undefined,
|
||||
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
|
||||
created_at: Math.round(scene.created_at.getTime() / 1000),
|
||||
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
|
||||
is_showcased: scene.showcased,
|
||||
entry_id: scene.entry_id || undefined,
|
||||
shoot_id: scene.shoot_id || undefined,
|
||||
channel_id: scene.channel_id,
|
||||
channel_slug: scene.channel_slug,
|
||||
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
|
||||
network_id: scene.network_id || undefined,
|
||||
network_slug: scene.network_slug || undefined,
|
||||
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
|
||||
studio_id: scene.studio_id || undefined,
|
||||
studio_slug: scene.studio_slug || undefined,
|
||||
studio_name: scene.studio_name || undefined,
|
||||
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
|
||||
actor_ids: scene.actors.map((actor) => actor.f1),
|
||||
actors: scene.actors.map((actor) => actor.f2).join(),
|
||||
tag_ids: scene.tags.map((tag) => tag.f1),
|
||||
tags: flatTags.join(' '),
|
||||
movie_ids: scene.movies.map((movie) => movie.f1),
|
||||
movies: scene.movies.map((movie) => movie.f2).join(' '),
|
||||
serie_ids: scene.series.map((serie) => serie.f1),
|
||||
series: scene.series.map((serie) => serie.f2).join(' '),
|
||||
fingerprints: scene.fingerprints.join(' '),
|
||||
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
|
||||
stashed: scene.stashed || 0,
|
||||
dupe_index: scene.dupe_index || 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
...scene.tags.map((tag) => ({
|
||||
replace: {
|
||||
index: 'scenes_tags',
|
||||
// id: scene.id,
|
||||
doc: {
|
||||
scene_id: scene.id,
|
||||
tag_id: tag.f1,
|
||||
actor_id: tag.f5,
|
||||
},
|
||||
},
|
||||
})),
|
||||
];
|
||||
});
|
||||
|
||||
// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
|
||||
|
||||
88
src/tools/manticore-stashes.js
Normal file
88
src/tools/manticore-stashes.js
Normal file
@@ -0,0 +1,88 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
|
||||
const knex = require('../knex');
|
||||
const chunk = require('../utils/chunk');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
||||
const indexApi = new manticore.IndexApi(mantiClient);
|
||||
|
||||
async function syncStashes(domain = 'scene') {
|
||||
await utilsApi.sql(`truncate table ${domain}s_stashed`);
|
||||
|
||||
const stashes = await knex(`stashes_${domain}s`)
|
||||
.select(
|
||||
`stashes_${domain}s.id as stashed_id`,
|
||||
`stashes_${domain}s.${domain}_id`,
|
||||
'stashes.id as stash_id',
|
||||
'stashes.user_id as user_id',
|
||||
`stashes_${domain}s.created_at as created_at`,
|
||||
)
|
||||
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
|
||||
|
||||
await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
|
||||
await chain;
|
||||
|
||||
const stashDocs = stashChunk.map((stash) => ({
|
||||
replace: {
|
||||
index: `${domain}s_stashed`,
|
||||
id: stash.stashed_id,
|
||||
doc: {
|
||||
[`${domain}_id`]: stash[`${domain}_id`],
|
||||
stash_id: stash.stash_id,
|
||||
user_id: stash.user_id,
|
||||
created_at: Math.round(stash.created_at.getTime() / 1000),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
|
||||
|
||||
console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
async function init() {
|
||||
await utilsApi.sql('drop table if exists scenes_stashed');
|
||||
|
||||
await utilsApi.sql(`create table if not exists scenes_stashed (
|
||||
scene_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await utilsApi.sql('drop table if exists movies_stashed');
|
||||
|
||||
await utilsApi.sql(`create table if not exists movies_stashed (
|
||||
movie_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await utilsApi.sql('drop table if exists actors_stashed');
|
||||
|
||||
await utilsApi.sql(`create table if not exists actors_stashed (
|
||||
actor_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await syncStashes('scene');
|
||||
await syncStashes('actor');
|
||||
await syncStashes('movie');
|
||||
|
||||
console.log('Done!');
|
||||
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -16,14 +16,14 @@ async function updateManticoreStashedScenes(docs) {
|
||||
await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
|
||||
await chain;
|
||||
|
||||
const sceneIds = docsChunk.map((doc) => doc.replace.id);
|
||||
const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
|
||||
|
||||
const stashes = await knex('stashes_scenes')
|
||||
.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
|
||||
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
|
||||
.whereIn('scene_id', sceneIds);
|
||||
|
||||
const stashDocs = docsChunk.flatMap((doc) => {
|
||||
const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
|
||||
const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
|
||||
|
||||
if (sceneStashes.length === 0) {
|
||||
@@ -50,6 +50,25 @@ async function updateManticoreStashedScenes(docs) {
|
||||
if (stashDocs.length > 0) {
|
||||
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
|
||||
}
|
||||
|
||||
const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
|
||||
|
||||
if (deleteSceneIds.length > 0) {
|
||||
await indexApi.callDelete({
|
||||
index: 'scenes_stashed',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
in: {
|
||||
scene_id: deleteSceneIds,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
@@ -128,9 +147,20 @@ async function updateManticoreSceneSearch(releaseIds) {
|
||||
studios.showcased
|
||||
`, releaseIds && [releaseIds]);
|
||||
|
||||
// console.log(scenes.rows);
|
||||
const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
|
||||
|
||||
const docs = releaseIds.map((sceneId) => {
|
||||
const scene = scenesById[sceneId];
|
||||
|
||||
if (!scene) {
|
||||
return {
|
||||
delete: {
|
||||
index: 'scenes',
|
||||
id: sceneId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const docs = scenes.rows.map((scene) => {
|
||||
const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
|
||||
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
|
||||
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
|
||||
@@ -291,7 +321,20 @@ async function updateManticoreMovieSearch(movieIds) {
|
||||
movies_covers.*
|
||||
`, movieIds && [movieIds]);
|
||||
|
||||
const docs = movies.rows.map((movie) => {
|
||||
const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
|
||||
|
||||
const docs = movieIds.map((movieId) => {
|
||||
const movie = moviesById[movieId];
|
||||
|
||||
if (!movie) {
|
||||
return {
|
||||
delete: {
|
||||
index: 'movies',
|
||||
id: movieId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
|
||||
id: tag.f1,
|
||||
name: tag.f2,
|
||||
|
||||
@@ -38,7 +38,7 @@ function filterLocalUniqueReleases(releases, accReleases) {
|
||||
};
|
||||
}
|
||||
|
||||
async function filterUniqueReleases(releases) {
|
||||
async function filterUniqueReleases(releases, options) {
|
||||
const releaseIdentifierChunks = chunk(releases.map((release) => [release.entity.id, release.entryId.toString()]));
|
||||
|
||||
const duplicateReleaseEntryChunks = await Promise.map(releaseIdentifierChunks, async (releaseIdentifiers) => {
|
||||
@@ -47,9 +47,13 @@ async function filterUniqueReleases(releases) {
|
||||
.leftJoin('entities', 'entities.id', 'releases.entity_id')
|
||||
.whereIn(['entity_id', 'entry_id'], releaseIdentifiers)
|
||||
.where((builder) => {
|
||||
if (!options?.forceDeepUpdate) {
|
||||
// we do not want to force deep rescrapes on release day, so if the scene is already deep-scraped, we add it to the duplicate list
|
||||
builder.where('deep', true); // scene is already deep scraped
|
||||
}
|
||||
|
||||
// check if previously upcoming scenes can be excluded from duplicates to be rescraped for release day updates
|
||||
builder
|
||||
.where('deep', true) // scene is already deep scraped
|
||||
.orWhereNull('date')
|
||||
.orWhereNotIn('date_precision', ['day', 'minute']) // don't worry about scenes without (accurate) dates for now
|
||||
.orWhere(knex.raw('date > NOW() - INTERVAL \'12 hours\'')) // scene is still upcoming, with a rough offset to wait for the end of the day west of UTC
|
||||
@@ -110,18 +114,23 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
|
||||
}
|
||||
|
||||
async function scrapeReleases(scraper, entity, preData, isUpcoming) {
|
||||
const options = {
|
||||
...config.options[scraper.slug],
|
||||
...include,
|
||||
...preData,
|
||||
parameters: getRecursiveParameters(entity),
|
||||
options: getRecursiveParameters(entity, 'options'),
|
||||
};
|
||||
|
||||
async function scrapeReleasesPage(page, accReleases, pageContext) {
|
||||
const options = {
|
||||
...config.options[scraper.slug],
|
||||
...include,
|
||||
...preData,
|
||||
const pageOptions = {
|
||||
...options,
|
||||
...pageContext,
|
||||
parameters: getRecursiveParameters(entity),
|
||||
};
|
||||
|
||||
const rawPageReleases = isUpcoming
|
||||
? await scraper.fetchUpcoming(entity, page, options, preData)
|
||||
: await scraper.fetchLatest(entity, page, options, preData);
|
||||
? await scraper.fetchUpcoming(entity, page, pageOptions, preData)
|
||||
: await scraper.fetchLatest(entity, page, pageOptions, preData);
|
||||
|
||||
const pageReleases = rawPageReleases?.scenes || rawPageReleases;
|
||||
|
||||
@@ -157,7 +166,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
|
||||
|
||||
const { uniqueReleases, duplicateReleases } = argv.force
|
||||
? { uniqueReleases: limitedReleases, duplicateReleases: [] }
|
||||
: await filterUniqueReleases(limitedReleases);
|
||||
: await filterUniqueReleases(limitedReleases, options);
|
||||
|
||||
return { uniqueReleases, duplicateReleases };
|
||||
}
|
||||
@@ -305,8 +314,6 @@ async function scrapeNetworkParallel(networkEntity) {
|
||||
async function fetchUpdates() {
|
||||
const includedNetworks = await fetchIncludedEntities();
|
||||
|
||||
// console.log(includedNetworks[0]);
|
||||
|
||||
const scrapedNetworks = await Promise.map(
|
||||
includedNetworks,
|
||||
async (networkEntity) => (networkEntity.parameters?.sequential
|
||||
|
||||
96
src/utils/batch-insert.js
Executable file
96
src/utils/batch-insert.js
Executable file
@@ -0,0 +1,96 @@
|
||||
'use strict';
|
||||
|
||||
const knex = require('../knex');
|
||||
const chunk = require('./chunk');
|
||||
const logger = require('../logger')(__filename);
|
||||
|
||||
// improved version of bulkInsert
|
||||
async function batchInsert(table, items, {
|
||||
conflict = true,
|
||||
update = false,
|
||||
chunkSize = 1000,
|
||||
concurrent = false,
|
||||
transaction,
|
||||
commit = false,
|
||||
} = {}) {
|
||||
if (!table) {
|
||||
throw new Error('No table specified for batch insert');
|
||||
}
|
||||
|
||||
if (!Array.isArray(items)) {
|
||||
throw new Error('Batch insert items are not an array');
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const chunks = chunk(items, chunkSize);
|
||||
const conflicts = [].concat(conflict).filter((column) => typeof column === 'string'); // conflict might be 'true'
|
||||
const trx = transaction || await knex.transaction();
|
||||
|
||||
try {
|
||||
const queries = chunks.map((chunkItems) => {
|
||||
const query = trx(table)
|
||||
.insert(chunkItems)
|
||||
.returning('*');
|
||||
|
||||
if (conflicts.length > 0) {
|
||||
if (Array.isArray(update)) {
|
||||
// udpate specified
|
||||
return query
|
||||
.onConflict(conflicts)
|
||||
.merge(update);
|
||||
}
|
||||
|
||||
if (update) {
|
||||
// update all
|
||||
return query
|
||||
.onConflict(conflicts)
|
||||
.merge();
|
||||
}
|
||||
|
||||
throw new Error('Batch insert conflict columns must be specified together with update');
|
||||
}
|
||||
|
||||
if (conflict && update) {
|
||||
throw new Error('Batch insert conflict must specify columns, or update must be disabled');
|
||||
}
|
||||
|
||||
// error on any conflict
|
||||
if (conflict) {
|
||||
return query;
|
||||
}
|
||||
|
||||
// ignore duplicates, keep old entries as-is
|
||||
return query
|
||||
.onConflict()
|
||||
.ignore();
|
||||
});
|
||||
|
||||
const results = concurrent
|
||||
? await Promise.all(queries)
|
||||
: await queries.reduce(async (chain, query) => {
|
||||
const acc = await chain;
|
||||
const result = await query;
|
||||
|
||||
return acc.concat(result);
|
||||
}, Promise.resolve([]));
|
||||
|
||||
if (!transaction || commit) {
|
||||
await trx.commit();
|
||||
}
|
||||
|
||||
return results;
|
||||
} catch (error) {
|
||||
if (!transaction || commit) {
|
||||
await trx.rollback();
|
||||
}
|
||||
|
||||
logger.error(`Failed batch insert: ${error.message} (${error.detail})`);
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = batchInsert;
|
||||
@@ -1,11 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
function getRecursiveParameters(entity, parameters) {
|
||||
function getRecursiveParameters(entity, property = 'parameters', parameters) {
|
||||
if (entity.parent) {
|
||||
return getRecursiveParameters(entity.parent, { ...entity.parameters, ...parameters });
|
||||
return getRecursiveParameters(entity.parent, property, { ...entity[property], ...parameters });
|
||||
}
|
||||
|
||||
return { ...entity.parameters, ...parameters };
|
||||
return { ...entity[property], ...parameters };
|
||||
}
|
||||
|
||||
module.exports = getRecursiveParameters;
|
||||
|
||||
@@ -7,6 +7,7 @@ const unprint = require('unprint');
|
||||
const omit = require('object.omit');
|
||||
|
||||
const argv = require('../src/argv');
|
||||
const logger = require('../src/logger')(__filename);
|
||||
const include = require('../src/utils/argv-include')(argv);
|
||||
const slugify = require('../src/utils/slugify');
|
||||
const scrapers = require('../src/scrapers/scrapers');
|
||||
@@ -32,6 +33,12 @@ unprint.options({
|
||||
proxy: config.proxy,
|
||||
});
|
||||
|
||||
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.isProxied ? ' proxied' : ''}${event.isBrowser ? ' browser' : ''}) ${event.url}`));
|
||||
unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.isProxied ? 'proxied ' : ''}${event.isBrowser ? 'browser ' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`));
|
||||
|
||||
unprint.on('browserOpen', (event) => logger.debug(`Unprint opened browsers ${event.keys} (${event.active}/${config.bypass.browser.clientRetirement} active, ${event.clients} clients)`));
|
||||
unprint.on('browserClose', (event) => logger.debug(`Unprint closed${event.retired ? ' retired' : ''} browsers ${event.keys} (${event.active}/${config.bypass.browser.clientRetirement} active, ${event.clients} clients)`));
|
||||
|
||||
const actors = [
|
||||
// vixen
|
||||
{ entity: 'vixen', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
|
||||
@@ -66,9 +73,11 @@ const actors = [
|
||||
{ entity: 'eyeontheguy', name: 'Tommy Gunn', fields: ['avatar'] },
|
||||
{ entity: 'interracialpovs', name: 'Nia Nacci', fields: ['avatar', 'aliases', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
|
||||
{ entity: 'povpornstars', name: 'Anna Bell Peaks', fields: ['avatar', 'aliases', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
|
||||
{ entity: 'seehimfuck', name: 'Sheem The Dream', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'penisLength', 'circumcised', 'socials'] },
|
||||
{ entity: 'seehimfuck', name: 'Sheem The Dream', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'penisLength', 'isCircumcised', 'socials'] },
|
||||
{ entity: 'hushpass', name: 'Dylan Ryder', fields: ['avatar'] },
|
||||
{ entity: 'interracialpass', name: 'Aidra Fox', fields: ['avatar', 'height', 'measurements'] },
|
||||
{ entity: 'ravebunnys', name: 'Lacey Jayne', fields: ['avatar', 'height', 'measurements', 'dateOfBirth', 'birthPlace', 'description', 'ethnicity', 'weight', 'naturalBoobs'] },
|
||||
{ entity: 'hotandtatted', name: 'Valerica Steele', url: 'https://hotandtatted.com/models/tattooed-pornstar-val-steele.html', fields: ['avatar', 'measurements', 'dateOfBirth', 'birthPlace', 'description', 'ethnicity', 'weight', 'foot', 'naturalBoobs', 'hasPiercings'] },
|
||||
// kelly madison / 8K
|
||||
{ entity: 'kellymadison', name: 'Ava Addams', fields: ['avatar', 'description', 'age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] },
|
||||
{ entity: '8kmembers', name: 'Angie Lynx', fields: ['age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] },
|
||||
@@ -81,7 +90,7 @@ const actors = [
|
||||
{ entity: 'letsdoeit', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] },
|
||||
{ entity: 'killergram', name: 'Clea Gaultier', fields: ['avatar', 'gender', 'hairColor', 'ethnicity'] },
|
||||
{ entity: 'men', name: 'Cade Maddox', fields: ['avatar', 'description', 'gender', 'height', 'ethnicity', 'penisLength', 'dateOfBirth', 'weight', 'hairColor', 'hasTattoos'] },
|
||||
{ entity: 'metrohd', name: 'April Olsen', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight'] },
|
||||
{ entity: 'metrohd', name: 'Vanna Bardot', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos'] },
|
||||
{ entity: 'mofos', name: 'Ariana Starr', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] },
|
||||
{ entity: 'propertysex', name: 'Desiree Dulce', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasPiercings'] },
|
||||
{ entity: 'sexyhub', name: 'Angie Lynx', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] },
|
||||
@@ -110,7 +119,7 @@ const actors = [
|
||||
{ entity: 'devilsfilm', name: 'Katrina Colt', fields: ['avatar', 'gender'] },
|
||||
{ entity: 'diabolic', name: 'Kira Noir', fields: ['avatar', 'gender'] },
|
||||
{ entity: 'evilangel', name: 'Francesca Le', fields: ['avatar', 'gender'] },
|
||||
{ entity: 'fantasymassage', name: 'Cherry Kiss', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] },
|
||||
{ entity: 'fantasymassage', name: 'Cherry Kiss', fields: ['avatar', 'gender'] },
|
||||
{ entity: 'filthykings', name: 'Armani Black', fields: ['avatar', 'gender'] },
|
||||
{ entity: 'gangbangcreampie', name: 'Luna Lovely', fields: ['avatar', 'gender', 'description'] },
|
||||
{ entity: 'girlsway', name: 'Adriana Chechik', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] },
|
||||
@@ -136,12 +145,12 @@ const actors = [
|
||||
// perv city
|
||||
{ entity: 'pervcity', name: 'Brooklyn Gray', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'dpdiva', name: 'Liz Jordan', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'bamvisions', name: 'Abella Danger', fields: ['avatar', 'height', 'measurements'] },
|
||||
{ entity: 'bamvisions', name: 'Abella Danger', fields: ['avatar', 'height', 'measurements'] }, // site offline as of 2026-02-25
|
||||
// radical
|
||||
{ entity: 'bjraw', name: 'Nikki Knightly', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'gotfilled', name: 'Alexa Chains', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'inserted', name: 'Anissa Kate', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||
{ entity: 'lucidflix', name: 'Ava Amira', fields: ['avatar', 'description', 'gender'] },
|
||||
// wankz
|
||||
@@ -163,10 +172,10 @@ const actors = [
|
||||
{ entity: 'thatsitcomshow', name: 'Casey Calvert', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] },
|
||||
{ entity: 'brattysis', name: 'Scarlett Alexis', fields: ['avatar', 'age', 'height', 'measurements', 'description', 'residencePlace', 'photos'] },
|
||||
// porndoe
|
||||
{ entity: 'vipsexvault', name: 'Amirah Adara', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'hairColor', 'description'] },
|
||||
{ entity: 'amateureuro', name: 'Luna Oara', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'description'] },
|
||||
{ entity: 'mamacitaz', name: 'Julia De Lucia', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'description', 'hairColor'] },
|
||||
{ entity: 'transbella', name: 'Kalena Rios', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'description', 'hairColor'] },
|
||||
{ entity: 'vipsexvault', name: 'Amirah Adara', fields: ['avatar', 'nationality', 'birthPlace', 'age', 'naturalBoobs', 'hairColor', 'description'] },
|
||||
{ entity: 'amateureuro', name: 'Luna Oara', fields: ['avatar', 'nationality', 'birthPlace', 'age', 'naturalBoobs', 'description'] },
|
||||
{ entity: 'mamacitaz', name: 'Julia De Lucia', fields: ['avatar', 'nationality', 'birthPlace', 'age', 'naturalBoobs', 'description', 'hairColor'] },
|
||||
{ entity: 'transbella', name: 'Kalena Rios', fields: ['avatar', 'nationality', 'birthPlace', 'age', 'naturalBoobs', 'description', 'hairColor'] },
|
||||
// snow valley group
|
||||
{ entity: 'spermmania', name: 'Lya Cutie', fields: ['avatar', 'age', 'height', 'cup', 'bust', 'waist', 'hip'] },
|
||||
{ entity: 'cospuri', name: 'Ria Kurumi', fields: ['avatar', 'birthPlace', 'description', 'height', 'cup', 'bust', 'waist', 'hip'] },
|
||||
@@ -207,7 +216,7 @@ const actors = [
|
||||
{ entity: 'amateurallure', name: 'Ava Amira', fields: ['avatar', 'description'] },
|
||||
{ entity: 'swallowsalon', name: 'Abella Danger', fields: ['avatar'] },
|
||||
// exploitedx
|
||||
{ entity: 'exploitedx', name: 'Amber Moore', url: 'https://exploitedcollegegirls.com/models/AmberMoore.html', fields: ['avatar', 'description', 'age', 'height', 'measurements'] },
|
||||
{ entity: 'exploitedx', name: 'Amber Moore', url: 'https://exploitedcollegegirls.com/models/Amber-Moore.html', fields: ['avatar', 'description', 'age', 'height', 'measurements'] },
|
||||
{ entity: 'exploitedx', name: 'Amber Moore', url: 'https://bbcsurprise.com/models/AmberMoore.html', fields: ['avatar', 'age'] },
|
||||
{ entity: 'exploitedx', name: 'Megan Marx', url: 'https://excogigirls.com/models/megan-marx.html', fields: ['avatar', 'description', 'age', 'height', 'measurements'] },
|
||||
{ entity: 'exploitedx', name: 'Sophie Hunt', url: 'https://www.backroomcastingcouch.com/models/Sophie-Hunt.html', fields: ['avatar', 'age'] },
|
||||
@@ -237,7 +246,7 @@ const actors = [
|
||||
{ entity: 'inthecrack', name: 'Vicki Chase', fields: ['dateOfBirth', 'height', 'weight', 'ethnicity', 'birthPlace'] },
|
||||
{ entity: 'karups', name: 'Peach Lollypop', fields: ['avatar'] },
|
||||
{ entity: 'littlecapricedreams', name: 'Littlecaprice', fields: ['avatar', 'nationality', 'cup', 'measurements', 'height', 'description'] }, // sic
|
||||
{ entity: 'mariskax', name: 'Honey Demon', fields: ['avatar', 'gender', 'dateOfBirth', 'placeOfBirth', 'measurements', 'height', 'weight', 'hairColor', 'eyes'] },
|
||||
{ entity: 'mariskax', name: 'Honey Demon', fields: ['avatar', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'hairColor', 'eyes'] },
|
||||
{ entity: 'meidenvanholland', name: 'Izzy Bizzy Bang Bang', fields: ['avatar', 'description'] },
|
||||
{ entity: 'nebraskacoeds', name: 'Mary Beth Haglin', fields: ['avatar'] },
|
||||
{ entity: 'pascalssubsluts', name: 'Zlata Shine', fields: ['avatar', 'gender', 'nationality', 'hairColor', 'height', 'description'] },
|
||||
@@ -272,7 +281,7 @@ async function validateUrl(url, mime = 'image/') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const res = await fetch(href, {
|
||||
const res = await unprint.get(href, {
|
||||
headers: {
|
||||
Referer: url.referer || new URL(href).origin,
|
||||
},
|
||||
@@ -301,9 +310,10 @@ const validators = {
|
||||
hasTattoos: (value) => typeof value === 'boolean',
|
||||
hasPiercings: (value) => typeof value === 'boolean',
|
||||
avatar: async (value) => [].concat(value).reduce(async (chain, url) => {
|
||||
// testing all avatar fallbacks is too time-consuming, just ensure one is valid
|
||||
const acc = await chain;
|
||||
|
||||
if (!acc) {
|
||||
if (acc) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
@@ -322,9 +332,9 @@ const validators = {
|
||||
|
||||
// profiler in this context is shorthand for profile scraper
|
||||
async function init() {
|
||||
const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: 'channel' });
|
||||
const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: 'options' });
|
||||
|
||||
Object.entries(actorScrapers).reduce(async (chain, [entitySlug, scraper]) => {
|
||||
await Object.entries(actorScrapers).reduce(async (chain, [entitySlug, scraper]) => {
|
||||
await chain;
|
||||
|
||||
const entity = entitiesBySlug[entitySlug] || null;
|
||||
@@ -332,7 +342,6 @@ async function init() {
|
||||
|
||||
const tests = actors.filter((actor) => actor.entity === entitySlug);
|
||||
|
||||
// TODO: remove when all tests are written
|
||||
if (tests.length === 0) {
|
||||
console.log('TODO', entitySlug);
|
||||
return;
|
||||
@@ -375,7 +384,7 @@ async function init() {
|
||||
validators[field]
|
||||
? await validators[field](profile[field])
|
||||
: typeof profile[field] !== 'undefined',
|
||||
`broken field ${field}, got ${profile[field]}`,
|
||||
`broken field ${field}, got ${profile[field]} for ${actor.name}`,
|
||||
);
|
||||
}));
|
||||
})));
|
||||
@@ -385,6 +394,7 @@ async function init() {
|
||||
console.log(actors.length);
|
||||
|
||||
await knex.destroy();
|
||||
await unprint.closeAllBrowsers();
|
||||
}
|
||||
|
||||
init();
|
||||
|
||||
Reference in New Issue
Block a user