152 Commits

Author SHA1 Message Date
DebaucheryLibrarian
0560fac1ff 1.250.38 2026-03-13 05:14:45 +01:00
DebaucheryLibrarian
108bf3b168 Integrated manticore stash sync tool. 2026-03-13 05:14:42 +01:00
DebaucheryLibrarian
155e235246 Fixed Aylo specifying wrong host for media. 2026-03-10 05:54:00 +01:00
DebaucheryLibrarian
bff665c6ec 1.250.37 2026-03-10 04:41:36 +01:00
DebaucheryLibrarian
c7111329dc Improved knex error reporting. 2026-03-10 04:41:30 +01:00
DebaucheryLibrarian
d7c1c0ae5c 1.250.36 2026-03-09 05:36:21 +01:00
DebaucheryLibrarian
ea298d7edb Fixed Aylo scraper ignoring session configuration. 2026-03-09 05:36:17 +01:00
DebaucheryLibrarian
99dfcae920 1.250.35 2026-03-08 04:03:57 +01:00
DebaucheryLibrarian
24cba1e1fa Deleting flushed scenes from manticore. 2026-03-08 04:03:55 +01:00
DebaucheryLibrarian
076bdad310 1.250.34 2026-03-06 04:25:55 +01:00
DebaucheryLibrarian
d432d291dd Added See Him Solo to Hussie Pass, added HP affiliates. 2026-03-06 04:25:51 +01:00
DebaucheryLibrarian
220f7e787d 1.250.33 2026-03-05 02:00:53 +01:00
DebaucheryLibrarian
f1caa77e4b Added scene tags table to manticore scenes tool. 2026-03-05 02:00:43 +01:00
DebaucheryLibrarian
ff633436cb 1.250.32 2026-03-04 02:53:21 +01:00
DebaucheryLibrarian
6860072a51 Added database support for actor-specific scene tags. 2026-03-04 02:53:17 +01:00
DebaucheryLibrarian
2c7b4cfc22 1.250.31 2026-03-04 01:57:39 +01:00
DebaucheryLibrarian
7d9e1be8d4 Added Lesbian Factor. 2026-03-04 01:57:33 +01:00
DebaucheryLibrarian
00db4b1b5b 1.250.30 2026-03-03 23:47:50 +01:00
DebaucheryLibrarian
9f1cf1575a Added ASG Max channel parameters. 2026-03-03 23:47:48 +01:00
DebaucheryLibrarian
4f13e4ed28 1.250.29 2026-03-03 23:11:33 +01:00
DebaucheryLibrarian
9805aa7b5b Added Deep Inside to Disruptive Films. Added Sodomy Squad affiliate. 2026-03-03 23:11:32 +01:00
DebaucheryLibrarian
0cc6ebc305 1.250.28 2026-03-03 22:40:52 +01:00
DebaucheryLibrarian
016c24af28 Added channel filter option to Gamma scraper, re-added Disruptive Films channel. 2026-03-03 22:40:49 +01:00
DebaucheryLibrarian
2158550091 1.250.27 2026-03-03 01:21:00 +01:00
DebaucheryLibrarian
68ddc8cb78 Added Wicked affiliate. Improved Gamma banner tool filename composition. 2026-03-03 01:20:58 +01:00
DebaucheryLibrarian
bc5693e44a 1.250.26 2026-03-02 23:55:11 +01:00
DebaucheryLibrarian
7276d90629 Disabled tags by default in Gamma banner tool filenames. Added Gangbang Creampie, Gloryhole Secrets and Taboo Heat affiliates. 2026-03-02 23:55:09 +01:00
DebaucheryLibrarian
1a1af95a10 1.250.25 2026-03-02 22:36:08 +01:00
DebaucheryLibrarian
bcb7a56588 Added alt descriptions and attributes columns to series. 2026-03-02 22:36:05 +01:00
DebaucheryLibrarian
16648d50f6 Re-enabled filename actors and tags in Gamma banner tool, improved disable argument. 2026-03-02 06:24:54 +01:00
DebaucheryLibrarian
062dc0e75e 1.250.24 2026-03-02 06:21:30 +01:00
DebaucheryLibrarian
42effd53fc Added Diabolic affiliate. Disabled filename actors and tags in Gamma banner tool, unreliable. 2026-03-02 06:21:27 +01:00
DebaucheryLibrarian
3a3403bb1f 1.250.23 2026-03-02 06:07:49 +01:00
DebaucheryLibrarian
6fb4989256 Added Chaos Men affiliate. 2026-03-02 06:07:45 +01:00
DebaucheryLibrarian
9750ca4b79 1.250.22 2026-03-02 05:52:48 +01:00
DebaucheryLibrarian
0500f7eda8 Added Burning Angel affiliate. Fixed Gamma banner tool breaking on invalid URL. 2026-03-02 05:52:46 +01:00
DebaucheryLibrarian
19beff7dbc 1.250.21 2026-03-02 05:38:52 +01:00
DebaucheryLibrarian
dfe1b84992 Explicitly unsetting channel parent in seed. 2026-03-02 05:38:50 +01:00
DebaucheryLibrarian
3d3b544cb4 1.250.20 2026-03-02 05:07:26 +01:00
DebaucheryLibrarian
65fa6027ee Prioritized pissing tag. 2026-03-02 05:07:23 +01:00
DebaucheryLibrarian
b3a0ba72eb 1.250.19 2026-03-02 04:01:41 +01:00
DebaucheryLibrarian
f3e2143b45 Fixed wrong date parse function call in Gamma scraper. Added Biphoria affiliate link. 2026-03-02 04:01:39 +01:00
DebaucheryLibrarian
d289f95d3d 1.250.18 2026-03-02 03:46:48 +01:00
DebaucheryLibrarian
d8b41ec9b5 Use request interface for Vixen deep fetch, seemingly less chance of a 403. 2026-03-02 03:46:46 +01:00
DebaucheryLibrarian
05f7d8b814 1.250.17 2026-03-02 03:27:27 +01:00
DebaucheryLibrarian
c2fc09fdaa Removed redundant program filter from Gamma banner tool. 2026-03-02 03:27:24 +01:00
DebaucheryLibrarian
8a7210a3b9 1.250.16 2026-03-02 03:08:28 +01:00
DebaucheryLibrarian
e029ca7fd0 Added Gamma banner downloader. 2026-03-02 03:08:26 +01:00
DebaucheryLibrarian
ffcfae69d5 1.250.15 2026-03-02 03:07:12 +01:00
DebaucheryLibrarian
dcaee01ce8 Using channel origin instead of URL for Gamma referer URL composition. 2026-03-02 03:07:10 +01:00
DebaucheryLibrarian
7561a4577e 1.250.14 2026-03-02 01:41:38 +01:00
DebaucheryLibrarian
98b735dbae Added Vivid and Zero Tolerance affiliate links. Restored BAM Visions profile scraper, site is back online. 2026-03-02 01:41:36 +01:00
DebaucheryLibrarian
d2daed788c 1.250.13 2026-03-02 01:14:27 +01:00
DebaucheryLibrarian
23257745a7 Fixed profile updated_at timestamp not updating. 2026-03-02 01:14:23 +01:00
DebaucheryLibrarian
156954553d 1.250.12 2026-03-02 01:06:31 +01:00
DebaucheryLibrarian
eb20af14a6 Improved Gamma scene URL composition. 2026-03-02 01:06:29 +01:00
DebaucheryLibrarian
ae247c7a91 1.250.11 2026-03-02 00:49:04 +01:00
DebaucheryLibrarian
d49e6ef488 Explicitly unsetting parameters in seed. 2026-03-02 00:49:01 +01:00
DebaucheryLibrarian
2b20d98ee0 Removed stray console log. 2026-03-01 23:53:22 +01:00
DebaucheryLibrarian
b8cf6a3e71 1.250.10 2026-03-01 23:52:46 +01:00
DebaucheryLibrarian
af57f412c9 Refactored Gamma scraper, only using API. 2026-03-01 23:52:41 +01:00
DebaucheryLibrarian
3696b81e69 1.250.9 2026-03-01 20:45:32 +01:00
DebaucheryLibrarian
5b6fefd43b Rounding actor profile values stored as integers to prevent database errors. 2026-03-01 20:45:30 +01:00
DebaucheryLibrarian
a863ab888d 1.250.8 2026-03-01 19:58:37 +01:00
DebaucheryLibrarian
209a81ef71 Removed Vivid wrapper, updated channel URLs. 2026-03-01 19:58:35 +01:00
DebaucheryLibrarian
bd91dcbc77 1.250.7 2026-03-01 04:49:04 +01:00
DebaucheryLibrarian
b89f25405a Using batch insert for various actor scraping inserts. 2026-03-01 04:49:01 +01:00
DebaucheryLibrarian
198f08cb3a Removed stray console log. 2026-03-01 04:28:09 +01:00
DebaucheryLibrarian
febaac3865 1.250.6 2026-03-01 04:27:32 +01:00
DebaucheryLibrarian
f82167656b Changed actor foot column to decimal. 2026-03-01 04:27:29 +01:00
DebaucheryLibrarian
6e20d7d216 1.250.5 2026-02-27 00:55:16 +01:00
DebaucheryLibrarian
612a489cdf Fixed actor scraper list reference. 2026-02-27 00:55:14 +01:00
DebaucheryLibrarian
db2e5b2da4 1.250.4 2026-02-27 00:51:13 +01:00
DebaucheryLibrarian
d81310ed25 Removed outdated profile source list. 2026-02-27 00:51:11 +01:00
DebaucheryLibrarian
ec86aa9286 1.250.3 2026-02-26 00:04:41 +01:00
DebaucheryLibrarian
5d58ddcd49 Disabled BAM Visions profile test while site is offline. 2026-02-26 00:04:39 +01:00
DebaucheryLibrarian
c515c8aeb3 1.250.2 2026-02-26 00:00:41 +01:00
DebaucheryLibrarian
debf92afd7 Changed MetroHD test actor to Vanna Bardot, April Olsen returns implausible weight 64, which seems to be a data error (too low for lbs, too high for kg). 2026-02-26 00:00:37 +01:00
DebaucheryLibrarian
601f930324 1.250.1 2026-02-25 01:09:53 +01:00
DebaucheryLibrarian
e77ced44c7 Added batch insert util to replace bulk insert. Fixed circular dependencies. 2026-02-25 01:09:49 +01:00
DebaucheryLibrarian
9f37f54634 1.250.0 2026-02-24 06:17:41 +01:00
DebaucheryLibrarian
dc7f325d13 Added scene media detach. 2026-02-24 06:17:38 +01:00
DebaucheryLibrarian
35c941488e 1.249.15 2026-02-24 05:50:06 +01:00
DebaucheryLibrarian
fc32843c5a Expanded title query in Hush scraper. 2026-02-24 05:50:04 +01:00
DebaucheryLibrarian
26b31fb10a 1.249.14 2026-02-24 05:39:07 +01:00
DebaucheryLibrarian
9aa6c9c6c5 Added Rave Bunnys and Hot and Tatted to Hussie Pass, improved scraper. Only looking for one valid avatar URL in profile tests. 2026-02-24 05:39:05 +01:00
DebaucheryLibrarian
855a15bc73 1.249.13 2026-02-24 05:08:08 +01:00
DebaucheryLibrarian
3329661135 Added profile referer parameter to Gamma, needed for Dogfart. 2026-02-24 05:07:23 +01:00
DebaucheryLibrarian
791bd6bf27 1.249.12 2026-02-24 04:46:16 +01:00
DebaucheryLibrarian
d6be985c4b Refactored Hush / Hussie Pass with unprint. 2026-02-24 04:46:12 +01:00
DebaucheryLibrarian
7286846308 1.249.11 2026-02-24 03:37:11 +01:00
DebaucheryLibrarian
81dfce8b3d Updated POV Pornstars parameter URLs to https. 2026-02-24 03:37:09 +01:00
DebaucheryLibrarian
aff0e27c55 1.249.10 2026-02-24 03:32:56 +01:00
DebaucheryLibrarian
68fe786cb7 Updated POV Pornstars URL to https. 2026-02-24 03:32:53 +01:00
DebaucheryLibrarian
9a0b0a8989 1.249.9 2026-02-24 03:15:20 +01:00
DebaucheryLibrarian
60b8271e4f Updated unprint to fix response OK. 2026-02-24 03:15:18 +01:00
DebaucheryLibrarian
a52042b56c 1.249.8 2026-02-24 02:32:42 +01:00
DebaucheryLibrarian
7a3dac865e Updated unprint for browser context close fix. 2026-02-24 02:32:40 +01:00
DebaucheryLibrarian
74e0fb721d 1.249.7 2026-02-24 02:12:19 +01:00
DebaucheryLibrarian
ba366df7a5 Added entity resolution prefer to entity options. 2026-02-24 02:12:16 +01:00
DebaucheryLibrarian
d4e6082d2e 1.249.6 2026-02-24 01:32:43 +01:00
DebaucheryLibrarian
ea325b8ec5 Removed unavailable profile details from Fantasy Massage profile test. 2026-02-24 01:32:41 +01:00
DebaucheryLibrarian
41b1f39752 1.249.5 2026-02-24 01:28:24 +01:00
DebaucheryLibrarian
c75c3e3ed9 Changed profile test to prefer network. Removed stray console from Gamma. 2026-02-24 01:28:22 +01:00
DebaucheryLibrarian
ee495a5cde 1.249.4 2026-02-24 01:18:22 +01:00
DebaucheryLibrarian
b52e871cfe Passing network channels as site scopes in Gamma API profile scraper. 2026-02-24 01:18:20 +01:00
DebaucheryLibrarian
0fc725873e 1.249.3 2026-02-24 00:43:25 +01:00
DebaucheryLibrarian
db14657101 Removed gender from Top Web Models profile test. 2026-02-24 00:43:23 +01:00
DebaucheryLibrarian
5524efb3ba 1.249.2 2026-02-24 00:04:20 +01:00
DebaucheryLibrarian
1397349058 Fixed circumcised field for See Him Fuck profile test. 2026-02-24 00:04:19 +01:00
DebaucheryLibrarian
25cac4d0ab 1.249.1 2026-02-23 23:48:03 +01:00
DebaucheryLibrarian
5a282cc372 Following redirects in Hush / Hussiepass scraper. 2026-02-23 23:48:01 +01:00
DebaucheryLibrarian
3e6592d1f3 1.249.0 2026-02-22 06:24:30 +01:00
DebaucheryLibrarian
b95e2fadf7 Added tag reassociation and dedupe migration. 2026-02-22 06:24:26 +01:00
DebaucheryLibrarian
e3b922da6c 1.248.70 2026-02-22 04:56:28 +01:00
DebaucheryLibrarian
9bf42ff6c0 Using API to fetch Aziani block IDs, fixed block IDs were incorrect interpretation. 2026-02-22 04:56:25 +01:00
DebaucheryLibrarian
ba127ee53d 1.248.69 2026-02-22 03:41:56 +01:00
DebaucheryLibrarian
3ee73d2f77 Fixed tags with parentheses not getting matched. 2026-02-22 03:41:54 +01:00
DebaucheryLibrarian
4e68a69e02 1.248.68 2026-02-22 03:36:23 +01:00
DebaucheryLibrarian
570be66a91 Added octet stream type expectation to Bang profile avatar. 2026-02-22 03:36:19 +01:00
DebaucheryLibrarian
0092aab579 1.248.67 2026-02-22 03:17:06 +01:00
DebaucheryLibrarian
78e9088ee5 Fixed WankzVR profile scraper selecting height in feet/inches when requested from North American IP. 2026-02-22 03:17:04 +01:00
DebaucheryLibrarian
cdb4644b42 1.248.66 2026-02-22 03:05:12 +01:00
DebaucheryLibrarian
f80f73d0d4 Added Lancelot Styles and Adventure Teens to AnalVids. 2026-02-22 03:05:09 +01:00
DebaucheryLibrarian
e70acdfe79 1.248.65 2026-02-22 02:58:28 +01:00
DebaucheryLibrarian
75c5a37567 Fixed Teen Mega World typo breaking profile avatar. 2026-02-22 02:58:26 +01:00
DebaucheryLibrarian
9aa7fd77c4 1.248.64 2026-02-21 00:56:17 +01:00
DebaucheryLibrarian
6b768b6828 Added entity option to bypass deep scraped check on release day update. 2026-02-21 00:56:15 +01:00
DebaucheryLibrarian
0f44b4d410 1.248.63 2026-02-17 01:55:31 +01:00
DebaucheryLibrarian
3368e2b343 Removed poster query that would select unrelated image from Score scraper. 2026-02-17 01:55:28 +01:00
DebaucheryLibrarian
8e77a5ef46 1.248.62 2026-02-17 00:43:03 +01:00
DebaucheryLibrarian
29dc94d77d Unshowcased Full Porn Network and most of Score, updated After School URL. 2026-02-17 00:43:00 +01:00
DebaucheryLibrarian
03b039c937 1.248.61 2026-02-12 01:31:43 +01:00
DebaucheryLibrarian
f4f1afdf3b Added extreme insertion tag, set as default on Sic Flics. 2026-02-12 01:31:41 +01:00
DebaucheryLibrarian
fefb165274 1.248.60 2026-02-09 06:28:21 +01:00
DebaucheryLibrarian
a3633c2f9f Added Snow Valley affiliate link. 2026-02-09 06:28:19 +01:00
DebaucheryLibrarian
a007593390 1.248.59 2026-02-09 05:23:21 +01:00
DebaucheryLibrarian
f31c69f255 Added poster fallback to WankzVR deep scrape, preventing failed trailer request from breaking. 2026-02-09 05:23:18 +01:00
DebaucheryLibrarian
9a088599e6 1.248.58 2026-02-09 05:10:08 +01:00
DebaucheryLibrarian
01341b80d6 Refactored WankzVR for unprint, added referers in hopes of fixing missing assets. 2026-02-09 05:10:06 +01:00
DebaucheryLibrarian
fec70db20b 1.248.57 2026-02-09 03:21:28 +01:00
DebaucheryLibrarian
7bbb7cb43d Broadened Love Her Films profile selectors. 2026-02-09 03:21:26 +01:00
DebaucheryLibrarian
e7a1ad42f5 1.248.56 2026-02-09 02:40:43 +01:00
DebaucheryLibrarian
be90982420 Added unprint logs to profile tests. 2026-02-09 02:40:41 +01:00
DebaucheryLibrarian
ce94456494 1.248.55 2026-02-09 02:33:14 +01:00
DebaucheryLibrarian
fe3fcd0741 Fixed ExploitedX not matching some age fields. 2026-02-09 02:33:11 +01:00
DebaucheryLibrarian
5ca7537e24 1.248.54 2026-02-09 02:16:18 +01:00
DebaucheryLibrarian
35449ef906 Using unprint for profile test URL probing. 2026-02-09 02:16:16 +01:00
DebaucheryLibrarian
bebebf8447 1.248.53 2026-02-09 02:10:43 +01:00
DebaucheryLibrarian
101c84763a Refactored Boobpedia for unprint. 2026-02-09 02:10:40 +01:00
DebaucheryLibrarian
36ca313a89 1.248.52 2026-02-09 01:48:50 +01:00
DebaucheryLibrarian
d07d3c2c8d Fixed birthPlace field in profile tests. 2026-02-09 01:48:48 +01:00
50 changed files with 1875 additions and 1992 deletions

View File

@@ -27,7 +27,7 @@
"require-await": "off",
"no-param-reassign": ["error", {
"props": true,
"ignorePropertyModificationsFor": ["state", "acc", "req"]
"ignorePropertyModificationsFor": ["state", "acc", "req", "error"]
}]
},
"globals": {

View File

@@ -188,6 +188,8 @@ module.exports = {
'wishescumtrue',
// hentaied
'somegore',
// digital playground
'digitalplayground', // no longer updates, produces a bunch of garbage for some reason
],
networks: [
// dummy network for testing
@@ -196,144 +198,7 @@ module.exports = {
'forbondage',
],
},
profiles: [
[
'evilangel',
'famedigital',
'devilsfilm',
'roccosiffredi',
],
[
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), (sometimes) Burning Angel and Wicked have their own assets
'xempire',
'blowpass',
],
[
// MindGeek; Mile High Media has its own assets
'brazzers',
'realitykings',
'mofos',
'digitalplayground',
'twistys',
'babes',
'fakehub',
'sexyhub',
'metrohd',
'iconmale',
'men',
'transangels',
],
'wicked',
'burningangel',
'milehighmedia',
[
'vixen',
'tushy',
'blacked',
'tushyraw',
'blackedraw',
'deeper',
],
[
// Nubiles
'nubiles',
'nubilesporn',
'deeplush',
'brattysis',
'nfbusty',
'anilos',
'hotcrazymess',
'thatsitcomshow',
],
'21sextury',
'dogfartnetwork',
'adultempire',
'julesjordan',
'dorcelclub',
'bang',
'pervcity',
'kink',
'peternorth',
'naughtyamerica',
'cherrypimps',
'pimpxxx',
'18vr',
'babevr',
'badoinkvr',
'realvr',
'vrcosplayx',
'teamskeet',
'mylf',
'spermmania',
[
'letsdoeit',
'mamacitaz',
'forbondage',
'amateureuro',
'vipsexvault',
'transbella',
],
[
'hussiepass',
'hushpass',
'interracialpass',
'interracialpovs',
'povpornstars',
'seehimfuck',
'eyeontheguy',
],
[
// Full Porn Network
'analized',
'hergape',
'jamesdeen',
'dtfsluts',
'analbbc',
'analviolation',
'baddaddypov',
'girlfaction',
'homemadeanalwhores',
'mugfucked',
'onlyprince',
'pervertgallery',
'povperverts',
],
'wankzvr',
'milfvr',
'tranzvr',
'topwebmodels',
'pascalssubsluts',
'kellymadison',
'5kporn',
'private',
'bangbros',
'hitzefrei',
'porncz',
'czechav',
'angelogodshackoriginal',
'littlecapricedreams',
'missyx',
'gangbangcreampie',
'gloryholesecrets',
'aziani',
[
'firstanalquest',
'doubleviewcasting',
],
[
'silverstonedvd',
'silviasaint',
],
[
'analvids',
'pornworld',
],
'pierrewoodman',
'score',
'boobpedia',
'pornhub',
'freeones',
],
profiles: null,
interpolation: {
excludeAvatarCredits: [ // never allow
'Pierre Woodman',
@@ -416,7 +281,7 @@ module.exports = {
trailerQuality: [540, 720, 960, 480, 1080, 360, 320, 1440, 1600, 1920, 2160, 270, 240, 180],
limit: 25, // max number of photos per release
attempts: 2,
flushOrphaned: true,
flushOrphaned: false,
flushWindow: 1000,
streams: {
enabled: true, // fetch streams

View File

@@ -0,0 +1,31 @@
exports.up = async (knex) => {
// dedupe
await knex.raw(`
DELETE
FROM releases_tags
WHERE ctid IN
(
SELECT ctid
FROM(
SELECT
*,
ctid,
row_number() OVER (PARTITION BY release_id, original_tag ORDER BY ctid)
FROM releases_tags
)s
WHERE row_number >= 2
)
`);
await knex.schema.alterTable('releases_tags', (table) => {
table.increments('id');
table.unique(['release_id', 'original_tag']);
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('releases_tags', (table) => {
table.dropColumn('id');
table.dropUnique(['release_id', 'original_tag']);
});
};

View File

@@ -0,0 +1,23 @@
exports.up = async function(knex) {
await knex.schema.alterTable('actors', (table) => {
table.decimal('foot')
.alter();
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.decimal('foot')
.alter();
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('actors', (table) => {
table.integer('foot')
.alter();
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.integer('foot')
.alter();
});
};

View File

@@ -0,0 +1,13 @@
exports.up = async function(knex) {
await knex.schema.alterTable('series', (table) => {
table.specificType('alt_descriptions', 'text ARRAY');
table.json('attributes');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('series', (table) => {
table.dropColumn('alt_descriptions');
table.dropColumn('attributes');
});
};

View File

@@ -0,0 +1,21 @@
exports.up = async function(knex) {
await knex.schema.alterTable('releases_tags', (table) => {
table.integer('actor_id')
.references('id')
.inTable('actors');
table.dropUnique(['tag_id', 'release_id']);
});
await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
};
exports.down = async function(knex) {
await knex.schema.alterTable('releases_tags', (table) => {
table.dropColumn('actor_id');
table.unique(['tag_id', 'release_id']);
});
await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
};

18
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "traxxx",
"version": "1.248.51",
"version": "1.250.38",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "traxxx",
"version": "1.248.51",
"version": "1.250.38",
"license": "ISC",
"dependencies": {
"@aws-sdk/client-s3": "^3.458.0",
@@ -94,7 +94,7 @@
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.37",
"undici": "^5.28.1",
"unprint": "^0.18.31",
"unprint": "^0.18.35",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3",
"video.js": "^8.6.1",
@@ -18822,6 +18822,11 @@
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
"integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw=="
},
"node_modules/set-cookie-parser": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.0.1.tgz",
"integrity": "sha512-n7Z7dXZhJbwuAHhNzkTti6Aw9QDDjZtm3JTpTGATIdNzdQz5GuFs22w90BcvF4INfnrL5xrX3oGsuqO5Dx3A1Q=="
},
"node_modules/set-function-length": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz",
@@ -20380,9 +20385,9 @@
}
},
"node_modules/unprint": {
"version": "0.18.31",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.31.tgz",
"integrity": "sha512-7z4kC7eyyyRUeGBGc+lByRv1WqWP1NkO5Fwh6RwOs60x6HW3mffYUxa2R02fjpUOsEaPRukZwEsikP5jOA/JAA==",
"version": "0.18.35",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.35.tgz",
"integrity": "sha512-oTCBE8pGzfTFlSb0QbYv/ctICTmcU/K81gOPfchn+efLHu48hq1S3582JHvwXAXCjiRKZYatJlEFzUTXVtfuvA==",
"license": "ISC",
"dependencies": {
"bottleneck": "^2.19.5",
@@ -20395,6 +20400,7 @@
"moment-timezone": "^0.5.34",
"object-hash": "^3.0.0",
"patchright": "^1.56.1",
"set-cookie-parser": "^3.0.1",
"srcset": "^4.0.0",
"tunnel": "^0.0.6",
"undici": "^7.18.2"

View File

@@ -1,6 +1,6 @@
{
"name": "traxxx",
"version": "1.248.51",
"version": "1.250.38",
"description": "All the latest porn releases in one place",
"main": "src/app.js",
"scripts": {
@@ -153,7 +153,7 @@
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.37",
"undici": "^5.28.1",
"unprint": "^0.18.31",
"unprint": "^0.18.35",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3",
"video.js": "^8.6.1",

View File

@@ -537,6 +537,10 @@ const tags = [
name: 'enhanced butt',
slug: 'enhanced-butt',
},
{
name: 'extreme insertion',
slug: 'extreme-insertion',
},
{
name: 'facefucking',
slug: 'facefucking',
@@ -834,6 +838,11 @@ const tags = [
slug: 'natural-boobs',
group: 'body',
},
{
name: 'natural butt',
slug: 'natural-butt',
group: 'body',
},
{
name: 'nipple clamps',
slug: 'nipple-clamps',
@@ -954,6 +963,11 @@ const tags = [
slug: 'pyjamas',
group: 'clothing',
},
{
name: 'rave',
slug: 'rave',
group: 'clothing',
},
{
name: 'redhead',
slug: 'redhead',
@@ -1612,6 +1626,10 @@ const aliases = [
name: 'big tits d-dd cup',
for: 'big-boobs',
},
{
name: 'busty',
for: 'big-boobs',
},
{
name: 'busty - big boobs',
for: 'big-boobs',
@@ -2155,6 +2173,18 @@ const aliases = [
name: 'natural tits',
for: 'natural-boobs',
},
{
name: 'natural butt',
for: 'natural-butt',
},
{
name: 'natural ass',
for: 'natural-butt',
},
{
name: 'real ass',
for: 'natural-butt',
},
{
name: 'oiled',
for: 'oil',
@@ -2994,6 +3024,8 @@ const priorities = [ // higher index is higher priority
['blowbang', 'orgy'],
['gangbang'],
['gay', 'transsexual', 'bisexual', 'hentai'],
['pissing'],
['compilation', 'bts'],
].reduce((acc, slugs, index) => {
slugs.forEach((slug) => { acc[slug] = index; });

View File

@@ -9,6 +9,9 @@ const grandParentNetworks = [
name: 'Gamma Entertainment',
url: 'https://www.gammaentertainment.com',
alias: ['gammaentertainment'],
options: {
preferNetwork: true,
},
},
{
slug: 'hush',
@@ -56,7 +59,6 @@ const parentNetworks = [
url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
parameters: {
layout: 'api',
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: 'gamma',
@@ -105,7 +107,6 @@ const parentNetworks = [
url: 'https://www.asgmax.com',
parent: 'gamma',
parameters: {
layout: 'api',
scene: 'https://www.asgmax.com/en/video/asgmax',
},
},
@@ -187,7 +188,6 @@ const networks = [
url: 'https://www.21sextreme.com',
description: 'Welcome to 21Sextreme.com, your portal to fisting porn, old and young lesbians, horny grannies & extreme BDSM featuring the best Euro & American Pornstars',
parameters: {
layout: 'api',
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: '21sextury',
@@ -198,7 +198,6 @@ const networks = [
url: 'https://www.21naturals.com',
description: 'Welcome to 21Naturals.com, the porn network featuring the hottest pornstars from all over the world in all natural porn and erotic sex videos. Watch thousands of girls with natural tits',
parameters: {
layout: 'api',
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: '21sextury',
@@ -228,7 +227,6 @@ const networks = [
description: 'Adult Time is a premium streaming service for adults! Watch adult movies, series, and channels from the top names in the industry.',
parent: 'gamma',
parameters: {
layout: 'api',
referer: 'https://freetour.adulttime.com/en/join',
// scene: false,
},
@@ -320,7 +318,6 @@ const networks = [
url: 'https://www.blowpass.com',
description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
parameters: {
layout: 'api',
referer: 'https://www.blowpass.com',
},
parent: 'gamma',
@@ -373,9 +370,6 @@ const networks = [
url: 'https://www.evilangel.com',
description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
parent: 'gamma',
parameters: {
layout: 'api',
},
},
{
slug: 'exploitedx',
@@ -419,7 +413,7 @@ const networks = [
description: 'The world famous Dogfart Interracial series. Online since 1996, we have the largest collection of Interracial videos, pictures and content on the web.',
parent: 'dfxtra',
parameters: {
layout: 'api',
profileReferer: 'https://www.dogfartnetwork.com',
},
},
{
@@ -432,9 +426,6 @@ const networks = [
slug: 'fantasymassage',
name: 'Fantasy Massage',
url: 'https://www.fantasymassage.com',
parameters: {
layout: 'api',
},
parent: 'gamma',
},
{
@@ -444,7 +435,6 @@ const networks = [
description: 'Watch and download thousands of the best porn videos at FameDigital.com, the largest porn network on the web! The hottest teens, MILFs and more pornstars are all here!',
parameters: {
mobile: 'https://m.dpfanatics.com/en/video',
layout: 'api',
},
parent: 'gamma',
},
@@ -464,7 +454,6 @@ const networks = [
url: 'https://www.filthykings.com',
parent: 'gamma',
parameters: {
layout: 'api',
queryChannel: true,
scene: 'https://www.filthykings.com/en/video/filthykings',
referer: 'https://www.filthykings.com',
@@ -491,6 +480,7 @@ const networks = [
slug: 'fullpornnetwork',
name: 'Full Porn Network',
url: 'https://www.fullpornnetwork.com',
showcased: false,
description: 'FullPornNetwork.com is the latest and greatest for one stop shop porn sites. Check out the expanding library of the multi-site network. All of fan\'s favorite content from ANALIZED.COM, DTFsluts.com, YourMomDoesPorn.com and many more. Give die hard porn fans access to an array of premium content available in 4k and 1080p. Full access included streaming hd and unlimited downloads. Be exclusive, be a member to FullPornNetwork.com Today.',
},
{
@@ -504,9 +494,6 @@ const networks = [
name: 'Girlsway',
url: 'https://www.girlsway.com',
description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!',
parameters: {
layout: 'api',
},
parent: 'gamma',
},
{
@@ -554,7 +541,6 @@ const networks = [
slug: 'jayrock',
name: 'JayRock Productions',
url: 'http://jayrockcontent.com',
parent: 'gamma',
},
{
slug: 'julesjordan',
@@ -702,6 +688,9 @@ const networks = [
name: 'Nubiles',
url: 'https://www.nubiles.com',
description: 'Welcome to the teen megasite that started it all! Browse our massive HD collection of fresh legal hotties at Nubiles.net.',
options: {
forceDeepUpdate: true,
},
},
{
slug: 'perfectgonzo',
@@ -755,7 +744,6 @@ const networks = [
description: 'PureTaboo.com is the ultimate site for family taboo porn, featuring submissive teens & virgins in rough sex videos in ultra 4k HD.',
parent: 'gamma',
parameters: {
layout: 'api',
scene: 'https://www.puretaboo.com/en/video',
referer: 'https://www.puretaboo.com',
},
@@ -775,6 +763,7 @@ const networks = [
slug: 'score',
name: 'SCORE',
url: 'https://www.scorepass.com',
showcased: false,
},
{
slug: 'sexyhub',
@@ -852,7 +841,7 @@ const networks = [
description: 'Home of the Kim Kardashian Sex Tape, Porn Parodies, and over 30,000 XXX Movies from The World Leader In Adult Entertainment.',
parent: 'gamma',
parameters: {
layout: 'api',
sceneMovies: false,
},
},
{
@@ -878,10 +867,8 @@ const networks = [
description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
parent: 'gamma',
parameters: {
layout: 'api',
sceneMovies: false,
actorScenes: 'https://www.xempire.com/en/videos/xempire/latest/{page}/All-Categories/0{actorPath}',
actorAvailableOnSites: ['allblackx', 'darkx', 'eroticax', 'hardx', 'lesbianx', 'xempire', 'xempirepartners'],
},
},
{
@@ -889,9 +876,6 @@ const networks = [
name: 'Zero Tolerance',
alias: ['ztod'],
url: 'https://www.zerotolerancefilms.com',
parameters: {
layout: 'api',
},
parent: 'gamma',
},
// ASG MAX
@@ -948,7 +932,8 @@ exports.seed = (knex) => Promise.resolve()
description: network.description,
has_logo: network.hasLogo ?? true,
showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
parameters: network.parameters,
parameters: network.parameters || null,
options: network.options,
parent_id: grandParentNetworksBySlug[network.parent] || null,
}));
@@ -964,7 +949,8 @@ exports.seed = (knex) => Promise.resolve()
description: network.description,
has_logo: network.hasLogo ?? true,
showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
parameters: network.parameters,
parameters: network.parameters || null,
options: network.options,
parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null,
}));

View File

@@ -579,7 +579,6 @@ const sites = [
tags: ['gay'],
independent: true,
parameters: {
layout: 'api',
scene: 'https://www.chaosmen.com/en/video/chaosmen',
},
},
@@ -601,7 +600,6 @@ const sites = [
parent: 'gamma',
independent: true,
parameters: {
layout: 'api',
scene: 'https://www.tabooheat.com/en/video/tabooheat',
},
},
@@ -611,6 +609,9 @@ const sites = [
slug: 'asgmaxoriginals',
url: 'https://www.asgmax.com/en/channel/asgmaxoriginals',
parent: 'asgmax',
parameters: {
queryChannel: 'asgmaxoriginals',
},
},
{
name: 'ASG Max Films',
@@ -618,6 +619,9 @@ const sites = [
url: 'https://www.asgmax.com/en/channel/asgmaxfilms',
parent: 'asgmax',
hasLogo: false,
parameters: {
queryChannel: 'asgmaxfilms',
},
},
{
name: 'ASG International',
@@ -625,6 +629,9 @@ const sites = [
url: 'https://www.asgmax.com/en/channel/asginternational',
parent: 'asgmax',
hasLogo: false,
parameters: {
queryChannel: 'asginternational',
},
},
{
name: 'ASG Massage',
@@ -633,6 +640,9 @@ const sites = [
parent: 'asgmax',
tags: ['massage'],
hasLogo: false,
parameters: {
queryChannel: 'asgmassage',
},
},
{
name: 'ASG Auditions',
@@ -641,6 +651,9 @@ const sites = [
parent: 'asgmax',
tags: ['audition'],
hasLogo: false,
parameters: {
queryChannel: 'asgauditions',
},
},
{
name: 'ASG Free Use',
@@ -649,6 +662,9 @@ const sites = [
parent: 'asgmax',
tags: ['free-use'],
hasLogo: false,
parameters: {
queryChannel: 'asgfreeuse',
},
},
{
name: 'Exeter Hill College',
@@ -657,6 +673,9 @@ const sites = [
parent: 'asgmax',
hasLogo: false,
tags: ['animated'],
parameters: {
queryChannel: 'asgexeterhillcollege',
},
},
// ASG MAX INDEPENDENT
{
@@ -1020,12 +1039,12 @@ const sites = [
{
name: 'Disruptive Films',
slug: 'disruptivefilms',
delete: true,
url: 'https://www.disruptivefilms.com',
parent: 'disruptivefilms',
tags: ['gay'],
parameters: {
queryChannel: 'asgmaxdisruptivefilms',
filterChannel: 'asgmaxdisruptivefilms',
},
},
{
@@ -1072,6 +1091,17 @@ const sites = [
queryChannel: 'asgmaxtruemale',
},
},
{
name: 'Deep Inside',
slug: 'deepinside',
url: 'https://www.asgmax.com/en/channel/asgmaxdeepinside',
parent: 'disruptivefilms',
tags: ['gay'],
hasLogo: false,
parameters: {
queryChannel: 'asgmaxdeepinside',
},
},
// AMATEUR ALLURE
{
name: 'Amateur Allure',
@@ -1325,7 +1355,6 @@ const sites = [
seriesId: 268,
// areaId: 3,
areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
blockId: 114458,
scene: 'https://aziani.com',
},
},
@@ -1338,9 +1367,7 @@ const sites = [
parameters: {
// areaId: 2,
areaId: '4f0a134f-8a1b-4bcb-8013-7e02fac4f61d',
blockId: 114064,
seriesId: 107,
modelBlockId: 114129,
},
},
{
@@ -1351,10 +1378,9 @@ const sites = [
independent: true,
parameters: {
// areaId: 11,
videos: '/creampiled-videos',
areaId: '29c61dae-db14-419b-93a4-d016b928dee9',
blockId: 115990,
seriesId: 436,
modelBlockId: 115345,
},
},
{
@@ -1366,7 +1392,6 @@ const sites = [
parameters: {
// areaId: 8,
areaId: 'ae0a26fe-7f08-433d-bb04-a9b6f358c48e',
blockId: 116531,
seriesId: 395,
},
},
@@ -1378,7 +1403,6 @@ const sites = [
parameters: {
// areaId: 3,
areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
blockId: 114458,
seriesId: 105,
scene: 'https://aziani.com',
},
@@ -1392,7 +1416,6 @@ const sites = [
parameters: {
// areaId: 3,
areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
blockId: 114458,
seriesId: 106,
scene: 'https://aziani.com',
},
@@ -1433,7 +1456,6 @@ const sites = [
independent: true,
parameters: {
scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie',
layout: 'api',
},
},
{
@@ -1445,7 +1467,6 @@ const sites = [
independent: true,
parameters: {
scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets',
layout: 'api',
},
},
/* different layout
@@ -1458,7 +1479,6 @@ const sites = [
independent: true,
parameters: {
scene: 'https://www.portagloryhole.com/scenes',
layout: 'api',
},
},
*/
@@ -2692,9 +2712,6 @@ const sites = [
url: 'https://www.biphoria.com',
independent: true,
tags: ['bisexual'],
parameters: {
layout: 'api',
},
parent: 'gamma',
},
// BLOWPASS
@@ -3187,9 +3204,6 @@ const sites = [
alias: ['burna'],
url: 'https://www.burningangel.com',
independent: true,
parameters: {
layout: 'api',
},
parent: 'gamma',
},
// CHERRY PIMPS
@@ -3835,7 +3849,6 @@ const sites = [
parent: 'gamma',
independent: true,
parameters: {
layout: 'api',
deep: 'https://www.diabolic.com/en/video/diabolic',
actors: 'https://www.diabolic.com/en/pornstar/view/{slug}/{id}',
},
@@ -3907,9 +3920,6 @@ const sites = [
name: 'DFXtra',
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
parameters: {
layout: 'api',
},
},
{
slug: 'dfxtraoriginals',
@@ -3917,9 +3927,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
{
slug: 'dfxtracompilations',
@@ -3927,9 +3934,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
{
slug: 'dfxbigbangz',
@@ -3937,9 +3941,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
{
slug: 'dfxsolemates',
@@ -3947,9 +3948,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
{
slug: 'cheatingwithmyex',
@@ -3957,9 +3955,6 @@ const sites = [
url: 'https://www.dfxtra.com',
parent: 'dfxtra',
hasLogo: false,
parameters: {
layout: 'api',
},
},
// DFXTRA DOGFART
{
@@ -5459,6 +5454,12 @@ const sites = [
referer: 'https://www.girlsway.com',
},
},
{
slug: 'lesbianfactor',
name: 'Lesbian Factor',
url: 'https://www.lesbianfactor.com',
parent: 'girlsway',
},
// HITZEFREI
{
slug: 'unleashed',
@@ -5565,6 +5566,19 @@ const sites = [
url: 'https://seehimfuck.com',
tags: ['male-focus'],
parent: 'hussiepass',
parameters: {
latest: 'https://seehimfuck.com',
},
},
{
slug: 'seehimsolo',
name: 'See Him Solo',
url: 'https://seehimsolo.com',
tags: ['male-focus', 'solo'],
parent: 'hussiepass',
parameters: {
latest: 'https://seehimsolo.com/categories/movies-2/{page}/latest/',
},
},
{
slug: 'interracialpovs',
@@ -5576,14 +5590,28 @@ const sites = [
{
slug: 'povpornstars',
name: 'POV Pornstars',
url: 'http://www.povpornstars.com',
url: 'https://www.povpornstars.com',
tags: ['pov'],
parent: 'hussiepass',
parameters: {
latest: 'http://www.povpornstars.com/tour/categories/movies_%d_d.html',
profile: 'http://www.povpornstars.com/tour/models/%s.html',
latest: 'https://www.povpornstars.com/tour/categories/movies_{page}_d.html',
profile: 'https://www.povpornstars.com/tour/models/{actor}.html',
},
},
{
slug: 'ravebunnys',
name: 'Rave Bunnys',
url: 'https://ravebunnys.com',
tags: ['rave'],
parent: 'hussiepass',
},
{
slug: 'hotandtatted',
name: 'Hot and Tatted',
url: 'https://hotandtatted.com',
tags: ['tattoos'],
parent: 'hussiepass',
},
// HUSH PASS
{
slug: 'shotherfirst',
@@ -5591,7 +5619,7 @@ const sites = [
url: 'https://shotherfirst.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/shot-her-first_%d_d.html',
latest: 'https://hushpass.com/t1/categories/shot-her-first_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5602,7 +5630,7 @@ const sites = [
url: 'https://whitezilla.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/whitezilla_%d_d.html',
latest: 'https://hushpass.com/t1/categories/whitezilla_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5613,7 +5641,7 @@ const sites = [
url: 'https://frathousefuckfest.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/frat-house-fuck-fest_%d_d.html',
latest: 'https://hushpass.com/t1/categories/frat-house-fuck-fest_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5624,7 +5652,7 @@ const sites = [
url: 'https://freakyfirsttimers.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/freaky-first-timers_%d_d.html',
latest: 'https://hushpass.com/t1/categories/freaky-first-timers_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5635,7 +5663,7 @@ const sites = [
url: 'https://milfinvaders.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/milf-invaders_%d_d.html',
latest: 'https://hushpass.com/t1/categories/milf-invaders_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5646,7 +5674,7 @@ const sites = [
url: 'https://housewivesneedcash.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/housewives-need-cash_%d_d.html',
latest: 'https://hushpass.com/t1/categories/housewives-need-cash_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5657,7 +5685,7 @@ const sites = [
url: 'https://bubblebuttbonanza.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/bubble-butt-bonanza_%d_d.html',
latest: 'https://hushpass.com/t1/categories/bubble-butt-bonanza_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5668,7 +5696,7 @@ const sites = [
url: 'https://suburbansexparty.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/suburban-sex-party_%d_d.html',
latest: 'https://hushpass.com/t1/categories/suburban-sex-party_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5679,7 +5707,7 @@ const sites = [
url: 'https://buttnakedinthestreets.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/ButtNakedInStreets_%d_d.html',
latest: 'https://hushpass.com/t1/categories/ButtNakedInStreets_{page}_d.html',
media: 'https://hushpass.com',
match: 'Butt Naked In Streets',
t1: true,
@@ -5691,7 +5719,7 @@ const sites = [
url: 'https://muffbumperpatrol.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/muff-bumper-patrol_%d_d.html',
latest: 'https://hushpass.com/t1/categories/muff-bumper-patrol_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5702,7 +5730,7 @@ const sites = [
url: 'https://biggathananigga.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/bigga-than-a-nigga_%d_d.html',
latest: 'https://hushpass.com/t1/categories/bigga-than-a-nigga_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5713,7 +5741,7 @@ const sites = [
url: 'https://bachelorpartyfuckfest.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/bachelor-party-fuck-fest_%d_d.html',
latest: 'https://hushpass.com/t1/categories/bachelor-party-fuck-fest_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5724,7 +5752,7 @@ const sites = [
url: 'https://teencumdumpsters.com',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/teen-cum-dumpsters_%d_d.html',
latest: 'https://hushpass.com/t1/categories/teen-cum-dumpsters_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5734,7 +5762,7 @@ const sites = [
name: 'POV Hunnies',
parent: 'hushpass',
parameters: {
latest: 'https://hushpass.com/t1/categories/POVHunnies_%d_d.html',
latest: 'https://hushpass.com/t1/categories/POVHunnies_{page}_d.html',
media: 'https://hushpass.com',
t1: true,
},
@@ -5872,7 +5900,7 @@ const sites = [
tags: ['interracial'],
parent: 'interracialpass',
parameters: {
latest: 'https://www.interracialpass.com/t1/categories/2-big-to-be-true_%d_d.html',
latest: 'https://www.interracialpass.com/t1/categories/2-big-to-be-true_{page}_d.html',
media: 'https://www.interracialpass.com',
t1: true,
},
@@ -5884,7 +5912,7 @@ const sites = [
tags: ['interracial'],
parent: 'interracialpass',
parameters: {
latest: 'https://www.interracialpass.com/t1/categories/abominable-black-man_%d_d.html',
latest: 'https://www.interracialpass.com/t1/categories/abominable-black-man_{page}_d.html',
media: 'https://www.interracialpass.com',
t1: true,
},
@@ -5896,7 +5924,7 @@ const sites = [
parent: 'interracialpass',
hasLogo: false,
parameters: {
latest: 'https://www.interracialpass.com/t1/categories/BootyAnnihilation_%d_d.html',
latest: 'https://www.interracialpass.com/t1/categories/BootyAnnihilation_{page}_d.html',
media: 'https://www.interracialpass.com',
t1: true,
},
@@ -5908,7 +5936,7 @@ const sites = [
tags: ['interracial'],
parent: 'interracialpass',
parameters: {
latest: 'https://www.interracialpass.com/t1/categories/daddys-worst-nightmare_%d_d.html',
latest: 'https://www.interracialpass.com/t1/categories/daddys-worst-nightmare_{page}_d.html',
media: 'https://www.interracialpass.com',
t1: true,
},
@@ -5920,7 +5948,7 @@ const sites = [
tags: ['interracial'],
parent: 'interracialpass',
parameters: {
latest: 'https://www.interracialpass.com/t1/categories/monster-cock-fuck-fest_%d_d.html',
latest: 'https://www.interracialpass.com/t1/categories/monster-cock-fuck-fest_{page}_d.html',
media: 'https://www.interracialpass.com',
t1: true,
},
@@ -5932,7 +5960,7 @@ const sites = [
tags: ['interracial'],
parent: 'interracialpass',
parameters: {
latest: 'https://www.interracialpass.com/t1/categories/my-daughters-fucking-a-black-dude_%d_d.html',
latest: 'https://www.interracialpass.com/t1/categories/my-daughters-fucking-a-black-dude_{page}_d.html',
media: 'https://www.interracialpass.com',
t1: true,
},
@@ -5944,7 +5972,7 @@ const sites = [
tags: ['interracial'],
parent: 'interracialpass',
parameters: {
latest: 'https://www.interracialpass.com/t1/categories/my-moms-fucking-blackzilla_%d_d.html',
latest: 'https://www.interracialpass.com/t1/categories/my-moms-fucking-blackzilla_{page}_d.html',
media: 'https://www.interracialpass.com',
t1: true,
},
@@ -5956,7 +5984,7 @@ const sites = [
tags: ['interracial'],
parent: 'interracialpass',
parameters: {
latest: 'https://www.interracialpass.com/t1/categories/my-wifes-first-monster-cock_%d_d.html',
latest: 'https://www.interracialpass.com/t1/categories/my-wifes-first-monster-cock_{page}_d.html',
media: 'https://www.interracialpass.com',
match: 'My Wifes First Monster Cock',
t1: true,
@@ -6069,7 +6097,6 @@ const sites = [
url: 'https://cospimps.com',
parent: 'jayrock',
parameters: {
layout: 'api',
/* Gamma scenes are out of date
referer: 'https://www.21sextury.com',
scene: false,
@@ -7279,6 +7306,29 @@ const sites = [
showcased: true,
hasLogo: false,
visible: true,
// merged into Christian Clay's studio, but that started rereleasing from other channels
},
{
name: 'Adventure Teens',
slug: 'adventureteens',
url: 'https://www.analvids.com/studios/deeplove',
alias: ['deeplove'],
parent: 'analvids',
hasLogo: false,
parameters: {
studioId: 1007,
},
},
{
name: 'Lancelot Styles',
slug: 'lancelotstyles',
url: 'https://www.analvids.com/studios/latinhot_studio',
alias: ['latinhot studio', 'latin hot studio'],
parent: 'analvids',
hasLogo: false,
parameters: {
studioId: 698,
},
},
// LETS DOE IT
{
@@ -8959,7 +9009,7 @@ const sites = [
{
slug: 'afterschool',
name: 'After School',
url: 'https://www.naughtyamerica.com/site/after-schooll', // sic
url: 'https://www.naughtyamerica.com/site/after-school',
parent: 'naughtyamericavr',
},
{
@@ -10509,10 +10559,11 @@ const sites = [
name: 'Look At Her Now',
url: 'https://www.lookathernow.com',
description: 'Look At Her Now brings you best HD reality porn videos every week. Check out these girls before and after they get some rough pounding.',
parameters: { native: true },
// parameters: { siteId: 300 },
slug: 'lookathernow',
parent: 'realitykings',
parameters: {
siteId: 364,
},
},
{
name: 'We Live Together',
@@ -11763,6 +11814,7 @@ const sites = [
slug: 'scoreland',
url: 'https://www.scoreland.com',
parent: 'score',
showcased: true,
parameters: {
path: '/big-boob-videos',
},
@@ -13349,9 +13401,11 @@ const sites = [
name: 'Sic Flics',
styled: 'SicFlics',
url: 'https://www.sicflics.com',
tags: ['extreme-insertion'],
parameters: {
layout: 'mobile', // has scene links
actorTags: {
// no dedicated actor fields, these tags are used instead
maria: 'Queen Maria',
crazywifeslut: 'CrazyWifeSlut',
kassiopeia: 'Kassiopeia',
@@ -15000,12 +15054,12 @@ const sites = [
name: 'Vivid Celeb',
url: 'https://www.vividceleb.com',
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2018-03-25'),
},
},
{
slug: 'vivid',
name: 'Vivid',
url: 'https://www.vivid.com/en/videos/sites/vivid',
parent: 'vivid',
},
{
slug: 'thebrats',
@@ -15022,122 +15076,72 @@ const sites = [
{
slug: 'nineteen',
name: 'Nineteen',
url: 'http://www.nineteen.com',
url: 'https://www.vivid.com/en/videos/sites/nineteen',
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-23'),
},
},
{
slug: 'nastystepfamily',
name: 'Nasty Step Family',
url: 'http://www.nastystepfamily.com',
url: 'https://www.vivid.com/en/videos/sites/nastystepfamily',
tags: ['family'],
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-29'),
},
},
{
slug: 'girlswhofuckgirls',
name: 'Girls Who Fuck Girls',
url: 'http://www.girlswhofuckgirls.com',
url: 'https://www.vivid.com/en/videos/sites/girlswhofuckgirls',
tags: ['lesbian'],
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-05-21'),
},
},
{
slug: 'petited',
name: 'Petited',
url: 'http://www.petited.com',
url: 'https://www.vivid.com/en/videos/sites/petited',
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-28'),
},
},
{
slug: 'orgytrain',
name: 'Orgy Train',
url: 'http://www.orgytrain.com',
url: 'https://www.vivid.com/en/videos/sites/orgytrain',
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-09'),
},
},
{
slug: 'momisamilf',
name: 'Mom Is A MILF',
url: 'http://www.momisamilf.com',
url: 'https://www.vivid.com/en/videos/sites/momisamilf',
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-25'),
},
},
{
slug: 'blackwhitefuckfest',
name: 'Black White Fuck Fest',
url: 'http://www.blackwhitefuckfest.com',
url: 'https://www.vivid.com/en/videos/sites/blackwhitefuckfest',
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-30'),
},
},
{
slug: '65inchhugeasses',
name: '65 Inch Huge Asses',
url: 'http://www.65inchhugeasses.com',
url: 'https://www.vivid.com/en/videos/sites/65inchhugeasses',
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-05-18'),
},
},
{
slug: 'brandnewfaces',
name: 'Brand New Faces',
url: 'http://www.brandnewfaces.com',
url: 'https://www.vivid.com/en/videos/sites/brandnewfaces',
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2018-02-28'),
},
},
{
slug: 'vividclassic',
name: 'Vivid Classic',
url: 'http://www.vividclassic.com',
url: 'https://www.vivid.com/en/videos/sites/vividclassic',
parent: 'vivid',
},
{
slug: 'tsdivas',
name: 'TS Divas',
url: 'https://www.vivid.com/en/videos/sites/tsdivas',
tags: ['transsexual'],
hasLogo: false,
parent: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2016-06-29'),
},
},
// VIXEN
{
@@ -15441,9 +15445,6 @@ const sites = [
url: 'https://www.wicked.com',
description: 'Welcome to the new Wicked.com! Watch over 25 years of Wicked Pictures\' brand of award-winning porn for couples and women in 4k HD movies & xxx videos',
independent: true,
parameters: {
layout: 'api',
},
parent: 'gamma',
},
// XEMPIRE
@@ -15580,9 +15581,9 @@ exports.seed = (knex) => Promise.resolve()
alias: site.alias,
description: site.description,
url: site.url,
parameters: site.parameters,
parameters: site.parameters || null,
options: site.options,
parent_id: networksMap[site.parent],
parent_id: networksMap[site.parent] || null,
priority: site.priority || 0,
independent: !!site.independent,
visible: site.visible,

View File

@@ -208,6 +208,11 @@ const affiliates = [
url: 'https://www.g2buddy.com/disruptivefilms/go.php?pr=9&su=2&si=119&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'sodomysquad',
url: 'https://www.g2buddy.com/sodomysquad/go.php?pr=9&su=2&si=137&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
// gamma > ags max > next door studios
// excluded affiliate links that link back to main site and don't seem to track properly
{
@@ -494,6 +499,92 @@ const affiliates = [
scene: false, // redirects to Adult Time
},
},
// gamma > vivid
{
network: 'vivid',
url: 'https://www.g2fame.com/vivid/go.php?pr=8&su=2&si=330&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
parameters: {
scene: false, // redirects to homepage
},
},
{
channel: 'wheretheboysarent',
url: 'https://www.g2fame.com/wheretheboysarent/go.php?pr=8&su=2&si=368&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'thebrats',
url: 'https://www.g2fame.com/thebrats/go.php?pr=8&su=2&si=369&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
// gamma > zero tolerance
{
network: 'zerotolerance',
url: 'https://www.g2fame.com/zerotolerancefilms/go.php?pr=8&su=2&si=507&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'zerotolerancefilms',
url: 'https://www.g2fame.com/zerotolerancefilms/go.php?pr=8&su=2&si=507&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: '3rddegreefilms',
url: 'https://www.g2fame.com/3rddegreefilms/go.php?pr=8&su=2&si=537&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'addicted2girls',
url: 'https://www.g2fame.com/addicted2girls/go.php?pr=8&su=2&si=477&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'genderxfilms',
url: 'https://www.g2fame.com/genderxfilms/go.php?pr=8&su=2&si=397&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'gangbangcreampie',
url: 'https://www.g2fame.com/gangbangcreampie/go.php?pr=8&su=2&si=656&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'gloryholesecrets',
url: 'https://www.g2fame.com/gloryholesecrets/go.php?pr=8&su=2&si=655&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'tabooheat',
url: 'https://www.g2fame.com/tabooheat/go.php?pr=8&su=2&si=552&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'wicked',
url: 'https://www.g2fame.com/wicked/go.php?pr=8&su=2&si=371&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
// gamma > independent channels
{
channel: 'biphoria',
url: 'https://www.g2fame.com/biphoria/go.php?pr=8&su=2&si=418&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'burningangel',
url: 'https://www.g2fame.com/burningangel/go.php?pr=8&su=2&si=174&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'chaosmen',
url: 'https://www.g2fame.com/chaosmen/go.php?pr=8&su=2&si=608&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
{
channel: 'diabolic',
url: 'https://www.g2fame.com/diabolic/go.php?pr=8&su=2&si=523&ad=277470&pa=index&ar=&buffer=',
comment: 'per signup',
},
// kelly madison / 8k
{
network: 'kellymadison',
@@ -847,6 +938,53 @@ const affiliates = [
channel: 'xangels',
url: 'https://secure.x-angels.com/track/MzAxNjcxLjUuMzguODEuMC4wLjAuMC4w',
},
// Snow Valley / Tsunami
{
network: 'snowvalley',
parameters: {
query: 'ref=4c331ef6',
},
},
// POV Porn Cash / HussiePass
{
network: 'hussiepass',
url: 'https://secure.hussiepass.com/track/MTk0NS4xLjUuNy4wLjAuMC4wLjA',
comment: '50% revshare',
parameters: {
// hussiepass website does not show network scenes
channelScenes: false,
},
},
{
channel: 'povpornstars',
url: 'https://join.povpornstars.com/track/MTk0NS4xLjMuNS4wLjAuMC4wLjA',
comment: '50% revshare',
},
{
channel: 'interracialpovs',
url: 'https://join.interracialpovs.com/track/MTk0NS4xLjYuOC4wLjAuMC4wLjA',
comment: '50% revshare',
},
{
channel: 'ravebunnys',
url: 'https://secure.ravebunnys.com/track/MTk0NS4xLjExLjI5LjAuMC4wLjAuMA',
comment: '50% revshare',
},
{
channel: 'hotandtatted',
url: 'https://join.hotandtatted.com/track/MTk0NS4xLjEwLjEyLjAuMC4wLjAuMA',
comment: '50% revshare',
},
{
channel: 'seehimfuck',
url: 'https://join.seehimfuck.com/track/MTk0NS4xLjcuOS4wLjAuMC4wLjA',
comment: '50% revshare',
},
{
channel: 'seehimsolo',
url: 'https://join.seehimsolo.com/track/MTk0NS4xLjguMTAuMC4wLjAuMC4w',
comment: '50% revshare',
},
// etc
{
network: 'bang',
@@ -855,6 +993,11 @@ const affiliates = [
},
comment: '$25 per signup',
},
{
channel: 'sicflics',
url: 'http://refer.ccbill.com/cgi-bin/clicks.cgi?CA=921613-0002&PA=2647683',
comment: '40% rebill',
},
];
const bannerTags = {

View File

@@ -17,11 +17,12 @@ const domPurify = DOMPurify(window);
// const logger = require('./logger')(__filename);
const knex = require('./knex');
const redis = require('./redis');
const scrapers = require('./scrapers/scrapers').actors;
const actorScrapers = require('./scrapers/scrapers').actors;
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const bulkInsert = require('./utils/bulk-insert');
const batchInsert = require('./utils/batch-insert');
const chunk = require('./utils/chunk');
const logger = require('./logger')(__filename);
@@ -46,6 +47,7 @@ const commonContext = {
slugify,
omit,
unprint,
batchInsert,
};
const hairColors = {
@@ -349,6 +351,7 @@ function curateProfileEntry(profile) {
tattoos: profile.tattoos,
blood_type: profile.bloodType,
avatar_media_id: profile.avatarMediaId || null,
updated_at: knex.raw('DEFAULT'), // default should be NOW(), this will update the column
};
return curatedProfileEntry;
@@ -438,35 +441,35 @@ async function curateProfile(profile, actor) {
|| null;
curatedProfile.dateOfDeath = Number.isNaN(Number(profile.dateOfDeath)) ? null : profile.dateOfDeath;
curatedProfile.age = Number(profile.age) || null;
curatedProfile.age = Math.round(profile.age) || null;
curatedProfile.height = Number(profile.height) || profile.height?.match?.(/\d+/)?.[0] || null;
curatedProfile.weight = Number(profile.weight) || profile.weight?.match?.(/\d+/)?.[0] || null;
curatedProfile.shoeSize = Number(profile.shoeSize) || profile.shoeSize?.match?.(/\d+/)?.[0] || null;
curatedProfile.height = Math.round(profile.height || profile.height?.match?.(/\d+/)?.[0]) || null;
curatedProfile.weight = Math.round(profile.weight || profile.weight?.match?.(/\d+/)?.[0]) || null;
// separate measurement values
curatedProfile.cup = profile.cup || (typeof profile.bust === 'string' && profile.bust?.match?.(/[a-zA-Z]+/)?.[0]) || null;
curatedProfile.bust = Number(profile.bust) || profile.bust?.match?.(/\d+/)?.[0] || null;
curatedProfile.waist = Number(profile.waist) || profile.waist?.match?.(/\d+/)?.[0] || null;
curatedProfile.hip = Number(profile.hip) || profile.hip?.match?.(/\d+/)?.[0] || null;
curatedProfile.bust = Math.round(profile.bust || profile.bust?.match?.(/\d+/)?.[0]) || null;
curatedProfile.waist = Math.round(profile.waist || profile.waist?.match?.(/\d+/)?.[0]) || null;
curatedProfile.hip = Math.round(profile.hip || profile.hip?.match?.(/\d+/)?.[0]) || null;
curatedProfile.leg = Number(profile.leg) || profile.leg?.match?.(/\d+/)?.[0] || null;
curatedProfile.thigh = Number(profile.thigh) || profile.thigh?.match?.(/\d+/)?.[0] || null;
curatedProfile.foot = Number(profile.foot) || profile.foot?.match?.(/\d+/)?.[0] || null;
curatedProfile.leg = Math.round(profile.leg || profile.leg?.match?.(/\d+/)?.[0]) || null;
curatedProfile.thigh = Math.round(profile.thigh || profile.thigh?.match?.(/\d+/)?.[0]) || null;
curatedProfile.foot = Number(profile.foot || profile.foot?.match?.(/\d+/)?.[0]) || null;
curatedProfile.shoeSize = Number(profile.shoeSize || profile.shoeSize?.match?.(/\d+/)?.[0]) || null;
// combined measurement value
// ExCoGi uses x, Jules Jordan has spaces between the dashes, SpermMenia/Cum Buffet sometimes misses cup
const measurements = profile.measurements?.match(/(\d+)([a-z]+)?(?:\s*[-x]\s*(\d+)\s*[-x]\s*(\d+))?/i);
if (measurements) {
curatedProfile.bust = Number(measurements[1]) || null;
curatedProfile.bust = Math.round(measurements[1]) || null;
curatedProfile.cup = measurements[2] || null;
curatedProfile.waist = Number(measurements[3]) || null;
curatedProfile.hip = Number(measurements[4]) || null;
curatedProfile.waist = Math.round(measurements[3]) || null;
curatedProfile.hip = Math.round(measurements[4]) || null;
}
curatedProfile.penisLength = Number(profile.penisLength) || profile.penisLength?.match?.(/\d+/)?.[0] || null;
curatedProfile.penisGirth = Number(profile.penisGirth) || profile.penisGirth?.match?.(/\d+/)?.[0] || null;
curatedProfile.penisLength = Math.round(profile.penisLength || profile.penisLength?.match?.(/\d+/)?.[0]) || null;
curatedProfile.penisGirth = Math.round(profile.penisGirth || profile.penisGirth?.match?.(/\d+/)?.[0]) || null;
curatedProfile.isCircumcised = getBoolean(profile.isCircumcised);
curatedProfile.naturalBoobs = getBoolean(profile.naturalBoobs);
@@ -544,7 +547,7 @@ async function curateProfile(profile, actor) {
async function insertProfiles(newProfiles) {
if (newProfiles.length > 0) {
const entries = await bulkInsert('actors_profiles', newProfiles);
const entries = await batchInsert('actors_profiles', newProfiles);
logger.info(`Saved ${newProfiles.length} actor profiles`);
@@ -606,10 +609,7 @@ async function upsertProfiles(profiles) {
}));
if (avatars.length > 0) {
await knex('actors_avatars')
.insert(avatars)
.onConflict()
.ignore();
await batchInsert('actors_avatars', avatars, { conflict: false });
}
}
}
@@ -624,7 +624,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
try {
const entity = entitiesBySlug[scraperSlug] || null;
const scraper = scrapers[scraperSlug];
const scraper = actorScrapers[scraperSlug];
const layoutScraper = resolveLayoutScraper(entity, scraper);
if (!layoutScraper?.fetchProfile) {
@@ -759,7 +759,8 @@ function curateSocials(socials, platformsByHostname) {
async function associateSocials(profiles) {
const { platformsByHostname } = await actorsCommon;
const profileEntries = await knex('actors_profiles').whereIn(['actor_id', 'entity_id'], profiles.map((profile) => [profile.actorId, profile.entity.id]));
const profileEntryChunks = await Promise.all(chunk(profiles).map((profilesChunk) => knex('actors_profiles').whereIn(['actor_id', 'entity_id'], profilesChunk.map((profile) => [profile.actorId, profile.entity.id]))));
const profileEntries = profileEntryChunks.flat();
const profileEntriesByActorIdAndEntityId = profileEntries.reduce((acc, profileEntry) => {
if (!acc[profileEntry.actor_id]) {
@@ -784,16 +785,14 @@ async function associateSocials(profiles) {
return;
}
await knex('actors_socials')
.insert(curateSocials(profile.social, platformsByHostname).map((social) => ({
platform: social.platform,
handle: social.handle,
url: social.url,
actor_id: profile.actorId,
// profile_id: profileId,
})))
.onConflict()
.ignore();
await batchInsert('actors_socials', curateSocials(profile.social, platformsByHostname).map((social) => ({
platform: social.platform,
handle: social.handle,
url: social.url,
actor_id: profile.actorId,
})), {
conflict: false,
});
}, Promise.resolve());
}
@@ -832,11 +831,11 @@ async function scrapeActors(argNames) {
logger.info(`Scraping profiles for ${actorNames.length} actors`);
const sources = argv.profileSources || config.profiles || Object.keys(scrapers.actors);
const sources = argv.profileSources || config.profiles || Object.keys(actorScrapers);
const entitySlugs = sources.flat();
const [entitiesBySlug, existingActorEntries] = await Promise.all([
fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'], prefer: argv.prefer || 'channel' }),
fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'], prefer: argv.prefer || 'options' }),
knex('actors')
.select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
.whereIn('actors.slug', baseActors.map((baseActor) => baseActor.slug))

View File

@@ -24,7 +24,8 @@ const { updateSceneSearch, updateMovieSearch } = require('./update-search');
const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
const { flushEntities } = require('./entities');
const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases');
const { flushOrphanedMedia } = require('./media');
const { flushOrphanedMedia, detachReleaseMedia, detachEntityReleaseMedia } = require('./media');
const { reassociateEntityReleaseTags, reassociateReleaseTags, reassociateOriginalTags } = require('./tags');
const getFileEntries = require('./utils/file-entries');
const inspector = new Inspector();
@@ -159,7 +160,8 @@ async function init() {
}
if (argv.flushNetworks || argv.flushChannels) {
await flushEntities(argv.flushNetworks, argv.flushChannels);
// inject flushOrphanedMedia to prevent circular dependency with entity media flush
await flushEntities(argv.flushNetworks, argv.flushChannels, flushOrphanedMedia);
}
if (argv.flushBatches) {
@@ -186,10 +188,30 @@ async function init() {
await deleteMovies(argv.deleteMovies);
}
if (argv.originalTags) {
await reassociateOriginalTags(argv.originalTags, argv.rematchTags);
}
if (argv.releaseTags) {
await reassociateReleaseTags(argv.releaseTags, argv.rematchTags);
}
if (argv.networkReleaseTags || argv.channelReleaseTags) {
await reassociateEntityReleaseTags(argv.networkReleaseTags, argv.channelReleaseTags, argv.rematchTags);
}
if (argv.flushOrphanedMedia) {
await flushOrphanedMedia();
}
if (argv.detachReleaseMedia) {
await detachReleaseMedia(argv.detachReleaseMedia);
}
if (argv.detachNetworkMedia || argv.detachChannelMedia) {
await detachEntityReleaseMedia(argv.detachNetworkMedia, argv.detachChannelMedia);
}
if (argv.request) {
const res = await http[argv.requestMethod](argv.request);

View File

@@ -349,7 +349,32 @@ const { argv } = yargs
describe: 'Remove files from storage when flushing media.',
type: 'boolean',
alias: 'flush-files',
default: true,
})
.option('detach-channel-media', {
describe: 'Remove media files from channel scenes.',
type: 'array',
})
.option('detach-network-media', {
describe: 'Remove media files from network scenes.',
type: 'array',
})
.option('detach-release-media', {
describe: 'Remove media files from network scenes.',
type: 'array',
alias: ['detach-scene-media'],
})
.option('detach-media-domains', {
describe: 'Only detach these types of media.',
type: 'array',
default: [
'posters',
'photos',
'caps',
'trailers',
'teasers',
'covers',
],
alias: ['detach-media'],
})
.option('flush-channels', {
describe: 'Delete all scenes and movies from channels.',
@@ -407,6 +432,32 @@ const { argv } = yargs
type: 'array',
alias: ['delete-movie', 'remove-movies', 'remove-movies'],
})
.option('original-tags', {
describe: 'Reassociate original tag names',
type: 'array',
alias: ['tags'],
})
.option('release-tags', {
describe: 'Reassociate tags for scene IDs',
type: 'array',
alias: ['scene-tags'],
})
.option('channel-release-tags', {
describe: 'Reassociate tags for all channel releases',
type: 'array',
alias: ['channel-scene-tags', 'channel-tags'],
})
.option('network-release-tags', {
describe: 'Reassociate tags for all network releases',
type: 'array',
alias: ['network-scene-tags', 'network-tags'],
})
.option('rematch-tags', {
describe: 'Reassociate tags that are already associated',
type: 'boolean',
alias: 'rematch',
default: false,
})
.option('request', {
describe: 'Make an arbitrary HTTP request',
type: 'string',

View File

@@ -69,6 +69,7 @@ async function fetchUnprintScene(scraper, url, entity, baseRelease, options, typ
entity,
baseRelease,
headers: res.headers,
cookies: res.cookies,
include,
beforeFetchScenes: options.beforeFetchScenes,
parameters: options.parameters,

View File

@@ -7,8 +7,9 @@ const logger = require('./logger')(__filename);
const argv = require('./argv');
const knex = require('./knex');
const { deleteScenes, deleteMovies, deleteSeries } = require('./releases');
const { flushOrphanedMedia } = require('./media');
const { resolveScraper, resolveLayoutScraper } = require('./scrapers/resolve');
const { fetchEntityReleaseIds } = require('./entity-releases');
const getRecursiveParameters = require('./utils/get-recursive-parameters');
function getRecursiveParent(entity) {
if (!entity) {
@@ -257,13 +258,18 @@ async function fetchEntitiesBySlug(entitySlugs, options = { prefer: 'channel', a
entitySlugs: entitySlugs.filter((slug) => !slug.includes('.')),
entityHosts: entitySlugs.filter((slug) => slug.includes('.')).map((hostname) => `%${hostname}`),
entityTypes: options.types || ['channel', 'network'],
sort: knex.raw(options.prefer === 'channel' ? 'asc' : 'desc'),
sort: knex.raw(options.prefer === 'channel' || options.prefer === 'options' ? 'asc' : 'desc'),
});
// channel entity will overwrite network entity
// by default channel entity will overwrite network entity
const entitiesBySlug = entities.rows.reduce((accEntities, { entity }) => {
const host = urlToHostname(entity.url);
const curatedEntity = accEntities[entity.slug] || accEntities[host] || curateEntity(entity, true);
const entityOptions = getRecursiveParameters(entity, 'options');
const accEntity = accEntities[entity.slug] || accEntities[host];
const curatedEntity = !accEntity || (options.prefer === 'options' && entity.type === 'network' && entityOptions.preferNetwork)
? curateEntity(entity, true)
: accEntity;
return {
...accEntities,
@@ -368,81 +374,10 @@ async function searchEntities(query, type, limit) {
return curateEntities(entities);
}
async function flushEntities(networkSlugs = [], channelSlugs = []) {
async function flushEntities(networkSlugs = [], channelSlugs = [], flushOrphanedMedia) {
const { sceneIds, movieIds, serieIds } = await fetchEntityReleaseIds(networkSlugs, channelSlugs);
const entitySlugs = networkSlugs.concat(channelSlugs).join(', ');
const entityQuery = knex
.withRecursive('selected_entities', knex.raw(`
SELECT entities.*
FROM entities
WHERE
entities.slug = ANY(:networkSlugs)
AND entities.type = 'network'
OR (entities.slug = ANY(:channelSlugs)
AND entities.type = 'channel')
UNION ALL
SELECT entities.*
FROM entities
INNER JOIN selected_entities ON selected_entities.id = entities.parent_id
`, {
networkSlugs,
channelSlugs,
}));
const sceneIds = await entityQuery
.clone()
.select('releases.id')
.distinct('releases.id')
.from('selected_entities')
.leftJoin('releases', 'releases.entity_id', 'selected_entities.id')
.whereNotNull('releases.id')
.modify((builder) => {
if (argv.flushAfter) {
builder.where('effective_date', '>=', argv.flushAfter);
}
if (argv.flushBefore) {
builder.where('effective_date', '<=', argv.flushBefore);
}
})
.pluck('releases.id');
const movieIds = await entityQuery
.clone()
.select('movies.id')
.distinct('movies.id')
.from('selected_entities')
.leftJoin('movies', 'movies.entity_id', 'selected_entities.id')
.whereNotNull('movies.id')
.modify((builder) => {
if (argv.flushAfter) {
builder.where('effective_date', '>=', argv.flushAfter);
}
if (argv.flushBefore) {
builder.where('effective_date', '<=', argv.flushBefore);
}
})
.pluck('movies.id');
const serieIds = await entityQuery
.clone()
.select('series.id')
.distinct('series.id')
.from('selected_entities')
.leftJoin('series', 'series.entity_id', 'selected_entities.id')
.whereNotNull('series.id')
.modify((builder) => {
if (argv.flushAfter) {
builder.where('date', '>=', argv.flushAfter);
}
if (argv.flushBefore) {
builder.where('date', '<=', argv.flushBefore);
}
})
.pluck('series.id');
if (sceneIds.length === 0 && movieIds.length === 0 && serieIds.length === 0) {
logger.info(`No scenes, movies or series found to remove for ${entitySlugs}`);
return;
@@ -479,6 +414,7 @@ module.exports = {
fetchIncludedEntities,
fetchReleaseEntities,
fetchEntitiesBySlug,
fetchEntityReleaseIds,
fetchEntity,
fetchEntities,
getRecursiveParent,

88
src/entity-releases.js Normal file
View File

@@ -0,0 +1,88 @@
'use strict';
const knex = require('./knex');
const argv = require('./argv');
async function fetchEntityReleaseIds(networkSlugs = [], channelSlugs = []) {
const entityQuery = knex
.withRecursive('selected_entities', knex.raw(`
SELECT entities.*
FROM entities
WHERE
entities.slug = ANY(:networkSlugs)
AND entities.type = 'network'
OR (entities.slug = ANY(:channelSlugs)
AND entities.type = 'channel')
UNION ALL
SELECT entities.*
FROM entities
INNER JOIN selected_entities ON selected_entities.id = entities.parent_id
`, {
networkSlugs,
channelSlugs,
}));
const sceneIds = await entityQuery
.clone()
.select('releases.id')
.distinct('releases.id')
.from('selected_entities')
.leftJoin('releases', 'releases.entity_id', 'selected_entities.id')
.whereNotNull('releases.id')
.modify((builder) => {
if (argv.flushAfter) {
builder.where('effective_date', '>=', argv.flushAfter);
}
if (argv.flushBefore) {
builder.where('effective_date', '<=', argv.flushBefore);
}
})
.pluck('releases.id');
const movieIds = await entityQuery
.clone()
.select('movies.id')
.distinct('movies.id')
.from('selected_entities')
.leftJoin('movies', 'movies.entity_id', 'selected_entities.id')
.whereNotNull('movies.id')
.modify((builder) => {
if (argv.flushAfter) {
builder.where('effective_date', '>=', argv.flushAfter);
}
if (argv.flushBefore) {
builder.where('effective_date', '<=', argv.flushBefore);
}
})
.pluck('movies.id');
const serieIds = await entityQuery
.clone()
.select('series.id')
.distinct('series.id')
.from('selected_entities')
.leftJoin('series', 'series.entity_id', 'selected_entities.id')
.whereNotNull('series.id')
.modify((builder) => {
if (argv.flushAfter) {
builder.where('date', '>=', argv.flushAfter);
}
if (argv.flushBefore) {
builder.where('date', '<=', argv.flushBefore);
}
})
.pluck('series.id');
return {
sceneIds,
movieIds,
serieIds,
};
}
module.exports = {
fetchEntityReleaseIds,
};

View File

@@ -3,7 +3,7 @@
const config = require('config');
const knex = require('knex');
module.exports = knex({
const knexInstance = knex({
client: 'pg',
connection: config.database.owner,
pool: config.database.pool,
@@ -11,3 +11,23 @@ module.exports = knex({
asyncStackTraces: process.env.NODE_ENV === 'development',
// debug: process.env.NODE_ENV === 'development',
});
knexInstance.on('query', function onQuery(query) {
const bindingCount = query.bindings?.length ?? 0;
if (bindingCount > 10000) {
const error = new Error(`[knex] Dangerous query: ${bindingCount} bindings detected: ${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`);
Error.captureStackTrace(error, onQuery);
// console.error(error);
throw error; // optionally hard-fail so you get a real stack trace
}
});
knexInstance.on('query-error', (error, query) => {
error.knexSql = `${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`;
error.knexBindingCount = query.bindings?.length;
});
module.exports = knexInstance;

View File

@@ -9,7 +9,7 @@ require('winston-daily-rotate-file');
const args = require('./argv');
function logger(filepath) {
const root = filepath.match(/src[/\\]|dist[/\\]/);
const root = filepath.match(/src[/\\]|dist[/\\]|tests[/\\]/);
const filename = filepath.slice(root.index + root[0].length)
.replace(path.extname(filepath), '');

View File

@@ -26,6 +26,7 @@ const http = require('./utils/http');
const bulkInsert = require('./utils/bulk-insert');
const chunk = require('./utils/chunk');
const { get } = require('./utils/qu');
const { fetchEntityReleaseIds } = require('./entity-releases');
// const pipeline = util.promisify(stream.pipeline);
const streamQueue = taskQueue();
@@ -646,6 +647,7 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
const res = await http.get(source.src, {
limits: 'media',
headers: {
host: new URL(source.src).hostname,
...(source.referer && { referer: source.referer }),
...(source.host && { host: source.host }),
},
@@ -1159,8 +1161,36 @@ async function flushOrphanedMedia(stage = 1) {
}
}
async function detachReleaseMedia(rawSceneIds) {
const sceneIds = rawSceneIds.map((sceneId) => Number(sceneId)).filter(Boolean);
await argv.detachMediaDomains.reduce(async (chain, domain) => {
await chain;
const mediaEntries = await knex(`releases_${domain}`).whereIn('release_id', sceneIds);
await knex(`releases_${domain}`)
.whereIn('release_id', sceneIds)
.delete();
logger.info(`Removed ${mediaEntries.length} ${domain} from ${new Set(mediaEntries.map((mediaEntry) => mediaEntry.release_id)).size} scenes`);
}, Promise.resolve());
if (argv.flushOrphanedMedia !== false) {
await flushOrphanedMedia();
}
}
async function detachEntityReleaseMedia(networkSlugs = [], channelSlugs = []) {
const { sceneIds } = await fetchEntityReleaseIds(networkSlugs, channelSlugs);
await detachReleaseMedia(sceneIds);
}
module.exports = {
associateAvatars,
associateReleaseMedia,
flushOrphanedMedia,
detachReleaseMedia,
detachEntityReleaseMedia,
};

View File

@@ -148,6 +148,8 @@ module.exports = {
interracialpovs: hush,
povpornstars: hush,
seehimfuck: hush,
ravebunnys: hush,
hotandtatted: hush,
// wankzvr
wankzvr,
tranzvr: wankzvr,

View File

@@ -57,7 +57,7 @@ function getCovers(images, target = 'cover') {
}
function getVideos(data) {
const teaserSources = data.videos.mediabook?.files;
const teaserSources = data.videos?.mediabook?.files;
const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
const teaser = teaserSources && Object.values(teaserSources).map((source) => ({
@@ -84,7 +84,7 @@ function scrapeLatestX(data, site, filterChannel, options) {
release.url = `${basepath}/${data.id}/${slugify(release.title)}`; // spartanId doesn't work in URLs
release.date = new Date(data.dateReleased);
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.duration = data.videos?.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map((tag) => tag.name);
@@ -127,6 +127,10 @@ async function scrapeLatest(items, site, filterChannel, options) {
}
function scrapeRelease(data, url, channel, networkName, options) {
if (Array.isArray(data)) {
return null;
}
const release = {};
const { title, description } = data;
@@ -136,7 +140,7 @@ function scrapeRelease(data, url, channel, networkName, options) {
release.description = description;
release.date = new Date(data.dateReleased);
release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.duration = data.videos?.mediabook?.length > 1 ? data.videos.mediabook.length : null;
release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map((tag) => tag.name);
@@ -144,7 +148,6 @@ function scrapeRelease(data, url, channel, networkName, options) {
[release.poster, ...release.photos] = getThumbs(data).map((src) => ({
src,
referer: url,
host: 'mediavault-private-fl.project1content.com',
}));
const { teaser, trailer } = getVideos(data);
@@ -270,7 +273,7 @@ async function fetchLatest(site, page = 1, options) {
return null;
}
const { instanceToken } = options.beforeNetwork?.instanceToken
const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
? options.beforeNetwork
: await getSession(site, options.parameters, url);

View File

@@ -71,10 +71,32 @@ function scrapeScene(data, channel, parameters) {
return release;
}
async function getBlockId(slug, dataSource, entity, parameters) {
const res = await unprint.get(`https://azianistudios.com/tour_api.php/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
headers: {
Referer: entity.url,
'x-nats-cms-area-id': parameters.areaId,
},
});
if (res.ok && res.data.success) {
// unsure how the blocks differ exactly, but type set_view is missing directors for some reason
return res.data.blocks?.find((block) => ['navigation', 'html'].includes(block.settings.type))?.cms_block_id || null;
}
return null;
}
async function fetchLatest(channel, page = 1, { parameters }) {
const blockId = await getBlockId(parameters.videos || '/videos', { page }, channel, parameters);
if (!blockId) {
return null;
}
const query = new URLSearchParams({
cms_area_id: parameters.areaId,
cms_block_id: parameters.blockId,
cms_block_id: blockId,
count: 100,
start: (page - 1) * 100,
orderby: 'published_desc',
@@ -106,10 +128,20 @@ async function fetchLatest(channel, page = 1, { parameters }) {
async function fetchScene(url, entity, _baseRelease, { parameters }) {
const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
if (!entryId) {
return null;
}
const blockId = await getBlockId('/video/:id', entryId, entity, parameters);
if (!blockId) {
return null;
}
const query = new URLSearchParams({
cms_set_ids: entryId,
cms_area_id: parameters.areaId,
cms_block_id: parameters.blockId,
cms_block_id: blockId,
content: 1,
orderby: 'published_desc',
content_type: 'video',
@@ -172,7 +204,7 @@ function scrapeProfile(data, entity, parameters) {
return profile;
}
async function fetchProfile({ url }, { entity, parameters }) {
async function fetchProfile({ name, url }, { entity, parameters }) {
if (!url) {
// no easy search option
return null;
@@ -180,9 +212,19 @@ async function fetchProfile({ url }, { entity, parameters }) {
const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
if (!actorId) {
return null;
}
const blockId = await getBlockId('/model/:id', { models: name, id: actorId }, entity, parameters);
if (!blockId) {
return null;
}
const query = new URLSearchParams({
cms_data_value_ids: actorId,
cms_block_id: entity.parameters.modelBlockId || entity.parameters.blockId,
cms_block_id: blockId,
cms_data_type_id: 4,
}).toString();

View File

@@ -41,7 +41,12 @@ function getAvatarFallback(url) {
return [
`${origin}${pathname}`,
url,
];
].map((src) => ({
src,
expectType: {
'application/octet-stream': 'image/jpeg',
},
}));
} catch (_error) {
return null;
}

View File

@@ -1,15 +1,15 @@
'use strict';
const { ex } = require('../utils/q');
const http = require('../utils/http');
const unprint = require('unprint');
function scrapeProfile(html) {
const { qu } = ex(html); /* eslint-disable-line object-curly-newline */
const slugify = require('../utils/slugify');
function scrapeProfile({ query }) {
const profile = {};
const bio = qu.all('.infobox tr[valign="top"]')
.map((detail) => qu.all(detail, 'td', true))
.reduce((acc, [key, value]) => ({ ...acc, [key.slice(0, -1).replace(/[\s+|/]/g, '_')]: value }), {});
const bio = query.all('.infobox tr[valign="top"]')
.map((detail) => unprint.query.contents(detail, 'td'))
.reduce((acc, [key, value]) => ({ ...acc, [slugify(key, '_')]: value }), {});
/* unreliable, see: Syren De Mer
const catlinks = qa('#mw-normal-catlinks a', true);
@@ -17,71 +17,51 @@ function scrapeProfile(html) {
profile.gender = isTrans ? 'transsexual' : 'female';
*/
profile.birthdate = qu.date('.bday', 'YYYY-MM-DD');
profile.dateOfBirth = query.date('.bday', 'YYYY-MM-DD');
profile.description = qu.q('#mw-content-text > p', true);
profile.description = query.content('.mw-parser-output > p');
if (bio.Born) profile.birthPlace = bio.Born.slice(bio.Born.lastIndexOf(')') + 1);
if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
profile.placeOfBirth = bio.born;
profile.ethnicity = bio.ethnicity;
profile.nationality = bio.nationality.split(',')[0];
if (bio.Measurements) {
const measurements = bio.Measurements
.match(/\d+(\w+)?-\d+-\d+/g)
?.slice(-1)[0] // allow for both '34C-25-36' and '86-64-94 cm / 34-25-37 in'
.split('-');
profile.measurements = bio.measurements;
// account for measuemrents being just e.g. '32EE'
if (measurements) {
const [bust, waist, hip] = measurements;
if (/[a-zA-Z]/.test(bust)) profile.bust = bust; // only use bust if cup size is included
profile.waist = Number(waist);
profile.hip = Number(hip);
}
if (/^\d+\w+$/.test(bio.Measurements)) profile.bust = bio.Measurements;
if (bio.bra_cup_size) {
[profile.bust] = bio.bra_cup_size.match(/^\d+\w+/) || [];
}
if (bio.Bra_cup_size) {
const bust = bio.Bra_cup_size.match(/^\d+\w+/);
if (bust) [profile.bust] = bust;
}
if (/enhanced/i.test(bio.boobs)) profile.naturalBoobs = false;
if (/natural/i.test(bio.boobs)) profile.naturalBoobs = true;
if (bio.Boobs === 'Enhanced') profile.naturalBoobs = false;
if (bio.Boobs === 'Natural') profile.naturalBoobs = true;
if (bio.height) profile.height = Number(bio.height.match(/\d+\.\d+/g).slice(-1)[0]) * 100;
if (bio.weight) profile.weight = Number(bio.weight.match(/\d+/g)[1]);
if (bio.Height) profile.height = Number(bio.Height.match(/\d+\.\d+/g).slice(-1)[0]) * 100;
if (bio.Weight) profile.weight = Number(bio.Weight.match(/\d+/g)[1]);
if (bio.eye_color) profile.eyes = bio.eye_color;
if (bio.hair) [profile.hairColor] = bio.hair.split(/(?=[A-Z])/); // field concatted to BrunetteLong
if (bio.Eye_color) profile.eyes = bio.Eye_color;
if (bio.Hair) [profile.hair] = bio.Hair.split(',');
if (bio.blood_group) profile.blood = bio.blood_group;
if (bio.also_known_as) profile.aliases = bio.also_known_as.split(', ');
if (bio.Blood_group) profile.blood = bio.Blood_group;
if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', ');
const avatarThumbPath = qu.img('.image img');
const avatarThumbPath = query.img('.infobox .mw-file-description img');
if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) {
const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', '');
profile.avatar = {
src: `http://www.boobpedia.com${avatarPath}`,
credit: null,
};
profile.avatar = unprint.prefixUrl(avatarPath, 'https://www.boobpedia.com');
}
profile.social = qu.urls('.infobox a.external');
profile.socials = query.urls('.infobox a.external');
return profile;
}
async function fetchProfile({ name: actorName }) {
const actorSlug = actorName.replace(/\s+/, '_');
const res = await http.get(`http://www.boobpedia.com/boobs/${actorSlug}`);
const actorSlug = slugify(actorName, '_', { lower: false });
const res = await unprint.get(`https://www.boobpedia.com/boobs/${actorSlug}`);
if (res.statusCode === 200) {
return scrapeProfile(res.body.toString());
if (res.ok) {
return scrapeProfile(res.context);
}
return null;

View File

@@ -81,7 +81,7 @@ function scrapeProfile({ query }, _entity) {
const bio = Object.fromEntries(keys.map((key, index) => [slugify(key, '_'), values[index]]));
profile.age = Number(bio.age) || null;
profile.age = unprint.extractNumber(bio.age);
profile.height = convert(bio.height, 'cm');
profile.measurements = bio.measurements;

View File

@@ -1,107 +0,0 @@
'use strict';
const {
fetchLatest,
fetchApiLatest,
fetchUpcoming,
fetchApiUpcoming,
fetchScene,
fetchProfile,
fetchApiProfile,
scrapeAll,
} = require('./gamma');
const { get } = require('../utils/qu');
const slugify = require('../utils/slugify');
function extractLowArtActors(release) {
const actors = release.title
.replace(/solo/i, '')
.split(/,|\band\b/ig)
.map((actor) => actor.trim());
return {
...release,
actors,
};
}
async function networkFetchLatest(site, page = 1) {
if (site.parameters?.api) return fetchApiLatest(site, page, false);
const releases = await fetchLatest(site, page);
if (site.slug === 'lowartfilms') {
return releases.map((release) => extractLowArtActors(release));
}
return releases;
}
async function networkFetchScene(url, site) {
const release = await fetchScene(url, site);
if (site.slug === 'lowartfilms') {
return extractLowArtActors(release);
}
return release;
}
async function networkFetchUpcoming(site, page = 1) {
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
return fetchUpcoming(site, page);
}
function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
}
function scrapeClassicProfile({ qu, html }, site) {
const profile = {};
profile.avatar = qu.img('.actorPicture');
profile.releases = scrapeAll(html, null, site.url, false);
return profile;
}
async function fetchClassicProfile(actorName, { site }) {
const actorSlug = slugify(actorName);
const url = `${site.url}/en/pornstars`;
const pornstarsRes = await get(url);
if (!pornstarsRes.ok) return null;
const actorPath = pornstarsRes.item.qa('option[value*="/pornstar"]')
.find((el) => slugify(el.textContent) === actorSlug)
?.value;
if (actorPath) {
const actorUrl = `${site.url}${actorPath}`;
const res = await get(actorUrl);
if (res.ok) {
return scrapeClassicProfile(res.item, site);
}
}
return null;
}
async function networkFetchProfile({ name: actorName }, context, include) {
const profile = await ((context.site.parameters?.api && fetchApiProfile(actorName, context, include))
|| (context.site.parameters?.classic && include.scenes && fetchClassicProfile(actorName, context, include)) // classic profiles only have scenes, no bio
|| fetchProfile({ name: actorName }, context, true, getActorReleasesUrl, include));
return profile;
}
module.exports = {
fetchLatest: networkFetchLatest,
fetchProfile: networkFetchProfile,
fetchScene: networkFetchScene,
fetchUpcoming: networkFetchUpcoming,
};

View File

@@ -91,11 +91,13 @@ function scrapeProfile({ query }, { url }) {
profile.description = [query.content('.model-bio-text, #performer-description'), bio.funfact].filter(Boolean).join(' ');
profile.aliases = bio.alias?.split(/[,\n]/).map((alias) => alias.trim());
profile.age = parseInt(bio.age, 10) || null;
profile.age = unprint.extractNumber(bio.age);
profile.dateOfBirth = unprint.extractDate(bio.age, 'MM/DD/YYYY');
profile.measurements = bio.measurements;
profile.height = Number(bio.height?.match(/(\d+)\s*cm/)?.[1]) || convert(bio.height, 'cm');
console.log(profile);
return profile;
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,20 +1,21 @@
'use strict';
const util = require('util');
const unprint = require('unprint');
const format = require('template-format');
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
const { feetInchesToCm, inchesToCm } = require('../utils/convert');
const tryUrls = require('../utils/try-urls');
const { convert } = require('../utils/convert');
function deriveEntryId(release) {
if (release.date && release.url) {
const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
}
if (release.date && release.title) {
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
}
return null;
@@ -22,7 +23,7 @@ function deriveEntryId(release) {
function extractPoster(posterPath, site, baseRelease) {
if (posterPath && !/400.jpg/.test(posterPath)) {
const poster = `${site.parameters?.media || site.url}${posterPath}`;
const poster = unprint.prefixUrl(posterPath, site.parameters?.media || site.url);
const posterSources = [
poster,
// upscaled
@@ -40,38 +41,38 @@ function extractPoster(posterPath, site, baseRelease) {
return [baseRelease?.poster || null, []];
}
function getImageWithFallbacks(q, selector, site, el) {
function getImageWithFallbacks(query, selector, site, el) {
const sources = el
? [
q(el, selector, 'src0_3x'),
q(el, selector, 'src0_2x'),
q(el, selector, 'src0_1x'),
unprint.query.attribute(el, selector, 'src0_3x'),
unprint.query.attribute(el, selector, 'src0_2x'),
unprint.query.attribute(el, selector, 'src0_1x'),
]
: [
q(selector, 'src0_3x'),
q(selector, 'src0_2x'),
q(selector, 'src0_1x'),
query.attribute(selector, 'src0_3x'),
query.attribute(selector, 'src0_2x'),
query.attribute(selector, 'src0_1x'),
];
return sources.filter(Boolean).map((src) => `${site.parameters?.media || site.url}${src}`);
return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url));
}
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.q('h4 a', true);
release.title = query.content('h4 a');
release.url = query.url('a');
release.date = query.date('.date', 'YYYY-MM-DD');
release.duration = query.duration('.time');
const count = query.number('a img', null, 'cnt');
const count = query.number('a img', { attribute: 'cnt' });
[release.poster, ...release.photos] = Array.from({ length: count }, (value, index) => [
query.img('a img', `src${index}_3x`, { origin: channel.url }),
query.img('a img', `src${index}_2x`, { origin: channel.url }),
query.img('a img', `src${index}_1x`, { origin: channel.url }),
[release.poster, ...release.photos] = Array.from({ length: count }, (_value, index) => [
query.img('a img', { attribute: `src${index}_3x`, origin: channel.url }),
query.img('a img', { attribute: `src${index}_2x`, origin: channel.url }),
query.img('a img', { attribute: `src${index}_1x`, origin: channel.url }),
]);
release.stars = query.count('img[src*="star_full"]') + (query.count('img[src*="star_half"]') * 0.5);
@@ -85,18 +86,18 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
release.title = query.attribute('h4 a', 'title') || query.content('h4 a');
release.url = query.url('h4 a');
release.date = query.date('.more-info-div', 'MMM D, YYYY');
release.duration = query.dur('.more-info-div');
release.duration = query.duration('.more-info-div');
if (/bts|behind the scenes/i.test(release.title)) release.tags = ['behind the scenes'];
const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
const posterPath = query.attribute('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${site.parameters?.media || site.url}${posterPath}`;
const poster = unprint.prefixUrl(posterPath, site.parameters?.media || site.url);
release.poster = [
poster.replace('-1x', '-3x'),
@@ -117,19 +118,40 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
}).filter(Boolean);
}
async function fetchLatest(site, page = 1, _include, { uniqueReleases = [], duplicateReleases = [] }) {
const url = (site.parameters?.latest && format(site.parameters.latest, { page }))
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|| `${site.url}/categories/movies_${page}_d.html`;
const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .updateItem' });
if (!res.ok) {
return res.status;
}
if (site.parameters?.t1) {
return scrapeAllT1(res.context, site, [...uniqueReleases, ...duplicateReleases]);
}
return scrapeAll(res.context, site, uniqueReleases);
}
function scrapeScene({ html, query }, channel, url) {
const release = { url }; // url used for entry ID
release.title = query.cnt('.videoDetails h3');
release.description = query.cnt('.videoDetails p');
release.title = query.content('.videoDetails h3, .videoDetails h1');
release.description = query.content('.videoDetails p');
release.date = query.date('.videoInfo p', ['MM/DD/YYYY', 'YYYY-MM-DD']);
release.duration = Number(query.cnt('.videoInfo p:nth-of-type(2)')?.match(/(\d+) min/i)?.[1]) * 60;
release.duration = Number(query.content('.videoInfo p:nth-of-type(2)')?.match(/(\d+) min/i)?.[1]) * 60;
release.actors = query.cnts('.update_models a');
release.actors = query.all('.update_models a').map((actorEl) => ({
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null, { origin: channel.origin }),
}));
const posterPath = html.match(/poster="([\w-/.]+)"/)?.[1];
const poster = qu.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
const poster = unprint.prefixUrl(posterPath, channel.url) || query.img('.update_thumb', 'src0_1x', { origin: channel.url }); // latter used when trailer requires signup
[release.poster, ...release.photos] = [poster, ...query.imgs('.item-thumb img', 'src0_1x', { origin: channel.url })]
.map((src) => src && [
@@ -141,10 +163,10 @@ function scrapeScene({ html, query }, channel, url) {
const trailerPath = html.match(/\/trailers?\/.*.mp4/);
if (trailerPath) {
release.trailer = qu.prefixUrl(trailerPath, channel.parameters?.media || channel.url);
release.trailer = unprint.prefixUrl(trailerPath, channel.parameters?.media || channel.url);
}
release.tags = query.cnts('.featuring a[href*="categories/"]');
release.tags = query.contents('.featuring a[href*="categories/"]');
release.stars = query.count('.stars img[src*="star_full"]') + (query.count('.stars img[src*="star_half"]') * 0.5);
release.entryId = deriveEntryId(release);
@@ -155,29 +177,34 @@ function scrapeScene({ html, query }, channel, url) {
function scrapeSceneT1({ html, query }, site, url, baseRelease) {
const release = { url };
release.title = query.q('.trailer-section-head .section-title', true);
release.title = query.content('.trailer-section-head .section-title');
release.description = query.text('.row .update-info-block');
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)');
release.duration = query.duration('.update-info-row:nth-child(2)');
release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.q(el, 'span', true),
avatar: getImageWithFallbacks(query.q, 'img', site, el),
name: unprint.query.content(el, 'span'),
url: unprint.query.url(el, null),
avatar: getImageWithFallbacks(query, 'img', site, el),
}));
release.tags = query.all('.tags a', true);
release.tags = query.contents('.tags a');
// const posterPath = html.match(/poster="(.*\.jpg)/)?.[1];
const posterPath = query.q('.player-thumb img', 'src0_1x');
const posterPath = query.img('.player-thumb img', { attribute: 'src0_1x' });
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
[release.poster, release.photos] = extractPoster(posterPath, site, baseRelease);
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
if (trailer && /^http/.test(trailer)) release.trailer = { src: trailer, referer: url };
else if (trailer) release.trailer = { src: `${site.parameters?.media || site.url}${trailer}`, referer: url };
if (trailer) {
release.trailer = {
src: unprint.prefixUrl(trailer, site.parameters?.media || site.url),
referer: url,
};
}
const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
if (stars) release.stars = Number(stars);
release.stars = query.number('.update-rating');
if (site.type === 'network') {
const channelRegExp = new RegExp(site.children.map((channel) => channel.parameters?.match || channel.name).join('|'), 'i');
@@ -188,16 +215,99 @@ function scrapeSceneT1({ html, query }, site, url, baseRelease) {
}
}
// release.entryId = q('.player-thumb img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
return release;
}
function scrapeProfileT1({ el, query }, site) {
const profile = {};
async function fetchScene(url, site, baseRelease) {
const res = await unprint.get(url);
const bio = query.all('.detail-div + .detail-div p, .detail-div p', true).reduce((acc, info) => {
if (!res.ok) {
return res.status;
}
if (site.parameters?.t1) {
return scrapeSceneT1(res.context, site, url, baseRelease);
}
return scrapeScene(res.context, site, url, baseRelease);
}
async function fetchActorScenes({ query }, channel, accScenes = []) {
const scenes = scrapeAll(unprint.initAll(query.all('.item-video')), channel);
const nextPage = query.url('.next a');
if (nextPage) {
const res = await unprint.get(nextPage);
if (res.ok) {
return fetchActorScenes(res.context, channel, scenes.concat(accScenes));
}
}
return accScenes.concat(scenes);
}
async function scrapeProfile({ query }, url, channel, options) {
const profile = { url };
const bio = query.all('.stats li').reduce((acc, bioEl) => {
const key = unprint.query.content(bioEl, 'strong');
const value = unprint.query.url(bioEl, null) || unprint.query.text(bioEl);
return {
...acc,
[slugify(key, '_')]: value,
};
}, {});
if (bio.date_of_birth) profile.dateOfBirth = unprint.extractDate(bio.date_of_birth, 'MMMM D, YYYY');
if (bio.birthplace) profile.birthPlace = bio.birthplace;
if (bio.fun_fact) profile.description = bio.fun_fact;
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
if (bio.height) profile.height = convert(bio.height, 'cm');
if (bio.weight) profile.weight = convert(bio.weight.match(/(\d+)\s*lb/i)?.[1], 'lb', 'kg');
if (bio.shoe_size && !/unknown/i.test(bio.shoe_size)) profile.foot = unprint.extractNumber(bio.shoe_size);
profile.measurements = bio.measurements;
if (bio.penis_length) profile.penisLength = Number(bio.penis_length.match(/(\d+)\s*cm/i)?.[1] || convert(bio.penis_length.match(/(\d+\.?\d+)\s*in/i)?.[1], 'cm')) || null;
if (bio.penis_girth) profile.penisGirth = Number(bio.penis_girth.match(/(\d+)\s*cm/i)?.[1] || convert(bio.penis_girth.match(/(\d+\.?\d+)\s*in/i)?.[1], 'cm')) || null;
if (bio.circumcised && /yes/i.test(bio.circumcised)) profile.isCircumcised = true;
if (bio.circumcised && /no/i.test(bio.circumcised)) profile.isCircumcised = false;
if (bio.natural_breasts && /yes/i.test(bio.natural_breasts)) profile.naturalBoobs = true;
if (bio.natural_breasts && /no/i.test(bio.natural_breasts)) profile.naturalBoobs = false;
if (bio.tattoos && /(yes)|(some)|(many)/i.test(bio.tattoos)) profile.hasTattoos = true;
if (bio.tattoos && /no/i.test(bio.tattoos)) profile.hasTattoos = false;
if (bio.piercings && /(yes)|(some)|(many)/i.test(bio.piercings)) profile.hasPiercings = true;
if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false;
if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim()).filter((alias) => !/known/i.test(alias)); // filter out "No known aliases"
profile.socials = [bio.onlyfans, bio.twitter, bio.instagram, bio.domain].filter(Boolean);
profile.avatar = [
query.img('.profile-pic img', { attribute: 'src0_3x', origin: channel.url }),
query.img('.profile-pic img', { attribute: 'src0_2x', origin: channel.url }),
query.img('.profile-pic img', { attribute: 'src0_1x', origin: channel.url }),
];
if (options.includeActorScenes) {
profile.releases = await fetchActorScenes({ query }, channel);
}
return profile;
}
function scrapeProfileT1({ query }, url, site) {
const profile = { url };
const bio = query.contents('.detail-div + .detail-div p, .detail-div p').reduce((acc, info) => {
const [key, value] = info.split(':');
if (!value) return acc;
@@ -215,142 +325,49 @@ function scrapeProfileT1({ el, query }, site) {
const heightMetric = bio.height?.match(/(\d{3})(\b|c)/);
const heightImperial = bio.height?.match(/\d{1}(\.\d)?/g);
if (heightMetric) profile.height = Number(heightMetric[1]);
if (heightImperial) profile.height = feetInchesToCm(Number(heightImperial[0]), Number(heightImperial[1]));
profile.avatar = getImageWithFallbacks(query.q, '.img-div img', site);
if (heightMetric) {
profile.height = Number(heightMetric[1]);
}
const qReleases = qu.initAll(el, '.item-video');
if (heightImperial) {
profile.height = convert(`${heightImperial[0]}' ${heightImperial[1]}"`, 'cm');
}
profile.avatar = getImageWithFallbacks(query, '.img-div img', site);
const qReleases = unprint.initAll(query.all('.item-video'));
profile.releases = scrapeAllT1(qReleases, site);
return profile;
}
async function fetchActorScenes({ query, el }, channel, accScenes = []) {
const scenes = scrapeAll(qu.initAll(el, '.item-video'), channel);
const nextPage = query.url('.next a');
if (nextPage) {
const res = await qu.get(nextPage);
if (res.ok) {
return fetchActorScenes(res.item, channel, scenes.concat(accScenes));
}
}
return accScenes.concat(scenes);
}
async function scrapeProfile({ query, el }, channel, options) {
const profile = {};
const bio = query.all('.stats li').reduce((acc, bioEl) => {
const key = query.cnt(bioEl, 'strong');
const value = query.url(bioEl) || query.text(bioEl);
return {
...acc,
[slugify(key, '_')]: value,
};
}, {});
if (bio.date_of_birth) profile.dateOfBirth = qu.extractDate(bio.date_of_birth, 'MMMM D, YYYY');
if (bio.birthplace) profile.birthPlace = bio.birthplace;
if (bio.fun_fact) profile.description = bio.fun_fact;
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
if (bio.height) profile.height = Number(bio.height.match(/^\d{2,3}/)?.[0]);
if (bio.weight) profile.weight = Number(bio.weight.match(/^\d{2,3}/)?.[0]);
if (bio.shoe_size) profile.foot = Number(bio.shoe_size);
profile.measurements = bio.measurements;
if (bio.penis_length) profile.penisLength = Number(bio.penis_length.match(/(\d+)\s*cm/i)?.[1] || inchesToCm(bio.penis_length.match(/(\d+\.?\d+)\s*in/i)?.[1])) || null;
if (bio.penis_girth) profile.penisGirth = Number(bio.penis_girth.match(/(\d+)\s*cm/i)?.[1] || inchesToCm(bio.penis_girth.match(/(\d+\.?\d+)\s*in/i)?.[1])) || null;
if (bio.circumcised && /yes/i.test(bio.circumcised)) profile.isCircumcised = true;
if (bio.circumcised && /no/i.test(bio.circumcised)) profile.isCircumcised = false;
if (bio.natural_breasts && /yes/i.test(bio.natural_breasts)) profile.naturalBoobs = true;
if (bio.natural_breasts && /no/i.test(bio.natural_breasts)) profile.naturalBoobs = false;
if (bio.tattoos && /(yes)|(some)|(many)/i.test(bio.tattoos)) profile.hasTattoos = true;
if (bio.tattoos && /no/i.test(bio.tattoos)) profile.hasTattoos = false;
if (bio.piercings && /(yes)|(some)|(many)/i.test(bio.piercings)) profile.hasPiercings = true;
if (bio.piercings && /no/i.test(bio.piercings)) profile.hasPiercings = false;
if (bio.aliases) profile.aliases = bio.aliases.split(',').map((alias) => alias.trim());
profile.socials = [bio.onlyfans, bio.twitter, bio.instagram, bio.domain].filter(Boolean);
profile.avatar = [
query.img('.profile-pic img', 'src0_3x', { origin: channel.url }),
query.img('.profile-pic img', 'src0_2x', { origin: channel.url }),
query.img('.profile-pic img', 'src0_1x', { origin: channel.url }),
];
if (options.includeActorScenes) {
profile.releases = await fetchActorScenes({ query, el }, channel);
}
return profile;
}
async function fetchLatest(site, page = 1, include, { uniqueReleases = [], duplicateReleases = [] }) {
const url = (site.parameters?.latest && util.format(site.parameters.latest, page))
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|| `${site.url}/categories/movies_${page}_d.html`;
const res = await qu.getAll(url, '.modelfeature, .item-video, .updateItem');
if (!res.ok) {
return res.status;
}
if (site.parameters?.t1) {
return scrapeAllT1(res.items, site, [...uniqueReleases, ...duplicateReleases]);
}
return scrapeAll(res.items, site, uniqueReleases);
}
async function fetchScene(url, site, baseRelease) {
const res = await qu.get(url);
if (!res.ok) {
return res.status;
}
if (site.parameters?.t1) {
return scrapeSceneT1(res.item, site, url, baseRelease);
}
return scrapeScene(res.item, site, url, baseRelease);
}
async function fetchProfile({ name: actorName }, { channel }, options) {
const actorSlugA = slugify(actorName, '');
async function fetchProfile({ name: actorName, url: actorUrl }, { channel }, options) {
const actorSlugA = slugify(actorName, '', { lower: false });
const actorSlugB = slugify(actorName);
const t1 = channel.parameters?.t1 ? 't1/' : '';
const res1 = channel.parameters?.profile
? await qu.get(util.format(channel.parameters.profile, actorSlugA))
: await qu.get(`${channel.url}/${t1}models/${actorSlugA}.html`, null, null, { followRedirects: false });
const res = (res1.ok && res1)
|| (channel.parameters?.profile && await qu.get(util.format(channel.parameters.profile, actorSlugB)))
|| await qu.get(`${channel.url}/${t1}models/${actorSlugB}.html`, null, null, { followRedirects: false });
const { res, url } = await tryUrls([
actorUrl,
...channel.parameters?.profile ? [
format(channel.parameters.profile, { actor: actorSlugA }),
format(channel.parameters.profile, { actor: actorSlugB }),
] : [
`${channel.url}/${t1}models/${actorSlugA}.html`,
`${channel.url}/${t1}models/${actorSlugB}.html`,
],
], { followRedirects: false });
if (!res.ok) {
return res.status;
}
if (channel.parameters?.t1) {
return scrapeProfileT1(res.item, channel);
return scrapeProfileT1(res.context, url, channel);
}
return scrapeProfile(res.item, channel, options);
return scrapeProfile(res.context, url, channel, options);
}
module.exports = {

View File

@@ -1,226 +0,0 @@
'use strict';
const Promise = require('bluebird');
const logger = require('../logger');
const { fetchApiLatest } = require('./gamma');
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
async function fetchActors(entryId, channel, { token, time }) {
const url = `${channel.url}/sapi/${token}/${time}/model.getModelContent?_method=model.getModelContent&tz=1&fields[0]=modelId.stageName&fields[1]=_last&fields[2]=modelId.upsellLink&fields[3]=modelId.upsellText&limit=25&transitParameters[contentId]=${entryId}`;
const res = await http.get(url);
if (res.statusCode === 200 && res.body.status === true) {
return Object.values(res.body.response.collection).map((actor) => Object.values(actor.modelId.collection)[0].stageName);
}
return [];
}
async function fetchTrailerLocation(entryId, channel) {
const url = `${channel.url}/api/download/${entryId}/hd1080/stream`;
try {
const res = await http.get(url, {
followRedirects: false,
});
if (res.statusCode === 302) {
return res.headers.location;
}
} catch (error) {
logger.warn(`${channel.name}: Unable to fetch trailer at '${url}': ${error.message}`);
}
return null;
}
function scrapeLatest(items, channel) {
return items.map(({ query }) => {
const release = {};
release.url = query.url('h5 a', null, { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
release.title = query.cnt('h5 a');
[release.poster, ...release.photos] = query.imgs('.screenshot').map((src) => [
// unnecessarily large
// src.replace(/\/\d+/, 3840),
// src.replace(/\/\d+/, '/2000'),
src.replace(/\/\d+/, '/1500'),
src.replace(/\/\d+/, '/1000'),
src,
]);
return release;
});
}
function scrapeScene({ query, html }, url, channel) {
const release = {};
release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1];
release.title = query.cnt('h1.description');
release.actors = query
.all('.video-performer')
.map((actorEl) => {
const actorUrl = query.url(actorEl, 'a', 'href', { origin: channel.url });
const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
const avatar = query.img(actorEl, 'img:not([data-bgsrc*="not-available"])', 'data-bgsrc');
return {
name: query.cnt(actorEl, '.video-performer-name'),
gender: 'female',
avatar: avatar && [
avatar.replace(/\/actor\/(\d+)/, '/actor/500'),
avatar,
],
url: actorUrl,
entryId,
};
})
.concat({ name: 'Jay Rock', gender: 'male' });
release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.number('.release-date:last-child') * 60;
release.studio = query.cnt('.studio span:nth-child(2)');
release.director = query.text('.director');
release.tags = query.cnts('.tags a');
const poster = html.match(/url\((https.+\.jpg)\)/)?.[1];
const photos = query.imgs('#moreScreenshots img');
[release.poster, ...release.photos] = [poster]
.concat(photos)
.filter(Boolean)
.map((src) => [
src.replace(/\/(\d+)\/\d+/, '/$1/1500'),
src.replace(/\/(\d+)\/\d+/, '/$1/1000'),
src,
]);
const videoId = html.match(/item: (\d+)/)?.[1];
if (videoId) {
release.trailer = { stream: `https://trailer.adultempire.com/hls/trailer/${videoId}/master.m3u8` };
}
return release;
}
async function scrapeSceneApi(scene, channel, tokens, deep) {
const release = {
entryId: scene.id,
title: scene.title,
duration: scene.length,
meta: {
tokens, // attach tokens to reduce number of requests required for deep fetching
},
};
release.url = `${channel.url}/scene/${release.entryId}/${slugify(release.title, { encode: true })}`;
release.date = new Date(scene.sites.collection[scene.id].publishDate);
release.poster = scene._resources.primary[0].url;
if (scene.tags) release.tags = Object.values(scene.tags.collection).map((tag) => tag.alias);
if (scene._resources.base) release.photos = scene._resources.base.map((resource) => resource.url);
if (deep) {
// don't make external requests during update scraping, as this would happen for every scene on the page
const [actors, trailer] = await Promise.all([
fetchActors(release.entryId, channel, tokens),
fetchTrailerLocation(release.entryId, channel),
]);
release.actors = actors;
if (trailer) {
release.trailer = { src: trailer, quality: 1080 };
}
}
return release;
}
function scrapeLatestApi(scenes, site, tokens) {
return Promise.map(scenes, async (scene) => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 });
}
async function fetchToken(channel) {
const res = await http.get(channel.url);
const html = res.body.toString();
const time = html.match(/"aet":\d+/)[0].split(':')[1];
const ah = html.match(/"ah":"[\w-]+"/)[0].split(':')[1].slice(1, -1);
const token = ah.split('').reverse().join('');
return { time, token };
}
async function fetchLatestApi(channel, page = 1) {
const { time, token } = await fetchToken(channel);
// transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory
const url = `${channel.url}/sapi/${token}/${time}/content.load?limit=50&offset=${(page - 1) * 50}&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`;
const res = await http.get(url);
if (res.ok && res.body.status) {
return scrapeLatestApi(res.body.response.collection, channel, { time, token });
}
return res.ok ? res.body.status : res.status;
}
async function fetchLatest(channel, page = 1, options, preData) {
if (channel.parameters?.useGamma) {
return fetchApiLatest(channel, page, preData, options, false);
}
const res = await qu.getAll(`https://jayspov.net/jays-pov-updates.html?view=list&page=${page}`, '.item-grid-list-view > .grid-item');
if (res.ok) {
return scrapeLatest(res.items, channel);
}
return res.status;
}
async function fetchSceneApi(url, channel, baseRelease) {
const { time, token } = baseRelease?.meta.tokens || await fetchToken(channel); // use attached tokens when deep fetching
const { pathname } = new URL(url);
const entryId = pathname.split('/')[2];
const apiUrl = `${channel.url}/sapi/${token}/${time}/content.load?filter[id][fields][0]=id&filter[id][values][0]=${entryId}&transitParameters[v1]=ykYa8ALmUD&transitParameters[preset]=scene`;
const res = await http.get(apiUrl);
if (res.ok && res.body.status) {
return scrapeSceneApi(res.body.response.collection[0], channel, { time, token }, true);
}
return res.ok ? res.body.status : res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
if (res.ok) {
return scrapeScene(res.item, url, channel);
}
return res.status;
}
module.exports = {
fetchLatest,
fetchScene,
api: {
fetchLatest: fetchLatestApi,
fetchScene: fetchSceneApi,
},
};

View File

@@ -125,12 +125,12 @@ async function fetchScene(sceneUrl, entity, _baseRelease, { parameters }) {
function scrapeProfile({ query }, url) {
const profile = { url };
const bio = Object.fromEntries(query.all('ul[class*="HeroModel_list"] li').map((bioEl) => [
const bio = Object.fromEntries(query.all('ul[class*=HeroModel][class*="__list"] li').map((bioEl) => [
slugify(unprint.query.content(bioEl, 'p:first-child'), '_'),
unprint.query.content(bioEl, 'p:last-child'),
]));
profile.description = query.content('p[class*="HeroModel_text"]');
profile.description = query.content('p[class*=HeroModel][class*="__text"]');
profile.birthPlace = bio.place_of_birth;
profile.dateOfBirth = unprint.extractDate(bio.date_of_birth, ['MMMM DD, YYYY', 'MM/DD/YYYY']);
@@ -163,7 +163,7 @@ function scrapeProfile({ query }, url) {
profile.piercings = profile.hasPiercings ? bio.piercings : null;
}
profile.avatar = query.sourceSet('picture[class*="modelImage"] img');
profile.avatar = query.sourceSet('picture[class*="__modelImage"] img');
return profile;
}

View File

@@ -19,7 +19,6 @@ const czechav = require('./czechav');
const modelmedia = require('./modelmedia');
const dorcel = require('./dorcel');
const fabulouscash = require('./fabulouscash');
// const famedigital = require('./famedigital');
const firstanalquest = require('./firstanalquest');
const elevatedx = require('./elevatedx');
const exploitedx = require('./exploitedx');
@@ -31,7 +30,6 @@ const hush = require('./hush');
const innofsin = require('./innofsin');
const insex = require('./insex');
const inthecrack = require('./inthecrack');
const jayrock = require('./jayrock');
const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials');
const julesjordan = require('./julesjordan');
const karups = require('./karups');
@@ -74,7 +72,6 @@ const tokyohot = require('./tokyohot');
// const topwebmodels = require('./topwebmodels');
const traxxx = require('./traxxx');
const virtualtaboo = require('./virtualtaboo');
const vivid = require('./vivid');
const vixen = require('./vixen');
const wankzvr = require('./wankzvr');
const whalemember = require('./whalemember');
@@ -138,7 +135,6 @@ module.exports = {
insex,
interracialpass: hush,
inthecrack,
jayrock,
jerkaoke: modelmedia,
jesseloadsmonsterfacials,
julesjordan,
@@ -180,7 +176,6 @@ module.exports = {
traxxx,
vipsexvault: porndoe,
virtualtaboo,
vivid,
vixen,
wankzvr,
westcoastproductions: adultempire,

View File

@@ -164,7 +164,8 @@ function scrapeScene({ query }, url) {
release.tags = query.contents('.p-desc a[href*="tag/"], .desc a[href*="tag/"]');
const style = query.content('.vp style');
const poster = query.img('#videos_page-page .item-img img') || style?.match(/background-image: url\('(http[\w.:/_-]+)'\);/)?.[1];
// const poster = query.img('#videos_page-page .item-img img') || style?.match(/background-image: url\('(http[\w.:/_-]+)'\);/)?.[1];
const poster = style?.match(/background-image: url\('(http[\w.:/_-]+)'\);/)?.[1];
const fallbackPoster = resizeSrc(query.img('meta[itemprop="image"]', { attribute: 'content' })); // usually a different image
const photos = query.all('.gallery .thumb').map((imgEl) => {

View File

@@ -103,7 +103,7 @@ function scrapeProfile({ query }, url, entity) {
profile.hairColor = bio.hair;
profile.eyes = bio.eyes;
profile.avatar = query.img('.model-profile-image-picture source', { origin: entity.oorigin, attribute: 'srcset' })
profile.avatar = query.img('.model-profile-image-picture source', { origin: entity.origin, attribute: 'srcset' })
|| query.img('.model-profile-image-picture img', { origin: entity.origin });
profile.scenes = scrapeAll(unprint.initAll(query.all('.video-list .thumb')), entity);

View File

@@ -1,134 +0,0 @@
'use strict';
/* eslint-disable no-unused-vars */
const { get, ed } = require('../utils/q');
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
function scrapeLatestNative(scenes, site) {
return scenes.map((scene) => {
const release = {};
release.entryId = scene.id;
release.url = `${site.url}${scene.url}`;
release.title = scene.name;
release.date = ed(scene.release_date, 'YYYY-MM-DD');
release.duration = parseInt(scene.runtime, 10) * 60;
release.actors = scene.cast?.map((actor) => ({
name: actor.stagename,
gender: actor.gender.toLowerCase(),
avatar: actor.placard,
})) || [];
release.stars = Number(scene.rating);
release.poster = scene.placard_800 || scene.placard;
return release;
});
}
function scrapeSceneNative({ html, q, qa }, url, _site) {
const release = { url };
release.entryId = new URL(url).pathname.split('/')[2]; // eslint-disable-line prefer-destructuring
release.title = q('.scene-h2-heading', true);
release.description = q('.indie-model-p', true);
const dateString = qa('h5').find((el) => /Released/.test(el.textContent)).textContent;
release.date = ed(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
const duration = qa('h5').find((el) => /Runtime/.test(el.textContent)).textContent;
const [hours, minutes] = duration.match(/\d+/g);
if (minutes) release.duration = (hours * 3600) + (minutes * 60);
else release.duration = hours * 60; // scene shorter that 1hr, hour match are minutes
release.actors = qa('h4 a[href*="/stars"], h4 a[href*="/celebs"]', true);
release.tags = qa('h5 a[href*="/categories"]', true);
const [poster, trailer] = html.match(/https:\/\/content.vivid.com(.*)(.jpg|.mp4)/g);
release.poster = poster;
if (trailer) {
release.trailer = {
src: trailer,
};
}
const channel = q('h5 a[href*="/sites"]', true);
if (channel) release.channel = channel.replace(/\.\w+/, '');
return release;
}
async function fetchLatestNative(site, page = 1) {
if (site.parameters?.useGamma) {
return fetchApiLatest(site, page);
}
const apiUrl = `${site.url}/videos/api/?limit=50&offset=${(page - 1) * 50}&sort=datedesc`;
const res = await http.get(apiUrl, {
decodeJSON: true,
});
if (res.statusCode === 200 && res.body.code === 200) {
return scrapeLatestNative(res.body.responseData, site);
}
return null;
}
async function fetchUpcomingNative(site) {
if (site.parameters?.useGamma) {
return fetchApiUpcoming(site);
}
return null;
}
async function fetchSceneNative(url, site, release) {
if (site.parameters?.useGamma) {
return fetchScene(url, site, release);
}
const res = await get(url);
return res.ok ? scrapeSceneNative(res.item, url, site) : res.status;
}
async function fetchSceneWrapper(url, site, release) {
const scene = await fetchScene(url, site, release);
if (scene.date - new Date(site.parameters?.lastNative) <= 0) {
// scene is probably still available on Vivid site, use search API to get URL and original date
const searchUrl = `${site.url}/videos/api/?limit=10&sort=datedesc&search=${encodeURI(scene.title)}`;
const searchRes = await http.get(searchUrl, {
decodeJSON: true,
});
if (searchRes.statusCode === 200 && searchRes.body.code === 200) {
const sceneMatch = searchRes.body.responseData.find((item) => slugify(item.name) === slugify(scene.title));
if (sceneMatch) {
return {
...scene,
url: `${site.url}${sceneMatch.url}`,
date: ed(sceneMatch.release_date, 'YYYY-MM-DD'),
};
}
}
}
return scene;
}
module.exports = {
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchUpcoming: fetchApiUpcoming,
fetchScene: fetchSceneWrapper,
};

View File

@@ -502,6 +502,7 @@ async function fetchScene(url, channel, baseRelease, options) {
}
const res = await unprint.get(url, {
interface: 'request',
useBrowser: !!options.parameters?.useBrowser,
});

View File

@@ -1,68 +1,43 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const unprint = require('unprint');
const slugify = require('../utils/slugify');
async function getTrailerUrl(release, channel, request) {
const csrfToken = request.cookie.match('csrfst=(.*?);')?.[1];
if (!csrfToken) {
return null;
}
const res = await http.post(`${channel.url}/ajax/player-config.json`, {
item_id: release.entryId,
}, {
headers: {
'X-CSRF-Token': csrfToken,
},
session: request.session,
encodeJSON: false,
});
if (res.ok) {
const trailers = res.body.streams.map((trailer) => ({
src: trailer.url,
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
vr: true,
}));
return {
trailers,
poster: qu.prefixUrl(res.body.poster, res.body.thumbCDN),
};
}
return null;
}
function scrapeAll(scenes, channel) {
function scrapeAll(scenes, channel, url) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.url = query.url('a', { origin: channel.origin });
release.entryId = new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.card__h');
release.date = query.date('.card__date', 'D MMMM, YYYY');
release.title = query.content('.card__h');
release.date = query.date('.card__date', 'D MMMM, YYYY', { match: null });
release.actors = query.all('.card__links a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
release.actors = query.all('.card__links a').map((actorEl) => ({
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null, { origin: channel.url }),
}));
const poster = query.srcset('picture source[type="image/jpeg"]', 'data-srcset')
|| query.srcset('picture source[type="image/jpeg"]', 'srcset')
|| query.srcset('.video__cover', 'srcset');
const poster = query.sourceSet('picture source[type="image/jpeg"]', 'data-srcset')
|| query.sourceSet('picture source[type="image/jpeg"]', 'srcset')
|| query.sourceSet('.video__cover', 'srcset');
if (poster?.[0]) {
release.poster = [
poster[0].replace(/small|tiny/, 'large'),
...poster,
];
].map((src) => ({
src,
referer: url,
}));
release.teaser = poster[0].replace(/\b(cover|hero|\d+)\/[a-z0-9_]+\.[a-z]+$/i, 'roll.webm'); // actually how site generates teaser URL
const teaser = poster[0].replace(/\b(cover|hero|\d+)\/[a-z0-9_]+\.[a-z]+$/i, 'roll.webm'); // actually how site generates teaser URL
release.teaser = {
src: teaser,
referer: url,
};
}
release.channel = channel.slug; // avoid being assigned to WankzVR network
@@ -71,31 +46,90 @@ function scrapeAll(scenes, channel) {
});
}
async function scrapeScene({ query }, url, channel, baseRelease, options, request) {
async function fetchLatest(channel, page) {
const url = `${channel.url}/videos?o=d&p=${page}`;
const res = await unprint.get(url, { selectAll: '.layout__content > .cards-list .card' }); // .cards-list is also used for hidden upcoming scenes
if (res.ok) {
return scrapeAll(res.context, channel, url);
}
return res.status;
}
async function getTrailerUrl(release, channel, cookies, referer) {
const csrfToken = cookies.csrfst;
if (!csrfToken) {
return null;
}
const res = await unprint.post(`${channel.url}/ajax/player-config.json`, {
item_id: release.entryId,
}, {
form: true,
headers: {
'X-Requested-With': 'XMLHttpRequest',
'X-CSRF-Token': csrfToken,
},
cookies,
});
if (res.ok) {
const trailers = res.data.streams.map((trailer) => ({
src: trailer.url,
quality: Number(trailer.id?.match(/\d+/)?.[0] || trailer?.name.match(/\d+/)?.[0]),
vr: true,
referer,
}));
const poster = unprint.prefixUrl(res.data.poster, res.data.thumbCDN);
return {
trailers,
poster: poster && {
src: poster,
referer,
},
};
}
return null;
}
async function scrapeScene({ query }, { url, entity, include, cookies }) {
const release = {};
const data = query.json('script[type="application/ld+json"]');
release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.detail__title');
release.description = query.cnt('.detail__txt');
release.title = query.content('.detail__title');
release.description = query.content('.detail__txt');
release.date = query.date('.detail__date', 'D MMMM, YYYY');
release.date = query.date('.detail__date', 'D MMMM, YYYY', { match: null });
release.duration = query.number('.time') * 60;
release.actors = (query.all('.detail__header-lg .detail__models a') || query.all('.detail__header-sm .detail__models a')).map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
name: unprint.query.content(el),
url: unprint.query.url(el, null, { origin: entity.origin }),
}));
release.tags = query.cnts('.tag-list .tag').concat(query.cnts('.detail__specs-list .detail__specs-item'));
release.tags = query.contents('.tag-list .tag').concat(query.contents('.detail__specs-list .detail__specs-item'));
release.photos = query.all('.photo-strip__slide').map((el) => ([
qu.query.img(el, null, 'data-src'),
qu.query.img(el, 'img', 'src'),
]));
unprint.query.img(el, null, 'data-src'),
unprint.query.img(el, 'img'),
].map((src) => ({ src, referer: url }))));
if (options.includePosters || options.includeTrailers) {
const { trailers, poster } = await getTrailerUrl(release, channel, request);
if (data?.thumbnailUrl) {
release.poster = [
data.thumbnailUrl.replace(/small|tiny/, 'large'),
data.thumbnailUrl,
].map((src) => ({ src, referer: url }));
}
if (include.trailers || (!release.poster && include.poster)) {
const { trailers, poster } = await getTrailerUrl(release, entity, cookies, url) || {};
release.trailer = trailers;
release.poster = poster;
@@ -105,17 +139,17 @@ async function scrapeScene({ query }, url, channel, baseRelease, options, reques
}
async function fetchActorScenes({ query }, url, entity, page = 1, accScenes = []) {
const scenes = scrapeAll(qu.initAll(query.all('.cards-list .card')), entity);
const scenes = scrapeAll(unprint.initAll(query.all('.cards-list .card')), entity);
const hasNextPage = !query.exists('.pagenav__link.inactive');
if (hasNextPage) {
const { origin, pathname, searchParams } = new URL(url);
searchParams.set('p', page + 1);
const res = await qu.get(`${origin}${pathname}?${searchParams}`);
const res = await unprint.get(`${origin}${pathname}?${searchParams}`);
if (res.ok) {
return fetchActorScenes(res.item, url, entity, page + 1, accScenes.concat(scenes));
return fetchActorScenes(res.context, url, entity, page + 1, accScenes.concat(scenes));
}
}
@@ -127,20 +161,24 @@ async function scrapeProfile({ query }, url, entity, options) {
const bio = query.all('.person__meta__item').reduce((acc, el) => ({
...acc,
[slugify(qu.query.cnt(el, '.person__meta__label'))]: qu.query.text(el),
[slugify(unprint.query.content(el, '.person__meta__label'))]: unprint.query.text(el),
}), {});
profile.description = query.cnt('.person__content');
profile.description = query.content('.person__content');
profile.gender = entity.slug === 'tranzvr' ? 'transsexual' : 'female';
profile.age = Number(bio.age) || null;
profile.birthPlace = bio.birthplace;
profile.height = parseInt(bio.height, 10);
// height shown in imperial with cm between brackets when requested from North American IP, but only in cm for European IPs
profile.height = unprint.extractNumber(bio.height, { match: /(\d+)cm/, matchIndex: 1 });
profile.measurements = bio.measurements;
profile.avatar = query.srcset('.person__avatar img');
profile.avatar = query.sourceSet('.person__avatar img').map((src) => ({
src,
referer: url,
}));
if (options.includeActorScenes) {
profile.scenes = await fetchActorScenes({ query }, url, entity);
@@ -149,22 +187,12 @@ async function scrapeProfile({ query }, url, entity, options) {
return profile;
}
async function fetchLatest(channel, page) {
const res = await qu.getAll(`${channel.url}/videos?o=d&p=${page}`, '.cards-list .card');
if (res.ok) {
return scrapeAll(res.items, channel);
}
return res.status;
}
async function fetchProfile(baseActor, { entity }, options) {
const url = `${entity.url}/${baseActor.slug}`;
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeProfile(res.item, url, entity, options);
return scrapeProfile(res.context, url, entity, options);
}
return res.status;
@@ -174,5 +202,4 @@ module.exports = {
fetchLatest,
scrapeScene,
fetchProfile,
deprecated: true,
};

View File

@@ -1,8 +1,12 @@
'use strict';
const logger = require('./logger')(__filename);
const knex = require('./knex');
const { fetchEntityReleaseIds } = require('./entities');
const slugify = require('./utils/slugify');
const bulkInsert = require('./utils/bulk-insert');
const batchInsert = require('./utils/batch-insert');
function curateTagMedia(media) {
if (!media) {
@@ -73,10 +77,9 @@ function withRelations(queryBuilder, withMedia) {
}
}
async function matchReleaseTags(releases) {
const tags = releases
.map((release) => release.tags).flat()
.map((tag) => tag?.trim().match(/[a-z0-9]+/ig)?.join(' ').toLowerCase())
async function matchTags(rawTags) {
const tags = rawTags
.map((tag) => tag?.trim().match(/[a-z0-9()]+/ig)?.join(' ').toLowerCase())
.filter(Boolean);
const tagEntries = await knex('tags')
@@ -153,12 +156,12 @@ async function associateReleaseTags(releases, type = 'release') {
return;
}
const tagIdsBySlug = await matchReleaseTags(releases);
const tagIdsBySlug = await matchTags(releases.flatMap((release) => release.tags));
const entityTagIdsByEntityId = await getEntityTags(releases);
const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type);
await bulkInsert(`${type}s_tags`, tagAssociations, false);
await batchInsert(`${type}s_tags`, tagAssociations, { conflict: false });
}
async function fetchTag(tagId) {
@@ -187,8 +190,58 @@ async function fetchTags(limit = 100) {
return tags.map((tag) => curateTag(tag));
}
async function reassociateTagEntries(tagEntries, rematch) {
const updatableTagEntries = tagEntries.filter((tagEntry) => (!tagEntry.tag_id || rematch) && tagEntry.original_tag);
const matchedTags = await matchTags(Array.from(new Set(updatableTagEntries.map((tagEntry) => tagEntry.original_tag))));
const updatedTagEntries = updatableTagEntries.map((tagEntry) => ({
...tagEntry,
tag_id: matchedTags[slugify(tagEntry.original_tag)],
})).filter((tagEntry) => tagEntry.tag_id);
if (updatedTagEntries.length > 0) {
const trx = await knex.transaction();
await trx('releases_tags')
.whereIn('id', updatedTagEntries.map((tagEntry) => tagEntry.id))
.delete();
await batchInsert('releases_tags', updatedTagEntries.map((tagEntry) => ({
...tagEntry,
id: undefined,
})), {
conflict: false,
transaction: trx,
});
}
logger.info(`Updated ${updatedTagEntries.length} tags in ${new Set(updatedTagEntries.map((tagEntry) => tagEntry.release_id)).size} scenes`);
}
async function reassociateReleaseTags(rawSceneIds, rematch) {
const sceneIds = rawSceneIds.map((sceneId) => Number(sceneId)).filter(Boolean);
const tagEntries = await knex('releases_tags').whereIn('release_id', sceneIds);
await reassociateTagEntries(tagEntries, rematch);
}
async function reassociateEntityReleaseTags(networkSlugs = [], channelSlugs = [], rematch = false) {
const { sceneIds } = await fetchEntityReleaseIds(networkSlugs, channelSlugs);
await reassociateReleaseTags(sceneIds, rematch);
}
async function reassociateOriginalTags(originalTags, rematch) {
const tagEntries = await knex('releases_tags').whereIn(knex.raw('lower(releases_tags.original_tag)'), originalTags.map((originalTag) => originalTag.toLowerCase()));
await reassociateTagEntries(tagEntries, rematch);
}
module.exports = {
associateReleaseTags,
fetchTag,
fetchTags,
reassociateEntityReleaseTags,
reassociateReleaseTags,
reassociateOriginalTags,
};

64
src/tools/batch-test.js Normal file
View File

@@ -0,0 +1,64 @@
'use strict';
const knex = require('../knex');
const batchInsert = require('../utils/batch-insert');
async function createTestTable() {
const tableExists = await knex.schema.hasTable('batch_test');
if (tableExists) {
// await knex('batch_test').delete();
return;
}
await knex.schema.createTable('batch_test', (table) => {
table.increments('id');
table.string('name')
.unique();
table.integer('age');
table.text('location');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
}
async function init() {
await createTestTable();
const transaction = await knex.transaction();
const entries = await batchInsert('batch_test', [
{
name: 'John',
age: 18,
location: 'Home',
},
{
name: 'Jack',
age: 38,
location: 'Work',
},
{
name: 'James',
age: 35,
location: 'Club',
},
], {
conflict: 'name',
update: true,
transaction,
commit: false,
});
await transaction.commit();
console.log('ENTRIES', entries);
// await knex.schema.dropTable('batch_test');
await knex.destroy();
}
init();

136
src/tools/gamma_banners.js Normal file
View File

@@ -0,0 +1,136 @@
'use strict';
const unprint = require('unprint');
const fs = require('fs');
const { Readable } = require('stream');
const { pipeline } = require('stream/promises');
const knex = require('../knex');
const argv = require('../argv');
const slugify = require('../utils/slugify');
const apiUrl = 'https://vjoc5ygk89-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20JavaScript%20(3.33.0)%3B%20Browser%20(lite)%3B%20react%20(16.8.6)%3B%20react-instantsearch%20(5.7.0)%3B%20JS%20Helper%20(2.28.1)&x-algolia-application-id=VJOC5YGK89&x-algolia-api-key=c5546bdfb4d3f31daf49ed3bb1463561';
async function fetchBanners() {
const res = await unprint.post(
apiUrl,
{
requests: [
{
indexName: 'creatives',
params: new URLSearchParams({
hitsPerPage: 1000,
maxValuesPerFacet: 100,
page: 0,
filters: '(ProgramType:Legacy OR ProgramType:Internal) AND NOT OverlayActive:false',
facets: '["SceneActors","SceneCategories","ProgramName","Size","Niche","MediaExt","SiteTag","OverlayName"]',
facetFilters: `[["SiteTag:${argv.site}"],["MediaExt:jpg", "MediaExt:png", "MediaExt:gif"]]`,
}).toString(),
},
],
},
{
headers: {
'content-type': 'application/x-www-form-urlencoded',
referer: 'https://creatives.gammae.com/',
},
},
);
if (res.ok && res.data.results[0]) {
return res.data.results[0].hits;
}
console.error(`Failed API request (${res.status}): ${res.body}`);
return null;
}
async function matchTags(rawTags) {
if (!rawTags) {
return [];
}
const tags = rawTags
.map((tag) => tag?.trim().match(/[a-z0-9()]+/ig)?.join(' ').toLowerCase())
.filter(Boolean);
const tagEntries = await knex('tags')
.select('tags.slug', 'aliases.slug as alias_slug')
.whereIn(knex.raw('lower(tags.name)'), tags)
.leftJoin('tags as aliases', 'aliases.id', 'tags.alias_for')
.orderByRaw('CASE WHEN tags.alias_for IS NOT NULL THEN aliases.priority ELSE tags.priority END DESC');
return tagEntries.map((tagEntry) => tagEntry.alias_slug || tagEntry.slug);
}
async function init() {
const banners = await fetchBanners();
if (!banners) {
return;
}
await banners.reduce(async (chain, banner) => {
await chain;
const channel = slugify(banner.SiteTag, '');
const url = unprint.prefixUrl(banner.MediaLocation || banner.CreativeURL, 'https://cdn.banhq.com');
if (!url) {
console.log('No URL found');
console.log(banner);
return;
}
const tags = await matchTags([
...banner.Tags?.map((tag) => tag.Value) || [],
...banner.SceneCategories || [],
banner.Niche,
].filter(Boolean));
const fileTags = tags.slice(0, 4).join('_');
const fileActors = banner.SceneActors?.slice(0, 2).map((actor) => slugify(actor, '_')).join('_');
// tags are unreliable and describe entire scene, not banner, don't include by default
const segments = [channel, banner.Width, banner.Height, banner.MediaID, argv.actors?.[0] !== false && fileActors].filter(Boolean);
const filename = `${segments.join('_')}${argv.tags && argv.tags ? `-${fileTags}` : ''}.${banner.MediaExt || 'jpg'}`;
const filepath = `/tmp/gamma/${channel}/${filename}`;
if (argv.inspect) {
console.log(banner);
}
if (argv.preview) {
console.log(`Preview ${url}: ${filepath}`);
return;
}
await fs.promises.mkdir(`/tmp/gamma/${channel}`, { recursive: true });
try {
const res = await fetch(url);
if (res.ok && res.body) {
const writer = fs.createWriteStream(filepath);
await pipeline(Readable.fromWeb(res.body), writer);
if (argv.actors) {
console.log(`Saved ${url} to ${filepath}`);
} else {
console.log(`Saved ${url} to ${filepath}, actors ${banner.SceneActors?.join(', ') || ''}`);
}
} else {
console.log(`Failed to fetch ${url} (${res.status})`);
}
} catch (error) {
console.log(`Failed to fetch ${url}: ${error.message}`);
}
}, Promise.resolve());
await knex.destroy();
}
init();

15
src/tools/huge-query.js Normal file
View File

@@ -0,0 +1,15 @@
'use strict';
const knex = require('../knex');
async function init() {
const data = Array.from({ length: 100_000 }, (value, index) => ({
id: `test_affiliate_${index}`,
}));
await knex('affiliates').insert(data);
console.log('Done!');
}
init();

View File

@@ -41,7 +41,7 @@ async function fetchScenes() {
studios.name as studio_name,
grandparents.id as parent_network_id,
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
@@ -136,6 +136,14 @@ async function init() {
dupe_index int
)`);
await utilsApi.sql('drop table if exists scenes_tags');
await utilsApi.sql(`create table scenes_tags (
id int,
scene_id int,
tag_id int,
actor_id int
)`);
console.log('Recreated scenes table');
console.log('Fetching scenes from primary database');
@@ -143,49 +151,62 @@ async function init() {
console.log('Fetched scenes from primary database');
const docs = scenes.map((scene) => {
const docs = scenes.flatMap((scene) => {
const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
return {
replace: {
index: 'scenes',
id: scene.id,
doc: {
title: scene.title || undefined,
title_filtered: filteredTitle || undefined,
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
created_at: Math.round(scene.created_at.getTime() / 1000),
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
is_showcased: scene.showcased,
entry_id: scene.entry_id || undefined,
shoot_id: scene.shoot_id || undefined,
channel_id: scene.channel_id,
channel_slug: scene.channel_slug,
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
network_id: scene.network_id || undefined,
network_slug: scene.network_slug || undefined,
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
studio_id: scene.studio_id || undefined,
studio_slug: scene.studio_slug || undefined,
studio_name: scene.studio_name || undefined,
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
actor_ids: scene.actors.map((actor) => actor.f1),
actors: scene.actors.map((actor) => actor.f2).join(),
tag_ids: scene.tags.map((tag) => tag.f1),
tags: flatTags.join(' '),
movie_ids: scene.movies.map((movie) => movie.f1),
movies: scene.movies.map((movie) => movie.f2).join(' '),
serie_ids: scene.series.map((serie) => serie.f1),
series: scene.series.map((serie) => serie.f2).join(' '),
fingerprints: scene.fingerprints.join(' '),
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
stashed: scene.stashed || 0,
dupe_index: scene.dupe_index || 0,
return [
{
replace: {
index: 'scenes',
id: scene.id,
doc: {
title: scene.title || undefined,
title_filtered: filteredTitle || undefined,
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
created_at: Math.round(scene.created_at.getTime() / 1000),
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
is_showcased: scene.showcased,
entry_id: scene.entry_id || undefined,
shoot_id: scene.shoot_id || undefined,
channel_id: scene.channel_id,
channel_slug: scene.channel_slug,
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
network_id: scene.network_id || undefined,
network_slug: scene.network_slug || undefined,
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
studio_id: scene.studio_id || undefined,
studio_slug: scene.studio_slug || undefined,
studio_name: scene.studio_name || undefined,
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
actor_ids: scene.actors.map((actor) => actor.f1),
actors: scene.actors.map((actor) => actor.f2).join(),
tag_ids: scene.tags.map((tag) => tag.f1),
tags: flatTags.join(' '),
movie_ids: scene.movies.map((movie) => movie.f1),
movies: scene.movies.map((movie) => movie.f2).join(' '),
serie_ids: scene.series.map((serie) => serie.f1),
series: scene.series.map((serie) => serie.f2).join(' '),
fingerprints: scene.fingerprints.join(' '),
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
stashed: scene.stashed || 0,
dupe_index: scene.dupe_index || 0,
},
},
},
};
...scene.tags.map((tag) => ({
replace: {
index: 'scenes_tags',
// id: scene.id,
doc: {
scene_id: scene.id,
tag_id: tag.f1,
actor_id: tag.f5,
},
},
})),
];
});
// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {

View File

@@ -0,0 +1,88 @@
'use strict';
const config = require('config');
const manticore = require('manticoresearch');
const knex = require('../knex');
const chunk = require('../utils/chunk');
const mantiClient = new manticore.ApiClient();
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
const utilsApi = new manticore.UtilsApi(mantiClient);
const indexApi = new manticore.IndexApi(mantiClient);
async function syncStashes(domain = 'scene') {
await utilsApi.sql(`truncate table ${domain}s_stashed`);
const stashes = await knex(`stashes_${domain}s`)
.select(
`stashes_${domain}s.id as stashed_id`,
`stashes_${domain}s.${domain}_id`,
'stashes.id as stash_id',
'stashes.user_id as user_id',
`stashes_${domain}s.created_at as created_at`,
)
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
await chain;
const stashDocs = stashChunk.map((stash) => ({
replace: {
index: `${domain}s_stashed`,
id: stash.stashed_id,
doc: {
[`${domain}_id`]: stash[`${domain}_id`],
stash_id: stash.stash_id,
user_id: stash.user_id,
created_at: Math.round(stash.created_at.getTime() / 1000),
},
},
}));
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
}, Promise.resolve());
}
async function init() {
await utilsApi.sql('drop table if exists scenes_stashed');
await utilsApi.sql(`create table if not exists scenes_stashed (
scene_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await utilsApi.sql('drop table if exists movies_stashed');
await utilsApi.sql(`create table if not exists movies_stashed (
movie_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await utilsApi.sql('drop table if exists actors_stashed');
await utilsApi.sql(`create table if not exists actors_stashed (
actor_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await syncStashes('scene');
await syncStashes('actor');
await syncStashes('movie');
console.log('Done!');
knex.destroy();
}
init();

View File

@@ -16,14 +16,14 @@ async function updateManticoreStashedScenes(docs) {
await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
await chain;
const sceneIds = docsChunk.map((doc) => doc.replace.id);
const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
const stashes = await knex('stashes_scenes')
.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
.whereIn('scene_id', sceneIds);
const stashDocs = docsChunk.flatMap((doc) => {
const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
if (sceneStashes.length === 0) {
@@ -50,6 +50,25 @@ async function updateManticoreStashedScenes(docs) {
if (stashDocs.length > 0) {
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
}
const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
if (deleteSceneIds.length > 0) {
await indexApi.callDelete({
index: 'scenes_stashed',
query: {
bool: {
must: [
{
in: {
scene_id: deleteSceneIds,
},
},
],
},
},
});
}
}, Promise.resolve());
}
@@ -128,9 +147,20 @@ async function updateManticoreSceneSearch(releaseIds) {
studios.showcased
`, releaseIds && [releaseIds]);
// console.log(scenes.rows);
const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
const docs = releaseIds.map((sceneId) => {
const scene = scenesById[sceneId];
if (!scene) {
return {
delete: {
index: 'scenes',
id: sceneId,
},
};
}
const docs = scenes.rows.map((scene) => {
const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
@@ -291,7 +321,20 @@ async function updateManticoreMovieSearch(movieIds) {
movies_covers.*
`, movieIds && [movieIds]);
const docs = movies.rows.map((movie) => {
const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
const docs = movieIds.map((movieId) => {
const movie = moviesById[movieId];
if (!movie) {
return {
delete: {
index: 'movies',
id: movieId,
},
};
}
const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
id: tag.f1,
name: tag.f2,

View File

@@ -38,7 +38,7 @@ function filterLocalUniqueReleases(releases, accReleases) {
};
}
async function filterUniqueReleases(releases) {
async function filterUniqueReleases(releases, options) {
const releaseIdentifierChunks = chunk(releases.map((release) => [release.entity.id, release.entryId.toString()]));
const duplicateReleaseEntryChunks = await Promise.map(releaseIdentifierChunks, async (releaseIdentifiers) => {
@@ -47,9 +47,13 @@ async function filterUniqueReleases(releases) {
.leftJoin('entities', 'entities.id', 'releases.entity_id')
.whereIn(['entity_id', 'entry_id'], releaseIdentifiers)
.where((builder) => {
if (!options?.forceDeepUpdate) {
// we do not want to force deep rescrapes on release day, so if the scene is already deep-scraped, we add it to the duplicate list
builder.where('deep', true); // scene is already deep scraped
}
// check if previously upcoming scenes can be excluded from duplicates to be rescraped for release day updates
builder
.where('deep', true) // scene is already deep scraped
.orWhereNull('date')
.orWhereNotIn('date_precision', ['day', 'minute']) // don't worry about scenes without (accurate) dates for now
.orWhere(knex.raw('date > NOW() - INTERVAL \'12 hours\'')) // scene is still upcoming, with a rough offset to wait for the end of the day west of UTC
@@ -110,18 +114,23 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
}
async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const options = {
...config.options[scraper.slug],
...include,
...preData,
parameters: getRecursiveParameters(entity),
options: getRecursiveParameters(entity, 'options'),
};
async function scrapeReleasesPage(page, accReleases, pageContext) {
const options = {
...config.options[scraper.slug],
...include,
...preData,
const pageOptions = {
...options,
...pageContext,
parameters: getRecursiveParameters(entity),
};
const rawPageReleases = isUpcoming
? await scraper.fetchUpcoming(entity, page, options, preData)
: await scraper.fetchLatest(entity, page, options, preData);
? await scraper.fetchUpcoming(entity, page, pageOptions, preData)
: await scraper.fetchLatest(entity, page, pageOptions, preData);
const pageReleases = rawPageReleases?.scenes || rawPageReleases;
@@ -157,7 +166,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const { uniqueReleases, duplicateReleases } = argv.force
? { uniqueReleases: limitedReleases, duplicateReleases: [] }
: await filterUniqueReleases(limitedReleases);
: await filterUniqueReleases(limitedReleases, options);
return { uniqueReleases, duplicateReleases };
}
@@ -305,8 +314,6 @@ async function scrapeNetworkParallel(networkEntity) {
async function fetchUpdates() {
const includedNetworks = await fetchIncludedEntities();
// console.log(includedNetworks[0]);
const scrapedNetworks = await Promise.map(
includedNetworks,
async (networkEntity) => (networkEntity.parameters?.sequential

96
src/utils/batch-insert.js Executable file
View File

@@ -0,0 +1,96 @@
'use strict';
const knex = require('../knex');
const chunk = require('./chunk');
const logger = require('../logger')(__filename);
// improved version of bulkInsert
async function batchInsert(table, items, {
conflict = true,
update = false,
chunkSize = 1000,
concurrent = false,
transaction,
commit = false,
} = {}) {
if (!table) {
throw new Error('No table specified for batch insert');
}
if (!Array.isArray(items)) {
throw new Error('Batch insert items are not an array');
}
if (items.length === 0) {
return [];
}
const chunks = chunk(items, chunkSize);
const conflicts = [].concat(conflict).filter((column) => typeof column === 'string'); // conflict might be 'true'
const trx = transaction || await knex.transaction();
try {
const queries = chunks.map((chunkItems) => {
const query = trx(table)
.insert(chunkItems)
.returning('*');
if (conflicts.length > 0) {
if (Array.isArray(update)) {
// udpate specified
return query
.onConflict(conflicts)
.merge(update);
}
if (update) {
// update all
return query
.onConflict(conflicts)
.merge();
}
throw new Error('Batch insert conflict columns must be specified together with update');
}
if (conflict && update) {
throw new Error('Batch insert conflict must specify columns, or update must be disabled');
}
// error on any conflict
if (conflict) {
return query;
}
// ignore duplicates, keep old entries as-is
return query
.onConflict()
.ignore();
});
const results = concurrent
? await Promise.all(queries)
: await queries.reduce(async (chain, query) => {
const acc = await chain;
const result = await query;
return acc.concat(result);
}, Promise.resolve([]));
if (!transaction || commit) {
await trx.commit();
}
return results;
} catch (error) {
if (!transaction || commit) {
await trx.rollback();
}
logger.error(`Failed batch insert: ${error.message} (${error.detail})`);
throw error;
}
}
module.exports = batchInsert;

View File

@@ -1,11 +1,11 @@
'use strict';
function getRecursiveParameters(entity, parameters) {
function getRecursiveParameters(entity, property = 'parameters', parameters) {
if (entity.parent) {
return getRecursiveParameters(entity.parent, { ...entity.parameters, ...parameters });
return getRecursiveParameters(entity.parent, property, { ...entity[property], ...parameters });
}
return { ...entity.parameters, ...parameters };
return { ...entity[property], ...parameters };
}
module.exports = getRecursiveParameters;

View File

@@ -7,6 +7,7 @@ const unprint = require('unprint');
const omit = require('object.omit');
const argv = require('../src/argv');
const logger = require('../src/logger')(__filename);
const include = require('../src/utils/argv-include')(argv);
const slugify = require('../src/utils/slugify');
const scrapers = require('../src/scrapers/scrapers');
@@ -32,6 +33,12 @@ unprint.options({
proxy: config.proxy,
});
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.isProxied ? ' proxied' : ''}${event.isBrowser ? ' browser' : ''}) ${event.url}`));
unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.isProxied ? 'proxied ' : ''}${event.isBrowser ? 'browser ' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`));
unprint.on('browserOpen', (event) => logger.debug(`Unprint opened browsers ${event.keys} (${event.active}/${config.bypass.browser.clientRetirement} active, ${event.clients} clients)`));
unprint.on('browserClose', (event) => logger.debug(`Unprint closed${event.retired ? ' retired' : ''} browsers ${event.keys} (${event.active}/${config.bypass.browser.clientRetirement} active, ${event.clients} clients)`));
const actors = [
// vixen
{ entity: 'vixen', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
@@ -66,9 +73,11 @@ const actors = [
{ entity: 'eyeontheguy', name: 'Tommy Gunn', fields: ['avatar'] },
{ entity: 'interracialpovs', name: 'Nia Nacci', fields: ['avatar', 'aliases', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
{ entity: 'povpornstars', name: 'Anna Bell Peaks', fields: ['avatar', 'aliases', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'measurements', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'naturalBoobs', 'socials'] },
{ entity: 'seehimfuck', name: 'Sheem The Dream', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'penisLength', 'circumcised', 'socials'] },
{ entity: 'seehimfuck', name: 'Sheem The Dream', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'hasTattoos', 'hasPiercings', 'penisLength', 'isCircumcised', 'socials'] },
{ entity: 'hushpass', name: 'Dylan Ryder', fields: ['avatar'] },
{ entity: 'interracialpass', name: 'Aidra Fox', fields: ['avatar', 'height', 'measurements'] },
{ entity: 'ravebunnys', name: 'Lacey Jayne', fields: ['avatar', 'height', 'measurements', 'dateOfBirth', 'birthPlace', 'description', 'ethnicity', 'weight', 'naturalBoobs'] },
{ entity: 'hotandtatted', name: 'Valerica Steele', url: 'https://hotandtatted.com/models/tattooed-pornstar-val-steele.html', fields: ['avatar', 'measurements', 'dateOfBirth', 'birthPlace', 'description', 'ethnicity', 'weight', 'foot', 'naturalBoobs', 'hasPiercings'] },
// kelly madison / 8K
{ entity: 'kellymadison', name: 'Ava Addams', fields: ['avatar', 'description', 'age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] },
{ entity: '8kmembers', name: 'Angie Lynx', fields: ['age', 'height', 'measurements', 'birthPlace', 'dateOfBirth', 'ethnicity'] },
@@ -81,7 +90,7 @@ const actors = [
{ entity: 'letsdoeit', name: 'Nicole Doshi', fields: ['avatar', 'description', 'gender', 'height', 'measurements', 'birthPlace', 'dateOfBirth'] },
{ entity: 'killergram', name: 'Clea Gaultier', fields: ['avatar', 'gender', 'hairColor', 'ethnicity'] },
{ entity: 'men', name: 'Cade Maddox', fields: ['avatar', 'description', 'gender', 'height', 'ethnicity', 'penisLength', 'dateOfBirth', 'weight', 'hairColor', 'hasTattoos'] },
{ entity: 'metrohd', name: 'April Olsen', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight'] },
{ entity: 'metrohd', name: 'Vanna Bardot', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasTattoos'] },
{ entity: 'mofos', name: 'Ariana Starr', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] },
{ entity: 'propertysex', name: 'Desiree Dulce', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth', 'weight', 'hairColor', 'ethnicity', 'hasPiercings'] },
{ entity: 'sexyhub', name: 'Angie Lynx', fields: ['avatar', 'description', 'gender', 'birthPlace', 'height', 'measurements', 'dateOfBirth'] },
@@ -110,7 +119,7 @@ const actors = [
{ entity: 'devilsfilm', name: 'Katrina Colt', fields: ['avatar', 'gender'] },
{ entity: 'diabolic', name: 'Kira Noir', fields: ['avatar', 'gender'] },
{ entity: 'evilangel', name: 'Francesca Le', fields: ['avatar', 'gender'] },
{ entity: 'fantasymassage', name: 'Cherry Kiss', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] },
{ entity: 'fantasymassage', name: 'Cherry Kiss', fields: ['avatar', 'gender'] },
{ entity: 'filthykings', name: 'Armani Black', fields: ['avatar', 'gender'] },
{ entity: 'gangbangcreampie', name: 'Luna Lovely', fields: ['avatar', 'gender', 'description'] },
{ entity: 'girlsway', name: 'Adriana Chechik', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] },
@@ -136,12 +145,12 @@ const actors = [
// perv city
{ entity: 'pervcity', name: 'Brooklyn Gray', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'dpdiva', name: 'Liz Jordan', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'bamvisions', name: 'Abella Danger', fields: ['avatar', 'height', 'measurements'] },
{ entity: 'bamvisions', name: 'Abella Danger', fields: ['avatar', 'height', 'measurements'] }, // site offline as of 2026-02-25
// radical
{ entity: 'bjraw', name: 'Nikki Knightly', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'gotfilled', name: 'Alexa Chains', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'inserted', name: 'Anissa Kate', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'lucidflix', name: 'Ava Amira', fields: ['avatar', 'description', 'gender'] },
// wankz
@@ -163,10 +172,10 @@ const actors = [
{ entity: 'thatsitcomshow', name: 'Casey Calvert', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] },
{ entity: 'brattysis', name: 'Scarlett Alexis', fields: ['avatar', 'age', 'height', 'measurements', 'description', 'residencePlace', 'photos'] },
// porndoe
{ entity: 'vipsexvault', name: 'Amirah Adara', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'hairColor', 'description'] },
{ entity: 'amateureuro', name: 'Luna Oara', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'description'] },
{ entity: 'mamacitaz', name: 'Julia De Lucia', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'description', 'hairColor'] },
{ entity: 'transbella', name: 'Kalena Rios', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'description', 'hairColor'] },
{ entity: 'vipsexvault', name: 'Amirah Adara', fields: ['avatar', 'nationality', 'birthPlace', 'age', 'naturalBoobs', 'hairColor', 'description'] },
{ entity: 'amateureuro', name: 'Luna Oara', fields: ['avatar', 'nationality', 'birthPlace', 'age', 'naturalBoobs', 'description'] },
{ entity: 'mamacitaz', name: 'Julia De Lucia', fields: ['avatar', 'nationality', 'birthPlace', 'age', 'naturalBoobs', 'description', 'hairColor'] },
{ entity: 'transbella', name: 'Kalena Rios', fields: ['avatar', 'nationality', 'birthPlace', 'age', 'naturalBoobs', 'description', 'hairColor'] },
// snow valley group
{ entity: 'spermmania', name: 'Lya Cutie', fields: ['avatar', 'age', 'height', 'cup', 'bust', 'waist', 'hip'] },
{ entity: 'cospuri', name: 'Ria Kurumi', fields: ['avatar', 'birthPlace', 'description', 'height', 'cup', 'bust', 'waist', 'hip'] },
@@ -207,7 +216,7 @@ const actors = [
{ entity: 'amateurallure', name: 'Ava Amira', fields: ['avatar', 'description'] },
{ entity: 'swallowsalon', name: 'Abella Danger', fields: ['avatar'] },
// exploitedx
{ entity: 'exploitedx', name: 'Amber Moore', url: 'https://exploitedcollegegirls.com/models/AmberMoore.html', fields: ['avatar', 'description', 'age', 'height', 'measurements'] },
{ entity: 'exploitedx', name: 'Amber Moore', url: 'https://exploitedcollegegirls.com/models/Amber-Moore.html', fields: ['avatar', 'description', 'age', 'height', 'measurements'] },
{ entity: 'exploitedx', name: 'Amber Moore', url: 'https://bbcsurprise.com/models/AmberMoore.html', fields: ['avatar', 'age'] },
{ entity: 'exploitedx', name: 'Megan Marx', url: 'https://excogigirls.com/models/megan-marx.html', fields: ['avatar', 'description', 'age', 'height', 'measurements'] },
{ entity: 'exploitedx', name: 'Sophie Hunt', url: 'https://www.backroomcastingcouch.com/models/Sophie-Hunt.html', fields: ['avatar', 'age'] },
@@ -237,7 +246,7 @@ const actors = [
{ entity: 'inthecrack', name: 'Vicki Chase', fields: ['dateOfBirth', 'height', 'weight', 'ethnicity', 'birthPlace'] },
{ entity: 'karups', name: 'Peach Lollypop', fields: ['avatar'] },
{ entity: 'littlecapricedreams', name: 'Littlecaprice', fields: ['avatar', 'nationality', 'cup', 'measurements', 'height', 'description'] }, // sic
{ entity: 'mariskax', name: 'Honey Demon', fields: ['avatar', 'gender', 'dateOfBirth', 'placeOfBirth', 'measurements', 'height', 'weight', 'hairColor', 'eyes'] },
{ entity: 'mariskax', name: 'Honey Demon', fields: ['avatar', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'hairColor', 'eyes'] },
{ entity: 'meidenvanholland', name: 'Izzy Bizzy Bang Bang', fields: ['avatar', 'description'] },
{ entity: 'nebraskacoeds', name: 'Mary Beth Haglin', fields: ['avatar'] },
{ entity: 'pascalssubsluts', name: 'Zlata Shine', fields: ['avatar', 'gender', 'nationality', 'hairColor', 'height', 'description'] },
@@ -272,7 +281,7 @@ async function validateUrl(url, mime = 'image/') {
return false;
}
const res = await fetch(href, {
const res = await unprint.get(href, {
headers: {
Referer: url.referer || new URL(href).origin,
},
@@ -301,9 +310,10 @@ const validators = {
hasTattoos: (value) => typeof value === 'boolean',
hasPiercings: (value) => typeof value === 'boolean',
avatar: async (value) => [].concat(value).reduce(async (chain, url) => {
// testing all avatar fallbacks is too time-consuming, just ensure one is valid
const acc = await chain;
if (!acc) {
if (acc) {
return acc;
}
@@ -322,9 +332,9 @@ const validators = {
// profiler in this context is shorthand for profile scraper
async function init() {
const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: 'channel' });
const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: 'options' });
Object.entries(actorScrapers).reduce(async (chain, [entitySlug, scraper]) => {
await Object.entries(actorScrapers).reduce(async (chain, [entitySlug, scraper]) => {
await chain;
const entity = entitiesBySlug[entitySlug] || null;
@@ -332,7 +342,6 @@ async function init() {
const tests = actors.filter((actor) => actor.entity === entitySlug);
// TODO: remove when all tests are written
if (tests.length === 0) {
console.log('TODO', entitySlug);
return;
@@ -375,7 +384,7 @@ async function init() {
validators[field]
? await validators[field](profile[field])
: typeof profile[field] !== 'undefined',
`broken field ${field}, got ${profile[field]}`,
`broken field ${field}, got ${profile[field]} for ${actor.name}`,
);
}));
})));
@@ -385,6 +394,7 @@ async function init() {
console.log(actors.length);
await knex.destroy();
await unprint.closeAllBrowsers();
}
init();