48 Commits
Author SHA1 Message Date
DebaucheryLibrarian 98121e6162 1.256.5 2026-10-22 00:23:23 +02:00
DebaucheryLibrarian e447a0b6da Fixed Team Skeet channel slug scraper not matching numeric digits, updated Mylf DOM URL. 2026-10-22 00:23:21 +02:00
DebaucheryLibrarian 530d45448a 1.256.4 2026-10-21 21:29:15 +02:00
DebaucheryLibrarian 6a0759693c Fixed double slash in Family Strokes network URL. 2026-10-21 21:29:13 +02:00
DebaucheryLibrarian 83c78ade5d 1.256.3 2026-10-21 20:42:51 +02:00
DebaucheryLibrarian eebe8eef89 Fixed Teamskeet Singles/Singels typo. 2026-10-21 20:42:49 +02:00
DebaucheryLibrarian a22a7288c1 1.256.2 2026-10-21 05:20:22 +02:00
DebaucheryLibrarian 40d56c515d Improved tag matching query efficiency, should fix some big scrape failures. 2026-10-21 05:20:19 +02:00
DebaucheryLibrarian b8ad6bb10b 1.256.1 2026-10-21 02:30:36 +02:00
DebaucheryLibrarian 131f744745 Fixed Team Skeet deep scrape and URL format, added duration. 2026-10-21 02:30:33 +02:00
DebaucheryLibrarian fef686f977 1.256.0 2026-10-19 07:35:06 +02:00
DebaucheryLibrarian 4ede0724a7 Migrated remaining scrapers from qu/http to unprint, likely fixing TeamSkeet and PinkyXXX (verification needed). 2026-10-19 07:35:03 +02:00
DebaucheryLibrarian 03a5ce5bb1 1.255.19 2026-10-19 06:23:09 +02:00
DebaucheryLibrarian 745aa023a8 Using unprint in Radical API, fixed updates broken due body being string instead of data. 2026-10-19 06:23:01 +02:00
DebaucheryLibrarian 5633aa2efa 1.255.18 2026-10-04 02:30:59 +02:00
DebaucheryLibrarian 6643185a81 Added RSGH Media sites. 2026-10-04 02:30:57 +02:00
DebaucheryLibrarian d0149e70a7 1.255.17 2026-10-03 06:32:07 +02:00
DebaucheryLibrarian 84ac366e36 Fixed beforeFetchScenes not running for update releases. Fixed Hookup Hotshot deep scrape broken due missing NATS cookie. 2026-10-03 06:32:03 +02:00
DebaucheryLibrarian b700e05c5e 1.255.16 2026-09-22 05:21:00 +02:00
DebaucheryLibrarian 9568691609 Added Nick Morris Studio to AnalVids. 2026-09-22 05:20:54 +02:00
DebaucheryLibrarian 35404bc7a4 1.255.15 2026-09-22 02:15:38 +02:00
DebaucheryLibrarian 6a95d16a4d Added support for series to Adult Empire scraper. 2026-09-22 02:15:36 +02:00
DebaucheryLibrarian 39f1a5c578 1.255.14 2026-09-22 01:41:16 +02:00
DebaucheryLibrarian 09eaf60a90 Added Jennifer White's new website. 2026-09-22 01:41:14 +02:00
DebaucheryLibrarian 4ad8ab4f1b 1.255.13 2026-09-21 21:17:01 +02:00
DebaucheryLibrarian 27743535c5 Using request interface for Vixen latest. 2026-09-21 21:16:59 +02:00
DebaucheryLibrarian 3f9695a0ed 1.255.12 2026-09-18 23:26:29 +02:00
DebaucheryLibrarian b1836b61c8 Added actor date of birth and death precision table. 2026-09-18 23:26:26 +02:00
DebaucheryLibrarian a62cfcac3d 1.255.11 2026-08-07 02:46:06 +02:00
DebaucheryLibrarian 5b9c635ed1 Fixed deprecated unprint cookie handling in Dorcel scraper. 2026-08-07 02:46:01 +02:00
DebaucheryLibrarian ad56e7b9e0 1.255.10 2026-07-29 06:22:49 +02:00
DebaucheryLibrarian 4eb6840c2a Added toy penetration and more detailed trans tags. 2026-07-29 06:22:42 +02:00
DebaucheryLibrarian 9f173d7aee 1.255.9 2026-07-26 17:59:39 +02:00
DebaucheryLibrarian 5f0320fde0 Handling inconsistent album data in Ricky's Room scraper. 2026-07-26 17:59:37 +02:00
DebaucheryLibrarian d21ba65a43 1.255.8 2026-07-26 03:28:31 +02:00
DebaucheryLibrarian ea61b45a87 Added archived column to stashes. 2026-07-26 03:28:27 +02:00
DebaucheryLibrarian f34de8d1b0 Merged migrations. 2026-07-25 03:46:43 +02:00
DebaucheryLibrarian f1898e6942 1.255.7 2026-07-25 03:22:08 +02:00
DebaucheryLibrarian 8970757a8e Checking social handle uniqueness case insensitively. 2026-07-25 03:22:06 +02:00
DebaucheryLibrarian c55953769f Renamed last stashed column. 2026-07-25 02:17:50 +02:00
DebaucheryLibrarian cf8f1a0f7b 1.255.6 2026-07-25 02:15:28 +02:00
DebaucheryLibrarian 1efa4715d7 Added last stash to stashes_meta to allow for sorting by recently used. 2026-07-25 02:15:25 +02:00
DebaucheryLibrarian ec307112a9 1.255.5 2026-07-24 16:57:18 +02:00
DebaucheryLibrarian 8f0609ade2 Improved Aziani profile scraper for Cosplayground, allowing image type override, added profile test for Cosplayground. 2026-07-24 16:57:15 +02:00
DebaucheryLibrarian 327f255c9d 1.255.4 2026-07-24 05:52:20 +02:00
DebaucheryLibrarian 06a5f5bec7 Extended Aziani scraper to support Cosplayground. 2026-07-24 05:52:17 +02:00
DebaucheryLibrarian 00ba8b9dd3 1.255.3 2026-07-23 23:55:37 +02:00
DebaucheryLibrarian d7e0171b2f Enforcing profile ID in actors_avatars. 2026-07-23 23:55:34 +02:00
61 changed files with 1276 additions and 2485 deletions
+1 -1
Submodule common updated: ec0812ad9d...7df69021ea
+270 -42
View File
@@ -40,6 +40,7 @@ exports.up = async (knex) => {
.index();
table.text('name');
table.string('name_stylized');
table.text('slug', 32);
table.text('type')
@@ -126,10 +127,15 @@ exports.up = async (knex) => {
.defaultTo(knex.fn.now());
});
await knex.schema.createMaterializedView('media_sfw', (view) => {
view.as(knex('media').select('id').where('is_sfw', true));
});
await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
await knex.raw(`
CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
SELECT id FROM media
WHERE is_sfw = true
SELECT id FROM media_sfw
ORDER BY random()
LIMIT 1;
$$ LANGUAGE sql STABLE;
@@ -143,6 +149,36 @@ exports.up = async (knex) => {
.defaultTo(knex.raw('get_random_sfw_media_id()'));
});
await knex.schema.createTable('media_biometrics', (table) => {
table.increments('id');
table.string('media_id', 21)
.references('id')
.inTable('media')
.notNullable()
.onDelete('cascade');
table.integer('width');
table.integer('height');
table.integer('face_index')
.notNullable()
.defaultTo(0);
table.json('biometrics');
table.specificType('embedding', 'vector(1024)');
table.datetime('updated_at')
.notNullable()
.defaultTo(knex.fn.now());
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
table.unique(['media_id', 'face_index']);
});
await knex.schema.createTable('tags_groups', (table) => {
table.increments('id');
@@ -335,7 +371,7 @@ exports.up = async (knex) => {
table.decimal('shoe_size');
table.integer('leg');
table.integer('foot');
table.decimal('foot');
table.integer('thigh');
table.string('blood_type');
@@ -357,6 +393,9 @@ exports.up = async (knex) => {
table.string('agency');
table.boolean('allow_global_match');
table.text('comment');
table.text('avatar_media_id', 21)
.references('id')
.inTable('media');
@@ -443,7 +482,7 @@ exports.up = async (knex) => {
table.decimal('shoe_size');
table.integer('leg');
table.integer('foot');
table.decimal('foot');
table.integer('thigh');
table.string('blood_type');
@@ -655,6 +694,7 @@ exports.up = async (knex) => {
.inTable('actors');
table.integer('profile_id', 12)
.notNullable()
.references('id')
.inTable('actors_profiles')
.onDelete('cascade');
@@ -672,8 +712,6 @@ exports.up = async (knex) => {
table.unique(['profile_id', 'media_id']);
});
await knex.raw('CREATE UNIQUE INDEX unique_main_avatars ON actors_avatars (actor_id) WHERE (profile_id IS NULL);');
await knex.schema.createTable('actors_photos', (table) => {
table.integer('actor_id', 12)
.notNullable()
@@ -713,12 +751,20 @@ exports.up = async (knex) => {
table.datetime('pinged_at');
table.datetime('verified_at');
table.unique(['actor_id', 'platform', 'handle']);
table.unique(['actor_id', 'url']);
});
await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_url_or_handle CHECK (num_nulls(handle, url) = 1);');
await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_handle_and_platform CHECK (num_nulls(platform, handle) = 2 or num_nulls(platform, handle) = 0);');
await knex.raw('CREATE UNIQUE INDEX actors_socials_actor_id_platform_handle_lower_unique ON actors_socials (actor_id, platform, lower(handle));');
await knex.schema.createTable('languages', (table) => {
table.string('alpha2')
.primary();
table.text('name');
table.text('name_native');
});
await knex.schema.createTable('releases', (table) => {
table.increments('id');
@@ -741,6 +787,7 @@ exports.up = async (knex) => {
table.text('url', 1000);
table.text('title');
table.specificType('alt_titles', 'text ARRAY');
table.specificType('alt_descriptions', 'text ARRAY');
table.text('slug');
table.timestamp('date');
@@ -758,6 +805,13 @@ exports.up = async (knex) => {
table.enum('date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
.defaultTo('day');
table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
.defaultTo('day');
table.string('language_alpha2')
.references('alpha2')
.inTable('languages');
table.text('description');
table.integer('duration')
@@ -773,6 +827,8 @@ exports.up = async (knex) => {
table.text('comment');
table.json('attributes');
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches')
@@ -937,6 +993,8 @@ exports.up = async (knex) => {
});
await knex.schema.createTable('releases_tags', (table) => {
table.increments('id');
table.integer('tag_id', 12)
.references('id')
.inTable('tags');
@@ -947,9 +1005,13 @@ exports.up = async (knex) => {
.inTable('releases')
.onDelete('cascade');
table.integer('actor_id')
.references('id')
.inTable('actors');
table.text('original_tag');
table.unique(['tag_id', 'release_id']);
table.unique(['release_id', 'original_tag']);
table.index('tag_id');
table.index('release_id');
@@ -958,12 +1020,7 @@ exports.up = async (knex) => {
.defaultTo('scraper');
});
await knex.schema.createTable('releases_search', (table) => {
table.integer('release_id', 16)
.references('id')
.inTable('releases')
.onDelete('cascade');
});
await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1));');
await knex.schema.createTable('movies', (table) => {
table.increments('id');
@@ -984,6 +1041,7 @@ exports.up = async (knex) => {
table.text('url', 1000);
table.text('title');
table.specificType('alt_titles', 'text ARRAY');
table.specificType('alt_descriptions', 'text ARRAY');
table.text('slug');
table.timestamp('date');
@@ -1000,6 +1058,8 @@ exports.up = async (knex) => {
table.text('comment');
table.json('attributes');
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches')
@@ -1134,13 +1194,6 @@ exports.up = async (knex) => {
table.unique(['movie_id', 'media_id']);
});
await knex.schema.createTable('movies_search', (table) => {
table.integer('movie_id', 16)
.references('id')
.inTable('movies')
.onDelete('cascade');
});
await knex.schema.createTable('series', (table) => {
table.increments('id');
@@ -1159,6 +1212,7 @@ exports.up = async (knex) => {
table.text('url', 1000);
table.text('title');
table.specificType('alt_titles', 'text ARRAY');
table.specificType('alt_descriptions', 'text ARRAY');
table.text('slug');
table.timestamp('date');
@@ -1175,6 +1229,8 @@ exports.up = async (knex) => {
table.text('comment');
table.json('attributes');
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches')
@@ -1288,13 +1344,6 @@ exports.up = async (knex) => {
table.unique(['serie_id', 'media_id']);
});
await knex.schema.createTable('series_search', (table) => {
table.integer('serie_id', 16)
.references('id')
.inTable('series')
.onDelete('cascade');
});
await knex.schema.createTable('chapters', (table) => {
table.increments('id');
@@ -1316,6 +1365,8 @@ exports.up = async (knex) => {
table.text('title');
table.text('description');
table.datetime('date');
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches')
@@ -1379,6 +1430,34 @@ exports.up = async (knex) => {
table.unique(['tag_id', 'chapter_id']);
});
await knex.schema.createTable('chapters_trailers', (table) => {
table.integer('chapter_id')
.notNullable()
.references('id')
.inTable('chapters')
.onDelete('cascade');
table.text('media_id')
.notNullable()
.references('id')
.inTable('media')
.onDelete('cascade');
});
await knex.schema.createTable('chapters_teasers', (table) => {
table.integer('chapter_id')
.notNullable()
.references('id')
.inTable('chapters')
.onDelete('cascade');
table.text('media_id')
.notNullable()
.references('id')
.inTable('media')
.onDelete('cascade');
});
await knex.schema.createTable('users_roles', (table) => {
table.string('role')
.primary();
@@ -1396,7 +1475,9 @@ exports.up = async (knex) => {
{ subject: 'actor', action: 'create' },
{ subject: 'actor', action: 'update' },
{ subject: 'actor', action: 'delete' },
{ plainUrls: true },
{ subject: 'actor', action: 'merge' },
{ subject: 'sync' },
{ subject: 'plainUrls' },
]),
},
{
@@ -1442,6 +1523,8 @@ exports.up = async (knex) => {
table.specificType('last_ip', 'cidr');
table.jsonb('settings');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
@@ -1660,6 +1743,98 @@ exports.up = async (knex) => {
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('feeds', (table) => {
table.increments('id');
table.integer('user_id')
.notNullable()
.references('id')
.inTable('users')
.onDelete('cascade');
table.string('name')
.notNullable();
table.string('slug')
.notNullable();
table.boolean('public');
table.boolean('primary');
table.text('comment');
table.json('meta');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('feeds_entities', (table) => {
table.increments('id');
table.integer('feed_id')
.notNullable()
.references('id')
.inTable('feeds')
.onDelete('cascade');
table.integer('entity_id')
.notNullable()
.references('id')
.inTable('entities')
.onDelete('cascade');
table.text('comment');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('feeds_actors', (table) => {
table.increments('id');
table.integer('feed_id')
.notNullable()
.references('id')
.inTable('feeds')
.onDelete('cascade');
table.integer('actor_id')
.notNullable()
.references('id')
.inTable('actors')
.onDelete('cascade');
table.text('comment');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('feeds_tags', (table) => {
table.increments('id');
table.integer('feed_id')
.notNullable()
.references('id')
.inTable('feeds')
.onDelete('cascade');
table.integer('tag_id')
.notNullable()
.references('id')
.inTable('tags')
.onDelete('cascade');
table.text('comment');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('alerts', (table) => {
table.increments('id');
@@ -1698,6 +1873,10 @@ exports.up = async (knex) => {
.notNullable()
.defaultTo(false);
table.boolean('is_expired')
.notNullable()
.defaultTo(false);
table.json('meta');
table.text('comment');
@@ -1878,6 +2057,31 @@ exports.up = async (knex) => {
);
});
await knex.schema.createMaterializedView('alerts_users_scenes', (view) => {
view.columns('user_id', 'scene_id', 'alert_ids');
view.as(
knex('alerts_scenes')
.select(
'alerts.user_id',
'alerts_scenes.scene_id',
knex.raw('array_agg(distinct alerts.id) as alert_ids'),
knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_scenes.id is null) as is_only'),
)
.leftJoin('alerts', 'alerts.id', 'alerts_scenes.alert_id')
.leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_scenes.alert_id')
.leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_scenes.alert_id')
.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_scenes.alert_id')
.leftJoin('alerts_scenes as related_scenes', (joinBuilder) => {
joinBuilder
.on('related_scenes.alert_id', 'alerts_scenes.alert_id')
.on('related_scenes.scene_id', '!=', 'alerts_scenes.scene_id');
})
.where('alerts.is_expired', false)
.groupBy(['user_id', 'alerts_scenes.scene_id', 'is_only']),
);
});
await knex.schema.createTable('notifications', (table) => {
table.increments('id');
@@ -1901,6 +2105,18 @@ exports.up = async (knex) => {
.notNullable()
.defaultTo(false);
table.datetime('seen_at');
table.boolean('notify')
.notNullable()
.defaultTo(true);
table.boolean('email')
.notNullable()
.defaultTo(false);
table.datetime('emailed_at');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
@@ -2177,12 +2393,20 @@ exports.up = async (knex) => {
.inTable('entities');
});
await knex.schema.createTable('sync', (table) => {
table.increments('id');
table.string('domain');
table.specificType('item_ids', 'integer array');
table.text('comment');
table.datetime('created_at')
.defaultTo(knex.fn.now());
});
// SEARCH AND SORT
await knex.raw(`
ALTER TABLE releases_search ADD COLUMN document tsvector;
ALTER TABLE movies_search ADD COLUMN document tsvector;
ALTER TABLE series_search ADD COLUMN document tsvector;
/* allow scenes without dates to be mixed inbetween scenes with dates */
ALTER TABLE releases
ADD COLUMN effective_date timestamptz
@@ -2200,19 +2424,12 @@ exports.up = async (knex) => {
// INDEXES
await knex.raw(`
CREATE UNIQUE INDEX unique_actor_slugs_network ON actors (slug, entity_id, entry_id);
CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL;
CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug, entry_id) WHERE entity_id IS NULL;
CREATE UNIQUE INDEX unique_entity_campaigns_banner_url ON campaigns (entity_id, url, banner_id) WHERE affiliate_id IS NULL;
CREATE UNIQUE INDEX unique_entity_campaigns_url ON campaigns (entity_id, url) WHERE banner_id IS NULL AND affiliate_id IS NULL;
CREATE UNIQUE INDEX unique_entity_campaigns_banner_affiliate ON campaigns (entity_id, affiliate_id, banner_id) WHERE url IS NULL;
CREATE UNIQUE INDEX unique_entity_campaigns_affiliate ON campaigns (entity_id, affiliate_id) WHERE banner_id IS NULL AND url IS NULL;
CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
CREATE INDEX releases_search_index ON releases_search USING GIN (document);
CREATE INDEX movies_search_index ON movies_search USING GIN (document);
CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
CREATE INDEX series_search_index ON series_search USING GIN (document);
`);
// FUNCTIONS
@@ -2275,7 +2492,8 @@ exports.up = async (knex) => {
stashes.id as stash_id,
COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes,
COUNT(DISTINCT stashes_movies)::integer as stashed_movies,
COUNT(DISTINCT stashes_actors)::integer as stashed_actors
COUNT(DISTINCT stashes_actors)::integer as stashed_actors,
MAX(GREATEST(stashes_scenes.created_at, stashes_movies.created_at, stashes_actors.created_at)) as last_stash_at
FROM stashes
LEFT JOIN stashes_scenes ON stashes_scenes.stash_id = stashes.id
LEFT JOIN stashes_movies ON stashes_movies.stash_id = stashes.id
@@ -2372,6 +2590,8 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS chapters_tags CASCADE;
DROP TABLE IF EXISTS chapters_posters CASCADE;
DROP TABLE IF EXISTS chapters_photos CASCADE;
DROP TABLE IF EXISTS chapters_trailers CASCADE;
DROP TABLE IF EXISTS chapters_teasers CASCADE;
DROP TABLE IF EXISTS banners_tags CASCADE;
DROP TABLE IF EXISTS banners CASCADE;
@@ -2381,6 +2601,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS batches CASCADE;
DROP TABLE IF EXISTS fingerprints_types CASCADE;
DROP TABLE IF EXISTS sync CASCADE;
DROP TABLE IF EXISTS actors_avatars CASCADE;
DROP TABLE IF EXISTS actors_photos CASCADE;
@@ -2412,8 +2633,10 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS sites CASCADE;
DROP TABLE IF EXISTS studios CASCADE;
DROP TABLE IF EXISTS media_sfw CASCADE;
DROP TABLE IF EXISTS media_biometrics CASCADE;
DROP TABLE IF EXISTS media CASCADE;
DROP TABLE IF EXISTS countries CASCADE;
DROP TABLE IF EXISTS languages CASCADE;
DROP TABLE IF EXISTS networks CASCADE;
DROP TABLE IF EXISTS entities_types CASCADE;
@@ -2426,6 +2649,11 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS stashes_actors CASCADE;
DROP TABLE IF EXISTS stashes CASCADE;
DROP TABLE IF EXISTS feeds_entities CASCADE;
DROP TABLE IF EXISTS feeds_actors CASCADE;
DROP TABLE IF EXISTS feeds_tags CASCADE;
DROP TABLE IF EXISTS feeds CASCADE;
DROP TABLE IF EXISTS alerts_scenes CASCADE;
DROP TABLE IF EXISTS alerts_actors CASCADE;
DROP TABLE IF EXISTS alerts_tags CASCADE;
@@ -1,19 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('releases', (table) => {
table.json('attributes');
});
await knex.schema.alterTable('movies', (table) => {
table.json('attributes');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('releases', (table) => {
table.dropColumn('attributes');
});
await knex.schema.alterTable('movies', (table) => {
table.dropColumn('attributes');
});
};
-11
View File
@@ -1,11 +0,0 @@
exports.up = async function(knex) {
await knex.schema.alterTable('releases', (table) => {
table.specificType('alt_descriptions', 'text ARRAY');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('releases', (table) => {
table.dropColumn('alt_descriptions');
});
};
@@ -1,58 +0,0 @@
exports.up = async function(knex) {
await knex.schema.alterTable('chapters', (table) => {
table.datetime('date');
});
await knex.schema.createTable('chapters_trailers', (table) => {
table.integer('chapter_id')
.notNullable()
.references('id')
.inTable('chapters')
.onDelete('cascade');
table.text('media_id')
.notNullable()
.references('id')
.inTable('media')
.onDelete('cascade');
});
await knex.schema.createTable('chapters_teasers', (table) => {
table.integer('chapter_id')
.notNullable()
.references('id')
.inTable('chapters')
.onDelete('cascade');
table.text('media_id')
.notNullable()
.references('id')
.inTable('media')
.onDelete('cascade');
});
await knex.schema.alterTable('entities', (table) => {
table.string('name_stylized');
});
await knex.schema.alterTable('movies', (table) => {
table.specificType('alt_descriptions', 'text ARRAY');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('chapters', (table) => {
table.dropColumn('date');
});
await knex.schema.alterTable('entities', (table) => {
table.dropColumn('name_stylized');
});
await knex.schema.dropTable('chapters_trailers');
await knex.schema.dropTable('chapters_teasers');
await knex.schema.alterTable('movies', (table) => {
table.dropColumn('alt_descriptions');
});
};
@@ -1,28 +0,0 @@
exports.up = async function(knex) {
await knex.schema.createMaterializedView('media_sfw', (view) => {
view.as(knex('media').select('id').where('is_sfw', true));
});
await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
await knex.raw(`
CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
SELECT id FROM media_sfw
ORDER BY random()
LIMIT 1;
$$ LANGUAGE sql STABLE;
`);
};
exports.down = async function(knex) {
await knex.raw(`
CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
SELECT id FROM media
WHERE is_sfw = true
ORDER BY random()
LIMIT 1;
$$ LANGUAGE sql STABLE;
`);
await knex.schema.dropMaterializedView('media_sfw');
};
@@ -1,31 +0,0 @@
exports.up = async (knex) => {
// dedupe
await knex.raw(`
DELETE
FROM releases_tags
WHERE ctid IN
(
SELECT ctid
FROM(
SELECT
*,
ctid,
row_number() OVER (PARTITION BY release_id, original_tag ORDER BY ctid)
FROM releases_tags
)s
WHERE row_number >= 2
)
`);
await knex.schema.alterTable('releases_tags', (table) => {
table.increments('id');
table.unique(['release_id', 'original_tag']);
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('releases_tags', (table) => {
table.dropColumn('id');
table.dropUnique(['release_id', 'original_tag']);
});
};
-23
View File
@@ -1,23 +0,0 @@
exports.up = async function(knex) {
await knex.schema.alterTable('actors', (table) => {
table.decimal('foot')
.alter();
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.decimal('foot')
.alter();
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('actors', (table) => {
table.integer('foot')
.alter();
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.integer('foot')
.alter();
});
};
@@ -1,13 +0,0 @@
exports.up = async function(knex) {
await knex.schema.alterTable('series', (table) => {
table.specificType('alt_descriptions', 'text ARRAY');
table.json('attributes');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('series', (table) => {
table.dropColumn('alt_descriptions');
table.dropColumn('attributes');
});
};
@@ -1,21 +0,0 @@
exports.up = async function(knex) {
await knex.schema.alterTable('releases_tags', (table) => {
table.integer('actor_id')
.references('id')
.inTable('actors');
table.dropUnique(['tag_id', 'release_id']);
});
await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
};
exports.down = async function(knex) {
await knex.schema.alterTable('releases_tags', (table) => {
table.dropColumn('actor_id');
table.unique(['tag_id', 'release_id']);
});
await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
};
@@ -1,27 +0,0 @@
exports.up = async function(knex) {
await knex.schema.createTable('languages', (table) => {
table.string('alpha2')
.primary();
table.text('name');
table.text('name_native');
});
await knex.schema.alterTable('releases', (table) => {
table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
.defaultTo('day');
table.string('language_alpha2')
.references('alpha2')
.inTable('languages');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('releases', (table) => {
table.dropColumn('production_date_precision');
table.dropColumn('language_alpha2');
});
await knex.schema.dropTable('languages');
};
-100
View File
@@ -1,100 +0,0 @@
exports.up = async function(knex) {
await knex.schema.createTable('feeds', (table) => {
table.increments('id');
table.integer('user_id')
.notNullable()
.references('id')
.inTable('users')
.onDelete('cascade');
table.string('name')
.notNullable();
table.string('slug')
.notNullable();
table.boolean('public');
table.boolean('primary');
table.text('comment');
table.json('meta');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('feeds_entities', (table) => {
table.increments('id');
table.integer('feed_id')
.notNullable()
.references('id')
.inTable('feeds')
.onDelete('cascade');
table.integer('entity_id')
.notNullable()
.references('id')
.inTable('entities')
.onDelete('cascade');
table.text('comment');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('feeds_actors', (table) => {
table.increments('id');
table.integer('feed_id')
.notNullable()
.references('id')
.inTable('feeds')
.onDelete('cascade');
table.integer('actor_id')
.notNullable()
.references('id')
.inTable('actors')
.onDelete('cascade');
table.text('comment');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('feeds_tags', (table) => {
table.increments('id');
table.integer('feed_id')
.notNullable()
.references('id')
.inTable('feeds')
.onDelete('cascade');
table.integer('tag_id')
.notNullable()
.references('id')
.inTable('tags')
.onDelete('cascade');
table.text('comment');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
};
exports.down = async function(knex) {
await knex.schema.dropTable('feeds_tags');
await knex.schema.dropTable('feeds_actors');
await knex.schema.dropTable('feeds_entities');
await knex.schema.dropTable('feeds');
};
@@ -1,59 +0,0 @@
exports.up = async function(knex) {
await knex.raw(`
DROP INDEX unique_actor_slugs;
CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug, entry_id) WHERE entity_id IS NULL;
`);
await knex.schema.alterTable('actors', (table) => {
table.boolean('allow_global_match');
table.text('comment');
});
await knex('users_roles')
.update('abilities', JSON.stringify([
{ subject: 'scene', action: 'create' },
{ subject: 'scene', action: 'update' },
{ subject: 'scene', action: 'delete' },
{ subject: 'actor', action: 'create' },
{ subject: 'actor', action: 'update' },
{ subject: 'actor', action: 'delete' },
{ subject: 'actor', action: 'merge' },
{ plainUrls: true },
]))
.where('role', 'admin');
};
exports.down = async function(knex) {
const dupes = await knex('actors')
.select('name', 'slug', knex.raw('count(*) as count'))
.whereNull('entity_id')
.groupBy('name', 'slug')
.havingRaw('count(*) > 1')
.orderBy('count', 'desc');
if (dupes.length > 0) {
console.log('DUPES\n', dupes.map((actor) => `${actor.name} ${actor.slug} ${actor.count}`).join('\n'));
}
await knex.raw(`
DROP INDEX unique_actor_slugs;
CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL;
`);
await knex.schema.alterTable('actors', (table) => {
table.dropColumn('allow_global_match');
table.dropColumn('comment');
});
await knex('users_roles')
.update('abilities', JSON.stringify([
{ subject: 'scene', action: 'create' },
{ subject: 'scene', action: 'update' },
{ subject: 'scene', action: 'delete' },
{ subject: 'actor', action: 'create' },
{ subject: 'actor', action: 'update' },
{ subject: 'actor', action: 'delete' },
{ plainUrls: true },
]))
.where('role', 'admin');
};
@@ -1,87 +0,0 @@
exports.up = async function(knex) {
await knex.schema.createTable('sync', (table) => {
table.increments('id');
table.string('domain');
table.specificType('item_ids', 'integer array');
table.text('comment');
table.datetime('created_at')
.defaultTo(knex.fn.now());
});
await knex('users_roles')
.update('abilities', JSON.stringify([
{ subject: 'scene', action: 'create' },
{ subject: 'scene', action: 'update' },
{ subject: 'scene', action: 'delete' },
{ subject: 'actor', action: 'create' },
{ subject: 'actor', action: 'update' },
{ subject: 'actor', action: 'delete' },
{ subject: 'actor', action: 'merge' },
{ subject: 'sync' },
{ subject: 'plainUrls' },
]))
.where('role', 'admin');
await knex.raw(`
DROP TABLE IF EXISTS releases_search CASCADE;
DROP TABLE IF EXISTS movies_search CASCADE;
DROP TABLE IF EXISTS series_search CASCADE;
DROP TABLE IF EXISTS releases_search_results CASCADE;
DROP TABLE IF EXISTS movies_search_results CASCADE;
`);
};
exports.down = async function(knex) {
await knex.schema.dropTable('sync');
await knex('users_roles')
.update('abilities', JSON.stringify([
{ subject: 'scene', action: 'create' },
{ subject: 'scene', action: 'update' },
{ subject: 'scene', action: 'delete' },
{ subject: 'actor', action: 'create' },
{ subject: 'actor', action: 'update' },
{ subject: 'actor', action: 'delete' },
{ subject: 'actor', action: 'merge' },
{ plainUrls: true },
]))
.where('role', 'admin');
await knex.schema.createTable('releases_search', (table) => {
table.integer('release_id', 16)
.references('id')
.inTable('releases')
.onDelete('cascade');
});
await knex.schema.createTable('movies_search', (table) => {
table.integer('movie_id', 16)
.references('id')
.inTable('movies')
.onDelete('cascade');
});
await knex.schema.createTable('series_search', (table) => {
table.integer('serie_id', 16)
.references('id')
.inTable('series')
.onDelete('cascade');
});
await knex.raw(`
ALTER TABLE releases_search ADD COLUMN document tsvector;
ALTER TABLE movies_search ADD COLUMN document tsvector;
ALTER TABLE series_search ADD COLUMN document tsvector;
CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
CREATE INDEX releases_search_index ON releases_search USING GIN (document);
CREATE INDEX movies_search_index ON movies_search USING GIN (document);
CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
CREATE INDEX series_search_index ON series_search USING GIN (document);
`);
};
-35
View File
@@ -1,35 +0,0 @@
exports.up = async function(knex) {
await knex.schema.createTable('media_biometrics', (table) => {
table.increments('id');
table.string('media_id', 21)
.references('id')
.inTable('media')
.notNullable()
.onDelete('cascade');
table.integer('width');
table.integer('height');
table.integer('face_index')
.notNullable()
.defaultTo(0);
table.json('biometrics');
table.specificType('embedding', 'vector(1024)');
table.datetime('updated_at')
.notNullable()
.defaultTo(knex.fn.now());
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
table.unique(['media_id', 'face_index']);
});
};
exports.down = async function(knex) {
await knex.schema.dropTable('media_biometrics');
};
-48
View File
@@ -1,48 +0,0 @@
exports.up = async function(knex) {
await knex.schema.alterTable('notifications', (table) => {
table.datetime('seen_at');
});
await knex.schema.alterTable('alerts', (table) => {
table.boolean('is_expired')
.notNullable()
.defaultTo(false);
});
await knex.schema.createMaterializedView('alerts_users_scenes', (view) => {
view.columns('user_id', 'scene_id', 'alert_ids');
view.as(
knex('alerts_scenes')
.select(
'alerts.user_id',
'alerts_scenes.scene_id',
knex.raw('array_agg(distinct alerts.id) as alert_ids'),
knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_scenes.id is null) as is_only'),
)
.leftJoin('alerts', 'alerts.id', 'alerts_scenes.alert_id')
.leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_scenes.alert_id')
.leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_scenes.alert_id')
.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_scenes.alert_id')
.leftJoin('alerts_scenes as related_scenes', (joinBuilder) => {
joinBuilder
.on('related_scenes.alert_id', 'alerts_scenes.alert_id')
.on('related_scenes.scene_id', '!=', 'alerts_scenes.scene_id');
})
.where('alerts.is_expired', false)
.groupBy(['user_id', 'alerts_scenes.scene_id', 'is_only']),
);
});
};
exports.down = async function(knex) {
await knex.schema.dropMaterializedView('alerts_users_scenes');
await knex.schema.alterTable('alerts', (table) => {
table.dropColumn('is_expired');
});
await knex.schema.alterTable('notifications', (table) => {
table.dropColumn('seen_at');
});
};
@@ -1,29 +0,0 @@
exports.up = async function(knex) {
await knex.schema.alterTable('users', (table) => {
table.jsonb('settings');
});
await knex.schema.alterTable('notifications', (table) => {
table.boolean('notify')
.notNullable()
.defaultTo(true);
table.boolean('email')
.notNullable()
.defaultTo(false);
table.datetime('emailed_at');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('users', (table) => {
table.dropColumn('settings');
});
await knex.schema.alterTable('notifications', (table) => {
table.dropColumn('notify');
table.dropColumn('email');
table.dropColumn('emailed_at');
});
};
@@ -0,0 +1,15 @@
exports.up = async function(knex) {
await knex.schema.alterTable('stashes', (table) => {
table.boolean('archived')
.notNullable()
.defaultTo(false);
table.check('NOT ("primary" AND archived)', [], 'stashes_not_primary_and_archived');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('stashes', (table) => {
table.dropColumn('archived');
});
};
@@ -0,0 +1,33 @@
exports.up = async function(knex) {
await knex.schema.alterTable('actors', (table) => {
table.string('date_of_birth_precision')
.notNullable()
.defaultTo('day');
table.string('date_of_death_precision')
.notNullable()
.defaultTo('day');
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.string('date_of_birth_precision')
.notNullable()
.defaultTo('day');
table.string('date_of_death_precision')
.notNullable()
.defaultTo('day');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('actors', (table) => {
table.dropColumn('date_of_birth_precision');
table.dropColumn('date_of_death_precision');
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.dropColumn('date_of_birth_precision');
table.dropColumn('date_of_death_precision');
});
};
@@ -0,0 +1,24 @@
exports.up = async function(knex) {
// we want to be able to assign unidentified actors
await knex.raw(`
ALTER TABLE releases_actors
ALTER COLUMN actor_id DROP NOT NULL
`);
await knex.schema.alterTable('releases_actors', (table) => {
table.increments('id');
table.string('gender');
});
};
exports.down = async function(knex) {
await knex.raw(`
ALTER TABLE releases_actors
ALTER COLUMN actor_id SET NOT NULL
`);
await knex.schema.alterTable('releases_actors', (table) => {
table.dropColumn('id');
table.dropColumn('gender');
});
};
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "traxxx",
"version": "1.255.2",
"version": "1.256.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "traxxx",
"version": "1.255.2",
"version": "1.256.5",
"license": "ISC",
"dependencies": {
"@aws-sdk/client-s3": "^3.458.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "traxxx",
"version": "1.255.2",
"version": "1.256.5",
"description": "All the latest porn releases in one place",
"main": "src/app.js",
"scripts": {
+92 -2
View File
@@ -2,6 +2,10 @@ const upsert = require('../src/utils/upsert');
const slugify = require('../src/utils/slugify');
const groups = [
{
slug: 'accessories',
name: 'Accessories',
},
{
slug: 'age',
name: 'Age',
@@ -344,10 +348,31 @@ const tags = [
slug: 'brunette',
group: 'hair',
},
{
name: 'body writing',
slug: 'body-writing',
description: 'Grab a marker or a lipstick and use your body as a canvas. Label all your holes and make it clear what they should call you.',
},
{
name: 'boots',
slug: 'boots',
group: 'clothing',
},
{
name: 'pleasers',
slug: 'pleasers',
group: 'clothing',
},
{
name: 'bukkake',
slug: 'bukkake',
description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang).',
description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang). If the mouth filled up directly or the cum is collected and drunk from a cup, it can be referred to as a [gokkun](/tag/gokkun).',
group: 'finish',
},
{
name: 'gokkun',
slug: 'gokkun',
description: 'Drinking cum from a group of three or more men, either ejaculated directly into the mouth or collected and drank from a cup, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang). It is essentially the swallowing equivalent of a [bukkake](/tag/bukkake).',
group: 'finish',
},
{
@@ -406,6 +431,10 @@ const tags = [
name: 'cum',
slug: 'cum',
},
{
name: 'cum covered fucking',
slug: 'ccf',
},
{
name: 'cum drunk',
slug: 'cum-drunk',
@@ -688,7 +717,12 @@ const tags = [
{
name: 'gangbang',
slug: 'gangbang',
description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a reverse gangbang, multiple women fuck one man.', /* eslint-disable-line max-len */
description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a [reverse gangbang](/tag/reverse-gangbang), multiple women fuck one man.', /* eslint-disable-line max-len */
group: 'group',
},
{
name: 'reverse gangbang',
slug: 'reverse-gangbang',
group: 'group',
},
{
@@ -1264,6 +1298,22 @@ const tags = [
slug: 'toy-dp',
description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).',
},
{
name: 'toy DVP',
slug: 'toy-dvp',
},
{
name: 'toy DAP',
slug: 'toy-dap',
},
{
name: 'toy TAP',
slug: 'toy-tap',
},
{
name: 'toy TVP',
slug: 'toy-tvp',
},
{
name: 'finger DP',
slug: 'finger-dp',
@@ -1272,6 +1322,16 @@ const tags = [
name: 'transsexual',
slug: 'transsexual',
},
{
name: 'trans man',
slug: 'trans-man',
description: 'A man who was assigned female at birth and has typically undergone a social transition, and may but not necessarily have had gender-affirming surgeries.',
},
{
name: 'trans woman',
slug: 'trans-woman',
description: 'woman man who was assigned male at birth and has typically undergone a social transition, and may but not necessarily have had gender-affirming surgeries.',
},
{
name: 'DA triple penetration',
slug: 'da-tp',
@@ -1465,6 +1525,16 @@ const tags = [
name: 'pregnant',
slug: 'pregnant',
},
{
name: 'collar',
slug: 'collar',
group: 'accessories',
},
{
name: 'choker',
slug: 'choker',
group: 'accessories',
},
];
const aliases = [
@@ -1715,6 +1785,10 @@ const aliases = [
name: 'busty - big boobs',
for: 'big-boobs',
},
{
name: 'big breasts',
for: 'big-boobs',
},
{
name: 'busty: big beautiful breast',
for: 'big-boobs',
@@ -3139,6 +3213,22 @@ const aliases = [
name: 'fist',
for: 'fisting',
},
{
name: 'boot',
for: 'boots',
},
{
name: 'pleaser boots',
for: 'pleasers',
},
{
name: 'pleaser shoes',
for: 'pleasers',
},
{
name: 'pleaser heels',
for: 'pleasers',
},
];
const priorities = [ // higher index is higher priority
+12 -1
View File
@@ -172,7 +172,7 @@ const networks = [
{
slug: 'familystrokes',
name: 'Family Strokes',
url: 'https://www.familystrokes.com/',
url: 'https://www.familystrokes.com',
parent: 'paperstreetmedia',
parameters: {
endpoint: 'familybundle',
@@ -284,6 +284,7 @@ const networks = [
areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
blockId: 114458,
scene: 'https://aziani.com',
api: 'https://azianistudios.com/tour_api.php',
cdn: 'https://c75c0c3063.mjedge.net',
},
},
@@ -593,6 +594,11 @@ const networks = [
concurrency: 1,
},
},
{
slug: 'ksm',
name: 'KSM',
hasLogo: false,
},
{
slug: 'letsdoeit',
name: 'LetsDoeIt',
@@ -777,6 +783,11 @@ const networks = [
parentSession: false,
},
},
{
slug: 'rsghmedia',
name: 'RSGH Media',
hasLogo: false,
},
{
slug: 'score',
name: 'SCORE',
+178 -12
View File
@@ -6127,7 +6127,7 @@ const sites = [
{
slug: 'blackforwife',
name: 'Black for Wife',
url: 'https://www.blackforwife.com',
url: 'https://www.blackforwife.org',
parent: 'jayrock',
parameters: {
useGamma: true,
@@ -6141,6 +6141,12 @@ const sites = [
name: 'Jax Slayher',
url: 'https://jaxslayher.com',
},
// JENNIFER WHITE
{
slug: 'jenniferwhite',
name: 'Jennifer White',
url: 'https://www.jenniferwhite.com',
},
// JESSE LOADS MONSTER FACIALS
{
slug: 'jesseloadsmonsterfacials',
@@ -7379,6 +7385,16 @@ const sites = [
studioId: 1214,
},
},
{
name: 'Nick Morris Studio',
slug: 'nickmorrisstudio',
url: 'https://www.analvids.com/studios/nick_morris_studio',
parent: 'analvids',
hasLogo: false,
parameters: {
studioId: 1038,
},
},
// LETS DOE IT
{
name: 'The White Boxxx',
@@ -8501,6 +8517,33 @@ const sites = [
id: 'hookup-pad',
},
},
{
slug: 'thepickupvan',
name: 'The Pickup Van',
url: 'https://www.mylf.com/series/the-pickup-van',
parent: 'mylf',
parameters: {
id: 'the-pickup-van',
},
},
{
slug: 'motherfucker',
name: 'Motherfucker',
url: 'https://www.mylf.com/series/motherfucker',
parent: 'mylf',
parameters: {
id: 'motherfucker',
},
},
{
slug: 'bluecollarbabes',
name: 'Blue Collar Babes',
url: 'https://www.mylf.com/series/blue-collar-babes',
parent: 'mylf',
parameters: {
id: 'blue-collar-babes',
},
},
/* premium */
{
slug: 'analmom',
@@ -8535,12 +8578,9 @@ const sites = [
{
slug: 'mylfdom',
name: 'MYLF Dom',
url: 'https://www.mylfdom.com',
url: 'https://www.mylf.com/series/mylfdom-ts',
tags: ['bdsm', 'milf'],
parent: 'mylf',
parameters: {
endpoint: 'mylf_dom',
},
},
// NAUGHTY AMERICA
{
@@ -11098,6 +11138,59 @@ const sites = [
slug: 'rickysroom',
url: 'https://rickysroom.com',
},
// RSGH MEDIA
{
name: 'Family Therapy XXX',
slug: 'familytherapyxxx',
url: 'https://familytherapyxxx.com',
tags: ['family'],
parent: 'rsghmedia',
independent: true,
},
{
name: 'Mom Comes First',
slug: 'momcomesfirst',
url: 'https://momcomesfirst.com',
tags: ['family'],
parent: 'rsghmedia',
independent: true,
},
{
name: 'Perfect Girlfriend',
slug: 'perfectgirlfriend',
url: 'https://perfectgirlfriend.com',
parent: 'rsghmedia',
independent: true,
},
{
name: 'Anal Therapy XXX',
slug: 'analtherapyxxx',
url: 'https://analtherapyxxx.com',
tags: ['anal'],
parent: 'rsghmedia',
independent: true,
},
{
name: 'Cock Ninja Studios',
slug: 'cockninjastudios',
url: 'https://cockninjastudios.com',
parent: 'rsghmedia',
independent: true,
},
{
name: 'Wife Loves Black',
slug: 'wifelovesblack',
url: 'https://wifelovesblack.com',
parent: 'rsghmedia',
independent: true,
},
{
name: 'Teen Loves Black',
slug: 'teenlovesblack',
url: 'https://teenlovesblack.com',
parent: 'rsghmedia',
independent: true,
},
// SCORE
{
name: '18 Eighteen',
@@ -12328,13 +12421,6 @@ const sites = [
url: 'https://www.teamskeet.com/series/bad-milfs',
parent: 'teamskeet',
},
{
slug: 'blackstepdad',
name: 'Black StepDad',
url: 'https://www.teamskeet.com/series/black-stepdad',
alias: ['bsd'],
parent: 'teamskeet',
},
{
slug: 'exxxtrasmall',
name: 'Exxxtra Small',
@@ -12714,6 +12800,44 @@ const sites = [
url: 'https://www.teamskeet.com/series/passport-bros',
parent: 'teamskeet',
},
{
slug: 'underthetable',
name: 'Under the Table',
url: 'https://www.teamskeet.com/series/under-the-table',
parent: 'teamskeet',
},
{
slug: 'mydirtyuncle',
name: 'My Dirty Uncle',
url: 'https://www.teamskeet.com/series/my-dirty-uncle',
tags: ['family'],
parent: 'teamskeet',
},
{
slug: 'mormongirlz',
name: 'Mormon Girlz',
url: 'https://www.teamskeet.com/series/mormon-girlz',
tags: ['religion'],
parent: 'teamskeet',
},
{
slug: 'breedingmaterial',
name: 'Breeding Material',
url: 'https://www.teamskeet.com/series/breeding-material',
parent: 'teamskeet',
},
{
slug: 'teamskeetsingles',
name: 'TeamSkeet Singles',
url: 'https://www.teamskeet.com/series/teamskeet-singles',
parent: 'teamskeet',
},
{
slug: 'sheseducedme',
name: 'She Seduced Me',
url: 'https://www.teamskeet.com/series/she-seduced-me',
parent: 'teamskeet',
},
// PERVZ
{
slug: 'pervzfeatures',
@@ -12790,6 +12914,12 @@ const sites = [
url: 'https://www.pervz.com/series/pervdriver',
parent: 'pervz',
},
{
slug: 'charmed',
name: 'Charmed',
url: 'https://www.pervz.com/series/charmed',
parent: 'pervz',
},
// SWAPPZ
{
slug: 'swappzfeatures',
@@ -12856,6 +12986,13 @@ const sites = [
parent: 'freeuse',
tags: ['free-use', 'pov'],
},
{
slug: 'freeusesingles',
name: 'Freeuse Singles',
url: 'https://www.freeuse.com/series/freeuse-singles',
parent: 'freeuse',
tags: ['free-use'],
},
// FAMILY STROKES
{
slug: 'familystrokes',
@@ -12937,6 +13074,14 @@ const sites = [
tags: ['family', 'milf'],
parent: 'familystrokes',
},
{
slug: 'blackstepdad',
name: 'Black StepDad',
url: 'https://www.familystrokes.com/series/black-stepdad',
alias: ['bsd'],
tags: ['family'],
parent: 'familystrokes',
},
// SAY UNCLE
{
slug: 'blackgodz',
@@ -13529,6 +13674,27 @@ const sites = [
},
},
},
// SKM
{
slug: 'cosplayground',
name: 'Cosplayground',
url: 'https://cosplayground.com',
tags: ['cosplay'],
parent: 'ksm',
independent: true,
parameters: {
// seems to use same CMS as Aziani
api: 'https://api.cosplayground.com/tour_api.php',
seriesId: null,
areaId: '2bc2444e-a053-40a2-bc4b-ca3618521563',
cdn: 'https://c880f36aa8.mjedge.net',
entryIdSlug: true,
videos: '/videos',
scenePath: '/videos',
modelPath: '/models',
imageType: 'photos',
},
},
// TEEN CORE CLUB
{
name: 'Analyzed Girls',
+1
View File
@@ -706,6 +706,7 @@ const tagMedia = [
['blowjob', 'azul_hermosa_realitykings', 'Azul Hermosa and Scott Nails in "Diva For A Day"', 'brazzers'],
['blowjob', 3, 'Rose Valie', 'handsonhardcore'],
['blowjob', 2, 'Luna Kitsuen in "Gag Reflex"', 'evilangel'],
['body-writing', 'emma_rosie_julesjordan', 'Emma Rosie', 'julesjordan'],
['bondage', 0, 'Veronica Leal', 'herlimit'],
['brunette', 0, 'Darcie Dolce', 'playboy'],
['bts', 'charlie_red_private', 'Charlie Red', 'private'],
+9 -2
View File
@@ -611,8 +611,15 @@ async function upsertProfiles(profiles) {
media_id: profile.avatarMediaId,
}));
if (avatars.length > 0) {
await batchInsert('actors_avatars', avatars, { conflict: false });
const resolvedAvatars = avatars.filter((avatar) => !!avatar.profile_id);
const unresolvedAvatars = avatars.filter((avatar) => !avatar.profile_id);
unresolvedAvatars.forEach((avatar) => {
logger.warn(`Skipping avatar for actor ${avatar.actor_id} (${avatar.media_id}), no profile could be resolved`);
});
if (resolvedAvatars.length > 0) {
await batchInsert('actors_avatars', resolvedAvatars, { conflict: false });
}
}
}
+32 -40
View File
@@ -9,7 +9,6 @@ const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const { fetchReleaseEntities, urlToHostname } = require('./entities');
const logger = require('./logger')(__filename);
const qu = require('./utils/qu');
const getRecursiveParameters = require('./utils/get-recursive-parameters');
const windows = require('./utils/http-windows');
@@ -85,35 +84,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
}
if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
/*
if (scraper.useUnprint || (type === 'scene' && scraper.scrapeScene?.unprint) || (type === 'movie' && scraper.scrapeMovie?.unprint)) {
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
}
*/
if (!scraper.deprecated) {
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
}
const session = qu.session();
const res = await qu.get(url, null, null, {
session,
rejectUnauthorized: false,
});
const cookie = await session._sessionOptions.cookieJar.get(url);
if (res.ok) {
return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.item, url, entity, baseRelease, options, {
session,
headers: res.headers,
cookieJar: session._sessionOptions.cookieJar,
cookie,
});
}
return res.status;
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
}
return null;
@@ -123,15 +94,21 @@ function fetchMovie(scraper, url, entity, baseRelease, options) {
return fetchScene(scraper, url, entity, baseRelease, options, 'movie');
}
async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
const entity = baseRelease.entity || entitiesByHostname[urlToHostname(baseRelease.url)];
function shouldDeepScrape(baseRelease) {
return argv.deep && !!(baseRelease.url || baseRelease.path || baseRelease.forceDeep);
}
async function scrapeRelease(baseRelease, entities, type = 'scene') {
const entity = entities.byId[baseRelease.entity?.id]
|| entities.byHostname[urlToHostname(baseRelease.url)]
|| baseRelease.entity;
if (!entity) {
logger.warn(`No entity available for ${baseRelease.url}`);
return baseRelease;
}
if ((!baseRelease.url && !baseRelease.path && !baseRelease.forceDeep) || !argv.deep) {
if (!shouldDeepScrape(baseRelease)) {
return {
...baseRelease,
entity,
@@ -247,22 +224,37 @@ async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
}
async function scrapeReleases(baseReleases, entitiesByHostname, type) {
const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesByHostname).map(async ([slug, entity]) => {
if (entity.scraper?.beforeFetchScenes) {
// entities attached to base releases (e.g. from updates) are not included in entitiesByHostname
const entitiesById = Object.fromEntries([
...Object.values(entitiesByHostname),
...baseReleases.map((baseRelease) => baseRelease.entity),
].filter(Boolean).map((entity) => [entity.id, entity]));
// only run before hook for entities that will actually be deep scraped
const deepEntityIds = new Set(baseReleases
.filter((baseRelease) => shouldDeepScrape(baseRelease))
.map((baseRelease) => baseRelease.entity?.id || entitiesByHostname[urlToHostname(baseRelease.url)]?.id)
.filter(Boolean));
const entitiesWithBeforeDataById = Object.fromEntries(await Promise.all(Object.values(entitiesById).map(async (entity) => {
if (deepEntityIds.has(entity.id) && entity.scraper?.beforeFetchScenes) {
const parameters = getRecursiveParameters(entity);
const preData = await entity.scraper.beforeFetchScenes(entity, parameters);
return [slug, { ...entity, preData }];
return [entity.id, { ...entity, preData }];
}
return [slug, entity];
}));
return [entity.id, entity];
})));
const entitiesWithBeforeDataBySlug = Object.fromEntries(entitiesWithBeforeDataEntries.filter(Boolean));
const entitiesWithBeforeData = {
byId: entitiesWithBeforeDataById,
byHostname: Object.fromEntries(Object.entries(entitiesByHostname).map(([hostname, entity]) => [hostname, entitiesWithBeforeDataById[entity.id]])),
};
return Promise.map(
baseReleases,
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeData, type),
{ concurrency: 1 },
);
}
+3 -3
View File
@@ -15,6 +15,7 @@ const sharp = require('sharp');
const blake2 = require('blake2');
const { format } = require('date-fns');
const taskQueue = require('promise-task-queue');
const unprint = require('unprint');
const { Upload } = require('@aws-sdk/lib-storage');
const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
@@ -25,7 +26,6 @@ const knex = require('./knex');
const http = require('./utils/http');
const batchInsert = require('./utils/batch-insert');
const chunk = require('./utils/chunk');
const { get } = require('./utils/qu');
const { fetchEntityReleaseIds } = require('./entity-releases');
const streamQueue = taskQueue();
@@ -383,10 +383,10 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
};
}
const res = await get(baseSource.url);
const res = await unprint.get(baseSource.url);
if (res.ok) {
const src = await baseSource.extract(res.item);
const src = await baseSource.extract(res.context);
return {
...baseSource,
+1 -3
View File
@@ -29,7 +29,6 @@ const karups = require('./karups');
const kellymadison = require('./kellymadison');
const kink = require('./kink');
const mariskax = require('./mariskax');
// const analvids = require('./analvids');
const pornbox = require('./pornbox');
const littlecapricedreams = require('./littlecapricedreams');
const loveherfilms = require('./loveherfilms');
@@ -59,7 +58,6 @@ const teenmegaworld = require('./teenmegaworld');
const testedefudelidade = require('./testedefudelidade');
const theflourish = require('./theflourish');
const tokyohot = require('./tokyohot');
// const topwebmodels = require('./topwebmodels');
const virtualtaboo = require('./virtualtaboo');
const vixen = require('./vixen');
const wankzvr = require('./wankzvr');
@@ -229,7 +227,6 @@ module.exports = {
boyfun: karups,
kellymadison,
'8kmembers': kellymadison,
// analvids,
analvids: pornbox,
littlecapricedreams,
mariskax,
@@ -279,6 +276,7 @@ module.exports = {
// etc
'18vr': badoink,
theflourishxxx: theflourish,
cosplayground: aziani,
pierrewoodman,
exploitedx, // only from known URL that will specify site
fullpornnetwork,
+13 -1
View File
@@ -62,7 +62,7 @@ async function scrapeRelease({ query, html, element }, { url, entity, baseReleas
};
});
release.tags = query.contents('.tags a, .categories a');
release.tags = query.contents('.tags a, .categories a, .tags + div a[data-label="Attribute"]');
release.studio = parameters?.studio === false ? null : slugify(query.content('.studio span:last-child, .studio a'), '');
if (type === 'scene') {
@@ -103,6 +103,18 @@ async function scrapeRelease({ query, html, element }, { url, entity, baseReleas
release.qualities = [2160];
}
const serieTitle = query.content('.series a');
const serieUrl = query.url('.series a');
const serieId = serieUrl && new URL(serieUrl).searchParams.get('series');
if (serieTitle && serieId) {
release.serie = {
entryId: serieId,
title: serieTitle,
url: serieUrl,
};
}
return release;
}
+14 -14
View File
@@ -1,45 +1,45 @@
'use strict';
const { fetchLatest, fetchProfile } = require('./julesjordan');
const unprint = require('unprint');
const qu = require('../utils/q');
const { fetchLatest, fetchProfile } = require('./julesjordan');
const slugify = require('../utils/slugify');
function scrapeScene({ query }, channel) {
const release = {};
const title = query.cnt('.update_title', false);
const title = query.content('.update_title', { trim: false });
release.title = title.trim();
release.description = query.cnt('.latest_update_description');
release.description = query.content('.latest_update_description');
release.date = query.date('.update_date', 'MM/DD/YYYY');
release.actors = query.all('.tour_update_models a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null),
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null),
}));
release.entryId = slugify(release.title);
release.duration = Number(query.cnt('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
release.duration = Number(query.content('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
const poster = query.img('.large_update_thumb', 'src', { origin: channel.url });
const trailer = query.q('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
const poster = query.img('.large_update_thumb', { attribute: 'src', origin: channel.url });
const trailer = query.attribute('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
release.poster = [
poster.replace(/.jpg$/, '-full.jpg'),
poster,
];
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
release.photos = query.imgs('.small_update_thumb', { attribute: 'src', origin: channel.url }).map((img) => [
img.replace(/.jpg$/, '-full.jpg'),
img,
]);
if (trailer) {
release.trailer = qu.prefixUrl(encodeURI(trailer), channel.url);
release.trailer = unprint.prefixUrl(encodeURI(trailer), channel.url);
}
release.tags = query.cnts('.tour_update_tags a');
release.tags = query.contents('.tour_update_tags a');
return release;
}
@@ -49,10 +49,10 @@ async function fetchLatestLocal(channel, page, include, preData) {
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, channel);
return scrapeScene(res.context, channel);
}
return res.status;
-174
View File
@@ -1,174 +0,0 @@
'use strict';
const unprint = require('unprint');
const slugify = require('../utils/slugify');
function extractTitle(originalTitle) {
const titleComponents = originalTitle.split(' ');
// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
return { shootId, title };
}
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('.card-scene__view > a');
release.entryId = query.dataset(null, 'content') || new URL(release.url).pathname.match(/watch\/(\d+)/)?.[1];
release.title = query.content('.card-scene__text');
release.shootId = extractTitle(release.title).shootId;
release.date = query.date('.label--time:nth-child(2)', 'YYYY-MM-DD'); // only available on front-page, not on studio page
release.duration = query.duration('.label--time:first-child');
release.poster = query.img('.card-scene__view img', { attribute: 'data-src' });
const caps = query.json('.card-scene__view > a', { attribute: 'data-casting' })?.map((timestamp) => `${channel.url}/casting/${release.entryId}/${timestamp}`);
if (caps && release.poster) {
release.caps = caps;
} else if (caps) {
[release.poster, ...release.caps] = caps;
}
release.teaser = query.video('.card-scene__view > a', { attribute: 'data-preview' });
return release;
});
}
async function fetchLatest(channel, page) {
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
// studios as channels
const url = `${channel.url}/latest/${page}`;
const res = await unprint.get(url, {
selectAll: '.card-scene',
headers: {
Referer: url,
},
});
if (res.ok) {
return scrapeAll(res.context, channel);
}
return res.status;
}
function scrapeScene({ query }, url) {
const release = {};
release.entryId = new URL(url).pathname.match(/watch\/(\d+)/)?.[1];
const featuring = query.content('.watch__title .watch__featuring_models');
release.title = query.content('.watch__title').replace(featuring, '');
release.description = query.content('.text-mob-more');
release.shootId = extractTitle(release.title).shootId;
release.date = query.date('.bi-calendar3', 'YYYY-MM-DD');
release.duration = query.duration('.bi-clock');
release.actors = query.all('.watch__title a[href*="/model"]').map((el) => ({
name: unprint.query.content(el),
url: unprint.query.url(el, null),
}));
release.tags = query.contents('.genres-list a[href*="/genre"]');
// release.studio = slugify(query.content('.genres-list a[href*="/studios"]'), '', { removePunctuation: true });
release.poster = query.img('.watch__video video', { attribute: 'data-poster' });
release.trailer = query.all('.watch__video source').map((el) => ({
src: unprint.query.video(el, null, { attribute: 'src' }),
quality: unprint.query.number(el, null, { attribute: 'size' }),
}));
return release;
}
async function fetchScene(url) {
const res = await unprint.get(url, {
headers: {
Referer: url,
},
});
if (res.ok) {
return scrapeScene(res.context, url);
}
return res.status;
}
function scrapeProfile({ query }, url, channel) {
const profile = { url };
profile.nationality = query.content('.model__info a[href*="/nationality"]');
profile.age = query.number('//td[contains(text(), "Age")]/following-sibling::td');
profile.avatar = query.img('.model__left img');
profile.scenes = scrapeAll(unprint.initAll(query.all('.card-scene')), channel);
return profile;
}
async function getActorUrl(actor, channel) {
if (actor.url) {
return actor.url;
}
const searchUrl = `${channel.url}/api/autocomplete/search?q=${slugify(actor.name, '+')}`;
const searchRes = await unprint.get(searchUrl, {
headers: {
Referer: actor.url,
},
});
if (!searchRes.ok) {
return searchRes.status;
}
const result = searchRes.body.terms.find((item) => item.type === 'model');
if (result) {
return result.url;
}
return null;
}
async function fetchProfile(actor, { channel }) {
const actorUrl = await getActorUrl(actor, channel);
if (typeof actorUrl !== 'string') {
return actorUrl;
}
const bioRes = await unprint.get(actorUrl, {
headers: {
Referer: actor.url,
},
});
if (bioRes.ok) {
return scrapeProfile(bioRes.context, actorUrl, channel);
}
return bioRes.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
};
-203
View File
@@ -1,203 +0,0 @@
'use strict';
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
function extractTitle(originalTitle) {
const titleComponents = originalTitle.split(' ');
// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
return { shootId, title };
}
function getPoster(posterElement, sceneId) {
const posterStyle = posterElement.attr('style');
if (posterStyle) {
return posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
}
const posterRange = posterElement.attr('data-casting');
const posterRangeData = posterRange ? JSON.parse(posterRange) : null;
const posterTimeRange = posterRangeData[Math.floor(Math.random() * posterRangeData.length)];
if (!posterTimeRange) {
return null;
}
if (typeof posterTimeRange === 'number') {
// poster time is already a single time value
return `https://analvids.com/casting/${sceneId}/${posterTimeRange}`;
}
const [max, min] = posterTimeRange.split('-');
const posterTime = Math.floor(Math.random() * (Number(max) - Number(min) + 1) + Number(min));
return `https://analvids.com/casting/${sceneId}/${posterTime}`;
}
function scrapeAll(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const scenesElements = $('.thumbnails > div').toArray();
return scenesElements.map((element) => {
const sceneLinkElement = $(element).find('.thumbnail-title a');
const url = sceneLinkElement.attr('href');
const originalTitle = sceneLinkElement.text().trim(); // title attribute breaks when they use \\ escaping
const { shootId, title } = extractTitle(originalTitle);
const entryId = new URL(url).pathname.split('/')[2];
const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
const sceneId = $(element).attr('data-content');
const posterElement = $(element).find('.thumbnail-avatar');
const poster = getPoster(posterElement, sceneId);
return {
url,
shootId,
entryId,
title,
date,
poster,
};
});
}
async function scrapeScene(html, url, site, useGallery) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const playerObject = $('script:contains("new WatchPage")').html();
const playerData = playerObject && playerObject.slice(playerObject.indexOf('{"swf":'), playerObject.lastIndexOf('},') + 1);
const data = playerData && JSON.parse(playerData);
const release = { url };
const originalTitle = $('h1.watchpage-title').text().trim();
const { shootId, title } = extractTitle(originalTitle);
release.shootId = shootId;
release.entryId = new URL(url).pathname.split('/')[2];
release.title = title;
release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
release.description = $('meta[name="description"]')?.attr('content')?.trim()
|| (descriptionElement && $(descriptionElement).find('dd').text().trim());
release.actors = $(actorsElement)
.find('a[href*="com/model"]')
.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const photos = useGallery
? $('.gallery a img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray()
: $('.screenshots img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray();
release.photos = photos.map((source) => {
// source without parameters sometimes serves larger preview photo
const { origin, pathname } = new URL(source);
return `${origin}${pathname}`;
});
const posterStyle = $('#player').attr('style');
const poster = posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
release.poster = poster || release.photos.slice(Math.floor(release.photos.length / 3) * -1); // poster unavailable, try last 1/3rd of high res photos as fallback
if (data) {
const qualityMap = {
web: 240,
vga: 480,
hd: 720,
'1080p': 1080,
};
release.trailer = data.clip.qualities.map((trailer) => ({
src: trailer.src,
type: trailer.type,
quality: qualityMap[trailer.quality] || trailer.quality,
}));
}
const studioName = $('.watchpage-studioname').first().text().trim();
release.studio = slugify(studioName, '', { removePunctuation: true });
return release;
}
async function scrapeProfile(html, _url, actorName) {
const { document } = new JSDOM(html).window;
const profile = {
name: actorName,
};
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
const bio = entries
.filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
profile.birthPlace = bio.Nationality;
if (bio.Age) profile.age = bio.Age;
if (avatarEl) profile.avatar = avatarEl.src;
profile.releases = scrapeAll(html);
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await http.get(`${site.url}/new-videos/${page}`);
return scrapeAll(res.body.toString(), site);
}
async function fetchScene(url, site) {
const useGallery = true;
// TODO: fall back on screenshots when gallery is not available
const res = useGallery
? await http.get(`${url}/gallery#gallery`)
: await http.get(`${url}/screenshots#screenshots`);
return scrapeScene(res.body.toString(), url, site, useGallery);
}
async function fetchProfile({ name: actorName }) {
const res = await http.get(`https://www.analvids.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
const data = res.body;
const result = data.terms.find((item) => item.type === 'model');
if (result) {
const bioRes = await http.get(result.url);
const html = bioRes.body.toString();
return scrapeProfile(html, result.url, actorName);
}
return null;
}
module.exports = {
fetchLatest,
fetchProfile,
fetchScene,
};
+30 -27
View File
@@ -1,6 +1,9 @@
'use strict';
const { get, geta, ctxa } = require('../utils/q');
const unprint = require('unprint');
// paths are relative, read raw attributes instead of resolved URLs
const raw = { forceGetAttribute: true };
function extractActors(actorString) {
return actorString
@@ -20,48 +23,48 @@ function matchActors(actorString, models) {
}
function scrapeLatest(scenes, site, models) {
return scenes.map(({ qu }) => {
return scenes.map(({ query }) => {
const release = {};
const pathname = qu.url('a.itemimg').slice(1);
const pathname = query.attribute('a.itemimg', 'href', raw).slice(1);
[release.entryId] = pathname.split('/').slice(-1);
release.url = `${site.url}${pathname}`;
release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
release.title = query.attribute('.itemimg img', 'alt') || query.content('h4 a');
release.description = query.content('.mas_longdescription');
release.date = query.date('.movie_info2', 'MM/DD/YY', { match: /\d{2}\/\d{2}\/\d{2}/ });
const actorString = qu.q('.mas_description', true);
const actorString = query.content('.mas_description');
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
const posterPath = qu.img('.itemimg img');
const posterPath = query.attribute('.itemimg img', 'src', raw);
release.poster = `${site.url}/${posterPath}`;
return release;
});
}
function scrapeScene({ html, qu }, url, site, include, models) {
function scrapeScene({ html, query }, url, site, include, models) {
const release = { url };
[release.entryId] = url.split('/').slice(-1);
release.title = qu.q('.mas_title', true);
release.description = qu.q('.mas_longdescription', true);
release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.title = query.content('.mas_title');
release.description = query.content('.mas_longdescription');
release.date = query.date('.mas_description', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
if (models) {
const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
const actorString = query.content('.mas_description').replace(/\w+ \d{1,2}, \d{4}/, '');
const actors = matchActors(actorString, models);
if (actors.length > 0) release.actors = actors;
else release.actors = extractActors(actorString);
}
release.tags = qu.all('.tags a', true);
release.tags = query.contents('.tags a');
release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
release.photos = query.attributes('.stills img', 'src', raw).map((photoPath) => `${site.url}/${photoPath}`);
const posterIndex = 'splash:';
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
@@ -74,20 +77,20 @@ function scrapeScene({ html, qu }, url, site, include, models) {
return release;
}
function extractModels({ el }, site) {
const models = ctxa(el, '.item');
function extractModels({ element }, site) {
const models = unprint.initAll(element, '.item');
return models.map(({ qu }) => {
return models.map(({ query }) => {
const actor = { gender: 'female' };
const avatar = qu.q('.itemimg img');
const avatar = query.element('.itemimg img');
actor.avatar = `${site.url}/${avatar.src}`;
actor.name = avatar.alt
.split(':').slice(-1)[0]
.replace(/xtreme girl|nurse/ig, '')
.trim();
const actorPath = qu.url('.itemimg');
const actorPath = query.attribute('.itemimg', 'href', raw);
actor.url = `${site.url}${actorPath.slice(1)}`;
return actor;
@@ -96,11 +99,11 @@ function extractModels({ el }, site) {
async function fetchModels(site, page = 1, accModels = []) {
const url = `${site.url}/?models/${page}`;
const res = await get(url);
const res = await unprint.get(url);
if (res.ok) {
const models = extractModels(res.item, site);
const nextPage = res.item.qa('.pagenumbers', true)
const models = extractModels(res.context, site);
const nextPage = res.context.query.contents('.pagenumbers')
.map((pageX) => Number(pageX))
.filter(Boolean) // remove << and >>
.includes(page + 1);
@@ -117,16 +120,16 @@ async function fetchModels(site, page = 1, accModels = []) {
async function fetchLatest(site, page = 1, include, { beforeFetchLatest }) {
const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
const res = await geta(url, '.item');
const res = await unprint.get(url, { selectAll: '.item' });
return res.ok ? scrapeLatest(res.items, site, beforeFetchLatest) : res.status;
return res.ok ? scrapeLatest(res.context, site, beforeFetchLatest) : res.status;
}
async function fetchScene(url, site, release, include, beforeFetchLatest) {
const models = beforeFetchLatest || await fetchModels(site);
const res = await get(url);
const res = await unprint.get(url);
return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
return res.ok ? scrapeScene(res.context, url, site, include, models) : res.status;
}
module.exports = {
+38 -21
View File
@@ -6,11 +6,13 @@ const { decode } = require('html-entities');
const slugify = require('../utils/slugify');
const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
const tagKeys = ['tags', 'category', 'categories'];
function scrapeScene(data, channel, parameters) {
const release = {};
release.entryId = data.cms_set_id;
release.url = `${parameters.scene || channel.url}/video/${data.cms_set_id}`;
release.url = `${parameters.scene || channel.url}${parameters.scenePath || '/video'}/${parameters.entryIdSlug ? data.slug : data.cms_set_id}`;
release.title = data.name;
release.description = data.description;
@@ -20,7 +22,7 @@ function scrapeScene(data, channel, parameters) {
release.actors = data.data_types.find((dataType) => dataType.data_type === 'Models' || dataType.data_type === 'Talent')?.data_values.map((actor) => ({
name: actor.name,
url: `${channel.url}/model/${actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
url: `${channel.url}${parameters.modelPath || '/model'}/${parameters.entryIdSlug && actor.slug ? actor.slug : actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
}));
release.directors = data.data_types.find((dataType) => dataType.data_type === 'Videographers')?.data_values.map((director) => ({
@@ -29,7 +31,7 @@ function scrapeScene(data, channel, parameters) {
}));
release.tags = data.data_types
.filter((dataType) => dataType.data_type === 'Tags' || dataType.data_type === 'Category')
.filter((dataType) => tagKeys.includes(dataType.data_type?.toLowerCase()))
.flatMap((tags) => tags.data_values.map((tag) => tag.name));
const poster = data.preview_formatted.thumb;
@@ -43,7 +45,7 @@ function scrapeScene(data, channel, parameters) {
.map((key) => unprint.prefixUrl(`${poster[key][0].fileuri}?${poster[key][0].signature}`, parameters.cdn));
}
release.caps = data.content
release[parameters.imageType || 'caps'] = data.content
?.filter((item) => item.content_type === 'image' && item.content?.[0])
.map((item) => [
item.content[0],
@@ -66,13 +68,14 @@ function scrapeScene(data, channel, parameters) {
release.trailer = unprint.prefixUrl(`${trailer.fileuri}?${trailer.signature}`, parameters.cdn);
}
release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '');
release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '') || null;
return release;
}
async function getBlockId(slug, dataSource, entity, parameters) {
const res = await unprint.get(`https://azianistudios.com/tour_api.php/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
const res = await unprint.get(`${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
headers: {
Referer: entity.url,
'x-nats-cms-area-id': parameters.areaId,
@@ -109,9 +112,10 @@ async function fetchLatest(channel, page = 1, { parameters }) {
data_type_search: parameters.seriesId && JSON.stringify({ 7: parameters.seriesId.toString() }), // doesn't seem relevant
}).toString();
const url = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
const url = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/sets?${query}`;
const res = await unprint.get(url, {
interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
headers: {
Referer: channel.url,
'x-nats-cms-area-id': parameters.areaId,
@@ -126,20 +130,21 @@ async function fetchLatest(channel, page = 1, { parameters }) {
}
async function fetchScene(url, entity, _baseRelease, { parameters }) {
const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
const entryId = new URL(url).pathname.match(/\/videos?\/([\w-]+)/)[1];
if (!entryId) {
return null;
}
const blockId = await getBlockId('/video/:id', entryId, entity, parameters);
const blockId = await getBlockId(`${parameters.videos || '/video'}/${parameters.entryIdSlug ? ':slug' : ':id'}`, entryId, entity, parameters);
if (!blockId) {
return null;
}
const query = new URLSearchParams({
cms_set_ids: entryId,
cms_set_ids: parameters.entryIdSlug ? '' : entryId,
...parameters.entryIdSlug ? { slug: entryId } : null,
cms_area_id: parameters.areaId,
cms_block_id: blockId,
content: 1,
@@ -150,9 +155,10 @@ async function fetchScene(url, entity, _baseRelease, { parameters }) {
content_count: 1,
}).toString();
const apiUrl = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
const apiUrl = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/sets?${query}`;
const res = await unprint.get(apiUrl, {
interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
headers: {
Referer: entity.url,
'x-nats-cms-area-id': parameters.areaId,
@@ -174,14 +180,18 @@ function scrapeProfile(data, entity, parameters) {
decode(detail.value || detail.content_formatted || detail.content),
]));
profile.url = `${entity.url}/model/${data.cms_data_value_id}`;
profile.url = `${entity.url}/model/${parameters.entryIdSlug ? data.slug : data.cms_data_value_id}`;
profile.entryId = data.cms_data_value_id;
profile.description = data.description;
profile.gender = bio.gender?.toLowerCase();
profile.age = bio.age;
profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ });
profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ })
|| unprint.extractDate(bio.date_of_birth, 'yyyy-MM-dd');
profile.birthPlace = bio.birthplace;
profile.ethnicity = bio.ethnicity;
profile.measurements = bio.measurements;
profile.height = feetInchesToCm(bio.height);
@@ -190,7 +200,9 @@ function scrapeProfile(data, entity, parameters) {
profile.hairColor = bio.hair_color;
profile.eyeColor = bio.eye_color;
const avatar = bio.thumbnail?.image[0];
const avatar = bio.thumbnail?.image?.[0] // aziani
|| bio.biopic?.image?.[0] // cosplayground profile
|| bio.headshot?.image?.[0]; // cosplayground headshot
if (avatar) {
profile.avatar = {
@@ -201,36 +213,41 @@ function scrapeProfile(data, entity, parameters) {
};
}
profile.socials = [bio.onlyfans, bio.instagram].filter(Boolean);
return profile;
}
async function fetchProfile({ name, url }, { entity, parameters }) {
if (!url) {
// no easy search option
if (!url && !parameters.entryIdSlug) {
// no easy search option and numeric ID is non-derivable
return null;
}
const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
const actorId = url && new URL(url).pathname.match(/models?\/(\d+)/)?.[1];
if (!actorId) {
if (!actorId && !parameters.entryIdSlug) {
return null;
}
const blockId = await getBlockId('/model/:id', { models: name, id: actorId }, entity, parameters);
const slug = slugify(name);
const blockId = await getBlockId(`${parameters.modelPath || '/model'}/${parameters.entryIdSlug ? ':slug' : 'id'}`, { models: name, id: actorId, slug }, entity, parameters);
if (!blockId) {
return null;
}
const query = new URLSearchParams({
cms_data_value_ids: actorId,
cms_data_value_ids: actorId || '',
...parameters.entryIdSlug ? { slug } : null,
cms_block_id: blockId,
cms_data_type_id: 4,
}).toString();
const apiUrl = `https://azianistudios.com/tour_api.php/content/data-values?${query}`;
const apiUrl = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/data-values?${query}`;
const res = await unprint.get(apiUrl, {
interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
headers: {
Referer: entity.url,
'x-nats-cms-area-id': entity.parameters.areaId,
+35 -31
View File
@@ -1,6 +1,7 @@
'use strict';
const qu = require('../utils/qu');
const unprint = require('unprint');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -21,23 +22,23 @@ function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.entryId = query.q(null, 'data-video-card-scene-id') || query.q('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.url = query.url('a', { origin: channel.url });
release.entryId = query.attribute(null, 'data-video-card-scene-id') || query.attribute('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.video-card-title span') || query.q('.video-card-title', 'title');
release.description = query.cnt('.video-card-description');
release.title = query.content('.video-card-title span') || query.attribute('.video-card-title', 'title');
release.description = query.content('.video-card-description');
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
release.duration = query.duration('.video-card-duration', { attribute: 'content' }) || query.number('.video-card-duration') * 60;
release.actors = query.all('.video-card-details--cast a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
name: unprint.query.content(el),
url: unprint.query.url(el, null, { origin: channel.url }),
}));
release.tags = query.cnts('.video-card-tags a');
release.tags = query.contents('.video-card-tags a');
const posterSources = query.srcset('picture source', 'data-srcset') || [query.img('.video-card-image', 'data-src')];
const posterSources = query.sourceSet('picture source', 'data-srcset') || [query.img('.video-card-image', { attribute: 'data-src' })];
release.poster = getPoster(posterSources);
release.teaser = query.video('video') || (channel.parameters?.teasers !== false ? `https://p.badoinkvr.com/d/scene_files/${release.entryId}/preview/preview.mp4` : null);
@@ -46,29 +47,29 @@ function scrapeAll(scenes, channel) {
});
}
function scrapeScene({ query }, url, channel) {
function scrapeScene({ query }, { url, entity: channel }) {
const release = {};
release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.video-title', '.breadcrumbs__item:last-child [itemprop=name]');
release.description = query.cnt('.video-description');
release.title = query.content(['.video-title', '.breadcrumbs__item:last-child [itemprop=name]']);
release.description = query.content('.video-description');
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-upload-date', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.duration = query.duration('.video-duration', { attribute: 'content' }) || query.number('.video-duration') * 60;
release.actors = query.all('.video-actors a').map((el) => ({
name: qu.query.cnt(el),
url: qu.query.url(el, null, 'href', { origin: channel.url }),
name: unprint.query.content(el),
url: unprint.query.url(el, null, { origin: channel.url }),
}));
release.tags = query.cnts('.video-tags a');
release.tags = query.contents('.video-tags a');
const posterSources = query.srcset('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
const posterSources = query.sourceSet('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
release.poster = getPoster(posterSources);
const photos = query.imgs('.gallery .gallery-item', 'data-big-image');
const extraPhotoCount = query.number('.gallery .gallery-item:last-child', /\d+/, 'data-desktop-text');
const photos = query.imgs('.gallery .gallery-item', { attribute: 'data-big-image' });
const extraPhotoCount = query.number('.gallery .gallery-item:last-child', { match: /\d+/, attribute: 'data-desktop-text' });
const lastPhoto = photos.slice(-1)[0];
const photosHaveIndex = /_\d+\./.test(lastPhoto);
@@ -87,10 +88,10 @@ function scrapeProfile({ query }, url, entity) {
const bio = query.all('.girl-details-stats-item').reduce((acc, el) => ({
...acc,
[slugify(qu.query.cnt(el, '.girl-details-stat'))]: qu.query.cnt(el, '.girl-details-stat-value'),
[slugify(unprint.query.content(el, '.girl-details-stat'))]: unprint.query.content(el, '.girl-details-stat-value'),
}), {});
profile.description = query.cnt('.girl-details-bio');
profile.description = query.content('.girl-details-bio');
profile.age = bio.age;
profile.birthPlace = bio.country;
@@ -104,23 +105,26 @@ function scrapeProfile({ query }, url, entity) {
profile.hairColor = bio.hair;
profile.eyes = bio.eyes;
const avatarSources = query.srcset('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
const avatarSources = query.sourceSet('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
profile.avatar = getPoster(avatarSources).slice(0, 3); // returns a metric ton of links, if first few don't work the rest presumably won't either
profile.social = query.urls('.girl-details-social-media-list a');
profile.scenes = scrapeAll(qu.initAll(query.all('.video-card')), entity);
profile.scenes = scrapeAll(unprint.initAll(query.all('.video-card')), entity);
return profile;
}
async function fetchLatest(channel, page) {
const res = await qu.getAll(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, '.video-card', {
Cookie: 'affsubid=12345-;', // required to show teaser video, exact number doesn't seem to matter
const res = await unprint.get(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, {
selectAll: '.video-card',
cookies: {
affsubid: '12345-', // required to show teaser video, exact number doesn't seem to matter
},
});
if (res.ok) {
return scrapeAll(res.items, channel);
return scrapeAll(res.context, channel);
}
return res.status;
@@ -128,10 +132,10 @@ async function fetchLatest(channel, page) {
async function fetchProfile(baseActor, { entity }) {
const url = `${entity.url}/${entity.parameters?.actor || 'pornstar'}/${slugify(baseActor.name, '-')}/`;
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeProfile(res.item, url, entity);
return scrapeProfile(res.context, url, entity);
}
return res.status;
+17 -17
View File
@@ -1,29 +1,29 @@
'use strict';
const qu = require('../utils/q');
const unprint = require('unprint');
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.url = query.url('a', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/tour\/preview\/(.+)/)[1];
release.title = query.cnt('.nice-title');
release.title = query.content('.nice-title');
release.poster = query.img('img.poster');
release.photos = query.q('img.poster', 'data-urls')?.split(' ');
release.teaser = query.video('img.poster', 'data-thumbnail');
release.photos = query.attribute('img.poster', 'data-urls')?.split(' ');
release.teaser = query.video('img.poster', { attribute: 'data-thumbnail' });
if (query.exists('.info')) {
release.date = query.date('.updated', 'MM/DD/YYYY');
release.actors = [{
name: query.cnt('.name'),
age: query.cnt('.age'),
name: query.content('.name'),
age: query.content('.age'),
}];
release.tags = query.cnts('.tags a');
release.tags = query.contents('.tags a');
}
return release;
@@ -35,38 +35,38 @@ function scrapeScene({ query }, url) {
release.entryId = new URL(url).pathname.match(/\/tour\/preview\/(.+)/)[1];
release.title = query.cnt('.nice-title');
release.description = query.cnt('.desc-text p');
release.title = query.content('.nice-title');
release.description = query.content('.desc-text p');
release.tags = query.cnts('.tags a');
release.tags = query.contents('.tags a');
release.poster = query.poster();
release.photos = query.imgs('.thumbnails img, .thumb img');
// some videos may not have unique trailers, but many do
release.trailer = query.video();
release.trailer = query.video('source');
release.formats = query.cnts('.download-quality option');
release.formats = query.contents('.download-quality option');
return release;
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/tour/videos/page-${page}`;
const res = await qu.getAll(url, '.episode-list .episode, .content .girl');
const res = await unprint.get(url, { selectAll: '.episode-list .episode, .content .girl' });
if (res.ok) {
return scrapeAll(res.items, channel);
return scrapeAll(res.context, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, url, channel);
return scrapeScene(res.context, url, channel);
}
return res.status;
+2 -5
View File
@@ -1,7 +1,6 @@
'use strict';
const unprint = require('unprint');
const cookie = require('cookie');
const slugify = require('../utils/slugify');
@@ -54,16 +53,14 @@ function scrapeAll(scenes, channel) {
async function beforeFetchLatest(channel) {
// scene page only seems to accept language preferences from session
const { res } = await unprint.get(`${channel.url}/en/news-videos-x-marc-dorcel`, {
const { cookies } = await unprint.get(`${channel.url}/en/news-videos-x-marc-dorcel`, {
headers: {
'X-Requested-With': 'XMLHttpRequest',
'Accept-Language': 'en-US,en', // fetch English rather than French titles
},
});
const sessionCookie = cookie.parse(res.headers['set-cookie'][0])?.dorcelclub;
return `dorcelclub=${sessionCookie}`;
return `dorcelclub=${cookies.dorcelclub}`;
}
async function fetchLatest(channel, page = 1, _options, { beforeFetchLatest: sessionCookie }) {
+50 -51
View File
@@ -1,8 +1,8 @@
'use strict';
const format = require('template-format');
const unprint = require('unprint');
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -10,11 +10,11 @@ function deriveEntryId(release) {
if (release.date && release.url) {
const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
}
if (release.date && release.title) {
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
}
return null;
@@ -22,7 +22,7 @@ function deriveEntryId(release) {
function extractPoster(posterPath, channel, baseRelease) {
if (posterPath && !/400.jpg/.test(posterPath)) {
const poster = qu.prefixUrl(posterPath, channel.parameters?.media || channel.url);
const poster = unprint.prefixUrl(posterPath, channel.parameters?.media || channel.url);
const posterSources = [
poster,
// upscaled
@@ -40,20 +40,20 @@ function extractPoster(posterPath, channel, baseRelease) {
return [baseRelease?.poster || null, []];
}
function getImageWithFallbacks(q, selector, site, el) {
function getImageWithFallbacks(query, selector, site, el) {
const sources = el
? [
q(el, selector, 'src0_3x'),
q(el, selector, 'src0_2x'),
q(el, selector, 'src0_1x'),
unprint.query.attribute(el, selector, 'src0_3x'),
unprint.query.attribute(el, selector, 'src0_2x'),
unprint.query.attribute(el, selector, 'src0_1x'),
]
: [
q(selector, 'src0_3x'),
q(selector, 'src0_2x'),
q(selector, 'src0_1x'),
query.attribute(selector, 'src0_3x'),
query.attribute(selector, 'src0_2x'),
query.attribute(selector, 'src0_1x'),
];
return sources.filter(Boolean).map((src) => qu.prefixUrl(src, site.parameters?.media || site.url));
return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url));
}
function scrapeAllClassic(scenes, channel) {
@@ -61,16 +61,16 @@ function scrapeAllClassic(scenes, channel) {
const release = {};
release.url = query.url('.updateInfo h5 a:not([href*="content/"]):not([href*="#coming"])');
release.entryId = query.url('.updateThumb img', 'alt');
release.entryId = query.attribute('.updateThumb img', 'alt');
release.title = query.cnt('.updateInfo h5 a');
release.title = query.content('.updateInfo h5 a');
release.actors = query.cnts('.tour_update_models a');
release.actors = query.contents('.tour_update_models a');
release.date = query.date('.availdate, .updateInfo p span:last-child', 'MM/DD/YYYY');
release.poster = query.img('.updateThumb img');
release.poster = query.img('.updateThumb img', { origin: channel.url }); // relative to site root, not listing page
const trailer = query.q('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
const trailer = query.attribute('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
if (trailer) {
release.trailer = `${channel.url}${trailer}`;
@@ -84,13 +84,14 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
release.title = query.attribute('h4 a', 'title') || query.content('h4 a');
release.url = query.url('h4 a');
release.date = query.date('.more-info-div', 'MMM D, YYYY');
release.duration = query.dur('.more-info-div');
release.duration = query.duration('.more-info-div');
const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
// raw path, prefixed with media URL below
const posterPath = query.attribute('.img-div img', 'src0_1x') || query.attribute('img.video_placeholder', 'data-src') || query.attribute('img.video_placeholder', 'src', { forceGetAttribute: true });
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${channel.parameters?.media || channel.url}${posterPath}`;
@@ -102,7 +103,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
];
}
release.teaser = query.video();
release.teaser = query.video('source');
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
@@ -116,11 +117,11 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
});
}
function scrapeSceneClassic({ query, html }, url, channel) {
function scrapeSceneClassic({ query, html }, { entity: channel }) {
const release = {};
release.title = query.q('.updatesBlock h2', true);
release.poster = query.meta('property="og:image"');
release.title = query.content('.updatesBlock h2');
release.poster = query.attribute('meta[property="og:image"]', 'content');
release.entryId = release.poster.match(/\/content\/(.*)\//)?.[1];
const trailer = html.match(/src="(.+\.mp4)"/)?.[1];
@@ -134,34 +135,34 @@ function scrapeSceneClassic({ query, html }, url, channel) {
return release;
}
function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
function scrapeSceneTubular({ query, html }, { entity, url, baseRelease }) {
const release = {};
release.title = query.q('.video-player .section-title', true) || query.q('.bio-section-head .section-title', true);
release.title = query.content('.video-player .section-title') || query.content('.bio-section-head .section-title');
release.description = query.text('.update-info-block .descriptionFull') || query.text('.update-info-block .description');
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)');
release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
release.date = query.date('.update-info-row', 'MMM D, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.duration = query.duration('.update-info-row:nth-child(2)');
release.photoCount = query.number('.update-info-row:nth-child(2)', { match: /(\d+) photos/i, matchIndex: 1 });
release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.cnt(el, 'span'),
avatar: getImageWithFallbacks(query.q, 'img', entity, el),
url: query.url(el, null),
name: unprint.query.content(el, 'span'),
avatar: getImageWithFallbacks(query, 'img', entity, el),
url: unprint.query.url(el, null),
}));
release.tags = query.all('.tags a', true);
release.tags = query.contents('.tags a');
const posterPath = query.q('.player-thumb img', 'src0_1x');
const posterPath = query.attribute('.player-thumb img', 'src0_1x');
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
[release.poster, release.photos] = extractPoster(posterPath, entity, baseRelease);
if (trailer) {
release.trailer = { src: qu.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
release.trailer = { src: unprint.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
}
const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
const stars = query.content('.update-rating').match(/\d.\d/)?.[0];
if (stars) release.stars = Number(stars);
if (entity.type === 'network') {
@@ -181,11 +182,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
async function scrapeProfile({ query }, entity, parameters) {
const profile = {};
const bio = query.cnt('.model_bio, .detail-div');
const avatarEl = query.q('.model_bio_pic img, .model_bio_thumb');
const bio = query.content('.model_bio, .detail-div');
const avatarEl = query.element('.model_bio_pic img, .model_bio_thumb');
profile.age = Number(bio?.match(/Age:\s*(\d{2})/)?.[1]) || null;
profile.dateOfBirth = qu.parseDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
profile.dateOfBirth = unprint.extractDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
profile.height = convert(bio?.match(/\d+\s*(feet|')\s*\d+\s*(inches|"|$)/)?.[0], 'cm');
profile.measurements = bio?.match(/\w+[-x]\d+[-x]\d+/)?.[0] || null;
@@ -201,17 +202,17 @@ async function scrapeProfile({ query }, entity, parameters) {
avatarEl.getAttribute('src'),
]
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map((avatar) => qu.prefixUrl(avatar, entity.url));
.map((avatar) => unprint.prefixUrl(avatar, entity.url));
if (avatarSources.length) profile.avatar = avatarSources;
}
if (parameters?.layout === 'classic') {
profile.scenes = scrapeAllClassic(qu.initAll(query.all('.bodyArea .updateItem')), entity);
profile.scenes = scrapeAllClassic(unprint.initAll(query.all('.bodyArea .updateItem')), entity);
}
if (parameters?.layout === 'tubular') {
profile.scenes = scrapeAllTubular(qu.initAll(query.all('.modelfeature, .item-video')), entity);
profile.scenes = scrapeAllTubular(unprint.initAll(query.all('.modelfeature, .item-video')), entity);
}
return profile;
@@ -221,20 +222,20 @@ async function fetchLatest(site, page = 1, options, preData, allScraper) {
const url = (site.parameters?.latest && format(site.parameters.latest, { page }))
|| `${site.url}/categories/movies_${page}_d.html`;
const res = await qu.getAll(url, '.modelfeature, .item-video, .bodyArea .updateItem');
const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .bodyArea .updateItem' });
if (!res.ok) {
return res.status;
}
return allScraper(res.items, site, preData?.uniqueReleases);
return allScraper(res.context, site, preData?.uniqueReleases);
}
async function fetchUpcomingClassic(channel) {
const res = await qu.getAll(channel.url, '#owl-upcomingScenes .updateItem');
const res = await unprint.get(channel.url, { selectAll: '#owl-upcomingScenes .updateItem' });
if (res.ok) {
return scrapeAllClassic(res.items, channel);
return scrapeAllClassic(res.context, channel);
}
return res.status;
@@ -273,10 +274,10 @@ async function fetchProfile({ name: actorName, url }, { entity, parameters }) {
return null;
}
const res = await qu.get(profileUrl);
const res = await unprint.get(profileUrl);
if (res.statusCode === 200) {
return scrapeProfile(res.item, entity, parameters);
if (res.status === 200) {
return scrapeProfile(res.context, entity, parameters);
}
return null;
@@ -290,14 +291,12 @@ module.exports = {
fetchProfile,
scrapeAll: scrapeAllClassic,
scrapeScene: scrapeSceneClassic,
deprecated: true,
},
tubular: {
fetchLatest: fetchLatestTubular,
fetchProfile,
scrapeAll: scrapeAllTubular,
scrapeScene: scrapeSceneTubular,
deprecated: true,
},
getImageWithFallbacks,
};
+23 -1
View File
@@ -1,6 +1,7 @@
'use strict';
const unprint = require('unprint');
const { nanoid } = require('nanoid');
function scrapeAll(scenes, _channel) {
return scenes.map(({ query }) => {
@@ -40,6 +41,12 @@ async function fetchLatest(channel, page = 1) {
return res.status;
}
async function beforeFetchScenes(channel) {
const res = await unprint.get(channel.url);
return { nats: res.cookies.nats || nanoid() };
}
function scrapeScene({ query, html }, { url, entity, baseRelease }) {
const release = {};
@@ -81,6 +88,20 @@ function scrapeScene({ query, html }, { url, entity, baseRelease }) {
return release;
}
async function fetchScene(url, entity, baseRelease, context) {
const res = await unprint.get(url, {
cookies: {
nats: context.beforeFetchScenes?.nats,
},
});
if (res.ok) {
return scrapeScene(res.context, { url, entity, baseRelease });
}
return res.status;
}
function scrapeProfile({ query }) {
const profile = {};
@@ -110,5 +131,6 @@ async function fetchProfile(actor, entity) {
module.exports = {
fetchLatest,
fetchProfile,
scrapeScene,
fetchScene,
beforeFetchScenes,
};
+44 -39
View File
@@ -1,25 +1,25 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const unprint = require('unprint');
const slugify = require('../utils/slugify');
function scrapeLatest(scenes, site) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
release.url = query.url('figure a', { origin: site.parameters.latest });
release.title = query.cnt('.has-text-weight-bold, .is-size-6');
release.title = query.content('.has-text-weight-bold, .is-size-6');
release.date = query.date('span.tag', 'YYYY-MM-DD');
release.actors = query.cnts('a.tag');
release.actors = query.contents('a.tag');
const cover = query.img('.image img');
release.poster = cover.replace('poster_noplay', 'trailer_noplay');
release.covers = [cover];
release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
return release;
});
@@ -30,11 +30,11 @@ function scrapeLatestLegacy(scenes, site) {
// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
const release = {};
const titleEl = query.q('.galleryTitleText, .articleTitleText');
const titleEl = query.element('.galleryTitleText, .articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
const url = query.url(titleEl, 'a');
const url = unprint.query.attribute(titleEl, 'a', 'href');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
@@ -44,20 +44,20 @@ function scrapeLatestLegacy(scenes, site) {
} else {
// title should contain date instead, not applicable in brief mode
release.title = title.slice(title.indexOf(':') + 1).trim();
release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
}
release.actors = actors.map((actor) => actor.trim());
const description = query.q('.articleCopyText', true);
const description = query.content('.articleCopyText');
if (description) release.description = description.slice(0, description.lastIndexOf('('));
const duration = query.dur('.articleCopyText a:nth-child(2)');
const duration = query.duration('.articleCopyText a:nth-child(2)');
if (duration) release.duration = duration;
release.likes = parseInt(query.q('.articlePostDateText td:nth-child(3)', true), 10);
release.likes = query.number('.articlePostDateText td:nth-child(3)');
const cover = query.img('a img');
const cover = query.img('a img', { origin: site.url });
release.covers = [[
cover.replace('_thumbnail', ''),
@@ -68,25 +68,26 @@ function scrapeLatestLegacy(scenes, site) {
});
}
async function scrapeScene({ query }, url, channel, parameters, session) {
async function scrapeScene({ query }, channel, include, cookies) {
const release = {};
release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
release.title = query.content('.columns div.is-size-5.has-text-weight-bold');
release.description = query.content('.has-background-black-ter > div:nth-child(4)');
release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
release.actors = query.contents('.has-text-white-ter a.tag[href*="home.php"]');
release.tags = query.contents('.has-background-black-ter > div:nth-child(6) > span');
release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
release.poster = query.img('#videoPlayer, #iodvideo', { attribute: 'poster' });
release.photos = Array.from((query.html('body > div:nth-child(6)') || '').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
release.trailer = query.video();
release.trailer = query.video('source');
if (!release.trailer && parameters.includeTrailers) {
const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
if (!release.trailer && include.includeTrailers) {
// play API relies on the session cookie set by the scene page
const trailerRes = await unprint.get(`${channel.url}/api/play-api.php`, { cookies, extract: false });
if (trailerRes.ok) {
release.trailer = trailerRes.body;
@@ -96,22 +97,22 @@ async function scrapeScene({ query }, url, channel, parameters, session) {
return release;
}
function scrapeSceneLegacy({ query }, site) {
function scrapeSceneLegacy({ query }, { entity: site }) {
const release = {};
const titleEl = query.q('.articleTitleText');
const titleEl = query.element('.articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const url = query.url(titleEl, 'a');
const url = unprint.query.attribute(titleEl, 'a', 'href');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
release.title = title.trim();
release.description = query.q('.articleCopyText', true);
release.description = query.content('.articleCopyText');
release.actors = actors.map((actor) => actor.trim());
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
release.duration = query.dur('.articlePostDateText a:nth-child(2)');
release.duration = query.duration('.articlePostDateText a:nth-child(2)');
const [cover, ...photos] = query.imgs('img[src*="images"]');
release.covers = [cover];
@@ -119,7 +120,7 @@ function scrapeSceneLegacy({ query }, site) {
release.poster = query.poster();
const trailer = query.trailer();
const trailer = query.video('source');
if (trailer) release.trailer = { src: trailer };
return release;
@@ -127,10 +128,13 @@ function scrapeSceneLegacy({ query }, site) {
async function fetchLatest(site, page = 1) {
const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
const res = await unprint.get(url, {
selectAll: 'body > .columns .column',
cookies: { consent: 'yes' },
});
if (res.ok) {
return scrapeLatest(res.items, site);
return scrapeLatest(res.context, site);
}
return res.status;
@@ -138,21 +142,22 @@ async function fetchLatest(site, page = 1) {
async function fetchLatestLegacy(site, page = 1) {
const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
const res = await qu.get(url); // JSON containing html as a property
const res = await unprint.get(url); // JSON containing html as a property
if (res.ok) {
return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
if (res.ok && res.data?.html) {
return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
}
return res.status;
}
async function fetchScene(url, site, baseRelease, parameters) {
const session = http.session();
const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
async function fetchScene(url, site, baseRelease, include) {
const res = await unprint.get(url, {
cookies: { consent: 'yes' },
});
if (res.ok) {
return scrapeScene(res.item, url, site, parameters, session);
return scrapeScene(res.context, site, include, { consent: 'yes', ...res.cookies });
}
return res.status;
+25 -22
View File
@@ -1,11 +1,14 @@
'use strict';
const { get, initAll } = require('../utils/qu');
const unprint = require('unprint');
// paths are relative to /visitors/, read raw attributes instead of resolved URLs
const raw = { forceGetAttribute: true };
function scrapeLatest(scenes, dates, site) {
return scenes.map(({ qu }, index) => {
return scenes.map(({ query }, index) => {
const release = {};
const path = qu.url('a[href*="videos/"]');
const path = query.attribute('a[href*="videos/"]', 'href', raw);
if (path) {
if (/\.wmv$/.test(path)) {
@@ -16,30 +19,30 @@ function scrapeLatest(scenes, dates, site) {
}
if (dates && dates[index]) {
release.date = dates[index].qu.date(null, 'MM/DD/YYYY');
release.date = dates[index].query.date(null, 'MM/DD/YYYY');
}
// release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
|| qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1];
|| query.attribute('img[src*="graphics/fft"]', 'src', raw)?.match(/fft_(\w+).gif/)?.[1];
release.description = qu.q('tbody tr:nth-child(3) font', true);
release.description = query.content('tbody tr:nth-child(3) font');
const infoLine = qu.q('font[color="#663366"]', true);
const infoLine = query.content('font[color="#663366"]');
if (infoLine) {
release.duration = Number(infoLine.match(/(\d+) min/i)?.[1] || infoLine.match(/video: (\d+)/i)?.[1]) * 60 || null;
}
const poster = qu.img('img[src*="photos/"][width="400"]');
const poster = query.attribute('img[src*="photos/"][width="400"]', 'src', raw);
release.poster = `${site.url}/visitors/${poster}`;
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
release.photos = query.attributes('img[src*="photos/"]:not([width="400"])', 'src', raw).map((source) => `${site.url}/visitors/${source}`);
return release;
});
}
function scrapeScene({ qu }, url, site) {
function scrapeScene({ query }, url, site) {
const release = { url };
const { pathname } = new URL(url);
@@ -52,47 +55,47 @@ function scrapeScene({ qu }, url, site) {
return release;
}
const actor = qu.q('font[color="#990033"] strong', true);
const actor = query.content('font[color="#990033"] strong');
release.actors = [actor];
const hdTrailer = qu.url('a[href*="hd_trailer.mp4"]');
const sdTrailer = qu.url('a[href*="hd_trailer_mobile.mp4"]');
const hdTrailer = query.attribute('a[href*="hd_trailer.mp4"]', 'href', raw);
const sdTrailer = query.attribute('a[href*="hd_trailer_mobile.mp4"]', 'href', raw);
release.trailer = [
{
hdTrailer && {
src: `${site.url}/visitors/videos/${hdTrailer}`,
quality: 1080,
},
{
sdTrailer && {
src: `${site.url}/visitors/videos/${sdTrailer}`,
quality: 270,
},
];
].filter(Boolean);
return release;
}
async function fetchLatest(site, page = 1) {
const url = `https://jesseloadsmonsterfacials.com/visitors/tour_${page.toString().padStart(2, '0')}.html`;
const res = await get(url);
const res = await unprint.get(url);
if (!res.ok) {
return res.status;
}
const { el } = res.item;
const { element } = res.context;
const scenes = initAll(el, 'table[width="880"]');
const dates = initAll(el, 'font[color="#000000"] strong:not(:empty)');
const scenes = unprint.initAll(element, 'table[width="880"]');
const dates = unprint.initAll(element, 'font[color="#000000"] strong:not(:empty)');
return scrapeLatest(scenes, dates, site);
}
async function fetchScene(url, site) {
const res = await get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, url, site);
return scrapeScene(res.context, url, site);
}
return res.status;
+15 -15
View File
@@ -2,7 +2,7 @@
const blake2 = require('blake2');
const qu = require('../utils/qu');
const unprint = require('unprint');
function getHash(identifier) {
const hash = blake2.createHash('blake2b', { digestLength: 8 });
@@ -45,8 +45,8 @@ async function scrapeLatest(scenes, site) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.q('a', 'title');
release.url = query.url('a', 'href', { origin: site.url });
release.title = query.attribute('a', 'title');
release.url = query.url('a', { origin: site.url });
release.date = query.date('.nm-date', 'MM/DD/YYYY');
const slug = new URL(release.url).pathname.split('/')[2];
@@ -56,7 +56,7 @@ async function scrapeLatest(scenes, site) {
[release.poster, ...release.photos] = query.imgs('.bloc-link img');
release.tags = query.cnts('.dropdown ul a').slice(1);
release.tags = query.contents('.dropdown ul a').slice(1);
release.duration = query.duration('.dropdown p:first-child');
return release;
@@ -66,22 +66,22 @@ async function scrapeLatest(scenes, site) {
async function scrapeScene({ query }, site, url) {
const release = { url, site };
release.title = query.cnt('#movie-header h2');
release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.title = query.content('#movie-header h2');
release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
release.description = query.cnt('.container .mg-md');
release.description = query.content('.container .mg-md');
release.duration = query.duration('#video-ribbon .container > div > span:nth-child(3)');
release.actors = query.cnts('#video-info a').concat(extractMaleModelsFromTags(query.q('.tag-container')));
release.tags = query.cnts('.tag-container a');
release.actors = query.contents('#video-info a').concat(extractMaleModelsFromTags(query.element('.tag-container')));
release.tags = query.contents('.tag-container a');
const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
const uhd = query.content('#video-ribbon .container > div > span:nth-child(2)');
if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
release.poster = query.poster();
const trailer = query.trailer();
const trailer = query.video('source');
if (trailer) release.trailer = { src: trailer };
if (release.photos.length > 0) release.channel = await extractChannelFromPhoto(release.photos[0], site);
@@ -99,20 +99,20 @@ async function scrapeScene({ query }, site, url) {
async function fetchLatest(site, page = 1) {
const url = `${site.url}/movies/page-${page}`;
const res = await qu.getAll(url, '#content-main [class^="item"]');
const res = await unprint.get(url, { selectAll: '#content-main [class^="item"]' });
if (res.ok) {
return scrapeLatest(res.items, site);
return scrapeLatest(res.context, site);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeScene(res.item, channel, url);
return scrapeScene(res.context, channel, url);
}
return res.status;
+46 -39
View File
@@ -1,24 +1,42 @@
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const unprint = require('unprint');
function parseJson(res) {
// admin-ajax responds with JSON labeled as text/html
if (typeof res.data === 'object') {
return res.data;
}
try {
return JSON.parse(res.body);
} catch (_error) {
return null;
}
}
async function getTrailer(entryId, channel) {
const videoLinksRes = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
const videoLinksRes = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
action: 'vidplay',
vid: entryId,
type: 'promo',
}, {
encodeJSON: false,
decodeJSON: true,
form: true,
});
if (!videoLinksRes.ok || !videoLinksRes.body.size1) {
const videoLinks = videoLinksRes.ok && parseJson(videoLinksRes);
if (!videoLinks?.size1) {
return null;
}
const scriptUrl = qu.extract(videoLinksRes.body.size1)?.query.url('script', 'src', { origin: channel.url });
const videoScriptRes = await http.get(scriptUrl);
const scriptUrl = unprint.init(videoLinks.size1, null, { origin: channel.url })?.query.url('script', { attribute: 'src' });
if (!scriptUrl) {
return null;
}
const videoScriptRes = await unprint.get(scriptUrl, { extract: false });
if (!videoScriptRes.ok) {
return null;
@@ -28,13 +46,11 @@ async function getTrailer(entryId, channel) {
const videoData = videoDataString && JSON.parse(videoDataString);
const posterUrlString = videoScriptRes.body.match(/src="(.*poster.*?)"/)?.[1];
const posterUrlEl = videoScriptRes.document.createElement('span');
posterUrlEl.innerHTML = posterUrlString;
const poster = posterUrlString && unprint.init(`<span>${posterUrlString}</span>`)?.query.content('span'); // decode HTML entities
return {
trailer: videoData?.url || null,
poster: posterUrlEl.textContent || null,
poster: poster || null,
};
}
@@ -48,7 +64,7 @@ function scrapeAll(scenes) {
release.title = scene.title;
release.description = scene.description;
release.date = qu.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || qu.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
release.date = unprint.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || unprint.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
release.poster = scene.poster[0];
release.photos = Object.entries(scene).reduce((acc, [key, data]) => {
@@ -63,20 +79,20 @@ function scrapeAll(scenes) {
});
}
async function scrapeScene({ query }, channel, baseScene) {
async function scrapeScene({ query }, { entity, baseRelease }) {
const release = {};
release.title = query.cnt('.title');
release.description = query.cnt('.description');
release.title = query.content('.title');
release.description = query.content('.description');
release.entryId = query.q('body', 'class').match(/postid-(\d+)/)?.[1];
release.entryId = query.attribute('body', 'class')?.match(/postid-(\d+)/)?.[1];
const { poster, trailer } = await getTrailer(release.entryId, channel) || {};
const { poster, trailer } = await getTrailer(release.entryId, entity) || {};
release.poster = baseScene?.poster || poster;
release.photos = poster && baseScene?.poster
? [poster, ...baseScene.photos]
: baseScene?.photos;
release.poster = baseRelease?.poster || poster;
release.photos = poster && baseRelease?.poster
? [poster, ...baseRelease.photos]
: baseRelease?.photos;
release.trailer = trailer;
@@ -84,33 +100,24 @@ async function scrapeScene({ query }, channel, baseScene) {
}
async function fetchLatest(channel, page) {
const res = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
const res = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
action: 'vls',
limit: 10,
offset: (page - 1) * 10,
}, {
encodeJSON: false,
decodeJSON: true,
form: true,
});
if (res.ok) {
return scrapeAll(res.body.listings);
const data = res.ok && parseJson(res);
if (data?.listings) {
return scrapeAll(data.listings);
}
return res.status;
}
async function fetchScene(url, channel, baseScene) {
const res = await qu.get(url);
if (res.ok) {
return scrapeScene(res.item, channel, baseScene);
}
return res.status;
return res.ok ? null : res.status;
}
module.exports = {
fetchLatest,
fetchScene,
scrapeScene,
};
+10 -11
View File
@@ -3,7 +3,6 @@
const unprint = require('unprint');
const mime = require('mime');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -132,10 +131,10 @@ async function fetchLatest(channel, page, { parameters }) {
? `${channel.parent.url}/_next/data/${endpoint}/sites/${parameters.site}.json?sitename=${parameters.site}&order_by=publish_date&sort_by=desc&per_page=30&page=${page}`
: `${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}.json?order_by=publish_date&sort_by=desc&per_page=30&page=${page}`;
const res = await http.get(url);
const res = await unprint.get(url);
if (res.ok && res.body.pageProps?.contents?.data) {
return scrapeAll(res.body.pageProps.contents.data, channel, parameters);
if (res.ok && res.data.pageProps?.contents?.data) {
return scrapeAll(res.data.pageProps.contents.data, channel, parameters);
}
return res.status;
@@ -170,12 +169,12 @@ async function fetchScene(url, channel, baseScene, { parameters }) {
const slug = new URL(url).pathname.split('/').at(-1);
const endpoint = await fetchEndpoint(channel);
const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
const res = await unprint.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
if (res.ok && res.body.pageProps?.content) {
if (res.ok && res.data.pageProps?.content) {
return scrapeScene({
...res.body.pageProps.content,
serie: res.body.pageProps.playlist?.data[0],
...res.data.pageProps.content,
serie: res.data.pageProps.playlist?.data[0],
}, channel, parameters);
}
@@ -215,10 +214,10 @@ function scrapeProfile(data, channel, scenes, parameters) {
async function fetchProfile(actor, { channel, parameters }) {
const endpoint = await fetchEndpoint(channel);
const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
const res = await unprint.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
if (res.ok && res.body.pageProps?.model) {
return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);
if (res.ok && res.data.pageProps?.model) {
return scrapeProfile(res.data.pageProps.model, channel, res.data.pageProps.model_contents, parameters);
}
return res.status;
+4 -2
View File
@@ -39,7 +39,6 @@ const karups = require('./karups');
const kellymadison = require('./kellymadison');
const kink = require('./kink');
const mariskax = require('./mariskax');
// const analvids = require('./analvids');
const pornbox = require('./pornbox');
const littlecapricedreams = require('./littlecapricedreams');
const loveherfilms = require('./loveherfilms');
@@ -62,6 +61,7 @@ const pornworld = require('./pornworld');
const privateNetwork = require('./private'); // reserved keyword
const radical = require('./radical');
const rickysroom = require('./rickysroom');
const rsghmedia = require('./rsghmedia');
const sexlikereal = require('./sexlikereal');
const score = require('./score');
const sicflics = require('./sicflics');
@@ -73,7 +73,6 @@ const teenmegaworld = require('./teenmegaworld');
const testedefudelidade = require('./testedefudelidade');
const theflourish = require('./theflourish');
const tokyohot = require('./tokyohot');
// const topwebmodels = require('./topwebmodels');
const traxxx = require('./traxxx');
const virtualtaboo = require('./virtualtaboo');
const vixen = require('./vixen');
@@ -102,6 +101,7 @@ module.exports = {
acam,
analvids: pornbox,
pornbox,
cosplayground: aziani,
kellymadison,
kink,
'8kmembers': kellymadison,
@@ -143,6 +143,7 @@ module.exports = {
interracialpass: hush,
inthecrack,
jerkaoke: modelmedia,
jenniferwhite: adultempire,
jesseloadsmonsterfacials,
julesjordan,
karups,
@@ -170,6 +171,7 @@ module.exports = {
private: privateNetwork,
radical,
rickysroom,
rsghmedia,
score,
sexlikereal,
sicflics,
+46 -44
View File
@@ -1,31 +1,36 @@
'use strict';
const qu = require('../utils/q');
const unprint = require('unprint');
const { lbsToKg, feetInchesToCm } = require('../utils/convert');
function scrapeScene(data, channel) {
const release = {};
release.entryId = data.id;
release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url);
release.url = unprint.prefixUrl(`/videos/${data.slug}`, channel.url);
release.title = data.title;
release.description = data.description;
release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
release.duration = qu.durationToSeconds(data.videos_duration);
release.date = unprint.extractDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
release.duration = unprint.extractDuration(data.videos_duration);
release.actors = data?.models_thumbs.map((model) => ({
name: model.name,
url: qu.prefixUrl(`/models/${model.slug}`, channel.url),
url: unprint.prefixUrl(`/models/${model.slug}`, channel.url),
avatar: model.thumb,
})) || data.models;
release.tags = data.tags;
release.poster = [data.trailer_screencap].concat(data.extra_thumbnails);
release.photos = data.previews?.full
.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
.concat(data.thumbs); // screenshots
// inconsistent data structure. if there is an album, it's previews: { full: [...] }, if it's empty, it's previews: []
if (Array.isArray(data.previews?.full)) {
release.photos = data.previews?.full
.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
.concat(data.thumbs); // screenshots
}
release.trailer = data.trailer_url;
release.teaser = data.special_thumbnails;
@@ -36,39 +41,14 @@ function scrapeScene(data, channel) {
return release;
}
function scrapeProfile(data, scenes, entity) {
const profile = {};
profile.entryId = data.id;
profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
profile.description = data.Bio || data.bio;
profile.birthPlace = data.Born || data.born;
profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
profile.measurements = data.Measurements || data.Measurements;
profile.height = feetInchesToCm(data.Height || data.height);
profile.weight = lbsToKg(data.Weight || data.weight);
profile.eyes = data.Eyes || data.eyes;
profile.hairColor = data.Hair || data.hair;
profile.avatar = data.thumb;
profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
return profile;
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.contents?.data) {
if (data?.props?.pageProps?.contents?.data) {
const scenes = data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel));
return scenes;
@@ -86,13 +66,12 @@ async function fetchScene(url, channel, baseRelease) {
return baseRelease;
}
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.content) {
if (data?.props?.pageProps?.content) {
return scrapeScene(data.props.pageProps.content, channel);
}
@@ -102,15 +81,38 @@ async function fetchScene(url, channel, baseRelease) {
return res.status;
}
function scrapeProfile(data, scenes, entity) {
const profile = {};
profile.entryId = data.id;
profile.url = unprint.prefixUrl(`/models/${data.slug}`, entity.url);
profile.description = data.Bio || data.bio;
profile.birthPlace = data.Born || data.born;
profile.dateOfBirth = unprint.extractDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
profile.measurements = data.Measurements || data.Measurements;
profile.height = feetInchesToCm(data.Height || data.height);
profile.weight = lbsToKg(data.Weight || data.weight);
profile.eyes = data.Eyes || data.eyes;
profile.hairColor = data.Hair || data.hair;
profile.avatar = data.thumb;
profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
return profile;
}
async function fetchProfile({ slug }, entity) {
const url = `${entity.url}/models/${slug}`;
const res = await qu.get(url);
const res = await unprint.get(url);
if (res.ok) {
const dataString = res.item.query.html('#__NEXT_DATA__');
const data = dataString && JSON.parse(dataString);
const data = res.context.query.json('#__NEXT_DATA__');
if (data.props?.pageProps?.model) {
if (data?.props?.pageProps?.model) {
return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity);
}
+158
View File
@@ -0,0 +1,158 @@
'use strict';
const unprint = require('unprint');
const escapeRegexp = require('escape-string-regexp');
const slugify = require('../utils/slugify');
const capitalize = require('../utils/capitalize');
function getDate(query, release) {
const date = query.date('.published', 'MMMM D, YYYY') || query.date('meta[itemprop="uploadDate"]', 'YYYY-MM-DD', { attribute: 'content' });
if (date) {
return {
date,
datePrecision: 'day',
};
}
const poster = Array.isArray(release.poster)
? release.poster[0]
: release.poster;
if (poster) {
const [, year, month] = new URL(poster).pathname.match(/\/uploads\/(\d{4})\/(\d{2})/) || [];
if (year && month) {
return {
date: new Date(Date.UTC(year, month - 1, 1)),
datePrecision: 'month',
};
}
}
return {};
}
function scrapeAll(scenes) {
return scenes.map(({ query, element }) => {
const release = {};
release.url = query.url('.entry-title a, a.entry-featured-image-url');
release.entryId = new URL(release.url).pathname.match(/\/([\w-]+)/)[1];
const postId = unprint.query.number(element, null, { attribute: 'id' });
if (postId) {
release.attributes = { postId };
}
release.title = query.content('.entry-title a');
// description is cut off
release.tags = query.contents('.post-meta a[href*="/category"], .post-meta a[href*="/tag"]'); // tag only seen on scene page, but just in case
release.poster = Array.from(new Set(query.sourceSet('.entry-featured-image-url img')));
const { date, datePrecision } = getDate(query, release);
release.date = date;
release.datePrecision = datePrecision;
return release;
}).toSorted((sceneA, sceneB) => {
// HTML is laid out in columns that does not align with chronology
if (sceneA.attributes?.postId && sceneB.attributes?.postId) {
return sceneB.attributes.postId - sceneA.attributes.postId;
}
if (sceneA.date && sceneB.date) {
return sceneB.date - sceneA.date;
}
return 0;
});
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.origin}/page/${page}`;
const res = await unprint.get(url, { selectAll: '.et_pb_blog_grid .et_pb_post' });
if (res.ok) {
return scrapeAll(res.context, channel);
}
return res.status;
}
function scrapeScene({ query }, { url }) {
const release = {};
release.entryId = new URL(url).pathname.match(/\/([\w-]+)/)[1];
const postId = query.number('article[id*="post"]', { attribute: 'id' });
if (postId) {
release.attributes = { postId };
}
release.title = query.content('.entry-title');
release.description = query.content('.entry-content p');
release.actors = query.all('.st-post-tags a[href*="/tag"]').map((actorEl) => ({
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null),
}));
release.poster = query.img('meta[property="og:image"], meta[property="twitter:image"]', { attribute: 'content' })
|| query.sourceSet('.et_post_meta_wrapper img');
release.trailer = query.video('.entry-content source');
const { date, datePrecision } = getDate(query, release);
release.date = date;
release.datePrecision = datePrecision;
const tags = query.all('.post-meta a[href*="/category"], .post-meta a[href*="/tag"]');
const starring = query.content('//div[contains(@class, "entry-content")]/p[contains(text(), "Starring")]')
|| query.content('//div[contains(@class, "entry-content")]/p[contains(text(), "***")]')
|| release.title?.split(/\s[-–]\s/).at(-1);
if (tags.length > 0) {
release.tags = tags.map((tagEl) => unprint.query.content(tagEl));
} else {
release.tags = query.content('article[id*="post"]', { attribute: 'class' })
?.match(/(category|tag)-[\w-]+/g)
?.map((tag) => slugify(tag.replace(/(category|tag)-/, ''), ' '))
|| [];
}
if (release.actors.length === 0 && release.tags.length > 0 && starring) {
// actors are always added as tags, sometimes they're clearly divided, sometimes they're mixed in with other tags
// the tail end of the description or title contains a *starring* section against which the tags can be matched
release.actors = release.tags
.map((tag) => {
if (tag === 'alex adams') {
// household male performer that is not usually mentioned in the starring section
return capitalize(tag);
}
// use description text instead of tag name for capitalization
const match = starring.match(new RegExp(`\\b${escapeRegexp(tag)}\\b`, 'i'))?.[0];
if (match) {
return match;
}
return null;
})
.filter(Boolean);
}
return release;
}
module.exports = {
fetchLatest,
scrapeScene,
};
+21 -25
View File
@@ -2,7 +2,6 @@
const unprint = require('unprint');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
@@ -34,7 +33,8 @@ async function fetchTrailerUrl(videoId, entity) {
const token = res.data.token;
if (token) {
return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.mpd?parentOrigin=${entity.origin}`;
// ffmpeg fails to read the DASH manifest over a single connection, HLS works
return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.m3u8?parentOrigin=${entity.origin}`;
}
}
@@ -47,17 +47,18 @@ async function scrapeScene(scene, channel, _parameters, includeTrailers) {
// release.entryId = scene.id; // legacy
release.entryId = scene.itemId;
release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${release.entryId}`;
release.url = `${channel.origin}/movies/${scene.id || release.entryId}`; // numeric item ID URL 404s, id is the slug
release.title = scene.title;
release.description = scene.description;
release.date = unprint.extractDate(scene.publishedDate, 'YYYY-MM-DD');
release.duration = scene.d;
release.actors = scene.models?.map((model) => ({
name: model.modelName || model.name || model.title,
avatar: model.img,
gender: model.gender,
url: `${channel.url}/models/${model.modelId || model.id}`,
url: `${channel.origin}/models/${model.modelId || model.id}`,
}));
if (scene.img) {
@@ -93,16 +94,16 @@ function scrapeAll(scenes, channel, parameters) {
async function fetchLatest(channel, page = 1, { parameters }) {
// url: 'https://www.pervz.com/series/pervz-features',
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([a-z-]+)/)?.[1];
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([\w-]+)/)?.[1];
const url = seriesId
? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
: `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
const res = await http.get(url);
const res = await unprint.get(url);
if (res.ok) {
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
if (res.ok && res.data?.hits) {
return scrapeAll(res.data.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
}
return res.status;
@@ -114,27 +115,22 @@ async function fetchScene(url, channel, baseScene, { parameters, includeTrailers
return baseScene;
}
const sceneSlug = new URL(url).pathname.match(/\/([\w-]+$)/)[1];
const sceneSlug = new URL(url).pathname.match(/\/([\w-]+)\/?$/)?.[1];
const res = await unprint.get(url, {
parser: {
runScripts: 'dangerously',
},
});
if (!sceneSlug) {
return null;
}
if (res.ok) {
const videos = res.context.window.__INITIAL_STATE__?.content?.videosContent;
// scene page no longer embeds state, look up the scene in the elastic API by its slug instead
// legacy URLs use the numeric item ID, which 404s on the site but can still be looked up
const isItemId = /^\d+$/.test(sceneSlug);
const res = await unprint.get(`https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(${isItemId ? `itemId:${sceneSlug}` : `id:"${sceneSlug}"`} AND type:video)&size=1`);
res.context.window.fetch = () => {}; // suppress fetch missing error
if (res.ok && res.data?.hits) {
const scene = res.data.hits.hits.find(({ _source: hit }) => (isItemId ? String(hit.itemId) : hit.id) === sceneSlug)?._source;
if (!videos) {
return null;
}
const video = videos?.[sceneSlug] || Object.values(videos)[0];
if (video) {
return scrapeScene(video, channel, parameters, includeTrailers);
if (scene) {
return scrapeScene(scene, channel, parameters, includeTrailers);
}
return null;
+1 -1
View File
@@ -31,7 +31,7 @@ function scrapeAll(scenes) {
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/${page}`;
const url = `${channel.origin}/${page}`;
const res = await unprint.get(url, { selectAll: '.scene' });
if (res.ok) {
-144
View File
@@ -1,144 +0,0 @@
'use strict';
const mime = require('mime');
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
function scrapeSceneX(scene) {
const release = {};
release.entryId = scene.id;
release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title, '-', { removePunctuation: true })}`;
release.title = scene.title;
release.description = scene.description;
release.duration = qu.durationToSeconds(scene.videos_duration);
release.date = new Date(scene.release_date);
release.actors = scene.models
.map((actor) => (/&/.test(actor.name)
? actor.name.split(/\s*&\s*/)
: {
name: actor.name,
gender: actor.gender || null,
avatar: actor.thumb,
url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
}))
.flat();
release.stars = scene.rating;
release.tags = scene.tags.map((tag) => tag.name);
if (mime.getType(scene.thumb) === 'image/gif') {
release.teaser = scene.thumb;
} else {
release.poster = scene.thumb;
}
release.channel = slugify(scene.sites[0]?.name, '');
return release;
}
function scrapeAll(scenes) {
return scenes.map(scrapeSceneX);
}
async function scrapeProfile(actor, options) {
const profile = {};
profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch
profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch
profile.gender = actor.attributes.gender || null;
profile.height = convert(actor.attributes.height.value, 'cm');
profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg');
const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || [];
profile.bust = Number(bust);
profile.cup = cup;
profile.waist = Number(waist);
profile.hip = Number(hip);
profile.ethnicity = actor.attributes.ethnicity.value;
profile.birthPlace = actor.attributes.born.value;
profile.eyes = actor.attributes.eyes.value;
profile.hairColor = actor.attributes.hair.value.split('/')[0];
profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`;
profile.avatar = actor.thumb;
if (options.includeActorScenes) {
const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } });
if (res.ok) {
profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items);
}
}
return profile;
}
async function fetchLatest(channel, page) {
const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
headers: {
Referer: 'https://tour.topwebmodels.com',
'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey,
'x-Requested-With': 'XMLHttpRequest',
},
});
if (res.ok) {
return scrapeAll(res.body.videos.items);
}
return res.status;
}
async function fetchScene(url) {
const res = await http.get(url, { extract: { runScripts: 'dangerously' } });
if (res.ok) {
return {
...scrapeSceneX(res.window.__DATA__.data.video),
...(/\.gif/.test(res.window.__DATA__.data.video.thumb) && { teaser: res.window.__DATA__.data.video.thumb }),
poster: res.window.__DATA__.data.file_poster,
};
}
return res.status;
}
async function fetchProfile(baseActor, entity, options) {
const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, {
headers: {
Referer: 'https://tour.topwebmodels.com',
'api-key': entity.parameters?.apiKey,
'x-Requested-With': 'XMLHttpRequest',
},
});
if (!searchRes.ok) {
return searchRes.status;
}
const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
if (actor) {
return scrapeProfile(actor, options);
}
return null;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
};
+3 -1
View File
@@ -74,7 +74,9 @@ function scrapeAll(scenes, channel) {
async function fetchLatest(site, page = 1) {
const url = `${site.url}/videos?page=${page}`;
const res = await unprint.get(url);
const res = await unprint.get(url, {
interface: 'request',
});
if (res.ok) {
const data = res.context.query.json('#__NEXT_DATA__');
+1 -2
View File
@@ -12,7 +12,6 @@ const slugify = require('./utils/slugify');
const bulkInsert = require('./utils/bulk-insert');
const chunk = require('./utils/chunk');
const redis = require('./redis');
const { formatDate } = require('./utils/qu');
const { associateActors, associateDirectors, scrapeActors, toBaseActors } = require('./actors');
const { associateReleaseTags } = require('./tags');
const { curateEntity } = require('./entities');
@@ -32,7 +31,7 @@ const commonContext = {
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
const slugBase = release.title
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
|| (release.date && `${release.entity.slug} ${unprint.formatDate(release.date, 'YYYY MM DD')}`)
|| null;
const slug = slugify(slugBase, '-', {
+5 -4
View File
@@ -79,13 +79,14 @@ function withRelations(queryBuilder, withMedia) {
}
async function matchTags(rawTags) {
const tags = rawTags
.map((tag) => tag?.trim().match(/[a-z0-9()-]+/ig)?.join(' ').toLowerCase())
.filter(Boolean);
const tags = Array.from(new Set(rawTags
.map((tag) => (typeof tag === 'string' ? tag : null)?.trim().match(/[a-z0-9()-]+/ig)?.join(' ').toLowerCase())
.filter(Boolean)));
// array binding, large batches can easily exceed the bindings limit otherwise
const tagEntries = await knex('tags')
.select('tags.id', 'tags.name', 'tags.alias_for', 'tags.implied_tag_ids')
.whereIn(knex.raw('lower(tags.name)'), tags);
.whereRaw('lower(tags.name) = ANY(:tags)', { tags });
const tagIdsBySlug = tagEntries
.reduce((acc, tag) => ({
-76
View File
@@ -1,76 +0,0 @@
'use strict';
const fs = require('fs').promises;
const Promise = require('bluebird');
const qu = require('../utils/qu');
const qualities = {
sm: 360,
med: 480,
big: 720,
};
/*
async function scrape() {
const urlsByPage = await Promise.map(Array.from({ length: 140 }), async (value, index) => {
const res = await qu.get(`https://www.dogfartnetwork.com/tour/scenes/?p=${index + 1}`);
if (res.ok) {
return res.item.query.urls('.recent-updates > a', 'href', { origin: 'https://www.dogfartnetwork.com' });
}
return [];
}, { concurrency: 1 });
const urls = urlsByPage.flat();
await fs.writeFile('./dogfart-links', urls.join('\n'));
console.log(`Saved ${urls.length} URLs to file`);
}
async function compare() {
const newLinksFile = await fs.readFile('./dogfart-links', 'utf8');
const oldLinksFile = await fs.readFile('./dogfart-got', 'utf8');
const newLinks = newLinksFile.split('\n').filter(Boolean);
const oldLinks = new Set(oldLinksFile.split('\n').filter(Boolean));
const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
console.log(getLinks);
}
*/
async function scrapeMembers() {
const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
if (res.ok) {
return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
actors: query.contents('a[href*="model.php"]'),
trailer: query.urls('.trailer-link a').map((url) => ({
src: url,
quality: qualities[url.match(/_([a-z]+).mp4/)[1]],
})),
}));
}
return [];
}, { concurrency: 1 });
const urls = titlesByPage.flat().map((data) => JSON.stringify(data));
console.log(urls);
await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
console.log(`Saved ${urls.length} URLs to file`);
}
scrapeMembers();
-110
View File
@@ -1,110 +0,0 @@
'use strict';
const config = require('config');
const fs = require('fs');
const fsPromises = require('fs').promises;
const Promise = require('bluebird');
const blake2 = require('blake2');
const sharp = require('sharp');
const { nanoid } = require('nanoid');
const { PassThrough } = require('stream');
const http = require('./http');
function getMemoryUsage() {
return process.memoryUsage().rss / (10 ** 6);
}
let peakMemoryUsage = getMemoryUsage();
async function fetchSource(link) {
const id = nanoid();
const hasher = new blake2.Hash('blake2b');
hasher.setEncoding('hex');
const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
const tempFileStream = fs.createWriteStream(tempFilePath);
const hashStream = new PassThrough();
hashStream.on('data', (chunk) => hasher.write(chunk));
try {
const res = await http.get(link, null, {
stream: true,
transforms: [hashStream],
destination: tempFileStream,
timeout: 5000,
});
if (!res.ok) {
throw new Error(res.status);
}
hasher.end();
const hash = hasher.read();
const memoryUsage = getMemoryUsage();
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
return {
id,
path: tempFilePath,
hash,
};
} catch (error) {
await fsPromises.unlink(tempFilePath);
throw error;
}
}
async function init() {
const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
const links = linksFile.split('\n').filter(Boolean);
await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
console.time('thumbs');
const files = await Promise.map(links, async (link) => {
try {
return await fetchSource(link);
} catch (error) {
console.log(`Failed to fetch ${link}: ${error.message}`);
return null;
}
});
await Promise.map(files.filter(Boolean), async (file) => {
const image = sharp(file.path).jpeg();
const [{ width, height }, { size }] = await Promise.all([
image.metadata(),
fsPromises.stat(file.path),
]);
await Promise.all([
image
.toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
image
.resize({
height: config.media.thumbnailSize,
withoutEnlargement: true,
})
.toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
]);
const memoryUsage = getMemoryUsage();
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
}, { concurrency: 10 });
console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
console.timeEnd('thumbs');
}
init();
-5
View File
@@ -1,5 +0,0 @@
'use strict';
const qu = require('./qu');
module.exports = qu;
-682
View File
@@ -1,682 +0,0 @@
'use strict';
const { JSDOM } = require('jsdom');
const moment = require('moment');
const http = require('./http');
const virtualConsole = require('./virtual-console')(__filename);
const { window: globalWindow } = new JSDOM('', { virtualConsole });
function trim(str) {
if (typeof str !== 'string') {
return str;
}
return str.trim().replace(/\s+/g, ' ');
}
function extractDate(dateString, format, match) {
if (!dateString) {
return null;
}
if (match) {
const dateStamp = trim(dateString).match(match);
if (dateStamp) {
const dateValue = moment.utc(dateStamp[0], format);
return dateValue.isValid() ? dateValue.toDate() : null;
}
return null;
}
const dateValue = moment.utc(trim(dateString), format);
return dateValue.isValid() ? dateValue.toDate() : null;
}
function formatDate(dateValue, format, inputFormat) {
if (inputFormat) {
return moment(dateValue, inputFormat).format(format);
}
return moment(dateValue).format(format);
}
function durationToSeconds(durationString, match) {
const durationMatch = durationString.match(match || /(\d+:)?\d+:\d+/);
if (durationMatch) {
const segments = ['00'].concat(durationMatch[0].split(/[:hm]/)).slice(-3);
return moment.duration(segments.join(':')).asSeconds();
}
return null;
}
function prefixUrl(urlValue, origin, protocol = 'https') {
if (!urlValue) {
return null;
}
if (/^http/.test(urlValue)) {
return urlValue;
}
if (protocol && /^\/\//.test(urlValue)) {
return `${protocol}:${urlValue}`;
}
if (origin && /^\//.test(urlValue)) {
return `${origin}${urlValue}`;
}
if (origin && /^\.\//.test(urlValue)) {
return `${origin}${urlValue.slice(1)}`;
}
if (origin) {
return `${origin}/${urlValue}`;
}
return urlValue;
}
function iterateXPathResult(iterator, results = []) {
const element = iterator.iterateNext();
if (element) {
return iterateXPathResult(iterator, [...results, element]);
}
return results;
}
function getElements(context, selector, first = false) {
if (!selector) {
return context;
}
if (/^\/\//.test(selector)) {
// XPath selector
const iterator = globalWindow.document.evaluate(selector, context, null, globalWindow.XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
if (first) {
return iterator.iterateNext();
}
return iterateXPathResult(iterator);
}
if (first) {
return context.querySelector(selector);
}
return Array.from(context.querySelectorAll(selector));
}
function q(context, selectors, attrArg, applyTrim = true) {
if (!selectors && context.nodeName === '#document') {
return null;
}
const attr = attrArg === true ? 'textContent' : attrArg;
const element = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, true), null);
if (!element) {
return null;
}
if (attr) {
const value = element[attr] || element.getAttribute(attr);
return applyTrim && typeof value === 'string' ? trim(value) : value;
}
return element;
}
function all(context, selectors, attrArg, applyTrim = true) {
const attr = attrArg === true ? 'textContent' : attrArg;
const elements = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector), null);
if (!Array.isArray(elements)) {
return [];
}
if (attr) {
return elements.map((el) => q(el, null, attr, applyTrim));
}
return elements;
}
function exists(context, selector) {
return !!q(context, selector);
}
function count(context, selector) {
return all(context, selector)?.length || 0;
}
function content(context, selector, applyTrim = true) {
return q(context, selector, 'textContent', applyTrim);
}
function contents(context, selector, applyTrim) {
return all(context, selector, 'textContent', applyTrim);
}
function html(context, selector) {
const el = q(context, selector, null, true);
return el && el.innerHTML;
}
function htmls(context, selector) {
const els = all(context, selector, null, true);
return els.map((el) => el.innerHTML);
}
function execute(context, selector = 'script') {
const scripts = htmls(context, selector);
const originalGlobal = Object.fromEntries(Object.entries(global));
const errors = scripts?.reduce((accErrors, script) => {
try {
Function(script)(); /* eslint-disable-line no-new-func */
return accErrors;
} catch (error) {
// the script failed
return [...accErrors, error];
}
}, []);
const data = Object.fromEntries(Object.entries(global).filter(([key, value]) => {
if (originalGlobal[key] !== value) {
delete global[key];
return true;
}
return false;
}));
return {
...data,
errors,
};
}
function json(context, selector) {
const el = q(context, selector, null, true);
try {
return JSON.parse(el?.innerHTML);
} catch (error) {
return null;
}
}
function jsons(context, selector) {
const els = all(context, selector, null, true);
return els.map((el) => {
try {
return JSON.parse(el?.innerHTML);
} catch (error) {
return null;
}
});
}
function texts(context, selector, applyTrim = true, filter = true) {
const el = q(context, selector, null, applyTrim);
if (!el) return null;
const nodes = Array.from(el.childNodes)
.filter((node) => node.nodeName === '#text')
.map((node) => (applyTrim ? trim(node.textContent) : node.textContent));
return filter ? nodes.filter(Boolean) : nodes;
}
function text(context, selector, applyTrim = true) {
const nodes = texts(context, selector, applyTrim, true);
if (!nodes) return null;
const textValue = nodes.join(' ');
return applyTrim ? trim(textValue) : textValue;
}
function removeStyleFunctionSpaces(el) {
// jsdom appears to have a bug where it ignores inline CSS attributes set to a function() containing spaces, e.g. url( image.png )
el.setAttribute('style', el.getAttribute('style').replace(/\(\s+(.*)\s+\)/g, (match, cssArgs) => `(${cssArgs})`));
}
function style(context, selector, styleAttr) {
const el = q(context, selector);
if (el?.hasAttribute('style')) {
removeStyleFunctionSpaces(el);
return styleAttr ? el.style[styleAttr] : el.style;
}
return null;
}
function styles(context, selector, styleAttr) {
const elStyles = Array.from(context.querySelectorAll(selector), (el) => {
removeStyleFunctionSpaces(el);
return styleAttr ? el.style[styleAttr] : el.style;
});
return elStyles;
}
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) {
const value = q(context, selector, attr);
if (value && match) {
return Number(value.match(match)?.[matchIndex]);
}
if (value) {
return Number(value);
}
return null;
}
function meta(context, selector, attrArg = 'content', applyTrim = true) {
if (/meta\[.*\]/.test(selector)) {
return q(context, selector, attrArg, applyTrim);
}
return q(context, `meta[${selector}]`, attrArg, applyTrim);
}
function date(context, selector, format, match, attr = 'textContent') {
const dateString = q(context, selector, attr, true);
if (!dateString) return null;
return extractDate(dateString, format, match);
}
function dateAgo(context, selector, match = /(\d+)\s*(\w+)/, attr = 'textContent') {
const timeString = q(context, selector, attr, 'textContent');
if (!timeString) {
return null;
}
const timeMatch = timeString.match(match);
if (timeMatch) {
const [n, period] = timeMatch.slice(1);
const thenDate = moment.utc().subtract(Number(n), period);
return {
date: thenDate.toDate(),
precision: period.replace(/s$/, ''),
};
}
return null;
}
function image(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
const imageEl = (attr && q(context, selector, attr))
|| q(context, selector, 'data-src')
|| q(context, selector, 'src');
return prefixUrl(imageEl, origin, protocol);
}
function images(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
const attribute = attr
|| (q(context, selector, 'data-src') && 'data-src')
|| (q(context, selector, 'src') && 'src');
const imageEls = all(context, selector, attribute);
return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol));
}
function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
const urlEl = q(context, selector, attr);
const prefixedUrl = prefixUrl(urlEl, origin, protocol);
if (prefixedUrl && object) {
return new URL(prefixedUrl);
}
return prefixedUrl;
}
function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
const urlEls = all(context, selector, attr);
return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls;
}
function sourceSet(context, selector, attr = 'srcset', options = {}) {
const srcset = q(context, selector, attr);
if (!srcset) {
return null;
}
const sources = srcset
.split(/\s*,\s*/)
.map((source) => {
const [link, descriptor] = source.split(' ');
if (link) {
return {
descriptor: descriptor || 'fallback',
url: prefixUrl(link, options.origin, options.protocol),
};
}
return null;
})
.filter(Boolean)
.sort((sourceA, sourceB) => {
if (sourceB.descriptor === 'fallback' || parseInt(sourceA.descriptor, 10) > parseInt(sourceB.descriptor, 10)) {
return -1;
}
if (parseInt(sourceA.descriptor, 10) < parseInt(sourceB.descriptor, 10)) {
return 1;
}
return 0;
});
if (options.includeDescriptor) {
return sources;
}
return sources.map((source) => source.url);
}
function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
const posterEl = q(context, selector, attr);
return attr ? prefixUrl(posterEl, origin, protocol) : posterEl;
}
function video(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
const trailerEl = q(context, selector, attr);
return attr ? prefixUrl(trailerEl, origin, protocol) : trailerEl;
}
function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
const trailerEls = all(context, selector, attr);
return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
}
function duration(context, selector, match, attr = 'textContent') {
const durationString = q(context, selector, attr);
if (!durationString) {
return null;
}
const durationMatch = durationToSeconds(durationString, match);
if (durationMatch) {
return durationMatch;
}
const timestampMatch = durationString.match(/(\d+H)?\s*(\d+M)?\s*\d+S?/i);
if (timestampMatch) {
const hours = timestampMatch[0].match(/(\d+)H/i)?.[1] || 0;
const minutes = timestampMatch[0].match(/(\d+)M/i)?.[1] || 0;
const seconds = timestampMatch[0].match(/(\d+)(S|$)/i)?.[1] || 0;
return (Number(hours) * 3600) + (Number(minutes) * 60) + Number(seconds);
}
return null;
}
const legacyFuncs = {
q,
qa: all,
qall: all,
qd: date,
qdate: date,
qh: html,
qhtml: html,
qi: image,
qimage: image,
qimages: images,
qis: images,
ql: duration,
qlength: duration,
qm: meta,
qmeta: meta,
qp: poster,
qposter: poster,
qs: all,
qt: video,
qtext: text,
qtexts: texts,
qtrailer: video,
qtrailers: videos,
qts: videos,
qtx: text,
qtxs: texts,
qtxt: text,
qtxts: texts,
// qu: url,
qurl: url,
qurls: urls,
qus: urls,
};
const quFuncs = {
all,
cnt: content,
cnts: contents,
content,
contents,
count,
date,
dateAgo,
dur: duration,
duration,
el: q,
element: q,
execute,
exec: execute,
exists,
html,
htmls,
image,
images,
img: image,
imgs: images,
json,
jsons,
length: duration,
meta,
num: number,
number,
poster,
q,
sourceSet,
sources: sourceSet,
srcs: sourceSet,
srcset: sourceSet,
style,
styles,
text,
texts,
trailer: video,
url,
urls,
video,
videos,
};
function init(context, selector, window) {
if (!context) {
return null;
}
const element = selector ? context.querySelector(selector) : context;
if (!element) {
return null;
}
const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context
.reduce((acc, [key, func]) => ({
...acc,
[key]: (...args) => (args[0] instanceof globalWindow.HTMLElement // allow for different context
? func(...args)
: func(element, ...args)),
}), {});
const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
.reduce((acc, [key, func]) => ({
...acc,
[key]: (...args) => (args[0]?.nodeType === undefined // allow for different context
? func(element, ...args)
: func(...args)),
}), {});
return {
element,
el: element,
html: element.outerHTML || element.body.outerHTML,
text: trim(element.textContent),
...(window && {
window,
document: window.document,
}),
...legacyContextFuncs,
qu: quContextFuncs,
query: quContextFuncs,
};
}
function initAll(context, selector, window) {
if (Array.isArray(context)) {
return context.map((element) => init(element, null, window));
}
return Array.from(context.querySelectorAll(selector))
.map((element) => init(element, null, window));
}
function extract(htmlValue, selector, options) {
const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
return init(window.document, selector, window);
}
function extractAll(htmlValue, selector, options) {
const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
return initAll(window.document, selector, window);
}
async function request(method = 'get', urlValue, body, selector, headers, options, queryAll = false) {
const res = await (method === 'post'
? http.post(urlValue, body, { ...options, headers })
: http[method](urlValue, {
...options,
headers,
parse: true,
}));
if (res.ok) {
const item = queryAll
? initAll(res.document, selector, res.window)
: init(res.document, selector, res.window);
return {
item,
items: all ? item : [item],
res,
ok: true,
status: res.statusCode,
...res,
};
}
return {
item: null,
items: [],
res,
ok: false,
status: res.statusCode,
...res,
};
}
async function get(urlValue, selector, headers, options) {
return request('get', urlValue, null, selector, headers, options, false);
}
async function post(urlValue, body, selector, headers, options) {
return request('post', urlValue, body, selector, headers, options, false);
}
async function getAll(urlValue, selector, headers, options) {
return request('get', urlValue, null, selector, headers, options, true);
}
async function postAll(urlValue, body, selector, headers, options) {
return request('post', urlValue, body, selector, headers, options, true);
}
function session(headers, options) {
return http.session(headers, options);
}
module.exports = {
extractDate,
extract,
extractAll,
durationToSeconds,
init,
initAll,
formatDate,
get,
getAll,
http,
fetch: get,
fetchAll: getAll,
context: init,
contextAll: initAll,
ed: extractDate,
ex: extract,
exa: extractAll,
fd: formatDate,
parseDate: extractDate,
ctx: init,
ctxa: initAll,
geta: getAll,
qu: quFuncs,
query: quFuncs,
post,
postAll,
prefixUrl,
session,
...legacyFuncs,
};
-20
View File
@@ -1,20 +0,0 @@
'use strict';
const http = require('./http');
const sleep = 5000;
const timeout = 1000;
async function init() {
try {
const res = await http.get(`https://httpstat.us/200?sleep=${sleep}`, {
timeout,
});
console.log(res.statusCode);
} catch (error) {
console.log(error);
}
}
init();
+1
View File
@@ -266,6 +266,7 @@ const actors = [
{ entity: 'wakeupnfuck', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] },
{ entity: 'darkkotv', name: 'Aidra Fox', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'measurements', 'naturalBoobs', 'hasTattoos', 'hasPiercings'] },
{ entity: 'jaxslayher', name: 'Eliza Ibarra', fields: ['avatar', 'description', 'eyes', 'height', 'weight', 'birthPlace', 'measurements', 'hairColor'] },
{ entity: 'cosplayground', name: 'Anna Claire Clouds', url: 'https://cosplayground.com/models/anna-claire-clouds?models=Anna%20Claire%20Clouds', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'height', 'measurements', 'hairColor', 'ethnicity'] },
];
const actorScrapers = scrapers.actors;